@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/runtime/routes/lifecycle-routes.ts

@@ -0,0 +1,199 @@
+/**
+ * Lifecycle drain endpoints backing `vellum sleep --wait`.
+ *
+ * The quiesce lease pauses the STARTING of new background LLM work (heartbeat
+ * beats, schedule/watcher/sequence claims, memory-job claims) across the
+ * daemon and its worker processes; drain-status reports what is still in
+ * flight so a client can wait for the assistant to go idle before stopping
+ * it. Enqueue paths stay open — work queued during a drain runs after the
+ * next start. The lease is TTL-bound and refreshed by the waiting client, so
+ * an abandoned drain can never leave background work paused.
+ */
+
+import { z } from "zod";
+
+import { listRunningHeartbeatRuns } from "../../heartbeat/heartbeat-run-store.js";
+import { listProcessingConversations } from "../../persistence/conversation-crud.js";
+import { listRunningMemoryJobs } from "../../persistence/jobs-store.js";
+import {
+  clearLifecycleQuiesce,
+  DEFAULT_QUIESCE_TTL_MS,
+  getLifecycleQuiesceUntil,
+  setLifecycleQuiesce,
+} from "../../persistence/lifecycle-quiesce.js";
+import { listRunningScheduleRuns } from "../../schedule/schedule-store.js";
+import { getLogger } from "../../util/logger.js";
+import { listRuns as listWorkflowRuns } from "../../workflows/journal-store.js";
+import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
+import { BadRequestError } from "./errors.js";
+import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
+
+const log = getLogger("lifecycle-routes");
+
+const quiesceResponseSchema = z.object({
+  quiescedUntil: z
+    .number()
+    .describe("Epoch ms when the quiesce lease expires."),
+});
+
+const activeConversationSchema = z.object({
+  conversationId: z.string(),
+  title: z.string().nullable(),
+  originChannel: z.string().nullable(),
+  originInterface: z.string().nullable(),
+  processingStartedAt: z.number(),
+});
+
+const runningMemoryJobSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  startedAt: z.number().nullable(),
+});
+
+const runningScheduleRunSchema = z.object({
+  runId: z.string(),
+  scheduleName: z.string().nullable(),
+  startedAt: z.number(),
+});
+
+const runningWorkflowRunSchema = z.object({
+  runId: z.string(),
+  name: z.string().nullable(),
+  startedAt: z.number().nullable(),
+});
+
+const runningHeartbeatRunSchema = z.object({
+  runId: z.string(),
+  startedAt: z.number(),
+});
+
+const drainStatusResponseSchema = z.object({
+  quiescedUntil: z.number().nullable(),
+  idle: z
+    .boolean()
+    .describe(
+      "True when no conversation turn, memory job, schedule run, workflow run, or heartbeat run is in flight.",
+    ),
+  activeConversations: z.array(activeConversationSchema),
+  memoryJobs: z.array(runningMemoryJobSchema),
+  scheduleRuns: z.array(runningScheduleRunSchema),
+  workflowRuns: z.array(runningWorkflowRunSchema),
+  heartbeatRuns: z.array(runningHeartbeatRunSchema),
+});
+
+/**
+ * Age cap for `running` heartbeat rows in the drain snapshot. A beat orphaned
+ * by a crash keeps its `running` status until a later boot's stale sweep, so
+ * an unbounded read would let a phantom row hold the drain open indefinitely.
+ * Real beats finish well inside this bound (the runner enforces its own
+ * timeout).
+ */
+const HEARTBEAT_RUN_MAX_AGE_MS = 15 * 60_000;
+
+export const ROUTES: RouteDefinition[] = [
+  {
+    operationId: "quiesceLifecycle",
+    endpoint: "lifecycle/quiesce",
+    method: "POST",
+    policy: {
+      requiredScopes: ["settings.write"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    summary: "Arm or refresh the drain quiesce lease",
+    description:
+      "Pauses the starting of new background work (heartbeat, schedules, watchers, sequences, memory jobs) for the lease TTL. Refresh by calling again; the lease self-expires so an abandoned drain never leaves background work paused.",
+    tags: ["system"],
+    requestBody: z.object({
+      ttlMs: z
+        .number()
+        .optional()
+        .describe("Lease TTL in ms (clamped to a sane range)."),
+    }),
+    responseBody: quiesceResponseSchema,
+    handler: (args: RouteHandlerArgs) => {
+      const raw = args.body?.ttlMs;
+      let ttlMs = DEFAULT_QUIESCE_TTL_MS;
+      if (raw !== undefined) {
+        if (typeof raw !== "number" || !Number.isFinite(raw)) {
+          throw new BadRequestError("ttlMs must be a finite number");
+        }
+        ttlMs = raw;
+      }
+      const quiescedUntil = setLifecycleQuiesce(ttlMs);
+      log.info({ quiescedUntil }, "Lifecycle quiesce lease armed");
+      return { quiescedUntil };
+    },
+  },
+  {
+    operationId: "releaseLifecycleQuiesce",
+    endpoint: "lifecycle/quiesce",
+    method: "DELETE",
+    policy: {
+      requiredScopes: ["settings.write"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    summary: "Release the drain quiesce lease",
+    description:
+      "Resumes background work immediately instead of waiting for the lease TTL to expire (e.g. when a drain is cancelled).",
+    tags: ["system"],
+    responseBody: z.object({ released: z.boolean() }),
+    handler: () => {
+      clearLifecycleQuiesce();
+      log.info("Lifecycle quiesce lease released");
+      return { released: true };
+    },
+  },
+  {
+    operationId: "getLifecycleDrainStatus",
+    endpoint: "lifecycle/drain-status",
+    method: "GET",
+    policy: {
+      requiredScopes: ["settings.read"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    summary: "In-flight background work",
+    description:
+      "Reports conversation turns, memory jobs, schedule runs, workflow runs, and heartbeat runs currently in flight, plus the active quiesce lease. `idle: true` means it is safe to stop the assistant without interrupting work.",
+    tags: ["system"],
+    responseBody: drainStatusResponseSchema,
+    handler: () => {
+      // Every read THROWS on failure (surfacing as a 5xx the client retries):
+      // for a drain decision, a failed read must not pass for proof of
+      // absence, or the client stops the assistant over unobservable
+      // in-flight work. Only the quiesce GATES are fail-open — this snapshot
+      // fails closed.
+      const activeConversations = listProcessingConversations();
+      const memoryJobs = listRunningMemoryJobs();
+      const scheduleRuns = listRunningScheduleRuns();
+      // Workflows launched by a schedule outlive their schedule run — the
+      // scheduler fires them and completes its own run row immediately — so
+      // a running workflow must hold the drain open in its own right.
+      const workflowRuns = listWorkflowRuns({
+        limit: 20,
+        status: "running",
+      }).map((run) => ({
+        runId: run.id,
+        name: run.name,
+        startedAt: run.createdAt,
+      }));
+      // A beat between startHeartbeatRun() and its background conversation
+      // existing (e.g. mid credential-health check) has a running row but no
+      // processing conversation yet.
+      const heartbeatRuns = listRunningHeartbeatRuns(HEARTBEAT_RUN_MAX_AGE_MS);
+      return {
+        quiescedUntil: getLifecycleQuiesceUntil(),
+        idle:
+          activeConversations.length === 0 &&
+          memoryJobs.length === 0 &&
+          scheduleRuns.length === 0 &&
+          workflowRuns.length === 0 &&
+          heartbeatRuns.length === 0,
+        activeConversations,
+        memoryJobs,
+        scheduleRuns,
+        workflowRuns,
+        heartbeatRuns,
+      };
+    },
+  },
+];

package/src/runtime/routes/platform-routes.ts

@@ -30,8 +30,7 @@ import {
   getSecureKeyAsync,
 } from "../../security/secure-keys.js";
 import { getExistingDeviceId } from "../../util/device-id.js";
-import { buildAssistantEvent } from "../assistant-event.js";
-import { assistantEventHub } from "../assistant-event-hub.js";
+import { broadcastMessage } from "../assistant-event-hub.js";
 import { ACTOR_PRINCIPALS, LOCAL_PRINCIPALS } from "../auth/route-policy.js";
 import {
   BadRequestError,
@@ -193,9 +192,7 @@ async function handlePlatformConnect(
   }
 
   // Emit signal for connected clients to show the platform login UI
-  await assistantEventHub.publish(
-    buildAssistantEvent({ type: "show_platform_login" }),
-  );
+  broadcastMessage({ type: "show_platform_login" });
 
   return { showPlatformLogin: true };
 }
@@ -259,9 +256,7 @@ async function handlePlatformDisconnect(
   }
 
   // Notify connected clients
-  await assistantEventHub.publish(
-    buildAssistantEvent({ type: "platform_disconnected" }),
-  );
+  broadcastMessage({ type: "platform_disconnected" });
 
   return {
     disconnected: true,

package/src/runtime/routes/playground/__tests__/inject-failures.test.ts

@@ -3,8 +3,8 @@
  */
 import { describe, expect, mock, test } from "bun:test";
 
+import type { AssistantEvent } from "../../../../api/index.js";
 import type { Conversation } from "../../../../daemon/conversation.js";
-import type { ServerMessage } from "../../../../daemon/message-protocol.js";
 
 let _mockConversation: MockConversation | undefined;
 
@@ -18,8 +18,8 @@ mock.module("../../../../context/token-estimator.js", () => ({
 
 mock.module("../helpers.js", () => ({
   getConversationById: async (id: string) => {
-    if (!_mockConversation) return undefined;
-    if (_mockConversation.conversationId !== id) return undefined;
+    if (!_mockConversation) {return undefined;}
+    if (_mockConversation.conversationId !== id) {return undefined;}
     return _mockConversation as unknown as Conversation;
   },
   listConversationsByTitlePrefix: () => [],
@@ -41,13 +41,13 @@ interface MockConversation {
   agentLoop: { compactionCircuit: MockCompactionCircuit };
   contextCompactedMessageCount: number;
   contextCompactedAt: number | null;
-  sentMessages: ServerMessage[];
-  sendToClient: (msg: ServerMessage) => void;
+  sentMessages: AssistantEvent[];
+  sendToClient: (msg: AssistantEvent) => void;
   getMessages: () => unknown[];
 }
 
 function makeConversation(id = "conv-playground-test"): MockConversation {
-  const sentMessages: ServerMessage[] = [];
+  const sentMessages: AssistantEvent[] = [];
   return {
     conversationId: id,
     agentLoop: {
@@ -68,7 +68,7 @@ function findRoute() {
   const route = ROUTES.find(
     (r) => r.operationId === "playgroundInjectCompactionFailures",
   );
-  if (!route) throw new Error("inject-failures route not registered");
+  if (!route) {throw new Error("inject-failures route not registered");}
   return route;
 }
 

package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts

@@ -19,8 +19,8 @@ mock.module("../helpers.js", () => ({
   addPlaygroundMessage: async () => ({ id: "msg-test" }),
 }));
 
+import type { AssistantEvent } from "../../../../api/index.js";
 import type { Conversation } from "../../../../daemon/conversation.js";
-import type { ServerMessage } from "../../../../daemon/message-protocol.js";
 import { RouteError } from "../../errors.js";
 import { ROUTES } from "../index.js";
 
@@ -33,7 +33,7 @@ interface FakeConversationState {
 
 interface FakeConversation {
   conversation: Conversation;
-  sent: ServerMessage[];
+  sent: AssistantEvent[];
   state: FakeConversationState;
 }
 
@@ -47,7 +47,7 @@ function makeFakeConversation(
     contextCompactedAt: null,
     ...overrides,
   };
-  const sent: ServerMessage[] = [];
+  const sent: AssistantEvent[] = [];
   const compactionCircuit = {
     get consecutiveCompactionFailures(): number {
       return state.consecutiveCompactionFailures;
@@ -72,7 +72,7 @@ function makeFakeConversation(
       return state.contextCompactedAt;
     },
     getMessages: () => [],
-    sendToClient: (msg: ServerMessage) => {
+    sendToClient: (msg: AssistantEvent) => {
       sent.push(msg);
     },
   } as unknown as Conversation;
@@ -84,7 +84,7 @@ function findRoute() {
   const route = ROUTES.find(
     (r) => r.operationId === "playgroundResetCompactionCircuit",
   );
-  if (!route) throw new Error("reset-circuit route not registered");
+  if (!route) {throw new Error("reset-circuit route not registered");}
   return route;
 }
 

package/src/runtime/routes/question-routes.ts

@@ -22,15 +22,16 @@
 import { z } from "zod";
 
 import {
-  buildBatchEntries,
-  type QuestionBatchMetadata,
   type QuestionBatchSubmission,
   QuestionBatchValidationError,
-  type QuestionPromptResult,
 } from "../../permissions/question-prompter.js";
 import { getLogger } from "../../util/logger.js";
 import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
 import * as pendingInteractions from "../pending-interactions.js";
+import {
+  readBatchMetadata,
+  resolvePendingQuestion,
+} from "../question-resolution.js";
 import { BadRequestError, NotFoundError } from "./errors.js";
 import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
 
@@ -111,35 +112,20 @@ function handleQuestionResponse({ body }: RouteHandlerArgs) {
   const response: QuestionResponseBody = parsed.data;
   const { requestId } = response;
 
-  const interaction = pendingInteractions.get(requestId);
-  if (!interaction || interaction.kind !== "question") {
-    log.warn(
-      { requestId, foundKind: interaction?.kind },
-      "Question response for unknown or wrong-kind requestId",
-    );
-    throw new NotFoundError(
-      "No pending question interaction found for this requestId",
-    );
-  }
-
-  // Build + validate the result BEFORE touching `pendingInteractions`, so a
-  // bad payload leaves the pending interaction (and its timer) intact and the
-  // user gets another chance to submit a correct batch.
-  let result: QuestionPromptResult;
+  // The legacy single-question shim needs the interaction's stashed metadata to
+  // synthesize a one-element batch; peek (don't consume) before resolving.
+  let input;
   try {
-    if (response.kind === "close") {
-      const { orderedIds } = readBatchMetadata(interaction);
-      result = {
-        entries: orderedIds.map((id) => ({
-          questionId: id,
-          decision: "skipped" as const,
-        })),
-        overall: "closed",
-      };
-    } else {
-      const submissions = buildSubmissions(response, interaction);
-      result = buildCompletedResult(submissions, interaction);
-    }
+    input =
+      response.kind === "close"
+        ? ({ kind: "close" } as const)
+        : ({
+            kind: "submit",
+            submissions: buildSubmissions(
+              response,
+              pendingInteractions.get(requestId),
+            ),
+          } as const);
   } catch (err) {
     if (err instanceof QuestionBatchValidationError) {
       throw new BadRequestError(err.message);
@@ -147,34 +133,36 @@ function handleQuestionResponse({ body }: RouteHandlerArgs) {
     throw err;
   }
 
-  // Validation passed — deregister now to clear the prompter timer, then
-  // hand the result to the prompter's caller via rpcResolve.
-  pendingInteractions.resolve(
-    requestId,
-    response.kind === "close" ? "cancelled" : "answered",
-  );
+  const outcome = resolvePendingQuestion(requestId, input);
+
+  if (outcome.status === "not_found") {
+    log.warn(
+      { requestId },
+      "Question response for unknown or wrong-kind requestId",
+    );
+    throw new NotFoundError(
+      "No pending question interaction found for this requestId",
+    );
+  }
+  if (outcome.status === "invalid") {
+    throw new BadRequestError(outcome.message);
+  }
 
   log.info(
     {
       requestId,
-      overall: result.overall,
-      conversationId: interaction.conversationId,
+      overall: outcome.result.overall,
+      conversationId: outcome.conversationId,
     },
     "Question resolved",
   );
 
-  (
-    interaction.rpcResolve as
-      | ((value: QuestionPromptResult) => void)
-      | undefined
-  )?.(result);
-
   return { success: true };
 }
 
 /**
  * Normalize the incoming body to a `QuestionBatchSubmission[]` for the
- * submit/legacy paths. Returns `null` for the `close` path (no submissions).
+ * submit/legacy paths.
  */
 function buildSubmissions(
   body: SubmitBody | LegacyOptionBody | LegacyFreeTextBody,
@@ -203,47 +191,6 @@ function buildSubmissions(
   return [{ questionId, kind: "free_text", text: body.text }];
 }
 
-/**
- * Build a `completed` QuestionPromptResult from a batched submission and the
- * per-question metadata the prompter stashed on the interaction. Delegates
- * the validation + ordering loop to {@link buildBatchEntries} so the
- * prompter and the route share a single implementation.
- */
-function buildCompletedResult(
-  submissions: QuestionBatchSubmission[],
-  interaction: ReturnType<typeof pendingInteractions.get>,
-): QuestionPromptResult {
-  const { orderedIds, optionsById } = readBatchMetadata(interaction);
-  if (orderedIds.length === 0) {
-    throw new QuestionBatchValidationError(
-      "No registered question ids for this batch",
-    );
-  }
-  const entries = buildBatchEntries(
-    orderedIds,
-    (qid, oid) => (optionsById[qid] ?? []).includes(oid),
-    new Set(Object.keys(optionsById)),
-    submissions,
-  );
-  return { entries, overall: "completed" };
-}
-
-/**
- * Pull the prompter-stashed batch bookkeeping off a pending interaction.
- * Returns empty defaults if the metadata is absent.
- */
-function readBatchMetadata(
-  interaction: ReturnType<typeof pendingInteractions.get>,
-): QuestionBatchMetadata {
-  const meta = interaction?.metadata as
-    | Partial<QuestionBatchMetadata>
-    | undefined;
-  return {
-    orderedIds: meta?.orderedIds ?? [],
-    optionsById: meta?.optionsById ?? {},
-  };
-}
-
 // ---------------------------------------------------------------------------
 // Route definitions
 // ---------------------------------------------------------------------------

package/src/runtime/routes/secret-routes.ts

@@ -32,13 +32,14 @@ import {
 } from "../../daemon/credential-transcript-scrub.js";
 import { syncManualTokenConnection } from "../../oauth/manual-token-connection.js";
 import { clearEmbeddingBackendCache } from "../../persistence/embeddings/embedding-backend.js";
-import { maybeReseedCapabilitiesAfterManagedCredential } from "../../plugins/defaults/memory/v3/substrate/memory-v2-startup.js";
+import { maybeReseedCapabilitiesAfterManagedCredential } from "../../plugins/defaults/memory/substrate/boot-maintenance.js";
 import { validateAnthropicApiKey } from "../../providers/anthropic/client.js";
 import { validateAtlasCloudApiKey } from "../../providers/atlascloud/client.js";
 import { validateBasetenApiKey } from "../../providers/baseten/client.js";
 import { validateGeminiApiKey } from "../../providers/gemini/client.js";
 import { validateMinimaxApiKey } from "../../providers/minimax/client.js";
 import { validateOpenAIApiKey } from "../../providers/openai/client.js";
+import { validatePoolsideApiKey } from "../../providers/poolside/client.js";
 import { API_KEY_PROVIDERS } from "../../providers/provider-secret-catalog.js";
 import { initializeProviders } from "../../providers/registry.js";
 import { credentialKey } from "../../security/credential-key.js";
@@ -235,6 +236,15 @@ async function handleAddSecret({ body }: RouteHandlerArgs) {
           );
           return { success: false, error: validation.reason };
         }
+      } else if (name === "poolside") {
+        const validation = await validatePoolsideApiKey(value);
+        if (!validation.valid) {
+          log.warn(
+            { provider: name, reason: validation.reason },
+            "API key validation failed",
+          );
+          return { success: false, error: validation.reason };
+        }
       }
 
       const stored = await setSecureKeyAsync(

package/src/runtime/routes/settings-routes.ts

@@ -69,8 +69,7 @@ import { generateAvatarImage } from "../../tools/system/avatar-generator.js";
 import { pathExists } from "../../util/fs.js";
 import { getLogger } from "../../util/logger.js";
 import { getAvatarImagePath, getWorkspaceDir } from "../../util/platform.js";
-import { buildAssistantEvent } from "../assistant-event.js";
-import { assistantEventHub } from "../assistant-event-hub.js";
+import { broadcastMessage } from "../assistant-event-hub.js";
 import { ACTOR_PRINCIPALS } from "../auth/route-policy.js";
 import { publishAvatarChanged } from "../sync/resource-sync-events.js";
 import { BadRequestError, InternalError, NotFoundError } from "./errors.js";
@@ -250,22 +249,13 @@ async function handleOAuthConnectStart({ body = {} }: RouteHandlerArgs) {
           // DB not ready — use orchestrator value
         }
 
-        assistantEventHub
-          .publish(
-            buildAssistantEvent({
-              type: "oauth_connect_result",
-              success: deferredResult.success,
-              service: deferredResult.service,
-              accountInfo,
-              error: deferredResult.error,
-            }),
-          )
-          .catch((err) => {
-            log.warn(
-              { err, service: deferredResult.service },
-              "Failed to publish oauth_connect_result event",
-            );
-          });
+        broadcastMessage({
+          type: "oauth_connect_result",
+          success: deferredResult.success,
+          service: deferredResult.service,
+          accountInfo,
+          error: deferredResult.error,
+        });
 
         if (!deferredResult.success) {
           log.warn(

package/src/runtime/routes/subagents-routes.ts

@@ -204,7 +204,7 @@ export const ROUTES: RouteDefinition[] = [
     },
     summary: "Reconcile subagent live status",
     description:
-      "Returns the live in-memory status of all subagents known to the daemon for a given parent conversation. Subagents not in the response are orphaned.",
+      "Returns the live in-memory status of all subagents known to the daemon for a given parent conversation, with the identity fields a client needs to rebuild a store entry it never saw spawn (label, the subagent's own conversation id, and the spawning tool-use id). Subagents not in the response are orphaned.",
     tags: ["subagents"],
     queryParams: [
       {
@@ -218,6 +218,10 @@ export const ROUTES: RouteDefinition[] = [
         z.string(),
         z.object({
           status: z.string(),
+          label: z.string().optional(),
+          /** The subagent's OWN conversation id (not the parent's). */
+          conversationId: z.string().optional(),
+          parentToolUseId: z.string().optional(),
         }),
       ),
     }),
@@ -230,9 +234,22 @@ export const ROUTES: RouteDefinition[] = [
       }
       const manager = getSubagentManager();
       const children = manager.getChildrenOf(parentConversationId);
-      const subagents: Record<string, { status: string }> = {};
+      const subagents: Record<
+        string,
+        {
+          status: string;
+          label?: string;
+          conversationId?: string;
+          parentToolUseId?: string;
+        }
+      > = {};
       for (const child of children) {
-        subagents[child.config.id] = { status: child.status };
+        subagents[child.config.id] = {
+          status: child.status,
+          label: child.config.label,
+          conversationId: child.conversationId,
+          parentToolUseId: child.config.parentToolUseId,
+        };
       }
       return { subagents };
     },
@@ -253,22 +270,32 @@ export const ROUTES: RouteDefinition[] = [
       {
         name: "conversationId",
         schema: { type: "string" },
-        description: "Parent conversation ID (required)",
+        description:
+          "The subagent's own conversation ID. Fallback only — when the " +
+          "daemon knows the subagent (live or rehydrated), it resolves the " +
+          "conversation itself and this parameter is ignored.",
       },
     ],
     responseBody: SubagentDetailResponseSchema,
     handler: ({ pathParams, queryParams }) => {
-      const conversationId = queryParams?.conversationId;
+      const manager = getSubagentManager();
+      const state = manager.getState(pathParams!.id);
+
+      // Prefer the authoritative child-conversation id from manager state.
+      // Clients recovering from a missed `subagent_spawned` only know the
+      // PARENT conversation id (that's what `subagent_event` carries), so a
+      // caller-supplied id may point at the wrong conversation entirely.
+      const conversationId =
+        state?.conversationId ?? queryParams?.conversationId;
       if (!conversationId) {
         throw new BadRequestError("conversationId query parameter is required");
       }
 
-      const manager = getSubagentManager();
-      const state = manager.getState(pathParams!.id);
-
       return {
         ...getSubagentDetail(pathParams!.id, conversationId),
         status: state?.status,
+        label: state?.config.label,
+        parentToolUseId: state?.config.parentToolUseId,
       };
     },
   },