@vellumai/assistant 0.12.2 → 0.12.3-staging.1

package/src/schedule/run-script.ts

@@ -1,6 +1,7 @@
 import { buildSanitizedEnv } from "../tools/terminal/safe-env.js";
 import {
   buildShellInvocation,
+  buildShellSpawnFlags,
   terminateProcessTree,
 } from "../util/host-process.js";
 import { getLogger } from "../util/logger.js";
@@ -64,10 +65,9 @@ export async function runScript(
   const shell = buildShellInvocation(command);
   const proc = Bun.spawn([shell.command, ...shell.args], {
     cwd,
-    detached: true,
+    ...buildShellSpawnFlags(),
     stdout: "pipe",
     stderr: "pipe",
-    windowsHide: true,
     env: {
       ...buildSanitizedEnv(),
       // __SCHEDULE_ID lets a saved command find its own dir; __SCHEDULE_RUN_ID

package/src/schedule/scheduler.ts

@@ -58,6 +58,13 @@ import {
   scheduleRetry,
   setScheduleRunConversationId,
 } from "./schedule-store.js";
+import {
+  getScheduleToolSurfaceSnapshot,
+  getScheduleToolSurfaceState,
+  isScheduleToolSurfaceReady,
+  MCP_NOT_READY_DEFER_MS,
+  scheduleModeNeedsToolSurface,
+} from "./tool-surface-readiness.js";
 import { buildWakeScheduleOptions } from "./wake-schedule-options.js";
 import {
   isScheduleWorkerAdministrativelyStopped,
@@ -703,6 +710,32 @@ export async function runDueSchedulesOnce(
       continue;
     }
 
+    if (
+      scheduleModeNeedsToolSurface(job.mode) &&
+      !isScheduleToolSurfaceReady()
+    ) {
+      log.info(
+        {
+          jobId: job.id,
+          name: job.name,
+          mode: job.mode,
+          toolSurfaceState: getScheduleToolSurfaceState(),
+          ...getScheduleToolSurfaceSnapshot(),
+        },
+        "Deferring tool-backed schedule until the MCP tool surface is ready",
+      );
+      try {
+        await deferClaimedSchedule(job.id, Date.now() + MCP_NOT_READY_DEFER_MS);
+      } catch (err) {
+        log.warn(
+          { err, jobId: job.id },
+          "Failed to defer claimed schedule while MCP tool surface is not ready",
+        );
+      }
+      result.skipped += 1;
+      continue;
+    }
+
     const isOneShot = job.expression == null;
 
     // ── Notify mode (one-shot or recurring) ─────────────────────────
@@ -1149,19 +1182,22 @@ export async function runDueSchedulesOnce(
       // that before the run is called done and its one result is produced.
       await awaitDelegatedWork(conversationId, runStartedAt, log);
       await completeScheduleRun(runId, { status: "ok" });
-      // The run succeeded; make sure it was not invisible. No-ops when the run
-      // already notified or produced nothing user-facing, so a schedule whose
-      // prompt ends in an explicit `assistant notifications send` is unaffected.
+      // Automatic completion notification for successful execute-mode runs.
+      // Quiet schedules skip this fallback so a clean tick stays silent.
+      // Explicit in-run delivery (e.g. `assistant notifications send`) is
+      // unaffected because it never goes through this producer.
       // Awaited rather than fired and forgotten: the schedule worker can exit
       // once the tick's loop drains, and a detached emit would race that exit.
-      await emitScheduleResultNotification({
-        scheduleId: job.id,
-        scheduleName: job.name,
-        conversationId,
-        runId,
-        runStartedAt,
-        rlog: log,
-      });
+      if (!job.quiet) {
+        await emitScheduleResultNotification({
+          scheduleId: job.id,
+          scheduleName: job.name,
+          conversationId,
+          runId,
+          runStartedAt,
+          rlog: log,
+        });
+      }
       if (isOneShot) {
         await completeOneShot(job.id);
       }

package/src/schedule/tool-surface-readiness.ts

@@ -0,0 +1,65 @@
+/**
+ * Process-local readiness of the MCP tool surface this process hosts.
+ *
+ * MCP tools reach the registry only after every configured server has been
+ * attempted. Until that pass settles, an execute/wake/workflow schedule that
+ * fires would record success while `mcp__*` calls fail as unknown tools.
+ * Notify and script schedules do not use that surface and keep firing.
+ *
+ * Default is ready so callers that never bootstrap MCP (existing scheduler
+ * tests, the daemon's unused tick) are not blocked. The schedule worker
+ * marks starting/reloading around its own connect.
+ */
+
+import {
+  EMPTY_MCP_STARTUP_SNAPSHOT,
+  type McpStartupSnapshot,
+} from "../mcp/tool-caps.js";
+import type { ScheduleMode } from "./schedule-store.js";
+
+export type ScheduleToolSurfaceState = "ready" | "starting" | "reloading";
+
+/** How far a claimed tool-backed schedule is pushed back while MCP is not ready. */
+export const MCP_NOT_READY_DEFER_MS = 5_000;
+
+let state: ScheduleToolSurfaceState = "ready";
+let snapshot: McpStartupSnapshot = EMPTY_MCP_STARTUP_SNAPSHOT;
+
+export function scheduleModeNeedsToolSurface(mode: ScheduleMode): boolean {
+  return mode === "execute" || mode === "wake" || mode === "workflow";
+}
+
+export function isScheduleToolSurfaceReady(): boolean {
+  return state === "ready";
+}
+
+export function getScheduleToolSurfaceState(): ScheduleToolSurfaceState {
+  return state;
+}
+
+export function getScheduleToolSurfaceSnapshot(): McpStartupSnapshot {
+  return snapshot;
+}
+
+export function markScheduleToolSurfaceStarting(
+  configuredServerCount = 0,
+): void {
+  state = "starting";
+  snapshot = { ...EMPTY_MCP_STARTUP_SNAPSHOT, configuredServerCount };
+}
+
+export function markScheduleToolSurfaceReloading(): void {
+  state = "reloading";
+}
+
+export function markScheduleToolSurfaceReady(
+  next: McpStartupSnapshot = EMPTY_MCP_STARTUP_SNAPSHOT,
+): void {
+  state = "ready";
+  snapshot = next;
+}
+
+export function resetScheduleToolSurfaceReadinessForTests(): void {
+  state = "ready";
+  snapshot = EMPTY_MCP_STARTUP_SNAPSHOT;
+}

package/src/schedule/worker-mcp.ts

@@ -0,0 +1,85 @@
+/**
+ * Schedule-worker MCP bootstrap and reload.
+ *
+ * The worker hosts execute/wake/workflow turns in its own process, so it
+ * has to connect MCP servers itself. Tool-backed schedules must not fire
+ * until that pass has settled: a 20s grace lets notify/script start, and
+ * the readiness latch defers the rest until registration finishes or
+ * fails closed.
+ */
+
+import {
+  restartConfiguredMcpServers,
+  startConfiguredMcpServers,
+} from "../mcp/startup.js";
+import {
+  EMPTY_MCP_STARTUP_SNAPSHOT,
+  type McpStartupSnapshot,
+} from "../mcp/tool-caps.js";
+import { getLogger } from "../util/logger.js";
+import {
+  markScheduleToolSurfaceReady,
+  markScheduleToolSurfaceReloading,
+  markScheduleToolSurfaceStarting,
+} from "./tool-surface-readiness.js";
+
+export { MCP_NOT_READY_DEFER_MS } from "./tool-surface-readiness.js";
+
+const log = getLogger("schedule-worker-mcp");
+
+/**
+ * How long startup waits before arming notify/script ticks.
+ *
+ * Connecting is worth waiting for. It is not worth waiting for without
+ * limit: each server may take 30s to connect and 30s to list tools. Past
+ * this deadline those ticks start. Tool-backed schedules stay deferred
+ * until {@link markScheduleToolSurfaceReady}.
+ */
+export const MCP_STARTUP_GRACE_MS = 20_000;
+
+export async function bootstrapScheduleWorkerMcp(options?: {
+  start?: () => Promise<McpStartupSnapshot>;
+  graceMs?: number;
+}): Promise<void> {
+  const start = options?.start ?? startConfiguredMcpServers;
+  const graceMs = options?.graceMs ?? MCP_STARTUP_GRACE_MS;
+  markScheduleToolSurfaceStarting();
+  const startup = start()
+    .then((snapshot) => {
+      markScheduleToolSurfaceReady(snapshot);
+      log.info(snapshot, "MCP tool surface ready in schedule worker");
+      return snapshot;
+    })
+    .catch((err: unknown) => {
+      markScheduleToolSurfaceReady(EMPTY_MCP_STARTUP_SNAPSHOT);
+      log.error(
+        { err },
+        "MCP tool surface failed in schedule worker; continuing without MCP tools",
+      );
+      return EMPTY_MCP_STARTUP_SNAPSHOT;
+    });
+
+  await Promise.race([
+    startup,
+    new Promise<void>((resolve) => {
+      setTimeout(resolve, graceMs).unref();
+    }),
+  ]);
+}
+
+export async function reloadScheduleWorkerMcp(options?: {
+  restart?: () => Promise<McpStartupSnapshot>;
+}): Promise<McpStartupSnapshot> {
+  const restart = options?.restart ?? restartConfiguredMcpServers;
+  markScheduleToolSurfaceReloading();
+  try {
+    const snapshot = await restart();
+    markScheduleToolSurfaceReady(snapshot);
+    log.info(snapshot, "MCP servers reloaded in schedule worker");
+    return snapshot;
+  } catch (err) {
+    markScheduleToolSurfaceReady(EMPTY_MCP_STARTUP_SNAPSHOT);
+    log.warn({ err }, "MCP reload failed in schedule worker");
+    throw err;
+  }
+}

package/src/schedule/worker.ts

@@ -22,10 +22,6 @@ import { rehydratePlatformCredentials } from "../config/platform-rehydration.js"
 import { startConversationEvictor } from "../daemon/conversation-evictor.js";
 import { stopMcpServerManager } from "../mcp/manager.js";
 import { MCP_RELOAD_SIGNAL_FILE } from "../mcp/reload-signal.js";
-import {
-  restartConfiguredMcpServers,
-  startConfiguredMcpServers,
-} from "../mcp/startup.js";
 import { resetDb } from "../persistence/db-connection.js";
 import { registerWorkerPluginSurface } from "../plugins/worker-plugin-surface.js";
 import { disableStreamSeqStamping } from "../runtime/assistant-stream-state.js";
@@ -37,6 +33,10 @@ import {
   startWorkerPidFileGuard,
 } from "../util/worker-process.js";
 import { runDueSchedulesOnce } from "./scheduler.js";
+import {
+  bootstrapScheduleWorkerMcp,
+  reloadScheduleWorkerMcp,
+} from "./worker-mcp.js";
 
 const log = getLogger("schedule-worker-process");
 
@@ -46,19 +46,6 @@ const TICK_INTERVAL_MS = 15_000;
 /** How often this process re-reads flag overrides from the gateway. */
 const FLAG_REFRESH_INTERVAL_MS = 60_000;
 
-/**
- * How long startup waits for MCP servers before arming the tick.
- *
- * Connecting is worth waiting for: a schedule that fires first would fail its
- * `mcp__*` calls as unknown tools. It is not worth waiting for without limit.
- * `McpServerManager.start()` walks servers one at a time and allows each 30s to
- * connect and 30s more to list its tools, so a handful of unreachable ones can
- * hold the first tick for minutes, including the notify and script schedules
- * that never touch MCP. Past this deadline the connect continues in the
- * background and the tick starts without it.
- */
-const MCP_STARTUP_GRACE_MS = 20_000;
-
 /**
  * Rebuild this process's MCP connections whenever the daemon reports a reload.
  *
@@ -85,10 +72,7 @@ function watchForMcpReload(): void {
       restartQueued = true;
       return;
     }
-    restarting = restartConfiguredMcpServers()
-      .then((toolCount) => {
-        log.info({ toolCount }, "MCP servers reloaded in schedule worker");
-      })
+    restarting = reloadScheduleWorkerMcp()
       .catch((err: unknown) => {
         log.warn({ err }, "MCP reload failed in schedule worker");
       })
@@ -193,16 +177,10 @@ async function main(): Promise<void> {
   // keeps this process independent of the daemon's event loop, which is the
   // reason it is a separate process at all.
   //
-  // Bounded by MCP_STARTUP_GRACE_MS: unreachable servers must not hold the
-  // schedules that do not use them. The connect runs on past the deadline.
-  const mcpStartup = startConfiguredMcpServers();
-  await Promise.race([
-    mcpStartup,
-    // Unref'd so a connect that beats the deadline leaves nothing pending.
-    new Promise((resolve) => {
-      setTimeout(resolve, MCP_STARTUP_GRACE_MS).unref();
-    }),
-  ]);
+  // Bounded by MCP_STARTUP_GRACE_MS so notify/script can fire. Execute, wake,
+  // and workflow schedules stay deferred until the connect settles and the
+  // tool surface is marked ready.
+  await bootstrapScheduleWorkerMcp();
 
   // React to an MCP reload in the daemon. The server set belongs to the config
   // and the daemon owns the watcher that notices it change; this process holds

package/src/security/secure-keys.ts

@@ -3,22 +3,24 @@
  * adapters.
  *
  * Backend selection (`resolveBackendAsync`) is the single async decision point:
- *   1. CES RPC (primary) — injected via `setCesClient()`: delegates credential
+ *   1. CES RPC (primary) - injected via `setCesClient()`: delegates credential
  *      operations to the CES process over Unix socket RPC. This is the default
- *      path for the daemon (which calls startCes() at boot) and for non-daemon
- *      processes that lazily connect via the CES_LOCAL_SOCKET path (see below).
- *   2. CES HTTP — containerized mode (IS_CONTAINERIZED + CES_CREDENTIAL_URL):
- *      delegates to the CES sidecar over HTTP. Used in Docker/managed mode,
- *      including failover when the bootstrap RPC transport dies later.
- *   3. Lazy CES RPC connect — non-daemon processes (workers, CLI subprocesses)
- *      that inherit CES_LOCAL_SOCKET but never call startCes(). On first
- *      credential resolution, a direct CES connection is established and
- *      cached. On failure, falls through to the encrypted file store.
- *   4. Encrypted file store (fallback) — used when CES is unavailable.
+ *      path for the assistant (which calls startCes() at boot) and for
+ *      child processes that lazily connect to the CES socket (see below).
+ *   2. Lazy CES RPC connect - non-assistant processes (workers, CLI
+ *      subprocesses) that never call startCes(). On first credential
+ *      resolution they discover the CES bootstrap socket
+ *      (`CES_BOOTSTRAP_SOCKET_DIR`) and cache
+ *      the connection.
+ *   3. CES HTTP - containerized failover when IPC is unavailable
+ *      (`IS_CONTAINERIZED` + `CES_CREDENTIAL_URL`). Used if the assistant's
+ *      bootstrap RPC transport is down, or if a process with HTTP env could
+ *      not open the socket.
+ *   4. Encrypted file store (fallback) - used when CES is unavailable locally.
  *
  * All operations (reads, writes, lists, deletes) go to exactly one backend.
  * There are no cross-store fallbacks or merges. The only transport failover is
- * CES RPC → CES HTTP in managed mode; both backends target the same CES
+ * CES RPC to CES HTTP in managed mode; both backends target the same CES
  * sidecar and credential data.
  */
 
@@ -36,6 +38,7 @@ import {
   type CesClient,
   createCesClient,
 } from "../credential-execution/client.js";
+import { discoverCes } from "../credential-execution/executable-discovery.js";
 import {
   CesUnavailableError,
   createCesProcessManager,
@@ -84,11 +87,11 @@ let _resolvePromise: Promise<CredentialBackend> | undefined;
 /**
  * In-flight lazy CES connection promise for non-daemon processes.
  *
- * Workers and CLI subprocesses inherit CES_LOCAL_SOCKET but never call
- * startCes(). When they hit resolveBackendAsync() with no _cesClient and
- * no _cesReconnect (daemon-only), this promise memoizes a direct CES
- * connection attempt so concurrent credential reads in the same process
- * share a single connect+handshake rather than racing.
+ * Workers and CLI subprocesses never call startCes(). When they hit
+ * resolveBackendAsync() with no _cesClient and no _cesReconnect
+ * (daemon-only), this promise memoizes a direct CES connection attempt so
+ * concurrent credential reads in the same process share a single
+ * connect+handshake rather than racing.
  */
 let _lazyConnectPromise: Promise<CesClient | undefined> | undefined;
 
@@ -203,14 +206,15 @@ function getEncryptedStoreBackend(): CredentialBackend {
  * Resolve the primary credential backend for this process (async).
  *
  * Priority:
- *   1. CES RPC client → primary path for all local modes.
- *   2. Containerized + CES_CREDENTIAL_URL → CES HTTP client (Docker/managed).
- *   3. Encrypted file store → fallback when CES is unavailable.
+ *   1. CES RPC client: primary path in every environment.
+ *   2. Lazy CES RPC connect: child processes discover the CES socket.
+ *   3. Containerized + CES_CREDENTIAL_URL: CES HTTP, only if IPC is down.
+ *   4. Encrypted file store: local fallback when CES is unavailable.
  *
  * Once resolved, the backend is cached. If it becomes unavailable (e.g. the
  * CES transport dies), we attempt to reconnect via `_cesReconnect` rather
  * than falling back to a different backend. In managed cloud mode CES is the
- * primary credential source — falling back to the encrypted file store would
+ * primary credential source. Falling back to the encrypted file store would
  * silently serve stale or empty data.
  *
  * In managed mode, if the CES bootstrap RPC transport dies, we first fail
@@ -413,10 +417,12 @@ export async function attemptCesReconnection(
 /**
  * Lazily connect to a CES sibling socket from a non-daemon process.
  *
- * Workers and CLI subprocesses inherit CES_LOCAL_SOCKET from the daemon's
- * environment but never call startCes(). This function establishes a direct
- * CES connection on first credential resolution, memoizing the in-flight
- * promise so concurrent callers share a single connect+handshake.
+ * Workers and CLI subprocesses never call startCes(). This function
+ * establishes a direct CES connection on first credential resolution,
+ * memoizing the in-flight promise so concurrent callers share a single
+ * connect+handshake. Discovery uses the shared CES bootstrap socket; a
+ * missing socket fails immediately so callers can fall through without
+ * polling.
  *
  * On success, the client is injected via setCesClient() so subsequent
  * resolveBackendAsync() calls take the fast CES RPC path (step 1). A
@@ -433,6 +439,14 @@ async function tryLazyCesConnect(): Promise<CesClient | undefined> {
 
   _lazyConnectPromise = (async () => {
     try {
+      const discovery = discoverCes();
+      if (discovery.mode === "unavailable") {
+        log.info(
+          { reason: discovery.reason },
+          "CES socket not reachable for lazy connect, falling back to encrypted file store",
+        );
+        return undefined;
+      }
       const pm = createCesProcessManager({});
       const transport = await pm.start();
       const client = createCesClient(transport);
@@ -498,7 +512,7 @@ async function tryLazyCesConnect(): Promise<CesClient | undefined> {
 }
 
 async function doResolveBackend(): Promise<CredentialBackend> {
-  // 1. CES RPC — primary credential backend for all local modes
+  // 1. CES RPC. Primary credential backend in every environment.
   if (_cesClient) {
     const cesRpc = new CesRpcCredentialBackend(_cesClient);
     if (cesRpc.isAvailable()) {
@@ -511,7 +525,24 @@ async function doResolveBackend(): Promise<CredentialBackend> {
     );
   }
 
-  // 2. CES HTTP — containerized / Docker / managed mode
+  // 2. Lazy CES RPC connect. Child processes never call startCes(). When
+  //    the assistant's setCesReconnect() is NOT registered, attempt a
+  //    direct connection to the CES bootstrap socket. On success, inject
+  //    the client via setCesClient()
+  //    and re-resolve through the CES RPC path. On failure, fall through.
+  if (!_cesClient && !_cesReconnect) {
+    const lazyClient = await tryLazyCesConnect();
+    if (lazyClient) {
+      const cesRpc = new CesRpcCredentialBackend(lazyClient);
+      if (cesRpc.isAvailable()) {
+        _resolvedBackend = cesRpc;
+        log.info("Resolved credential backend: ces-rpc (lazy connect)");
+        return cesRpc;
+      }
+    }
+  }
+
+  // 3. CES HTTP. Managed failover when IPC is unavailable.
   if (getIsContainerized() && process.env.CES_CREDENTIAL_URL) {
     const ces = createCesCredentialBackend();
     if (ces.isAvailable()) {
@@ -525,26 +556,7 @@ async function doResolveBackend(): Promise<CredentialBackend> {
     );
   }
 
-  // 2.5. Lazy CES RPC connect — non-daemon processes (workers, CLI
-  //      subprocesses) inherit CES_LOCAL_SOCKET but never call startCes().
-  //      When the daemon's setCesReconnect() is NOT registered, attempt a
-  //      direct connection to the CES sibling socket. On success, inject
-  //      the client via setCesClient() and re-resolve through the CES RPC
-  //      path. On failure, fall through to the encrypted file store.
-  if (!_cesClient && !_cesReconnect && process.env.CES_LOCAL_SOCKET) {
-    const lazyClient = await tryLazyCesConnect();
-    if (lazyClient) {
-      const cesRpc = new CesRpcCredentialBackend(lazyClient);
-      if (cesRpc.isAvailable()) {
-        _resolvedBackend = cesRpc;
-        log.info("Resolved credential backend: ces-rpc (lazy connect)");
-        return cesRpc;
-      }
-    }
-    // Lazy connect failed; fall through.
-  }
-
-  // 3. On a containerized pod the local encrypted store does not exist and CES
+  // 4. On a containerized pod the local encrypted store does not exist and CES
   //    owns credentials. Never resolve to the encrypted store here; it would
   //    report a provisioned credential as absent. Return an unreachable backend
   //    (presence indeterminate, which callers retry) WITHOUT caching it, so the
@@ -556,7 +568,7 @@ async function doResolveBackend(): Promise<CredentialBackend> {
     return createUnavailableBackend();
   }
 
-  // 4. Encrypted file store: the legitimate backend for local / self-hosted
+  // 5. Encrypted file store: the legitimate backend for local / self-hosted
   //    mode when CES is unavailable.
   _resolvedBackend = getEncryptedStoreBackend();
   log.info("Resolved credential backend: encrypted-store (local mode)");