@vellumai/assistant 0.11.6 → 0.11.7-staging.2

package/src/schedule/__tests__/worker-feature-flags.test.ts

@@ -0,0 +1,58 @@
+/**
+ * Guards over the schedule worker entrypoint's feature flag bootstrap.
+ *
+ * The override cache is process-local, so this worker has to populate its own.
+ * When it does not, every flag check inside it resolves to the registry default
+ * and ignores remote values and local overrides, and this is the process that
+ * decides at fire time whether a schedule runs. The entrypoint installs signal
+ * handlers and calls `process.exit`, so it cannot be imported into a test
+ * process; these assertions read its source the same way
+ * `src/__tests__/worker-entrypoint-guards.test.ts` does.
+ */
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+
+const INIT_CALL = "await initFeatureFlagOverrides(";
+const REFRESH_CALL = "refreshOverridesFromGateway(";
+const REFRESH_INTERVAL = "FLAG_REFRESH_INTERVAL_MS";
+
+/** The worker's first call that can execute a schedule. */
+const WORK_START = "void tick()";
+
+function readWorkerSource(): string {
+  return readFileSync(join(process.cwd(), "src/schedule/worker.ts"), "utf8");
+}
+
+describe("schedule worker feature flags", () => {
+  test("loads flag overrides before the first schedule tick", () => {
+    const source = readWorkerSource();
+    const initAt = source.indexOf(INIT_CALL);
+    const workAt = source.indexOf(WORK_START);
+
+    expect(
+      initAt,
+      `The schedule worker must \`${INIT_CALL})\` at startup. Without it ` +
+        "its process-local override cache stays unset and fire-time flag " +
+        "checks fall back to registry defaults.",
+    ).toBeGreaterThanOrEqual(0);
+    expect(workAt).toBeGreaterThanOrEqual(0);
+    expect(
+      initAt < workAt,
+      "The flag load must complete before the first tick, so the schedules " +
+        "that tick fires are gated on real flag values.",
+    ).toBe(true);
+  });
+
+  test("re-reads the flag cache on a timer while running", () => {
+    const source = readWorkerSource();
+
+    expect(
+      source.includes(REFRESH_CALL) && source.includes(REFRESH_INTERVAL),
+      `This process outlives any single flag value, so it must call ` +
+        `${REFRESH_CALL}) on the ${REFRESH_INTERVAL} timer rather than ` +
+        "holding whatever was cached at startup.",
+    ).toBe(true);
+  });
+});

package/src/schedule/retry-policy.ts

@@ -41,15 +41,19 @@ export function decideRetry(
 export async function applyRetryDecision(params: {
   job: RetryPolicyJob;
   isOneShot: boolean;
-  errorMsg: string;
   decision: RetryDecision;
   scheduleRetry: (id: string, nextRetryAt: number) => void | Promise<void>;
   failOneShotPermanently: (id: string) => void | Promise<void>;
   resetRetryCount: (id: string) => void | Promise<void>;
-  emitAlert: (title: string, summary: string, dedupKey: string) => void;
+  /**
+   * Fired exactly once, when retries are exhausted. This is the single
+   * user-facing alert for a failed schedule (per-attempt failures are
+   * retried silently), and the caller owns its content and dedupe key.
+   */
+  emitAlert: () => void;
   log: Logger;
 }): Promise<void> {
-  const { job, isOneShot, errorMsg, decision } = params;
+  const { job, isOneShot, decision } = params;
 
   if (decision.action === "retry") {
     await params.scheduleRetry(job.id, decision.nextRetryAt);
@@ -69,11 +73,7 @@ export async function applyRetryDecision(params: {
     } else {
       await params.resetRetryCount(job.id);
     }
-    params.emitAlert(
-      `${job.name}: Retries exhausted`,
-      `Failed after ${job.retryCount + 1} attempt(s): ${errorMsg}`,
-      `schedule-retries-exhausted:${job.id}:${Date.now()}`,
-    );
+    params.emitAlert();
     params.log.warn(
       { jobId: job.id, name: job.name, attempts: job.retryCount + 1 },
       "Schedule retries exhausted",

package/src/schedule/run-script.ts

@@ -1,4 +1,8 @@
 import { buildSanitizedEnv } from "../tools/terminal/safe-env.js";
+import {
+  buildShellInvocation,
+  terminateProcessTree,
+} from "../util/host-process.js";
 import { getLogger } from "../util/logger.js";
 import { getWorkspaceDir } from "../util/platform.js";
 
@@ -34,14 +38,13 @@ export interface ScriptResult {
 /**
  * Run a shell command and capture its output.
  *
- * Uses Bun.spawn with /bin/sh so the command string supports pipes,
- * redirects, and shell builtins. Output is truncated to
- * {@link MAX_OUTPUT_BYTES} to keep schedule_runs rows bounded.
+ * Uses the platform shell so the command string supports pipes, redirects,
+ * and shell builtins. Windows uses PowerShell and POSIX hosts use Bash. Output
+ * is truncated to {@link MAX_OUTPUT_BYTES} to keep schedule_runs rows bounded.
  *
- * The command runs in its own process group, and the whole group is killed
- * on timeout and swept after exit, so background children cannot outlive the
- * run. A child that daemonizes itself with setsid leaves the group and
- * deliberately survives.
+ * The command runs in its own process group or Windows process tree. The tree
+ * is terminated on timeout and swept after exit so background children cannot
+ * outlive the run.
  */
 export async function runScript(
   command: string,
@@ -57,7 +60,8 @@ export async function runScript(
 
   log.info({ command, cwd, timeoutMs }, "Running script");
 
-  const proc = Bun.spawn(["sh", "-c", command], {
+  const shell = buildShellInvocation(command);
+  const proc = Bun.spawn([shell.command, ...shell.args], {
     cwd,
     detached: true,
     stdout: "pipe",
@@ -82,7 +86,7 @@ export async function runScript(
   const timeoutPromise = new Promise<never>((_, reject) => {
     const timer = setTimeout(() => {
       timedOut = true;
-      killProcessGroup(proc.pid);
+      terminateProcessTree(proc);
       reject(new Error(`Script timed out after ${timeoutMs}ms`));
     }, timeoutMs);
     timer.unref();
@@ -116,7 +120,7 @@ export async function runScript(
   // Sweep anything the script left running. This reaps orphaned background
   // children and closes their copies of the pipe fds so the stream reads
   // below can reach EOF.
-  killProcessGroup(proc.pid);
+  terminateProcessTree(proc);
 
   const [stdoutStr, stderrStr] = await drainStreams(
     stdoutCollector,
@@ -133,19 +137,6 @@ export async function runScript(
   return { exitCode, stdout, stderr };
 }
 
-/**
- * SIGKILL every process in the script's process group. The group is often
- * already empty, in which case the signal fails with ESRCH and there is
- * nothing to reap.
- */
-function killProcessGroup(pid: number): void {
-  try {
-    process.kill(-pid, "SIGKILL");
-  } catch {
-    // Process group may have already exited.
-  }
-}
-
 interface StreamCollector {
   /** Resolves with the text read so far once the stream ends or is cancelled. */
   promise: Promise<string>;

package/src/schedule/schedule-recovery.ts

@@ -52,7 +52,6 @@ export async function recoverStaleSchedules(): Promise<number> {
       await applyRetryDecision({
         job,
         isOneShot,
-        errorMsg,
         decision,
         scheduleRetry,
         failOneShotPermanently,

package/src/schedule/scheduler.ts

@@ -1,4 +1,5 @@
 import { refreshBackgroundWakeIntent } from "../background-wake/publisher.js";
+import { resolveSingleRouteProfileKey } from "../config/llm-resolver.js";
 import { getConfig } from "../config/loader.js";
 import {
   checkDiskPressureBackgroundGate,
@@ -7,13 +8,18 @@ import {
 } from "../daemon/disk-pressure-background-gate.js";
 import { processMessage } from "../daemon/process-message.js";
 import { INTERNAL_GUARDIAN_TRUST_CONTEXT } from "../daemon/trust-context.js";
+import { emitBackgroundFailureSignal } from "../notifications/background-failure-signal.js";
 import { emitNotificationSignal } from "../notifications/emit-signal.js";
 import { getConversation } from "../persistence/conversation-crud.js";
 import { isLifecycleQuiesced } from "../persistence/lifecycle-quiesce.js";
 import { invalidateAssistantInferredItemsForConversation } from "../plugins/defaults/memory/task-memory-cleanup.js";
+import { dispatchProviderResolvable } from "../providers/provider-resolvability.js";
 import { wakeAgentForOpportunity } from "../runtime/agent-wake.js";
 import { broadcastMessage } from "../runtime/assistant-event-hub.js";
-import { runBackgroundJob } from "../runtime/background-job-runner.js";
+import {
+  type BackgroundJobErrorKind,
+  runBackgroundJob,
+} from "../runtime/background-job-runner.js";
 import { publishConversationListChanged } from "../runtime/sync/resource-sync-events.js";
 import { runSequencesOnce } from "../sequence/engine.js";
 import type { TurnFailure } from "../telemetry/turn-outcome.js";
@@ -223,27 +229,84 @@ async function handleExecutionFailure(params: {
   job: ScheduleJob;
   errorMsg: string;
   isOneShot: boolean;
+  /**
+   * Classified failure cause from the runner, when the failing path had one.
+   * Keys the exhaustion alert's dedupe so same-cause failures across jobs
+   * collapse; absent means a generic exception, which keys per schedule.
+   */
+  errorKind?: BackgroundJobErrorKind;
+  /**
+   * The failing turn's carried classification, when the failure was a
+   * non-throwing turn failure. The exhaustion alert renders its authored
+   * message and scopes by the route that actually served the runs.
+   */
+  turnFailure?: TurnFailure;
 }): Promise<void> {
   const decision = decideRetry(params.job);
   await applyRetryDecision({
     job: params.job,
     isOneShot: params.isOneShot,
-    errorMsg: params.errorMsg,
     decision,
     scheduleRetry,
     failOneShotPermanently,
     resetRetryCount,
-    emitAlert: (_title, _summary, dedupKey) =>
-      emitScheduleActivityFailed({
-        jobId: params.job.id,
-        jobName: params.job.name,
+    emitAlert: () => {
+      const fallbackProviderScope = resolveScheduleProviderScope(params.job);
+      const carried = params.turnFailure;
+      emitBackgroundFailureSignal({
+        jobName: `schedule:${params.job.id}`,
+        displayName: `schedule:${params.job.name}`,
+        sourceChannel: "scheduler",
+        sourceContextId: params.job.id,
+        errorKind: params.errorKind ?? "exception",
         errorMessage: params.errorMsg,
-        dedupKey,
-      }),
+        ...(carried?.failureCode !== undefined
+          ? { failureCode: carried.failureCode }
+          : {}),
+        ...(carried?.userMessage !== undefined
+          ? { failureSummary: carried.userMessage }
+          : {}),
+        ...(carried?.errorCategory !== undefined
+          ? { errorCategory: carried.errorCategory }
+          : {}),
+        ...(carried?.connectionName !== undefined
+          ? { connectionName: carried.connectionName }
+          : {}),
+        ...(carried?.profileName !== undefined
+          ? { profileName: carried.profileName }
+          : {}),
+        ...(fallbackProviderScope !== undefined
+          ? { fallbackProviderScope }
+          : {}),
+      });
+    },
     log,
   });
 }
 
+/**
+ * The provider scope a schedule's failing runs resolved through: its pinned
+ * profile when that names a single provider route, else the route the
+ * mainAgent winner actually served (a deleted pin falls through to it). A
+ * mix pin, no named winner, or an unreadable config yield no scope, so the
+ * exhaustion alert keys per schedule rather than claiming one scope for
+ * arms that can span providers.
+ */
+function resolveScheduleProviderScope(job: ScheduleJob): string | undefined {
+  try {
+    return resolveSingleRouteProfileKey("mainAgent", getConfig().llm, {
+      ...(job.inferenceProfile
+        ? { overrideProfile: job.inferenceProfile }
+        : {}),
+      // Dispatch's own resolvability predicate, so a stale pin dispatch
+      // rejected (force-deleted connection) is not accepted as the scope.
+      isResolvableProvider: dispatchProviderResolvable,
+    });
+  } catch {
+    return undefined;
+  }
+}
+
 /** The running scheduler, retained so shutdown can stop it. */
 let instance: SchedulerHandle | null = null;
 
@@ -896,6 +959,8 @@ export async function runDueSchedulesOnce(
     let conversationId: string;
     let ok: boolean;
     let errorMsg: string | undefined;
+    let errorKind: BackgroundJobErrorKind | undefined;
+    let failedTurn: TurnFailure | undefined;
     const conversationReused = reusedConversationId != null;
     let runConversationId = reusedConversationId;
     const runId = await createScheduleRun(job.id, reusedConversationId);
@@ -929,17 +994,20 @@ export async function runDueSchedulesOnce(
         if (turnFailure) {
           ok = false;
           errorMsg = describeTurnFailure(turnFailure);
+          errorKind = "model_provider";
+          failedTurn = turnFailure;
         } else {
           ok = true;
         }
       } catch (err) {
         ok = false;
         errorMsg = err instanceof Error ? err.message : String(err);
+        errorKind = "exception";
       }
     } else {
-      // Fresh-bootstrap path: route through the shared runner so failures
-      // surface via `activity.failed` and we get the standard timeout +
-      // error-classification policy applied to every background producer.
+      // Fresh-bootstrap path: route through the shared runner for the
+      // standard timeout + error-classification policy applied to every
+      // background producer.
       // The runner fires `onConversationCreated` synchronously after bootstrap
       // (before `processMessage` starts) so the macOS sidebar gets the new
       // conversation immediately rather than after the up-to-30-min job ends.
@@ -962,7 +1030,12 @@ export async function runDueSchedulesOnce(
         groupId: resolveScheduleConversationGroupId(job),
         conversationType: "scheduled",
         scheduleJobId: job.id,
-        suppressFailureNotifications: job.quiet === true,
+        // The retry policy owns the single user-facing alert for a failing
+        // schedule (fired once, when retries are exhausted). Letting the
+        // runner also emit per attempt would notify for failures that a
+        // retry may yet recover, and pair every exhaustion alert with a
+        // duplicate.
+        suppressFailureNotifications: true,
         onConversationCreated: async (newConversationId) => {
           runConversationId = newConversationId;
           await setScheduleRunConversationId(runId, newConversationId);
@@ -988,6 +1061,8 @@ export async function runDueSchedulesOnce(
       }
       ok = result.ok;
       errorMsg = result.error?.message;
+      errorKind = result.errorKind;
+      failedTurn = result.turnFailure;
     }
 
     if (ok) {
@@ -1016,6 +1091,8 @@ export async function runDueSchedulesOnce(
         job,
         errorMsg: errorMsg ?? "Schedule run failed",
         isOneShot,
+        ...(errorKind !== undefined ? { errorKind } : {}),
+        ...(failedTurn !== undefined ? { turnFailure: failedTurn } : {}),
       });
 
       // Only skip invalidation when the conversation was *actually* reused,
@@ -1039,42 +1116,3 @@ export async function runDueSchedulesOnce(
 
   return result;
 }
-
-/**
- * Emit an `activity.failed` notification for a schedule whose retries have
- * been exhausted. Fires once when the retry policy has given up, so the
- * dedupeKey caller is the per-attempt key passed in by `applyRetryDecision`
- * (already includes the job id and a timestamp).
- */
-function emitScheduleActivityFailed(args: {
-  jobId: string;
-  jobName: string;
-  errorMessage: string;
-  dedupKey: string;
-}): void {
-  emitNotificationSignal({
-    sourceChannel: "scheduler",
-    sourceContextId: args.jobId,
-    sourceEventName: "activity.failed",
-    dedupeKey: args.dedupKey,
-    contextPayload: {
-      jobName: `schedule:${args.jobName}`,
-      errorMessage: args.errorMessage,
-      errorKind: "exception",
-    },
-    attentionHints: {
-      requiresAction: false,
-      urgency: "medium",
-      isAsyncBackground: true,
-      visibleInSourceNow: false,
-    },
-  }).catch((emitErr) => {
-    log.warn(
-      {
-        err: emitErr instanceof Error ? emitErr.message : String(emitErr),
-        jobId: args.jobId,
-      },
-      "Failed to emit activity.failed notification for exhausted schedule",
-    );
-  });
-}

package/src/schedule/worker.ts

@@ -13,8 +13,13 @@
  */
 import { writeFileSync } from "node:fs";
 
+import {
+  initFeatureFlagOverrides,
+  refreshOverridesFromGateway,
+} from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
 import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
+import { startConversationEvictor } from "../daemon/conversation-evictor.js";
 import { resetDb } from "../persistence/db-connection.js";
 import { registerWorkerPluginSurface } from "../plugins/worker-plugin-surface.js";
 import { disableStreamSeqStamping } from "../runtime/assistant-stream-state.js";
@@ -32,6 +37,9 @@ const log = getLogger("schedule-worker-process");
 /** Same cadence as the daemon scheduler's tick. */
 const TICK_INTERVAL_MS = 15_000;
 
+/** How often this process re-reads flag overrides from the gateway. */
+const FLAG_REFRESH_INTERVAL_MS = 60_000;
+
 async function main(): Promise<void> {
   // Only the daemon stamps SSE seqs and writes the shared reservation file.
   disableStreamSeqStamping();
@@ -44,6 +52,29 @@ async function main(): Promise<void> {
   writeFileSync(pidPath, String(process.pid), { flag: "w" });
   log.info({ pid: process.pid, pidPath }, "Schedule worker process started");
 
+  // The override cache is per-process, so this worker has to populate its own
+  // even though the daemon already populated the daemon's. Without it every
+  // flag check here resolves to its registry default and ignores both remote
+  // values and local overrides, which decides at fire time whether a schedule
+  // runs at all. Awaited so the first tick reads real values, and best-effort:
+  // a gateway that never answers leaves the cache unset and the worker falls
+  // back to registry defaults rather than failing to start.
+  // Bounded to one short attempt: a gateway that accepts the socket but never
+  // answers would otherwise hold the first tick for the full production retry
+  // schedule. A miss here leaves the cache unset (registry defaults) and the
+  // periodic refresh below converges to real values within a minute.
+  try {
+    await initFeatureFlagOverrides({
+      retryBackoffsMs: [],
+      callTimeoutMs: 2_000,
+    });
+  } catch (err) {
+    log.warn(
+      { err },
+      "Failed to load feature flag overrides in schedule worker; continuing on registry defaults",
+    );
+  }
+
   // Rehydrate the platform base URL and IDs from the credential store before
   // the first tick. The daemon does this in initializeProvidersAndTools(); this
   // standalone process must do it itself so getPlatformBaseUrl() resolves to
@@ -71,9 +102,15 @@ async function main(): Promise<void> {
     );
   }
 
+  // Sweep idle conversations out of the in-memory pool. The daemon starts
+  // this at startup; this process hosts conversations too, so without it
+  // every scheduled run's conversation is retained for the process lifetime.
+  startConversationEvictor();
+
   let stopped = false;
   let tickRunning = false;
   let timer: ReturnType<typeof setInterval> | null = null;
+  let flagRefreshTimer: ReturnType<typeof setInterval> | null = null;
   let disposePidGuard: (() => void) | null = null;
   const shutdown = (signal: string) => {
     log.info({ signal }, "Schedule worker process shutting down");
@@ -81,6 +118,9 @@ async function main(): Promise<void> {
     if (timer != null) {
       clearInterval(timer);
     }
+    if (flagRefreshTimer != null) {
+      clearInterval(flagRefreshTimer);
+    }
     disposePidGuard?.();
     cleanupWorkerPidFile(pidPath);
     process.exit(0);
@@ -127,6 +167,20 @@ async function main(): Promise<void> {
   }, TICK_INTERVAL_MS);
   void tick();
 
+  // This process outlives any single flag value, so poll the gateway instead of
+  // holding whatever was cached at startup. The daemon's gateway flag listener
+  // is not reusable here: it also reconciles managed profiles and broadcasts
+  // sync events, both of which belong to the daemon alone. The refresh swaps the
+  // cache atomically, so a tick reading a flag mid-refresh still sees the last
+  // known values. Unref'd so the tick interval above stays the one thing
+  // keeping this process alive.
+  flagRefreshTimer = setInterval(() => {
+    void refreshOverridesFromGateway().catch((err) => {
+      log.warn({ err }, "Failed to refresh feature flag overrides");
+    });
+  }, FLAG_REFRESH_INTERVAL_MS);
+  flagRefreshTimer.unref();
+
   process.on("SIGUSR1", () => {
     log.info("Received SIGUSR1 — refreshing database connections");
     resetDb();
@@ -155,6 +209,9 @@ async function main(): Promise<void> {
     if (timer != null) {
       clearInterval(timer);
     }
+    if (flagRefreshTimer != null) {
+      clearInterval(flagRefreshTimer);
+    }
     cleanupWorkerPidFile(getScheduleWorkerPidPath());
   });
 }