@vellumai/assistant 0.7.0 → 0.7.2

package/src/schedule/retry-backoff.ts

@@ -0,0 +1,18 @@
+export const DEFAULT_MAX_BACKOFF_MS = 30 * 60 * 1000; // 30 minutes
+
+/**
+ * Compute retry delay with exponential backoff and jitter.
+ * The exponential term is clamped to maxMs BEFORE jitter to prevent
+ * Infinity/NaN for large attempt values, and capped again after jitter
+ * so the result never exceeds maxMs.
+ */
+export function computeRetryDelay(
+  attempt: number,
+  baseMs: number,
+  maxMs: number = DEFAULT_MAX_BACKOFF_MS,
+  random: () => number = Math.random,
+): number {
+  const exponential = Math.min(baseMs * Math.pow(2, attempt), maxMs);
+  const jitter = exponential * 0.2 * (2 * random() - 1);
+  return Math.max(0, Math.min(Math.round(exponential + jitter), maxMs));
+}

package/src/schedule/retry-policy.ts

@@ -0,0 +1,82 @@
+import type { Logger } from "pino";
+
+import { computeRetryDelay } from "./retry-backoff.js";
+
+export interface RetryPolicyJob {
+  id: string;
+  name: string;
+  retryCount: number;
+  maxRetries: number;
+  retryBackoffMs: number;
+}
+
+export type RetryDecision =
+  | { action: "retry"; delayMs: number; nextRetryAt: number }
+  | { action: "exhaust" };
+
+/**
+ * Pure decision function: given a job's retry state, decide whether
+ * to retry with backoff or give up.
+ *
+ * `retryCount` is the PRE-increment value (before completeScheduleRun
+ * bumped it). So `retryCount < maxRetries` allows exactly maxRetries
+ * retries: retryCount 0, 1, …, maxRetries−1 all pass the check.
+ */
+export function decideRetry(
+  job: RetryPolicyJob,
+  now: number = Date.now(),
+): RetryDecision {
+  if (job.retryCount < job.maxRetries) {
+    const delayMs = computeRetryDelay(job.retryCount, job.retryBackoffMs);
+    return { action: "retry", delayMs, nextRetryAt: now + delayMs };
+  }
+  return { action: "exhaust" };
+}
+
+/**
+ * Apply the retry decision to a schedule: schedule a retry or exhaust.
+ * Calls the provided store operations so this module stays decoupled
+ * from direct DB imports.
+ */
+export function applyRetryDecision(params: {
+  job: RetryPolicyJob;
+  isOneShot: boolean;
+  errorMsg: string;
+  decision: RetryDecision;
+  scheduleRetry: (id: string, nextRetryAt: number) => void;
+  failOneShotPermanently: (id: string) => void;
+  resetRetryCount: (id: string) => void;
+  emitAlert: (title: string, summary: string, dedupKey: string) => void;
+  log: Logger;
+}): void {
+  const { job, isOneShot, errorMsg, decision } = params;
+
+  if (decision.action === "retry") {
+    params.scheduleRetry(job.id, decision.nextRetryAt);
+    params.log.info(
+      {
+        jobId: job.id,
+        name: job.name,
+        attempt: job.retryCount + 1,
+        maxRetries: job.maxRetries,
+        delayMs: decision.delayMs,
+      },
+      "Scheduling retry with backoff",
+    );
+  } else {
+    if (isOneShot) {
+      params.failOneShotPermanently(job.id);
+    } else {
+      params.resetRetryCount(job.id);
+    }
+    params.emitAlert(
+      `${job.name}: Retries exhausted`,
+      `Failed after ${job.retryCount + 1} attempt(s): ${errorMsg}`,
+      `schedule-retries-exhausted:${job.id}:${Date.now()}`,
+    );
+    params.log.warn(
+      { jobId: job.id, name: job.name, attempts: job.retryCount + 1 },
+      "Schedule retries exhausted",
+    );
+  }
+}

package/src/schedule/schedule-recovery.ts

@@ -0,0 +1,64 @@
+import { getLogger } from "../util/logger.js";
+import { applyRetryDecision, decideRetry } from "./retry-policy.js";
+import {
+  completeScheduleRun,
+  createScheduleRun,
+  failOneShotPermanently,
+  findStaleInFlightJobs,
+  getSchedule,
+  resetRetryCount,
+  scheduleRetry,
+} from "./schedule-store.js";
+
+const log = getLogger("schedule-recovery");
+
+/**
+ * Recover schedules left in an inconsistent state by a prior process crash.
+ * Called once at daemon startup, before the scheduler tick loop starts,
+ * so all "firing" / "running" rows are definitively stale.
+ */
+export function recoverStaleSchedules(): number {
+  const stale = findStaleInFlightJobs(0);
+  if (stale.length === 0) return 0;
+
+  log.info({ count: stale.length }, "Recovering stale in-flight schedules");
+
+  let recovered = 0;
+  for (const { jobId, staleRunId } of stale) {
+    try {
+      const job = getSchedule(jobId);
+      if (!job) continue;
+
+      const errorMsg =
+        "Process terminated during execution (recovered on restart)";
+
+      if (staleRunId) {
+        completeScheduleRun(staleRunId, { status: "error", error: errorMsg });
+      } else {
+        const runId = createScheduleRun(jobId, `recovery:${jobId}`);
+        completeScheduleRun(runId, { status: "error", error: errorMsg });
+      }
+
+      // Use the same retry-or-exhaust path as the scheduler
+      const isOneShot = job.expression == null;
+      const decision = decideRetry(job);
+      applyRetryDecision({
+        job,
+        isOneShot,
+        errorMsg,
+        decision,
+        scheduleRetry,
+        failOneShotPermanently,
+        resetRetryCount,
+        emitAlert: () => {}, // no feed event on startup recovery
+        log,
+      });
+      recovered++;
+    } catch (err) {
+      log.error({ err, jobId }, "Failed to recover stale schedule");
+    }
+  }
+
+  log.info({ recovered }, "Stale schedule recovery complete");
+  return recovered;
+}

package/src/schedule/schedule-store.ts

@@ -33,6 +33,8 @@ export interface ScheduleJob {
   lastRunAt: number | null;
   lastStatus: string | null;
   retryCount: number;
+  maxRetries: number;
+  retryBackoffMs: number;
   createdBy: string;
   mode: ScheduleMode;
   routingIntent: RoutingIntent;
@@ -66,22 +68,6 @@ export function isValidCronExpression(expr: string): boolean {
   }
 }
 
-export function computeNextRunAt(
-  cronExpression: string,
-  timezone?: string | null,
-): number {
-  const cron = new Cron(cronExpression, {
-    timezone: timezone ?? undefined,
-  });
-  const next = cron.nextRun();
-  if (!next) {
-    throw new Error(
-      `Schedule expression "${cronExpression}" has no upcoming runs`,
-    );
-  }
-  return next.getTime();
-}
-
 export function createSchedule(params: {
   name: string;
   cronExpression?: string | null;
@@ -99,6 +85,8 @@ export function createSchedule(params: {
   routingHints?: Record<string, unknown>;
   quiet?: boolean;
   reuseConversation?: boolean;
+  maxRetries?: number;
+  retryBackoffMs?: number;
 }): ScheduleJob {
   const expression = params.expression ?? params.cronExpression ?? null;
   const isOneShot = expression == null;
@@ -132,6 +120,8 @@ export function createSchedule(params: {
   const routingHints = params.routingHints ?? {};
   const quiet = params.quiet ?? false;
   const reuseConversation = params.reuseConversation ?? false;
+  const maxRetries = params.maxRetries ?? 3;
+  const retryBackoffMs = params.retryBackoffMs ?? 60000;
 
   let nextRunAt: number;
   if (isOneShot) {
@@ -156,6 +146,8 @@ export function createSchedule(params: {
     lastRunAt: null as number | null,
     lastStatus: null as string | null,
     retryCount: 0,
+    maxRetries,
+    retryBackoffMs,
     createdBy: params.createdBy ?? "agent",
     mode,
     routingIntent,
@@ -251,6 +243,8 @@ export function updateSchedule(
     quiet?: boolean;
     reuseConversation?: boolean;
     wakeConversationId?: string | null;
+    maxRetries?: number;
+    retryBackoffMs?: number;
   },
 ): ScheduleJob | null {
   const db = getDb();
@@ -273,12 +267,13 @@ export function updateSchedule(
 
   const isOneShot = newExpr == null;
 
-  // Validate if expression or syntax changed (only for recurring schedules)
+  // Validate if expression, syntax, or timezone changed (only for recurring schedules)
   if (
     !isOneShot &&
     (updates.expression !== undefined ||
       updates.cronExpression !== undefined ||
-      updates.syntax !== undefined)
+      updates.syntax !== undefined ||
+      updates.timezone !== undefined)
   ) {
     const spec = {
       syntax: newSyntax,
@@ -311,6 +306,9 @@ export function updateSchedule(
     set.reuseConversation = updates.reuseConversation;
   if (updates.wakeConversationId !== undefined)
     set.wakeConversationId = updates.wakeConversationId;
+  if (updates.maxRetries !== undefined) set.maxRetries = updates.maxRetries;
+  if (updates.retryBackoffMs !== undefined)
+    set.retryBackoffMs = updates.retryBackoffMs;
 
   // Recompute nextRunAt if schedule timing may have changed (only for recurring)
   if (
@@ -833,6 +831,94 @@ export function describeCronExpression(expr: string | null): string {
   }
 }
 
+/**
+ * Set the next retry time for a schedule and revert one-shot status from
+ * "firing" to "active" so the scheduler will claim it again when nextRetryAt
+ * arrives. No-op for recurring schedules (they stay in their current status).
+ */
+export function scheduleRetry(id: string, nextRetryAt: number): void {
+  const db = getDb();
+  const now = Date.now();
+  db.update(scheduleJobs)
+    .set({ nextRunAt: nextRetryAt, updatedAt: now })
+    .where(eq(scheduleJobs.id, id))
+    .run();
+  // Revert one-shot status from "firing" to "active" so the scheduler
+  // will claim it again when nextRetryAt arrives. No-op for recurring.
+  db.update(scheduleJobs)
+    .set({ status: "active", updatedAt: now })
+    .where(and(eq(scheduleJobs.id, id), eq(scheduleJobs.status, "firing")))
+    .run();
+}
+
+/**
+ * Reset the retry count for a schedule back to zero (e.g. after a successful run).
+ */
+export function resetRetryCount(id: string): void {
+  const db = getDb();
+  db.update(scheduleJobs)
+    .set({ retryCount: 0, updatedAt: Date.now() })
+    .where(eq(scheduleJobs.id, id))
+    .run();
+}
+
+/**
+ * Find schedules stuck in an in-flight state (one-shots in "firing",
+ * cron runs in "running"). Used at daemon startup to recover from
+ * a prior process crash.
+ *
+ * @param staleThresholdMs If >0, only consider rows whose lastRunAt
+ *   (for one-shots) or startedAt (for runs) is older than `now - staleThresholdMs`.
+ *   Pass 0 at startup (the previous process is definitely dead).
+ */
+export function findStaleInFlightJobs(staleThresholdMs: number = 0): Array<{
+  jobId: string;
+  staleRunId: string | null;
+}> {
+  const db = getDb();
+  const cutoff = Date.now() - staleThresholdMs;
+
+  // One-shots stuck in "firing" where lastRunAt is older than cutoff
+  const staleOneShots = db
+    .select({ id: scheduleJobs.id })
+    .from(scheduleJobs)
+    .where(
+      and(
+        isNull(scheduleJobs.cronExpression),
+        eq(scheduleJobs.status, "firing"),
+        eq(scheduleJobs.enabled, true),
+        staleThresholdMs > 0 ? lte(scheduleJobs.lastRunAt, cutoff) : undefined,
+      ),
+    )
+    .all();
+
+  // Cron runs stuck in "running" where startedAt is older than cutoff
+  const staleRuns = db
+    .select({ id: scheduleRuns.id, jobId: scheduleRuns.jobId })
+    .from(scheduleRuns)
+    .where(
+      and(
+        eq(scheduleRuns.status, "running"),
+        staleThresholdMs > 0 ? lte(scheduleRuns.startedAt, cutoff) : undefined,
+      ),
+    )
+    .all();
+
+  const result: Array<{ jobId: string; staleRunId: string | null }> = [];
+  const seenJobIds = new Set<string>();
+
+  for (const run of staleRuns) {
+    result.push({ jobId: run.jobId, staleRunId: run.id });
+    seenJobIds.add(run.jobId);
+  }
+  for (const job of staleOneShots) {
+    if (!seenJobIds.has(job.id)) {
+      result.push({ jobId: job.id, staleRunId: null });
+    }
+  }
+  return result;
+}
+
 function parseJobRow(row: typeof scheduleJobs.$inferSelect): ScheduleJob {
   return {
     id: row.id,
@@ -849,6 +935,8 @@ function parseJobRow(row: typeof scheduleJobs.$inferSelect): ScheduleJob {
     lastRunAt: row.lastRunAt,
     lastStatus: row.lastStatus,
     retryCount: row.retryCount,
+    maxRetries: row.maxRetries ?? 3,
+    retryBackoffMs: row.retryBackoffMs ?? 60000,
     createdBy: row.createdBy,
     mode: (row.mode ?? "execute") as ScheduleMode,
     routingIntent: (row.routingIntent ?? "all_channels") as RoutingIntent,

package/src/schedule/scheduler-types.ts

@@ -0,0 +1,25 @@
+/**
+ * Types extracted from scheduler.ts to break the scheduler ↔ engine cycle.
+ * `sequence/engine.ts` needs `ScheduleMessageProcessor` but scheduler.ts
+ * imports from engine — extracting the type here breaks the back-edge.
+ */
+
+import type { LLMCallSite } from "../config/schemas/llm.js";
+
+export interface ScheduleMessageOptions {
+  trustClass?: "guardian" | "trusted_contact" | "unknown";
+  taskRunId?: string;
+  /**
+   * Optional LLM call-site identifier propagated to the per-call provider
+   * config. Schedule and sequence callers will start passing their own call-site
+   * (e.g. for a future scheduled-agent profile) once PRs 7-11 migrate them off
+   * the default `mainAgent` route.
+   */
+  callSite?: LLMCallSite;
+}
+
+export type ScheduleMessageProcessor = (
+  conversationId: string,
+  message: string,
+  options?: ScheduleMessageOptions,
+) => Promise<unknown>;

package/src/schedule/scheduler.ts

@@ -1,4 +1,3 @@
-import type { LLMCallSite } from "../config/schemas/llm.js";
 import { emitFeedEvent } from "../home/emit-feed-event.js";
 import { bootstrapConversation } from "../memory/conversation-bootstrap.js";
 import { getConversation } from "../memory/conversation-crud.js";
@@ -12,40 +11,26 @@ import {
   type WatcherNotifier,
 } from "../watcher/engine.js";
 import { hasSetConstructs } from "./recurrence-engine.js";
+import { applyRetryDecision, decideRetry } from "./retry-policy.js";
 import { runScript, type ScriptResult } from "./run-script.js";
 import {
   claimDueSchedules,
   completeOneShot,
   completeScheduleRun,
   createScheduleRun,
-  failOneShot,
   failOneShotPermanently,
   getLastScheduleConversationId,
+  resetRetryCount,
   retryOneShot,
   type RoutingIntent,
+  type ScheduleJob,
+  scheduleRetry,
 } from "./schedule-store.js";
 
 const log = getLogger("scheduler");
 
-export interface ScheduleMessageOptions {
-  trustClass?: "guardian" | "trusted_contact" | "unknown";
-  taskRunId?: string;
-  /**
-   * Optional LLM call-site identifier propagated to the per-call provider
-   * config. Schedule and sequence callers will start passing their own call-site
-   * (e.g. for a future scheduled-agent profile) once PRs 7-11 migrate them off
-   * the default `mainAgent` route.
-   */
-  callSite?: LLMCallSite;
-}
-
-export type ScheduleMessageProcessor = (
-  conversationId: string,
-  message: string,
-  options?: ScheduleMessageOptions,
-) => Promise<unknown>;
-
-export type ScheduleNotifyModeNotifier = (payload: {
+import type { ScheduleMessageProcessor } from "./scheduler-types.js";
+type ScheduleNotifyModeNotifier = (payload: {
   id: string;
   label: string;
   message: string;
@@ -53,7 +38,7 @@ export type ScheduleNotifyModeNotifier = (payload: {
   routingHints: Record<string, unknown>;
 }) => void | Promise<void>;
 
-export type ScheduleConversationCreatedNotifier = (info: {
+type ScheduleConversationCreatedNotifier = (info: {
   conversationId: string;
   scheduleJobId: string;
   title: string;
@@ -73,6 +58,26 @@ const TICK_INTERVAL_MS = 15_000;
  */
 const WAKE_MAX_RETRIES = 20;
 
+function handleExecutionFailure(params: {
+  job: ScheduleJob;
+  errorMsg: string;
+  isOneShot: boolean;
+}): void {
+  const decision = decideRetry(params.job);
+  applyRetryDecision({
+    job: params.job,
+    isOneShot: params.isOneShot,
+    errorMsg: params.errorMsg,
+    decision,
+    scheduleRetry,
+    failOneShotPermanently,
+    resetRetryCount,
+    emitAlert: (title, summary, dedupKey) =>
+      emitScheduleFeedEvent({ title, summary, dedupKey }),
+    log,
+  });
+}
+
 export function startScheduler(
   processMessage: ScheduleMessageProcessor,
   notifyScheduleOneShot: ScheduleNotifyModeNotifier,
@@ -154,6 +159,8 @@ async function runScheduleOnce(
           routingHints: job.routingHints,
         });
         if (isOneShot) {
+          const successRunId = createScheduleRun(job.id, `notify-ok:${job.id}`);
+          completeScheduleRun(successRunId, { status: "ok" });
           completeOneShot(job.id);
           emitScheduleFeedEvent({
             title: job.name,
@@ -176,15 +183,10 @@ async function runScheduleOnce(
           { err, jobId: job.id, name: job.name, isOneShot },
           "Schedule notification failed",
         );
-        if (isOneShot) {
-          failOneShot(job.id);
-        } else {
-          // Track recurring notify-mode failures via a schedule run so the
-          // occurrence isn't silently lost and lastStatus/retryCount update.
-          const errorMsg = err instanceof Error ? err.message : String(err);
-          const runId = createScheduleRun(job.id, `notify-error:${job.id}`);
-          completeScheduleRun(runId, { status: "error", error: errorMsg });
-        }
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        const errorRunId = createScheduleRun(job.id, `notify-error:${job.id}`);
+        completeScheduleRun(errorRunId, { status: "error", error: errorMsg });
+        handleExecutionFailure({ job, errorMsg, isOneShot });
       }
       processed += 1;
       continue;
@@ -222,7 +224,9 @@ async function runScheduleOnce(
           }
           if (isOneShot) completeOneShot(job.id);
         } else {
-          if (isOneShot) failOneShot(job.id);
+          const errorMsg =
+            result.stderr || "Script exited with non-zero status";
+          handleExecutionFailure({ job, errorMsg, isOneShot });
         }
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
@@ -231,7 +235,7 @@ async function runScheduleOnce(
           "Script schedule execution failed",
         );
         completeScheduleRun(runId, { status: "error", error: errorMsg });
-        if (isOneShot) failOneShot(job.id);
+        handleExecutionFailure({ job, errorMsg, isOneShot });
       }
       processed += 1;
       continue;
@@ -309,7 +313,11 @@ async function runScheduleOnce(
           continue;
         }
 
-        if (isOneShot) completeOneShot(job.id);
+        if (isOneShot) {
+          const successRunId = createScheduleRun(job.id, `wake-ok:${job.id}`);
+          completeScheduleRun(successRunId, { status: "ok" });
+          completeOneShot(job.id);
+        }
         if (!job.quiet) {
           emitScheduleFeedEvent({
             title: job.name,
@@ -322,7 +330,16 @@ async function runScheduleOnce(
           { err, jobId: job.id, name: job.name, wakeConversationId, isOneShot },
           "Wake schedule execution failed",
         );
-        if (isOneShot) failOneShot(job.id);
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        const wakeErrorRunId = createScheduleRun(
+          job.id,
+          `wake-error:${job.id}`,
+        );
+        completeScheduleRun(wakeErrorRunId, {
+          status: "error",
+          error: errorMsg,
+        });
+        handleExecutionFailure({ job, errorMsg, isOneShot });
       }
       processed += 1;
       continue;
@@ -380,7 +397,11 @@ async function runScheduleOnce(
             status: "error",
             error: result.error ?? "Task run failed",
           });
-          if (isOneShot) failOneShot(job.id);
+          handleExecutionFailure({
+            job,
+            errorMsg: result.error ?? "Task run failed",
+            isOneShot,
+          });
         } else {
           completeScheduleRun(runId, { status: "ok" });
           if (!job.quiet) {
@@ -424,7 +445,11 @@ async function runScheduleOnce(
         });
         const runId = createScheduleRun(job.id, fallbackConversation.id);
         completeScheduleRun(runId, { status: "error", error: message });
-        if (isOneShot) failOneShot(job.id);
+        handleExecutionFailure({
+          job,
+          errorMsg: message,
+          isOneShot,
+        });
       }
       continue;
     }
@@ -507,7 +532,7 @@ async function runScheduleOnce(
           : "Schedule execution failed",
       );
       completeScheduleRun(runId, { status: "error", error: message });
-      if (isOneShot) failOneShot(job.id);
+      handleExecutionFailure({ job, errorMsg: message, isOneShot });
 
       // Only skip invalidation when the conversation was *actually* reused,
       // i.e. it contains prior successful context worth preserving. When

package/src/security/oauth-callback-registry.ts

@@ -4,6 +4,14 @@
  * back to the runtime code that initiated the OAuth handshake.
  */
 
+/**
+ * Sibling: `assistant/src/mcp/mcp-auth-state.ts`. The MCP auth state map sits
+ * one layer up from this registry — it tracks polling-visible status per
+ * MCP server, while this registry resolves the OAuth code-arrival promise
+ * keyed by OAuth `state`. The callback registry is shared by all
+ * gateway-transport OAuth flows (MCP and otherwise); the MCP state map is
+ * MCP-specific.
+ */
 interface PendingCallback {
   resolve: (code: string) => void;
   reject: (error: Error) => void;