agent-relay-server 0.88.2 → 0.89.0

package/src/agent-lifecycle-events.ts

@@ -28,6 +28,34 @@ function diagnosisSuffix(diagnosis?: AgentDeathDiagnosis): string {
 // falls back to the persisted stamp, so a post-restart clean exit still reads as `agent.exited`.
 const readyFired = new Set<string>();
 
+// #648 — terminal-outcome guard per spawnRequestId, enforcing the invariant "exactly one terminal
+// event per spawn". `agent.ready` and `agent.spawn_failed` are mutually exclusive for one request,
+// yet two paths can race: the orchestrator can report the `agent.spawn` COMMAND failed (a transient
+// launch/registration timeout) at the same moment the child actually registers and becomes ready.
+// Observed live (issue #648): spawn_failed delivered, then ready for the SAME spawnRequestId — a
+// false negative a coordinator acts on (double-spawn or false-abandon).
+//
+// We record the first terminal seen, keyed on spawnRequestId (the idempotency key shared by the
+// command's `params.spawnRequestId` and the child's `meta.spawnRequestId`):
+//   - a late spawn_failed after ready  → SUPPRESSED (the agent is alive, the failure is wrong)
+//   - a ready after a delivered spawn_failed → CORRECTED with `agent.spawn_recovered` so a
+//     coordinator that already acted on the false failure can reconcile (do not retry / un-abandon).
+// `ready` is sticky — once a request is ready it never downgrades to failed. In-memory like
+// `readyFired`: the race it guards is same-moment, so a relay restart between the two is not a concern.
+type SpawnTerminal = "agent.ready" | "agent.spawn_failed";
+const spawnTerminalBySpawnRequest = new Map<string, SpawnTerminal>();
+
+function spawnTerminalOf(spawnRequestId: string | undefined): SpawnTerminal | undefined {
+  return spawnRequestId === undefined ? undefined : spawnTerminalBySpawnRequest.get(spawnRequestId);
+}
+
+function recordSpawnTerminal(spawnRequestId: string | undefined, outcome: SpawnTerminal): void {
+  if (spawnRequestId === undefined) return;
+  // ready is terminal-sticky: never let a (stale/contradicting) spawn_failed overwrite a live ready.
+  if (spawnTerminalBySpawnRequest.get(spawnRequestId) === "agent.ready") return;
+  spawnTerminalBySpawnRequest.set(spawnRequestId, outcome);
+}
+
 /** A child counts as "had become ready" if the live set saw it OR the durable stamp survives a restart. */
 function everReady(child: AgentCard): boolean {
   return typeof child.meta?.readyNotifiedAt === "number";
@@ -63,15 +91,48 @@ export function notifyAgentReady(childId: string): void {
   // (the in-memory set is wiped on restart). Idempotent — only stamped on first ready.
   markAgentReadyNotified(childId);
 
+  const spawnRequestId = spawnRequestIdOf(child);
+  // #648 — if a spawn_failed was already emitted for this request, this ready RETRACTS it: the agent
+  // actually came up, so the earlier failure was a false negative. Emit a corrective recovery instead
+  // of a plain ready notice so a coordinator that acted on the failure can reconcile.
+  const recovered = spawnTerminalOf(spawnRequestId) === "agent.spawn_failed";
+  recordSpawnTerminal(spawnRequestId, "agent.ready");
+
   emitRelayEvent({
     type: "agent.ready",
     source: "server",
     subject: childId,
-    data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId: spawnRequestIdOf(child) },
+    data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId },
   });
 
+  if (recovered) {
+    emitRelayEvent({
+      type: "agent.spawn_recovered",
+      source: "server",
+      subject: childId,
+      data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId, supersedes: "agent.spawn_failed" },
+    });
+  }
+
   const config = getNotificationsConfig();
-  if (!config.enabled || !config.agentReady) return;
+  if (!config.enabled) return;
+  if (recovered) {
+    // A correction must always reach a parent that got the (false) failure — gate only on the global
+    // switch, NOT the agentReady sub-toggle, so the retraction lands even if ready pushes are off.
+    routeNotification({
+      type: "agent.spawn_recovered",
+      scope: { agentId: childId, parent },
+      recipients: [parent],
+      message: {
+        subject: "Spawn recovered",
+        body: `♻️ Correction: your earlier spawn failure${spawnRequestId ? ` for \`${spawnRequestId}\`` : ""} was a false alarm — ${describe(child)} actually registered and is ready and idle. Disregard the failure and do NOT retry; send it work with relay_send_message to \`${childId}\`.`,
+        payload: { kind: "agent.spawn_recovered", agentId: childId, parent, provider: providerOf(child), spawnRequestId, supersedes: "agent.spawn_failed" },
+        replyExpected: false,
+      },
+    });
+    return;
+  }
+  if (!config.agentReady) return;
   routeNotification({
     type: "agent.ready",
     scope: { agentId: childId, parent },
@@ -79,7 +140,7 @@ export function notifyAgentReady(childId: string): void {
     message: {
       subject: "Spawned agent ready",
       body: `✅ Your spawned agent ${describe(child)} is ready and idle — send it work with relay_send_message to \`${childId}\`.`,
-      payload: { kind: "agent.ready", agentId: childId, parent, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child) },
+      payload: { kind: "agent.ready", agentId: childId, parent, provider: providerOf(child), spawnRequestId },
       replyExpected: false,
     },
   });
@@ -108,7 +169,7 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
   // #645 — persist a terminal marker for ANY offline terminus (clean exit or spawn-fail) keyed by
   // agentId so it outlives the reaper. The #634 watchdog reads this to skip a benign shutdown; it
   // must fire ONLY on a genuine silent death that emitted no terminal event at all.
-  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", createdAt: Date.now() });
+  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", epoch: child.epoch, createdAt: Date.now() });
 
   if (wasReady) {
     emitRelayEvent({
@@ -132,6 +193,9 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
     return;
   }
 
+  // #648 — record the terminal outcome so a (defensive) later ready for the same request is detected
+  // as a recovery and corrected, keeping the one-terminal-per-spawn invariant across both fail paths.
+  recordSpawnTerminal(spawnRequestIdOf(child), "agent.spawn_failed");
   emitRelayEvent({
     type: "agent.spawn_failed",
     source: "server",
@@ -158,6 +222,11 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
  * to the spawning parent threaded on the command, since there is no child agent record to resolve.
  */
 export function notifyAgentSpawnFailed(input: { parent: string; spawnRequestId?: string; provider?: string; reason: string }): void {
+  // #648 — if the spawn already became ready, the orchestrator's command-failure report is a false
+  // negative (the child registered despite the spawn command reporting failure). Suppress it: the
+  // ready already delivered the authoritative terminal outcome for this request.
+  if (spawnTerminalOf(input.spawnRequestId) === "agent.ready") return;
+  recordSpawnTerminal(input.spawnRequestId, "agent.spawn_failed");
   emitRelayEvent({
     type: "agent.spawn_failed",
     source: "server",
@@ -182,4 +251,5 @@ export function notifyAgentSpawnFailed(input: { parent: string; spawnRequestId?:
 /** Test seam: clear the in-memory ready-tracking set between tests. */
 export function __resetReadyTracking(): void {
   readyFired.clear();
+  spawnTerminalBySpawnRequest.clear();
 }

package/src/branch-landed.ts

@@ -64,7 +64,7 @@ function landedCommitSubjects(input: BranchLandedInput, log: BranchLandedLogFn):
   const tipSha = cleanRef(input.mergedSha);
   if (!repoRoot || !baseSha || !tipSha || baseSha === tipSha) return undefined;
   try {
-    const proc = Bun.spawnSync(["git", "-C", repoRoot, "log", "--format=%s", `${baseSha}..${tipSha}`], {
+    const proc = Bun.spawnSync(["git", "-C", repoRoot, "log", "--format=%B%x00", `${baseSha}..${tipSha}`], {
       stdin: "ignore",
       stdout: "pipe",
       stderr: "pipe",
@@ -74,7 +74,7 @@ function landedCommitSubjects(input: BranchLandedInput, log: BranchLandedLogFn):
       log(`issue lifecycle commit range scan failed for ${input.workspace.id} (${baseSha.slice(0, 12)}..${tipSha.slice(0, 12)}): ${error}; falling back to tip subject`);
       return undefined;
     }
-    const subjects = Buffer.from(proc.stdout).toString("utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    const subjects = Buffer.from(proc.stdout).toString("utf8").split("\x00").map((msg) => msg.trim()).filter(Boolean);
     return subjects.length > 0 ? subjects : undefined;
   } catch (error) {
     log(`issue lifecycle commit range scan failed for ${input.workspace.id} (${baseSha.slice(0, 12)}..${tipSha.slice(0, 12)}): ${errMessage(error)}; falling back to tip subject`);

package/src/commands-db.ts

@@ -139,13 +139,16 @@ function defaultExpiresAt(type: string, now: number): number | undefined {
 function recordShutdownTerminalEventOnSuccess(previousStatus: CommandStatus, command: Command): void {
   if (command.type !== "agent.shutdown" || command.status !== "succeeded" || previousStatus === "succeeded") return;
   const agentId = stringValue(command.result?.agentId) ?? stringValue(command.params?.agentId);
-  if (!agentId || getTerminalEvent(agentId)) return;
+  if (!agentId) return;
   const agent = getAgent(agentId);
+  const epoch = agent?.epoch ?? 0;
+  if (getTerminalEvent(agentId, epoch)) return;
   const hadReadySignal = !agent || agent.ready || typeof agent.meta?.readyNotifiedAt === "number";
   recordTerminalEvent({
     agentId,
     reason: stringValue(command.result?.reason) ?? stringValue(command.params?.reason) ?? "shutdown",
     kind: hadReadySignal ? "agent.exited" : "agent.spawn_failed",
+    epoch,
     createdAt: Date.now(),
   });
 }

package/src/config.ts

@@ -80,6 +80,13 @@ export const MAX_BODY_BYTES = 64 * 1024;
 export const INTEGRATION_RATE_LIMIT_PER_MINUTE = envPositiveInt("AGENT_RELAY_INTEGRATION_RATE_LIMIT_PER_MINUTE", 120);
 
 export const AUTOMATION_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_AUTOMATION_INTERVAL_MS", 30 * 1000);
+export const SCHEDULER_TICK_MS = envNumberOrDefault("AGENT_RELAY_SCHEDULER_TICK_MS", 5 * 1000);
+export const SCHEDULER_CLAIM_LEASE_MS = envNumberOrDefault("AGENT_RELAY_SCHEDULER_CLAIM_LEASE_MS", 60 * 1000);
+export const SCHEDULER_RETRY_MS = envNumberOrDefault("AGENT_RELAY_SCHEDULER_RETRY_MS", 60 * 1000);
+export const SCHEDULER_MAX_FAILURES = envNumberOrDefault("AGENT_RELAY_SCHEDULER_MAX_FAILURES", 3);
+export const SCHEDULER_MIN_RECURRING_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_SCHEDULER_MIN_RECURRING_INTERVAL_MS", 60 * 1000);
+export const SCHEDULER_MAX_ACTIVE_TIMERS = envNumberOrDefault("AGENT_RELAY_SCHEDULER_MAX_ACTIVE_TIMERS", 100);
+export const SCHEDULER_MAX_PAYLOAD_BYTES = envNumberOrDefault("AGENT_RELAY_SCHEDULER_MAX_PAYLOAD_BYTES", 16 * 1024);
 export const IDLE_TIMEOUT_SECONDS = envNumberOrDefault("AGENT_RELAY_IDLE_TIMEOUT_SECONDS", 255);
 export const LOG_REQUESTS = process.env.AGENT_RELAY_LOG_REQUESTS === "1";
 

package/src/db/index.ts

@@ -37,3 +37,4 @@ export * from "./project-entries.ts";
 export * from "./provisioning-registry.ts";
 export * from "./provider-quotas.ts";
 export * from "./provider-quota-leases.ts";
+export * from "./scheduled-timers.ts";

package/src/db/migrations.ts

@@ -838,6 +838,15 @@ export function applyMigrations(): void {
   // writes it), so it adds nothing to the legacy flow.
   ensureDriveLeaseSchema();
 
+  // #652 — epoch column on terminal_events: a stale event from a prior epoch must NOT suppress the
+  // heartbeat-gap alert for a genuine silent death in a later epoch (agent_id reuse across epochs).
+  // Default 0 preserves the suppression behaviour for all existing rows written before this migration.
+  const terminalEventCols = (getDb().query("PRAGMA table_info(terminal_events)").all() as any[]).map((c: any) => c.name);
+  if (!terminalEventCols.includes("epoch")) {
+    getDb().run("ALTER TABLE terminal_events ADD COLUMN epoch INTEGER NOT NULL DEFAULT 0");
+    getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
+  }
+
   // Built-in agents — registered unconditionally so sends from these ids
   // pass the sendMessage validation. The reaper exempts these by checking
   // meta.builtin (or by id for "user").
@@ -865,4 +874,47 @@ export function applyMigrations(): void {
     `);
   }
 
+  // #652 follow-up — the epoch migration above used CREATE INDEX IF NOT EXISTS, but existing prod
+  // DBs already had idx_terminal_events_agent under the old (agent_id, created_at DESC) shape from
+  // #645; that made the new three-column definition a silent no-op. Drop-and-recreate forces the
+  // correct shape on all DB vintages; fresh DBs already carry the right shape and are unaffected.
+  getDb().run("DROP INDEX IF EXISTS idx_terminal_events_agent");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
+
+  // #361 — persistent Relay scheduler. Keep this append-only at the end: older DBs may
+  // have reached this point through many schema vintages, and the table references messages.
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS scheduled_timers (
+      id TEXT PRIMARY KEY,
+      target TEXT NOT NULL,
+      created_by TEXT NOT NULL,
+      created_by_kind TEXT NOT NULL,
+      created_by_scopes TEXT NOT NULL DEFAULT '[]',
+      created_by_constraints TEXT,
+      delivery_from TEXT NOT NULL,
+      created_at INTEGER NOT NULL,
+      next_run_at INTEGER NOT NULL,
+      schedule_kind TEXT NOT NULL,
+      schedule_def TEXT NOT NULL,
+      subject TEXT,
+      body TEXT NOT NULL,
+      payload TEXT NOT NULL DEFAULT '{}',
+      enabled INTEGER NOT NULL DEFAULT 1,
+      run_count INTEGER NOT NULL DEFAULT 0,
+      max_runs INTEGER,
+      expires_at INTEGER,
+      last_run_at INTEGER,
+      last_error TEXT,
+      consecutive_failures INTEGER NOT NULL DEFAULT 0,
+      idempotency_key TEXT,
+      correlation_id TEXT NOT NULL,
+      last_message_id INTEGER REFERENCES messages(id),
+      claimed_by TEXT,
+      claim_expires_at INTEGER
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_scheduled_timers_due ON scheduled_timers(enabled, next_run_at, claim_expires_at)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_scheduled_timers_creator ON scheduled_timers(created_by, enabled, next_run_at)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_scheduled_timers_correlation ON scheduled_timers(correlation_id)");
+
 }

package/src/db/scheduled-timers.ts

@@ -0,0 +1,297 @@
+import { randomUUID } from "node:crypto";
+import { getDb } from "./connection.ts";
+import type { TokenConstraints } from "../types";
+
+export type TimerScheduleKind = "once" | "interval";
+
+export interface TimerScheduleDef {
+  afterMs?: number;
+  at?: string;
+  everyMs?: number;
+}
+
+export interface ScheduledTimer {
+  id: string;
+  target: string;
+  createdBy: string;
+  createdByKind: string;
+  createdByScopes: string[];
+  createdByConstraints?: TokenConstraints;
+  deliveryFrom: string;
+  createdAt: number;
+  nextRunAt: number;
+  scheduleKind: TimerScheduleKind;
+  scheduleDef: TimerScheduleDef;
+  subject?: string;
+  body: string;
+  payload: Record<string, unknown>;
+  enabled: boolean;
+  runCount: number;
+  maxRuns?: number;
+  expiresAt?: number;
+  lastRunAt?: number;
+  lastError?: string;
+  consecutiveFailures: number;
+  idempotencyKey?: string;
+  correlationId: string;
+  lastMessageId?: number;
+  claimedBy?: string;
+  claimExpiresAt?: number;
+}
+
+interface ScheduledTimerRow {
+  id: string;
+  target: string;
+  created_by: string;
+  created_by_kind: string;
+  created_by_scopes: string;
+  created_by_constraints: string | null;
+  delivery_from: string;
+  created_at: number;
+  next_run_at: number;
+  schedule_kind: string;
+  schedule_def: string;
+  subject: string | null;
+  body: string;
+  payload: string;
+  enabled: number;
+  run_count: number;
+  max_runs: number | null;
+  expires_at: number | null;
+  last_run_at: number | null;
+  last_error: string | null;
+  consecutive_failures: number;
+  idempotency_key: string | null;
+  correlation_id: string;
+  last_message_id: number | null;
+  claimed_by: string | null;
+  claim_expires_at: number | null;
+}
+
+export interface CreateScheduledTimerRowInput {
+  target: string;
+  createdBy: string;
+  createdByKind: string;
+  createdByScopes: string[];
+  createdByConstraints?: TokenConstraints;
+  deliveryFrom: string;
+  nextRunAt: number;
+  scheduleKind: TimerScheduleKind;
+  scheduleDef: TimerScheduleDef;
+  subject?: string;
+  body: string;
+  payload?: Record<string, unknown>;
+  maxRuns?: number;
+  expiresAt?: number;
+  idempotencyKey?: string;
+}
+
+export function insertScheduledTimer(input: CreateScheduledTimerRowInput, now = Date.now()): ScheduledTimer {
+  const id = randomUUID();
+  const correlationId = `timer:${id}`;
+  getDb().query(`
+    INSERT INTO scheduled_timers (
+      id, target, created_by, created_by_kind, created_by_scopes, created_by_constraints, delivery_from,
+      created_at, next_run_at, schedule_kind, schedule_def, subject, body, payload, enabled,
+      run_count, max_runs, expires_at, idempotency_key, correlation_id
+    )
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 0, ?, ?, ?, ?)
+  `).run(
+    id,
+    input.target,
+    input.createdBy,
+    input.createdByKind,
+    JSON.stringify(input.createdByScopes),
+    input.createdByConstraints ? JSON.stringify(input.createdByConstraints) : null,
+    input.deliveryFrom,
+    now,
+    input.nextRunAt,
+    input.scheduleKind,
+    JSON.stringify(input.scheduleDef),
+    input.subject ?? null,
+    input.body,
+    JSON.stringify(input.payload ?? {}),
+    input.maxRuns ?? null,
+    input.expiresAt ?? null,
+    input.idempotencyKey ?? null,
+    correlationId,
+  );
+  return getScheduledTimer(id)!;
+}
+
+export function getScheduledTimer(id: string): ScheduledTimer | null {
+  const row = getDb().query("SELECT * FROM scheduled_timers WHERE id = ?").get(id) as ScheduledTimerRow | undefined;
+  return row ? rowToTimer(row) : null;
+}
+
+export function listScheduledTimers(input: {
+  createdBy?: string;
+  includeDisabled?: boolean;
+  limit?: number;
+} = {}): ScheduledTimer[] {
+  const where: string[] = [];
+  const params: Array<string | number> = [];
+  if (input.createdBy) {
+    where.push("created_by = ?");
+    params.push(input.createdBy);
+  }
+  if (!input.includeDisabled) where.push("enabled = 1");
+  const limit = Math.min(Math.max(input.limit ?? 100, 1), 500);
+  const sql = `SELECT * FROM scheduled_timers${where.length ? ` WHERE ${where.join(" AND ")}` : ""} ORDER BY next_run_at ASC, created_at ASC LIMIT ?`;
+  const rows = getDb().query(sql).all(...params, limit) as ScheduledTimerRow[];
+  return rows.map(rowToTimer);
+}
+
+export function countActiveScheduledTimers(createdBy: string): number {
+  const row = getDb()
+    .query("SELECT count(*) AS count FROM scheduled_timers WHERE created_by = ? AND enabled = 1")
+    .get(createdBy) as { count: number };
+  return row.count;
+}
+
+export function listDueScheduledTimerIds(now: number, limit = 25): string[] {
+  const rows = getDb().query(`
+    SELECT id FROM scheduled_timers
+    WHERE enabled = 1
+      AND next_run_at <= ?
+      AND (claim_expires_at IS NULL OR claim_expires_at <= ?)
+    ORDER BY next_run_at ASC, created_at ASC
+    LIMIT ?
+  `).all(now, now, limit) as Array<{ id: string }>;
+  return rows.map((row) => row.id);
+}
+
+export function claimScheduledTimer(id: string, owner: string, now: number, leaseMs: number): ScheduledTimer | null {
+  const claimExpiresAt = now + leaseMs;
+  const result = getDb().query(`
+    UPDATE scheduled_timers
+    SET claimed_by = ?, claim_expires_at = ?
+    WHERE id = ?
+      AND enabled = 1
+      AND next_run_at <= ?
+      AND (claim_expires_at IS NULL OR claim_expires_at <= ?)
+  `).run(owner, claimExpiresAt, id, now, now);
+  return result.changes === 0 ? null : getScheduledTimer(id);
+}
+
+export function completeScheduledTimerRun(input: {
+  timer: ScheduledTimer;
+  messageId: number;
+  now: number;
+}): ScheduledTimer {
+  const runCount = input.timer.runCount + 1;
+  const everyMs = input.timer.scheduleKind === "interval" ? input.timer.scheduleDef.everyMs : undefined;
+  const candidateNextRunAt = everyMs ? Math.max(input.now + everyMs, input.timer.nextRunAt + everyMs) : null;
+  const reachedMaxRuns = input.timer.maxRuns !== undefined && runCount >= input.timer.maxRuns;
+  const expiredBeforeNext = input.timer.expiresAt !== undefined &&
+    (candidateNextRunAt === null || candidateNextRunAt > input.timer.expiresAt);
+  const enabled = Boolean(candidateNextRunAt && !reachedMaxRuns && !expiredBeforeNext);
+
+  getDb().query(`
+    UPDATE scheduled_timers
+    SET run_count = ?,
+        last_run_at = ?,
+        last_error = NULL,
+        consecutive_failures = 0,
+        last_message_id = ?,
+        next_run_at = COALESCE(?, next_run_at),
+        enabled = ?,
+        claimed_by = NULL,
+        claim_expires_at = NULL
+    WHERE id = ?
+  `).run(runCount, input.now, input.messageId, candidateNextRunAt, enabled ? 1 : 0, input.timer.id);
+  return getScheduledTimer(input.timer.id)!;
+}
+
+export function recordScheduledTimerFailure(input: {
+  id: string;
+  error: string;
+  now: number;
+  retryMs: number;
+  maxFailures: number;
+}): ScheduledTimer | null {
+  const current = getScheduledTimer(input.id);
+  if (!current) return null;
+  const failures = current.consecutiveFailures + 1;
+  const enabled = failures < input.maxFailures;
+  getDb().query(`
+    UPDATE scheduled_timers
+    SET last_error = ?,
+        consecutive_failures = ?,
+        next_run_at = ?,
+        enabled = ?,
+        claimed_by = NULL,
+        claim_expires_at = NULL
+    WHERE id = ?
+  `).run(input.error, failures, input.now + input.retryMs, enabled ? 1 : 0, input.id);
+  return getScheduledTimer(input.id);
+}
+
+export function cancelScheduledTimer(id: string, now = Date.now()): ScheduledTimer | null {
+  const result = getDb().query(`
+    UPDATE scheduled_timers
+    SET enabled = 0, claimed_by = NULL, claim_expires_at = NULL, last_error = COALESCE(last_error, 'cancelled')
+    WHERE id = ? AND enabled = 1
+  `).run(id);
+  if (result.changes === 0) return getScheduledTimer(id);
+  return getScheduledTimer(id);
+}
+
+export function disableScheduledTimer(id: string, reason: string): ScheduledTimer | null {
+  getDb().query(`
+    UPDATE scheduled_timers
+    SET enabled = 0, last_error = ?, claimed_by = NULL, claim_expires_at = NULL
+    WHERE id = ?
+  `).run(reason, id);
+  return getScheduledTimer(id);
+}
+
+function rowToTimer(row: ScheduledTimerRow): ScheduledTimer {
+  return {
+    id: row.id,
+    target: row.target,
+    createdBy: row.created_by,
+    createdByKind: row.created_by_kind,
+    createdByScopes: parseStringArray(row.created_by_scopes),
+    createdByConstraints: parseRecord(row.created_by_constraints) as TokenConstraints | undefined,
+    deliveryFrom: row.delivery_from,
+    createdAt: row.created_at,
+    nextRunAt: row.next_run_at,
+    scheduleKind: row.schedule_kind === "interval" ? "interval" : "once",
+    scheduleDef: parseRecord(row.schedule_def) as TimerScheduleDef,
+    subject: row.subject ?? undefined,
+    body: row.body,
+    payload: parseRecord(row.payload) ?? {},
+    enabled: row.enabled === 1,
+    runCount: row.run_count,
+    maxRuns: row.max_runs ?? undefined,
+    expiresAt: row.expires_at ?? undefined,
+    lastRunAt: row.last_run_at ?? undefined,
+    lastError: row.last_error ?? undefined,
+    consecutiveFailures: row.consecutive_failures,
+    idempotencyKey: row.idempotency_key ?? undefined,
+    correlationId: row.correlation_id,
+    lastMessageId: row.last_message_id ?? undefined,
+    claimedBy: row.claimed_by ?? undefined,
+    claimExpiresAt: row.claim_expires_at ?? undefined,
+  };
+}
+
+function parseRecord(raw: string | null): Record<string, unknown> | undefined {
+  if (!raw) return undefined;
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function parseStringArray(raw: string): string[] {
+  try {
+    const parsed = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : [];
+  } catch {
+    return [];
+  }
+}

package/src/db/schema.ts

@@ -164,6 +164,37 @@ export function initDb(path: string = "agent-relay.db"): Database {
     -- per candidate row (O(n^2)): ~8.6s at 4k rows, which blew the 5s Stop-hook
     -- timeout and wedged turns in "busy" (#199).
     CREATE INDEX IF NOT EXISTS idx_msg_reply_to ON messages(reply_to, from_agent);
+    CREATE TABLE IF NOT EXISTS scheduled_timers (
+      id TEXT PRIMARY KEY,
+      target TEXT NOT NULL,
+      created_by TEXT NOT NULL,
+      created_by_kind TEXT NOT NULL,
+      created_by_scopes TEXT NOT NULL DEFAULT '[]',
+      created_by_constraints TEXT,
+      delivery_from TEXT NOT NULL,
+      created_at INTEGER NOT NULL,
+      next_run_at INTEGER NOT NULL,
+      schedule_kind TEXT NOT NULL,
+      schedule_def TEXT NOT NULL,
+      subject TEXT,
+      body TEXT NOT NULL,
+      payload TEXT NOT NULL DEFAULT '{}',
+      enabled INTEGER NOT NULL DEFAULT 1,
+      run_count INTEGER NOT NULL DEFAULT 0,
+      max_runs INTEGER,
+      expires_at INTEGER,
+      last_run_at INTEGER,
+      last_error TEXT,
+      consecutive_failures INTEGER NOT NULL DEFAULT 0,
+      idempotency_key TEXT,
+      correlation_id TEXT NOT NULL,
+      last_message_id INTEGER REFERENCES messages(id),
+      claimed_by TEXT,
+      claim_expires_at INTEGER
+    );
+    CREATE INDEX IF NOT EXISTS idx_scheduled_timers_due ON scheduled_timers(enabled, next_run_at, claim_expires_at);
+    CREATE INDEX IF NOT EXISTS idx_scheduled_timers_creator ON scheduled_timers(created_by, enabled, next_run_at);
+    CREATE INDEX IF NOT EXISTS idx_scheduled_timers_correlation ON scheduled_timers(correlation_id);
     CREATE TABLE IF NOT EXISTS teams (
       id TEXT PRIMARY KEY,
       coordinator_id TEXT,

package/src/db/terminal-events.ts

@@ -10,6 +10,7 @@ interface TerminalEventRow {
   agent_id: string;
   reason: string;
   kind: string;
+  epoch: number;
   created_at: number;
 }
 
@@ -20,10 +21,11 @@ export function ensureTerminalEventsSchema(): void {
       agent_id TEXT NOT NULL,
       reason TEXT NOT NULL,
       kind TEXT NOT NULL,
+      epoch INTEGER NOT NULL DEFAULT 0,
       created_at INTEGER NOT NULL
     )
   `);
-  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, created_at DESC)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
 }
 
 export interface TerminalEvent {
@@ -32,25 +34,29 @@ export interface TerminalEvent {
   reason: string;
   /** Relay event kind that terminated the agent, e.g. "agent.exited", "agent.spawn_failed". */
   kind: string;
+  /** Agent epoch at the time of the terminal event. A stale event from an older epoch must not
+   *  suppress a death alert in a later epoch (#652). */
+  epoch: number;
   createdAt: number;
 }
 
 export function recordTerminalEvent(event: TerminalEvent): void {
   getDb()
-    .query("INSERT INTO terminal_events (agent_id, reason, kind, created_at) VALUES (?, ?, ?, ?)")
-    .run(event.agentId, event.reason, event.kind, event.createdAt);
+    .query("INSERT INTO terminal_events (agent_id, reason, kind, epoch, created_at) VALUES (?, ?, ?, ?, ?)")
+    .run(event.agentId, event.reason, event.kind, event.epoch, event.createdAt);
 }
 
-/** The most recent terminal event for an agent, or null if none was ever recorded. */
-export function getTerminalEvent(agentId: string): TerminalEvent | null {
+/** The most recent terminal event for an agent at the given epoch, or null if none was recorded. */
+export function getTerminalEvent(agentId: string, epoch: number): TerminalEvent | null {
   const row = getDb()
-    .query("SELECT * FROM terminal_events WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
-    .get(agentId) as TerminalEventRow | undefined;
+    .query("SELECT * FROM terminal_events WHERE agent_id = ? AND epoch = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId, epoch) as TerminalEventRow | undefined;
   if (!row) return null;
-  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, createdAt: row.created_at };
+  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, epoch: row.epoch, createdAt: row.created_at };
 }
 
-/** True when any terminal event (clean exit, spawn-fail, reap) was recorded for the agent. */
-export function hasTerminalEvent(agentId: string): boolean {
-  return getTerminalEvent(agentId) !== null;
+/** True when a terminal event (clean exit, spawn-fail, reap) was recorded for the agent at the given epoch.
+ *  A terminal event from an older epoch does NOT suppress a death alert for the current epoch (#652). */
+export function hasTerminalEvent(agentId: string, epoch: number): boolean {
+  return getTerminalEvent(agentId, epoch) !== null;
 }

package/src/index.ts

@@ -13,6 +13,7 @@ import { startConnectorStatusPoller } from "./connectors";
 import { installNotificationSubscriptionRule } from "./notification-subscription-rules";
 import { installQuotaAdvisoryHandler } from "./quota-advisory";
 import { startPlanLiveBindings } from "./services/plan-live-bindings";
+import { startRelayScheduler } from "./services/scheduler";
 import { resolve, sep } from "path";
 import { gzipSync, brotliCompressSync, constants as zlibConstants } from "node:zlib";
 import {
@@ -85,6 +86,7 @@ function startServer(): void {
   reconcileRelayVersion();
   startConnectorStatusPoller();
   startMaintenanceScheduler();
+  startRelayScheduler();
 
   setInterval(() => {
     emitAutomationReconciliations(reconcileAutomationRuns());