agent-relay-server 0.88.2 → 0.88.3

package/docs/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "Agent Relay API",
-    "version": "0.88.2",
+    "version": "0.88.3",
     "description": "Real-time message bus for inter-agent communication. Agent-first: this spec is designed for machine consumption — agents can self-discover the full API surface via GET /api/spec.",
     "license": {
       "name": "MIT",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-server",
-  "version": "0.88.2",
+  "version": "0.88.3",
   "description": "Lightweight HTTP message relay for inter-agent communication across machines",
   "module": "src/index.ts",
   "type": "module",

package/runner/src/adapter.ts

@@ -25,6 +25,10 @@ export interface ProviderStatusEvent {
   label?: string;
   role?: string;
   parentId?: string;
+  // OS pids of the process(es) this busy work tracks (#650). When supplied, the
+  // runner reconciles them against actual liveness and clears the busy marker if
+  // every pid is dead — recovering from a tracked process killed out-of-band.
+  pids?: number[];
   providerState?: Record<string, unknown>;
   metadata?: Record<string, unknown>;
 }

package/src/agent-lifecycle-events.ts

@@ -28,6 +28,34 @@ function diagnosisSuffix(diagnosis?: AgentDeathDiagnosis): string {
 // falls back to the persisted stamp, so a post-restart clean exit still reads as `agent.exited`.
 const readyFired = new Set<string>();
 
+// #648 — terminal-outcome guard per spawnRequestId, enforcing the invariant "exactly one terminal
+// event per spawn". `agent.ready` and `agent.spawn_failed` are mutually exclusive for one request,
+// yet two paths can race: the orchestrator can report the `agent.spawn` COMMAND failed (a transient
+// launch/registration timeout) at the same moment the child actually registers and becomes ready.
+// Observed live (issue #648): spawn_failed delivered, then ready for the SAME spawnRequestId — a
+// false negative a coordinator acts on (double-spawn or false-abandon).
+//
+// We record the first terminal seen, keyed on spawnRequestId (the idempotency key shared by the
+// command's `params.spawnRequestId` and the child's `meta.spawnRequestId`):
+//   - a late spawn_failed after ready  → SUPPRESSED (the agent is alive, the failure is wrong)
+//   - a ready after a delivered spawn_failed → CORRECTED with `agent.spawn_recovered` so a
+//     coordinator that already acted on the false failure can reconcile (do not retry / un-abandon).
+// `ready` is sticky — once a request is ready it never downgrades to failed. In-memory like
+// `readyFired`: the race it guards is same-moment, so a relay restart between the two is not a concern.
+type SpawnTerminal = "agent.ready" | "agent.spawn_failed";
+const spawnTerminalBySpawnRequest = new Map<string, SpawnTerminal>();
+
+function spawnTerminalOf(spawnRequestId: string | undefined): SpawnTerminal | undefined {
+  return spawnRequestId === undefined ? undefined : spawnTerminalBySpawnRequest.get(spawnRequestId);
+}
+
+function recordSpawnTerminal(spawnRequestId: string | undefined, outcome: SpawnTerminal): void {
+  if (spawnRequestId === undefined) return;
+  // ready is terminal-sticky: never let a (stale/contradicting) spawn_failed overwrite a live ready.
+  if (spawnTerminalBySpawnRequest.get(spawnRequestId) === "agent.ready") return;
+  spawnTerminalBySpawnRequest.set(spawnRequestId, outcome);
+}
+
 /** A child counts as "had become ready" if the live set saw it OR the durable stamp survives a restart. */
 function everReady(child: AgentCard): boolean {
   return typeof child.meta?.readyNotifiedAt === "number";
@@ -63,15 +91,48 @@ export function notifyAgentReady(childId: string): void {
   // (the in-memory set is wiped on restart). Idempotent — only stamped on first ready.
   markAgentReadyNotified(childId);
 
+  const spawnRequestId = spawnRequestIdOf(child);
+  // #648 — if a spawn_failed was already emitted for this request, this ready RETRACTS it: the agent
+  // actually came up, so the earlier failure was a false negative. Emit a corrective recovery instead
+  // of a plain ready notice so a coordinator that acted on the failure can reconcile.
+  const recovered = spawnTerminalOf(spawnRequestId) === "agent.spawn_failed";
+  recordSpawnTerminal(spawnRequestId, "agent.ready");
+
   emitRelayEvent({
     type: "agent.ready",
     source: "server",
     subject: childId,
-    data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId: spawnRequestIdOf(child) },
+    data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId },
   });
 
+  if (recovered) {
+    emitRelayEvent({
+      type: "agent.spawn_recovered",
+      source: "server",
+      subject: childId,
+      data: { agentId: childId, parent, provider: providerOf(child), label: child.label, spawnRequestId, supersedes: "agent.spawn_failed" },
+    });
+  }
+
   const config = getNotificationsConfig();
-  if (!config.enabled || !config.agentReady) return;
+  if (!config.enabled) return;
+  if (recovered) {
+    // A correction must always reach a parent that got the (false) failure — gate only on the global
+    // switch, NOT the agentReady sub-toggle, so the retraction lands even if ready pushes are off.
+    routeNotification({
+      type: "agent.spawn_recovered",
+      scope: { agentId: childId, parent },
+      recipients: [parent],
+      message: {
+        subject: "Spawn recovered",
+        body: `♻️ Correction: your earlier spawn failure${spawnRequestId ? ` for \`${spawnRequestId}\`` : ""} was a false alarm — ${describe(child)} actually registered and is ready and idle. Disregard the failure and do NOT retry; send it work with relay_send_message to \`${childId}\`.`,
+        payload: { kind: "agent.spawn_recovered", agentId: childId, parent, provider: providerOf(child), spawnRequestId, supersedes: "agent.spawn_failed" },
+        replyExpected: false,
+      },
+    });
+    return;
+  }
+  if (!config.agentReady) return;
   routeNotification({
     type: "agent.ready",
     scope: { agentId: childId, parent },
@@ -79,7 +140,7 @@ export function notifyAgentReady(childId: string): void {
     message: {
       subject: "Spawned agent ready",
       body: `✅ Your spawned agent ${describe(child)} is ready and idle — send it work with relay_send_message to \`${childId}\`.`,
-      payload: { kind: "agent.ready", agentId: childId, parent, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child) },
+      payload: { kind: "agent.ready", agentId: childId, parent, provider: providerOf(child), spawnRequestId },
       replyExpected: false,
     },
   });
@@ -108,7 +169,7 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
   // #645 — persist a terminal marker for ANY offline terminus (clean exit or spawn-fail) keyed by
   // agentId so it outlives the reaper. The #634 watchdog reads this to skip a benign shutdown; it
   // must fire ONLY on a genuine silent death that emitted no terminal event at all.
-  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", createdAt: Date.now() });
+  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", epoch: child.epoch, createdAt: Date.now() });
 
   if (wasReady) {
     emitRelayEvent({
@@ -132,6 +193,9 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
     return;
   }
 
+  // #648 — record the terminal outcome so a (defensive) later ready for the same request is detected
+  // as a recovery and corrected, keeping the one-terminal-per-spawn invariant across both fail paths.
+  recordSpawnTerminal(spawnRequestIdOf(child), "agent.spawn_failed");
   emitRelayEvent({
     type: "agent.spawn_failed",
     source: "server",
@@ -158,6 +222,11 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
  * to the spawning parent threaded on the command, since there is no child agent record to resolve.
  */
 export function notifyAgentSpawnFailed(input: { parent: string; spawnRequestId?: string; provider?: string; reason: string }): void {
+  // #648 — if the spawn already became ready, the orchestrator's command-failure report is a false
+  // negative (the child registered despite the spawn command reporting failure). Suppress it: the
+  // ready already delivered the authoritative terminal outcome for this request.
+  if (spawnTerminalOf(input.spawnRequestId) === "agent.ready") return;
+  recordSpawnTerminal(input.spawnRequestId, "agent.spawn_failed");
   emitRelayEvent({
     type: "agent.spawn_failed",
     source: "server",
@@ -182,4 +251,5 @@ export function notifyAgentSpawnFailed(input: { parent: string; spawnRequestId?:
 /** Test seam: clear the in-memory ready-tracking set between tests. */
 export function __resetReadyTracking(): void {
   readyFired.clear();
+  spawnTerminalBySpawnRequest.clear();
 }

package/src/commands-db.ts

@@ -139,13 +139,16 @@ function defaultExpiresAt(type: string, now: number): number | undefined {
 function recordShutdownTerminalEventOnSuccess(previousStatus: CommandStatus, command: Command): void {
   if (command.type !== "agent.shutdown" || command.status !== "succeeded" || previousStatus === "succeeded") return;
   const agentId = stringValue(command.result?.agentId) ?? stringValue(command.params?.agentId);
-  if (!agentId || getTerminalEvent(agentId)) return;
+  if (!agentId) return;
   const agent = getAgent(agentId);
+  const epoch = agent?.epoch ?? 0;
+  if (getTerminalEvent(agentId, epoch)) return;
   const hadReadySignal = !agent || agent.ready || typeof agent.meta?.readyNotifiedAt === "number";
   recordTerminalEvent({
     agentId,
     reason: stringValue(command.result?.reason) ?? stringValue(command.params?.reason) ?? "shutdown",
     kind: hadReadySignal ? "agent.exited" : "agent.spawn_failed",
+    epoch,
     createdAt: Date.now(),
   });
 }

package/src/db/migrations.ts

@@ -838,6 +838,15 @@ export function applyMigrations(): void {
   // writes it), so it adds nothing to the legacy flow.
   ensureDriveLeaseSchema();
 
+  // #652 — epoch column on terminal_events: a stale event from a prior epoch must NOT suppress the
+  // heartbeat-gap alert for a genuine silent death in a later epoch (agent_id reuse across epochs).
+  // Default 0 preserves the suppression behaviour for all existing rows written before this migration.
+  const terminalEventCols = (getDb().query("PRAGMA table_info(terminal_events)").all() as any[]).map((c: any) => c.name);
+  if (!terminalEventCols.includes("epoch")) {
+    getDb().run("ALTER TABLE terminal_events ADD COLUMN epoch INTEGER NOT NULL DEFAULT 0");
+    getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
+  }
+
   // Built-in agents — registered unconditionally so sends from these ids
   // pass the sendMessage validation. The reaper exempts these by checking
   // meta.builtin (or by id for "user").
@@ -865,4 +874,11 @@ export function applyMigrations(): void {
     `);
   }
 
+  // #652 follow-up — the epoch migration above used CREATE INDEX IF NOT EXISTS, but existing prod
+  // DBs already had idx_terminal_events_agent under the old (agent_id, created_at DESC) shape from
+  // #645; that made the new three-column definition a silent no-op. Drop-and-recreate forces the
+  // correct shape on all DB vintages; fresh DBs already carry the right shape and are unaffected.
+  getDb().run("DROP INDEX IF EXISTS idx_terminal_events_agent");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
+
 }

package/src/db/terminal-events.ts

@@ -10,6 +10,7 @@ interface TerminalEventRow {
   agent_id: string;
   reason: string;
   kind: string;
+  epoch: number;
   created_at: number;
 }
 
@@ -20,10 +21,11 @@ export function ensureTerminalEventsSchema(): void {
       agent_id TEXT NOT NULL,
       reason TEXT NOT NULL,
       kind TEXT NOT NULL,
+      epoch INTEGER NOT NULL DEFAULT 0,
       created_at INTEGER NOT NULL
     )
   `);
-  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, created_at DESC)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, epoch, created_at DESC)");
 }
 
 export interface TerminalEvent {
@@ -32,25 +34,29 @@ export interface TerminalEvent {
   reason: string;
   /** Relay event kind that terminated the agent, e.g. "agent.exited", "agent.spawn_failed". */
   kind: string;
+  /** Agent epoch at the time of the terminal event. A stale event from an older epoch must not
+   *  suppress a death alert in a later epoch (#652). */
+  epoch: number;
   createdAt: number;
 }
 
 export function recordTerminalEvent(event: TerminalEvent): void {
   getDb()
-    .query("INSERT INTO terminal_events (agent_id, reason, kind, created_at) VALUES (?, ?, ?, ?)")
-    .run(event.agentId, event.reason, event.kind, event.createdAt);
+    .query("INSERT INTO terminal_events (agent_id, reason, kind, epoch, created_at) VALUES (?, ?, ?, ?, ?)")
+    .run(event.agentId, event.reason, event.kind, event.epoch, event.createdAt);
 }
 
-/** The most recent terminal event for an agent, or null if none was ever recorded. */
-export function getTerminalEvent(agentId: string): TerminalEvent | null {
+/** The most recent terminal event for an agent at the given epoch, or null if none was recorded. */
+export function getTerminalEvent(agentId: string, epoch: number): TerminalEvent | null {
   const row = getDb()
-    .query("SELECT * FROM terminal_events WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
-    .get(agentId) as TerminalEventRow | undefined;
+    .query("SELECT * FROM terminal_events WHERE agent_id = ? AND epoch = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId, epoch) as TerminalEventRow | undefined;
   if (!row) return null;
-  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, createdAt: row.created_at };
+  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, epoch: row.epoch, createdAt: row.created_at };
 }
 
-/** True when any terminal event (clean exit, spawn-fail, reap) was recorded for the agent. */
-export function hasTerminalEvent(agentId: string): boolean {
-  return getTerminalEvent(agentId) !== null;
+/** True when a terminal event (clean exit, spawn-fail, reap) was recorded for the agent at the given epoch.
+ *  A terminal event from an older epoch does NOT suppress a death alert for the current epoch (#652). */
+export function hasTerminalEvent(agentId: string, epoch: number): boolean {
+  return getTerminalEvent(agentId, epoch) !== null;
 }

package/src/maintenance/teams.ts

@@ -66,6 +66,7 @@ function detectSilentMembers(
     watch.set(member.id, {
       lastActiveAt: Math.max(member.lastSeen, existing?.lastActiveAt ?? 0),
       pinged: recovered ? false : existing?.pinged ?? false,
+      epoch: member.epoch,
       ...(member.label ? { label: member.label } : {}),
     });
   }
@@ -77,7 +78,9 @@ function detectSilentMembers(
     entry.pinged = true;
     // A terminal event already fired — a #633 crash report (death) or a #645 clean terminus
     // (transient reap after land / graceful shutdown). Either way this is not a silent death.
-    if (getCrashReport(memberId) || hasTerminalEvent(memberId)) continue;
+    // #652: only suppress when the terminal event's epoch matches the member's observed epoch —
+    // a stale event from a prior epoch must not mask a real death in the current epoch.
+    if (getCrashReport(memberId) || hasTerminalEvent(memberId, entry.epoch)) continue;
     if (!coordinatorId) continue;
     const minutes = Math.floor(gapMs / 60_000);
     notifyTarget(
@@ -124,6 +127,7 @@ function detectGoneCoordinator(
       coordinatorId,
       lastActiveAt: Math.max(coord.lastSeen, prior?.lastActiveAt ?? 0),
       escalated: false, // alive again ⇒ re-arm the escalation
+      epoch: coord.epoch,
       ...(coord.label ? { label: coord.label } : {}),
     });
     return;
@@ -138,7 +142,8 @@ function detectGoneCoordinator(
 
   // A terminal marker means a #645 clean terminus (graceful shutdown) or a #633 crash the operator was
   // already surfaced on — not a SILENT death. Suppress.
-  if (getCrashReport(coordinatorId) || hasTerminalEvent(coordinatorId)) {
+  // #652: only suppress when the terminal event's epoch matches the coordinator's observed epoch.
+  if (getCrashReport(coordinatorId) || hasTerminalEvent(coordinatorId, watch.epoch)) {
     out.suppressedTeamIds.push(teamId);
     return;
   }

package/src/notification-types.ts

@@ -4,6 +4,7 @@ export type NotificationType =
   | "agent.ready"
   | "agent.exited"
   | "agent.spawn_failed"
+  | "agent.spawn_recovered"
   | "agent.context_threshold"
   | "agent.stuck_busy"
   | "agent.quota_threshold"
@@ -59,6 +60,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.ready": null,
   "agent.exited": null,
   "agent.spawn_failed": null,
+  "agent.spawn_recovered": null,
   "agent.context_threshold": 15 * MINUTES,
   "agent.stuck_busy": null,
   "agent.quota_threshold": 15 * MINUTES,

package/src/quota-advisory.ts

@@ -7,15 +7,17 @@ import { parseJson } from "./utils";
 import { setProviderQuotaAdvisoryHandler, type ProviderQuotaAdvisoryInput } from "./db/provider-quotas.ts";
 import type { QuotaState } from "./types";
 
+type QuotaAdvisoryStage = "warn" | "critical" | "reset";
+
 interface QuotaAdvisoryWindowState {
   lastQuotaAdvisoryBand?: number;
+  lastQuotaAdvisoryStage?: QuotaAdvisoryStage;
   lastQuotaAdvisoryAt?: number;
   lastRemainingPercent?: number;
   lastResetsAt?: number;
 }
 
 type QuotaAdvisoryState = Record<string, QuotaAdvisoryWindowState>;
-type QuotaAdvisoryStage = "warn" | "critical" | "reset";
 
 type AgentRecipientRow = {
   id: string;
@@ -45,7 +47,7 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
     const remaining = quotaRemaining(window.utilization);
     const remainingPercent = percent(remaining);
     const previous = nextState[window.name];
-    const reset = previous?.lastQuotaAdvisoryBand !== undefined && isQuotaReset(remaining, previousWindow, window, config.resetRemaining);
+    const reset = hasLastQuotaAdvisory(previous) && isQuotaReset(remaining, previousWindow, window, config.resetRemaining);
     if (reset) {
       notifyQuotaAdvisory({
         provider: input.provider,
@@ -69,8 +71,8 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
     }
 
     const band = quotaRemainingBand(remaining, config.bandSize);
-    if (previous?.lastQuotaAdvisoryBand === band) continue;
     const stage: QuotaAdvisoryStage = remaining - config.criticalRemaining <= QUOTA_THRESHOLD_EPSILON ? "critical" : "warn";
+    if (lastQuotaAdvisoryStage(previous, config.criticalRemaining) === stage) continue;
     notifyQuotaAdvisory({
       provider: input.provider,
       accountKey: input.accountKey,
@@ -85,6 +87,7 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
     });
     nextState[window.name] = {
       lastQuotaAdvisoryBand: band,
+      lastQuotaAdvisoryStage: stage,
       lastQuotaAdvisoryAt: input.now,
       lastRemainingPercent: remainingPercent,
       ...(window.resetsAt !== undefined ? { lastResetsAt: window.resetsAt } : {}),
@@ -94,6 +97,27 @@ function handleProviderQuotaAdvisory(input: ProviderQuotaAdvisoryInput): string
   return JSON.stringify(nextState);
 }
 
+function hasLastQuotaAdvisory(previous: QuotaAdvisoryWindowState | undefined): boolean {
+  return previous?.lastQuotaAdvisoryStage !== undefined || previous?.lastQuotaAdvisoryBand !== undefined;
+}
+
+function lastQuotaAdvisoryStage(
+  previous: QuotaAdvisoryWindowState | undefined,
+  criticalRemaining: number,
+): QuotaAdvisoryStage | undefined {
+  if (!previous) return undefined;
+  if (previous.lastQuotaAdvisoryStage === "warn" || previous.lastQuotaAdvisoryStage === "critical") {
+    return previous.lastQuotaAdvisoryStage;
+  }
+  if (typeof previous.lastRemainingPercent === "number") {
+    return previous.lastRemainingPercent <= percent(criticalRemaining) ? "critical" : "warn";
+  }
+  if (typeof previous.lastQuotaAdvisoryBand === "number") {
+    return previous.lastQuotaAdvisoryBand <= percent(criticalRemaining) ? "critical" : "warn";
+  }
+  return undefined;
+}
+
 function notifyQuotaAdvisory(input: {
   provider: string;
   accountKey: string;

package/src/services/transient-agent-reaper.ts

@@ -137,7 +137,7 @@ export function reapTransientAgents(): Record<string, unknown> {
     emitRelayEvent({ type: "agent.exited", source: "server", subject: agent.id, data: { agentId: agent.id, parent: agent.spawnedBy, reason: "transient-complete", workspaceId: land.workspaceId, workspaceStatus: land.status } });
     // #645 — persist a clean terminal marker so the #634 heartbeat-gap watchdog reads this benign
     // completion as a terminus (not a silent death) once the reaped worker stops heartbeating.
-    recordTerminalEvent({ agentId: agent.id, reason: "transient-complete", kind: "agent.exited", createdAt: now });
+    recordTerminalEvent({ agentId: agent.id, reason: "transient-complete", kind: "agent.exited", epoch: agent.epoch, createdAt: now });
     createActivityEvent({
       clientId: `transient-agent-reaped-${agent.id}-${now}`,
       kind: "state",

package/src/team-idle-tracker.ts

@@ -20,6 +20,9 @@ export interface TeamMemberWatch {
   lastActiveAt: number;
   /** Whether the member's CURRENT silence has already pinged the coordinator (re-armed on recovery). */
   pinged: boolean;
+  /** Agent epoch at the time the member was last observed alive. Used to epoch-scope the terminal-event
+   *  suppression check (#652): a stale event from an older epoch must not suppress a current-epoch alert. */
+  epoch: number;
   /** Member label for a readable ping. */
   label?: string;
 }
@@ -39,6 +42,9 @@ export interface TeamCoordinatorWatch {
   lastActiveAt: number;
   /** Whether the coordinator's CURRENT gone-spell already escalated (re-armed when it recovers). */
   escalated: boolean;
+  /** Agent epoch at the time the coordinator was last observed alive. Used to epoch-scope the terminal-event
+   *  suppression check (#652): a stale event from an older epoch must not suppress a current-epoch alert. */
+  epoch: number;
   /** Coordinator label for a readable escalation. */
   label?: string;
 }