agent-relay-server 0.84.0 → 0.86.0

package/src/agent-death-diagnosis.ts

@@ -0,0 +1,255 @@
+// #636 — crash/exit post-mortem classifier. A self-contained, pure mapping from the evidence
+// captured at an agent's death (orchestrator exit diagnostics + the server's own agent state)
+// to a single likely-cause label + supporting tail. It does NO IO: every host-side probe (OOM
+// cgroup/journal, tmux pane status, log tail) is gathered upstream (orchestrator
+// `diagnoseSessionExit`, runner provider-exit diagnostics) and handed in here, so the verdict is
+// deterministic and unit-testable per failure mode. The output rides the #633 terminal event and
+// is persisted as a CrashReport so the WHY survives the reaper.
+
+import { extractClaudeModelUnavailableMessage } from "agent-relay-sdk";
+import type { AgentCard, AgentDeathCause, AgentDeathDiagnosis, CrashReport, ManagedSessionExitDiagnostics } from "./types";
+
+export interface DeathEvidence {
+  /** The offline cause the caller already has (e.g. "heartbeat lost", "shutdown", exit reason). */
+  reason: string;
+  /** Orchestrator/runner-captured exit diagnostics, when the death was observed as a session exit. */
+  exit?: ManagedSessionExitDiagnostics;
+  /** The dead agent's `meta` blob (lastError, lastExit, terminalProviderExit, terminalFailureReason, auth…). */
+  agentMeta?: Record<string, unknown>;
+  /** Pre-death agent status, for distinguishing a busy stall from an idle one. */
+  status?: string;
+  /** Server's last-seen heartbeat for the agent. */
+  lastHeartbeatAt?: number;
+  /** Last time the agent reported busy (turn activity). */
+  lastBusyAt?: number;
+  /** Active work titles at death. */
+  activeWork?: string[];
+  /** Evaluation clock (defaults to Date.now()). */
+  now?: number;
+}
+
+const SIGNAL_RE = /\bSIG(KILL|TERM|SEGV|ABRT|BUS|INT|HUP|QUIT)\b/i;
+const OOM_RE = /\b(out of memory|oom[- ]?kill(?:ed|er)?|killed process \d+|cannot allocate memory|javascript heap out of memory)\b/i;
+const TOKEN_401_RE = /\b(401|403\b.*token|unauthorized|invalid[_ ]token|token (?:expired|revoked|invalid)|authentication failed|please run\s+\/login|run \/login|oauth token)\b/i;
+const ONBOARDING_RE = /(do you trust|trust this folder|onboarding|first[- ]run|select a theme|choose your theme|accept the terms|welcome to claude)/i;
+const CRASH_RE = /\b(panic|segfault|segmentation fault|core dumped|traceback \(most recent|unhandled (?:exception|rejection)|fatal error|abort(?:ed)?\b|stack overflow)\b/i;
+const TRANSPORT_RE = /\b(heartbeat|transport|websocket|connection (?:lost|closed|reset)|disconnect)\b/i;
+
+function pushUnique(list: string[], value: string | undefined): void {
+  if (value && !list.includes(value)) list.push(value);
+}
+
+function record(meta: Record<string, unknown> | undefined, key: string): Record<string, unknown> | undefined {
+  const value = meta?.[key];
+  return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : undefined;
+}
+
+function str(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function num(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+/** Resolve a termination signal name + exit code from the orchestrator exit diagnostics or agent meta. */
+function resolveSignalAndCode(ev: DeathEvidence): { signal?: string; exitCode?: number } {
+  const exit = ev.exit;
+  let signal = exit?.signal;
+  let exitCode = exit?.exitCode;
+  const systemd = exit?.systemd;
+  if (!signal && systemd?.execMainCode && /killed|dumped|signal/i.test(systemd.execMainCode)) {
+    // systemd ExecMainStatus carries the signal name/number when ExecMainCode is killed/dumped.
+    signal = systemd.execMainStatus ? `SIG${systemd.execMainStatus.replace(/^SIG/i, "")}` : undefined;
+  }
+  if (exitCode === undefined && systemd?.execMainCode === "exited" && systemd.execMainStatus) {
+    const n = Number(systemd.execMainStatus);
+    if (Number.isFinite(n)) exitCode = n;
+  }
+  const tpe = record(ev.agentMeta, "terminalProviderExit");
+  if (!signal) signal = str(tpe?.signal);
+  if (exitCode === undefined) exitCode = num(tpe?.exitCode);
+  return { signal, exitCode };
+}
+
+/** Build the haystack of free text scanned by the known-pattern probes. */
+function haystack(ev: DeathEvidence): string {
+  const parts: string[] = [ev.reason];
+  if (ev.exit?.logTail) parts.push(...ev.exit.logTail);
+  pushUnique(parts, ev.exit?.lastError);
+  pushUnique(parts, str(ev.agentMeta?.lastError));
+  pushUnique(parts, str(ev.agentMeta?.terminalFailureReason));
+  pushUnique(parts, str(ev.agentMeta?.terminalFailureMessage));
+  const lastExit = record(ev.agentMeta, "lastExit");
+  pushUnique(parts, str(lastExit?.lastError));
+  const tpe = record(ev.agentMeta, "terminalProviderExit");
+  pushUnique(parts, str(tpe?.reason));
+  return parts.join("\n");
+}
+
+function logTailOf(ev: DeathEvidence): string[] | undefined {
+  if (ev.exit?.logTail?.length) return ev.exit.logTail;
+  const lastExit = record(ev.agentMeta, "lastExit");
+  const tail = lastExit?.logTail;
+  if (Array.isArray(tail) && tail.length) return tail.filter((x): x is string => typeof x === "string");
+  return undefined;
+}
+
+/**
+ * Classify an agent death into a single likely-cause label with supporting evidence (#636).
+ * Pure — all host probes are pre-gathered into {@link DeathEvidence}. Probe order is by
+ * conclusiveness: a clean exit code never masks a token-401 / onboarding gate that also exits 0.
+ */
+export function classifyAgentDeath(ev: DeathEvidence): AgentDeathDiagnosis {
+  const now = ev.now ?? Date.now();
+  const text = haystack(ev);
+  const logTail = logTailOf(ev);
+  const { signal, exitCode } = resolveSignalAndCode(ev);
+  const evidence: string[] = [];
+
+  const systemdResult = ev.exit?.systemd?.result;
+  const resumeId = str(record(ev.agentMeta, "terminalProviderExit")?.claudeResumeId);
+  const heartbeatGapMs = ev.lastHeartbeatAt !== undefined ? Math.max(0, now - ev.lastHeartbeatAt) : undefined;
+
+  const base: Pick<AgentDeathDiagnosis, "signal" | "exitCode" | "logTail" | "lastHeartbeatAt" | "heartbeatGapMs" | "lastBusyAt" | "activeWork" | "detectedAt"> = {
+    ...(signal ? { signal } : {}),
+    ...(exitCode !== undefined ? { exitCode } : {}),
+    ...(logTail ? { logTail } : {}),
+    ...(ev.lastHeartbeatAt !== undefined ? { lastHeartbeatAt: ev.lastHeartbeatAt } : {}),
+    ...(heartbeatGapMs !== undefined ? { heartbeatGapMs } : {}),
+    ...(ev.lastBusyAt !== undefined ? { lastBusyAt: ev.lastBusyAt } : {}),
+    ...(ev.activeWork?.length ? { activeWork: ev.activeWork } : {}),
+    detectedAt: now,
+  };
+
+  const verdict = (cause: AgentDeathCause, confidence: AgentDeathDiagnosis["confidence"], summary: string, oom?: boolean): AgentDeathDiagnosis => ({
+    cause,
+    confidence,
+    summary,
+    evidence,
+    ...(oom !== undefined ? { oom } : {}),
+    ...base,
+  });
+
+  // 1) OOM — most conclusive: positive cgroup/journal/systemd signal, then log text.
+  if (ev.exit?.oom) {
+    pushUnique(evidence, `OOM detected via ${ev.exit.oom.source}${ev.exit.oom.detail ? `: ${ev.exit.oom.detail}` : ""}`);
+    return verdict("oom", "high", "Out-of-memory kill", true);
+  }
+  if (systemdResult === "oom-kill") {
+    pushUnique(evidence, "systemd Result=oom-kill");
+    return verdict("oom", "high", "Out-of-memory kill (systemd Result=oom-kill)", true);
+  }
+  if (OOM_RE.test(text)) {
+    pushUnique(evidence, "log/error text matched an out-of-memory pattern");
+    return verdict("oom", "medium", "Likely out-of-memory (log pattern)", true);
+  }
+
+  // 2) token-401 — auth expiry/revocation (cf. managed-token-revocation). Can exit 0, so before clean-exit.
+  if (TOKEN_401_RE.test(text)) {
+    pushUnique(evidence, "log/error text matched a token-401 / auth-failure pattern");
+    return verdict("token-401", "high", "Token expired/revoked (401 auth failure)");
+  }
+
+  // 3) onboarding-gate — first-run/trust gate crash-loop (cf. isolated-claude onboarding gate).
+  if (ONBOARDING_RE.test(text)) {
+    const fast = (ev.exit?.runtimeMs ?? Infinity) < 60_000;
+    if (fast) pushUnique(evidence, `died fast (${Math.round((ev.exit?.runtimeMs ?? 0) / 1000)}s) at a first-run/onboarding gate`);
+    else pushUnique(evidence, "log/error text matched an onboarding/trust-gate pattern");
+    return verdict("onboarding-gate", fast ? "high" : "medium", "Stuck at a first-run onboarding/trust gate");
+  }
+
+  // 4) provider-crash — CLI panic/segfault/model-unavailable, or a crash signal.
+  const modelUnavailable = extractClaudeModelUnavailableMessage(text);
+  if (modelUnavailable) {
+    pushUnique(evidence, modelUnavailable);
+    return verdict("provider-crash", "high", "Provider unavailable (model/CLI error)");
+  }
+  if (signal && /SEGV|ABRT|BUS/i.test(signal)) {
+    pushUnique(evidence, `provider terminated by ${signal}`);
+    return verdict("provider-crash", "high", `Provider crashed (${signal})`);
+  }
+  if (CRASH_RE.test(text)) {
+    pushUnique(evidence, "log/error text matched a crash pattern (panic/segfault/traceback)");
+    return verdict("provider-crash", "medium", "Likely provider crash (log pattern)");
+  }
+
+  // 5) killed — external SIGKILL/SIGTERM that isn't OOM and isn't a parent-initiated shutdown.
+  if (signal && /KILL|TERM|INT|HUP|QUIT/i.test(signal)) {
+    pushUnique(evidence, `process terminated by ${signal}`);
+    return verdict("killed", "high", `Process killed (${signal})`);
+  }
+
+  // 6) clean-exit — exit 0 / systemd success / Claude native auto-compaction resume handoff.
+  if (resumeId) {
+    pushUnique(evidence, `clean self-exit with a Claude resume id (${resumeId}) — native context auto-compaction`);
+    return verdict("clean-exit", "high", "Clean self-exit (Claude auto-compaction / resume handoff)");
+  }
+  if (exitCode === 0 || systemdResult === "success") {
+    pushUnique(evidence, exitCode === 0 ? "process exited 0" : "systemd Result=success");
+    return verdict("clean-exit", "high", "Clean exit (exit code 0)");
+  }
+  if (exitCode !== undefined && exitCode > 0) {
+    pushUnique(evidence, `process exited non-zero (code ${exitCode})`);
+    return verdict("provider-crash", "medium", `Non-zero exit (code ${exitCode})`);
+  }
+
+  // 7) transport-loss — heartbeat/bus gap with no exit diagnostics: the process may even be alive,
+  //    but the relay lost the agent. This is the silent-stall backstop (#634).
+  if (TRANSPORT_RE.test(ev.reason) && !ev.exit) {
+    pushUnique(evidence, `lost contact: ${ev.reason}${heartbeatGapMs !== undefined ? ` (heartbeat gap ${Math.round(heartbeatGapMs / 1000)}s)` : ""}`);
+    return verdict("transport-loss", "medium", "Lost heartbeat/transport (no exit captured)");
+  }
+
+  pushUnique(evidence, ev.reason || "no conclusive signal");
+  return verdict("unknown", "low", "Cause undetermined");
+}
+
+function activeWorkTitles(meta: Record<string, unknown> | undefined): string[] | undefined {
+  const work = meta?.activeWork;
+  if (!Array.isArray(work) || work.length === 0) return undefined;
+  const titles = work
+    .map((item) => (item && typeof item === "object" ? str((item as Record<string, unknown>).title) ?? str((item as Record<string, unknown>).kind) : undefined))
+    .filter((x): x is string => Boolean(x));
+  return titles.length ? titles : undefined;
+}
+
+/**
+ * Assemble {@link DeathEvidence} from a live agent card + the offline reason (and any orchestrator
+ * exit diagnostics), then classify. The single seam every death-detection site calls so evidence
+ * extraction (heartbeat, active work, lastExit) stays DRY.
+ */
+export function diagnoseAgentDeath(input: {
+  agent?: AgentCard | null;
+  reason: string;
+  exit?: ManagedSessionExitDiagnostics;
+  now?: number;
+}): AgentDeathDiagnosis {
+  const meta = input.agent?.meta;
+  return classifyAgentDeath({
+    reason: input.reason,
+    exit: input.exit,
+    agentMeta: meta,
+    status: input.agent?.status,
+    lastHeartbeatAt: input.agent?.lastSeen,
+    activeWork: activeWorkTitles(meta),
+    now: input.now,
+  });
+}
+
+/** Build a persistable {@link CrashReport} from a dead agent's card + its diagnosis. */
+export function crashReportFor(agent: AgentCard, reason: string, diagnosis: AgentDeathDiagnosis): CrashReport {
+  const provider = str(agent.meta?.provider) ?? agent.providerCapabilities?.model?.provider;
+  const spawnRequestId = str(agent.meta?.spawnRequestId);
+  return {
+    agentId: agent.id,
+    ...(agent.spawnedBy ? { parentId: agent.spawnedBy } : {}),
+    ...(agent.teamId ? { teamId: agent.teamId } : {}),
+    ...(spawnRequestId ? { spawnRequestId } : {}),
+    ...(provider ? { provider } : {}),
+    ...(agent.label ? { label: agent.label } : {}),
+    reason,
+    diagnosis,
+    createdAt: diagnosis.detectedAt,
+  };
+}

package/src/agent-lifecycle-events.ts

@@ -1,8 +1,14 @@
 import { emitRelayEvent } from "./events";
 import { getNotificationsConfig } from "./config-store";
 import { routeNotification } from "./notification-router";
-import { getAgent } from "./db";
-import type { AgentCard } from "./types";
+import { getAgent, markAgentReadyNotified, recordCrashReport, recordTerminalEvent } from "./db";
+import { crashReportFor } from "./agent-death-diagnosis";
+import type { AgentCard, AgentDeathDiagnosis } from "./types";
+
+/** One-line cause tag appended to a death notification body, e.g. " · why: Clean exit (exit code 0)". */
+function diagnosisSuffix(diagnosis?: AgentDeathDiagnosis): string {
+  return diagnosis ? ` · why: ${diagnosis.summary}` : "";
+}
 
 // #308/#239 — async spawn lifecycle signals. A spawn-capable agent fires a spawn, gets its
 // spawnRequestId back immediately, and is WOKEN on ready/exit/failure instead of burning its
@@ -11,15 +17,22 @@ import type { AgentCard } from "./types";
 
 // Spawned children that have fired `agent.ready` during this relay lifetime. Serves two jobs:
 // (1) dedup the ready push — it must fire exactly once per child, not on every heartbeat
-// re-registration; and (2) classify a later offline transition: a child IN this set exited
-// cleanly (`agent.exited`); one that never entered it went offline without ever becoming ready,
+// re-registration; and (2) classify a later offline transition: a child that HAD become ready
+// exited cleanly (`agent.exited`); one that never did went offline without ever becoming ready,
 // i.e. a spawn that failed to come up (`agent.spawn_failed` — the "process started ≠ ready"
-// crash-loop / onboarding-gate mode the isolated-Claude failure proves). In-memory by design: a
-// child still alive across a relay restart was ready long ago and its parent already knows, so a
-// post-restart exit reads as the benign `agent.exited` default — a restart must never manufacture
-// a spurious failure alert.
+// crash-loop / onboarding-gate mode the isolated-Claude failure proves).
+//
+// This set is in-memory, so a relay restart wipes it. A restart must never manufacture a spurious
+// failure alert for a child that was ready long ago — so readiness is ALSO persisted as a durable
+// `readyNotifiedAt` meta stamp (markAgentReadyNotified). Classification consults the set first and
+// falls back to the persisted stamp, so a post-restart clean exit still reads as `agent.exited`.
 const readyFired = new Set<string>();
 
+/** A child counts as "had become ready" if the live set saw it OR the durable stamp survives a restart. */
+function everReady(child: AgentCard): boolean {
+  return typeof child.meta?.readyNotifiedAt === "number";
+}
+
 function describe(child: AgentCard): string {
   return child.label ? `${child.label} (\`${child.id}\`)` : `\`${child.id}\``;
 }
@@ -46,6 +59,9 @@ export function notifyAgentReady(childId: string): void {
   if (!child || !parent) return;
   if (readyFired.has(childId)) return;
   readyFired.add(childId);
+  // Persist a durable readiness stamp so a post-restart offline edge still classifies as exited
+  // (the in-memory set is wiped on restart). Idempotent — only stamped on first ready.
+  markAgentReadyNotified(childId);
 
   emitRelayEvent({
     type: "agent.ready",
@@ -70,24 +86,36 @@ export function notifyAgentReady(childId: string): void {
 }
 
 /**
- * A spawned child went offline (heartbeat-reaped, or shut down). Classifies via `readyFired`: a
- * child that had become ready exited (`agent.exited`); one that never did failed to come up
- * (`agent.spawn_failed`). Routes to the `spawned_by` parent. No-op for non-spawned agents.
+ * A spawned child went offline (heartbeat-reaped, or shut down). Classifies via readiness (live set
+ * or the durable `readyNotifiedAt` stamp): a child that had become ready exited (`agent.exited`);
+ * one that never did failed to come up (`agent.spawn_failed`). Routes to the `spawned_by` parent.
+ * No-op for non-spawned agents.
  * `reason` is the offline cause (e.g. "heartbeat lost", "shutdown").
  */
-export function notifyAgentOffline(childId: string, reason: string): void {
+export function notifyAgentOffline(childId: string, reason: string, diagnosis?: AgentDeathDiagnosis): void {
   const child = getAgent(childId);
   const parent = child?.spawnedBy;
   if (!child || !parent) return;
-  const wasReady = readyFired.delete(childId);
+  // Classify from the live set OR the durable readiness stamp. The `delete` always runs (cleanup of
+  // the in-memory set); the persisted stamp covers a child that became ready before a relay restart.
+  const wasReady = readyFired.delete(childId) || everReady(child);
   const config = getNotificationsConfig();
 
+  // #636 — persist the post-mortem keyed by agentId so the WHY survives the reaper, regardless of
+  // whether the parent push is enabled. Recorded once per terminal-event emission.
+  if (diagnosis) recordCrashReport(crashReportFor(child, reason, diagnosis));
+
+  // #645 — persist a terminal marker for ANY offline terminus (clean exit or spawn-fail) keyed by
+  // agentId so it outlives the reaper. The #634 watchdog reads this to skip a benign shutdown; it
+  // must fire ONLY on a genuine silent death that emitted no terminal event at all.
+  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", createdAt: Date.now() });
+
   if (wasReady) {
     emitRelayEvent({
       type: "agent.exited",
       source: "server",
       subject: childId,
-      data: { agentId: childId, parent, reason, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child) },
+      data: { agentId: childId, parent, reason, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child), ...(diagnosis ? { diagnosis } : {}) },
     });
     if (!config.enabled || !config.agentExited) return;
     routeNotification({
@@ -96,8 +124,8 @@ export function notifyAgentOffline(childId: string, reason: string): void {
       recipients: [parent],
       message: {
         subject: "Spawned agent exited",
-        body: `⏹ Your spawned agent ${describe(child)} exited — ${reason}.`,
-        payload: { kind: "agent.exited", agentId: childId, parent, reason },
+        body: `⏹ Your spawned agent ${describe(child)} exited — ${reason}${diagnosisSuffix(diagnosis)}.`,
+        payload: { kind: "agent.exited", agentId: childId, parent, reason, ...(diagnosis ? { diagnosis } : {}) },
         replyExpected: false,
       },
     });
@@ -108,7 +136,7 @@ export function notifyAgentOffline(childId: string, reason: string): void {
     type: "agent.spawn_failed",
     source: "server",
     subject: childId,
-    data: { agentId: childId, parent, reason, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child), neverReady: true },
+    data: { agentId: childId, parent, reason, provider: providerOf(child), spawnRequestId: spawnRequestIdOf(child), neverReady: true, ...(diagnosis ? { diagnosis } : {}) },
   });
   if (!config.enabled || !config.agentSpawnFailed) return;
   routeNotification({
@@ -117,8 +145,8 @@ export function notifyAgentOffline(childId: string, reason: string): void {
     recipients: [parent],
     message: {
       subject: "Spawn failed",
-      body: `❌ Your spawned agent ${describe(child)} went offline before ever becoming ready — ${reason}. Likely a crash-loop, onboarding gate, or bad launch; call relay_spawn_targets to re-check provider availability before retrying.`,
-      payload: { kind: "agent.spawn_failed", agentId: childId, parent, reason },
+      body: `❌ Your spawned agent ${describe(child)} went offline before ever becoming ready — ${reason}${diagnosisSuffix(diagnosis)}. Likely a crash-loop, onboarding gate, or bad launch; call relay_spawn_targets to re-check provider availability before retrying.`,
+      payload: { kind: "agent.spawn_failed", agentId: childId, parent, reason, ...(diagnosis ? { diagnosis } : {}) },
       replyExpected: false,
     },
   });

package/src/branch-landed.ts

@@ -257,6 +257,87 @@ export function notifyBranchLandFailed(input: {
   });
 }
 
+/**
+ * Steward-handoff awareness wake (#641/#642). When the relay can't auto-merge a branch and
+ * hands the workspace to a steward, the two ends used to be mutually blind: the branch agent
+ * was told only that land failed (never which steward holds it), and its coordinator — the
+ * agent responsible for the merge landing — was never looped in at all. This fires BOTH
+ * notices off one handoff event:
+ *   - the OWNER branch agent learns which steward took over (#641), and
+ *   - the branch agent's COORDINATOR (owner's `spawnedBy`, resolved in steward-identity.ts)
+ *     is proactively notified so it keeps the big picture (#642).
+ * The steward id is the managed steward's policy name (reachable via `policy:<name>`) or an
+ * elected steward's agent id, whichever the caller has in scope.
+ */
+export function notifyStewardHandoff(input: {
+  workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "baseRef" | "ownerAgentId">;
+  /** The steward now holding the merge — managed steward policy name or elected agent id. */
+  stewardId: string;
+  /** The branch agent's coordinator (owner's `spawnedBy` lineage), or null when none resolves. */
+  coordinatorAgentId?: string | null;
+  /** Why it was handed over ("conflict", "merge state unknown", auto-merge back-off, …). */
+  reason?: string;
+}): void {
+  const { workspace } = input;
+  const base = workspace.baseRef ?? "base";
+  const owner = workspace.ownerAgentId;
+  const coordinator = input.coordinatorAgentId?.trim() || undefined;
+  const branchLabel = workspace.branch ? `\`${workspace.branch}\`` : "Your branch";
+  const reasonLabel = input.reason ? ` (${input.reason})` : "";
+  const payload = {
+    kind: "workspace.steward-handoff",
+    workspaceId: workspace.id,
+    repoRoot: workspace.repoRoot,
+    branch: workspace.branch,
+    base,
+    author: owner,
+    stewardId: input.stewardId,
+    coordinatorAgentId: coordinator,
+    reason: input.reason,
+  };
+
+  emitRelayEvent({ type: "workspace.steward_handoff", source: "server", subject: workspace.id, data: payload });
+
+  const config = getNotificationsConfig();
+  if (!config.enabled) return;
+
+  // #641 — tell the owner branch agent which steward now holds its merge, so it stops
+  // thinking the branch silently failed. Durable (ttl null): the owner must learn this
+  // even if it went idle. Mirror the "don't touch main" guardrail of the landed-failure wake.
+  if (owner) {
+    routeNotification({
+      type: "workspace.steward_handoff",
+      scope: { author: owner, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
+      recipients: [owner],
+      ttlMs: null,
+      message: {
+        subject: "Your merge is now with a steward",
+        body: `🛟 ${branchLabel} couldn't be auto-merged to \`${base}\` and is now being handled by steward agent \`${input.stewardId}\`${reasonLabel}. The steward will rebase/resolve and land it. Do NOT push, rebase, merge, or touch the main checkout yourself; stay idle until you're woken with the result.`,
+        payload: { ...payload, notification: "steward-handoff" },
+        replyExpected: false,
+      },
+    });
+  }
+
+  // #642 — proactively loop in the branch agent's coordinator so it keeps the big picture
+  // (it spawned the branch agent and owns the merge landing). Skip when the coordinator IS
+  // the owner (no real lineage) to avoid a duplicate ping.
+  if (coordinator && coordinator !== owner) {
+    routeNotification({
+      type: "workspace.steward_handoff",
+      scope: { author: owner, parent: coordinator, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
+      recipients: [coordinator],
+      ttlMs: null,
+      message: {
+        subject: "A branch agent's merge is now with a steward",
+        body: `🛟 Your branch agent \`${owner ?? "?"}\`'s merge (${branchLabel} → \`${base}\`) couldn't be auto-merged and is now with steward agent \`${input.stewardId}\`${reasonLabel}. The steward will land it or escalate to you if it can't.`,
+        payload: { ...payload, notification: "steward-handoff", audience: "coordinator" },
+        replyExpected: false,
+      },
+    });
+  }
+}
+
 /**
  * Finalize a workspace whose work landed in base out-of-band — a pr-strategy land whose
  * PR merged (by anyone, incl. `gh pr merge`), or a squash/cherry-pick already in base

package/src/bus.ts

@@ -7,6 +7,7 @@ import { getCommand, updateCommand } from "./commands-db";
 import { emitCommandEvent } from "./command-events";
 import { getLifecycleManager } from "./lifecycle-manager";
 import { noteAgentTimelineEvent, noteCompactionCommandCompleted } from "./compaction-watch";
+import { reconcileTerminalProviderExit } from "./services/terminal-provider-exit";
 import { registerAgent } from "./services/register-agent";
 import { authContextFromBus } from "./services/auth-context";
 import { commandAuthorizationResource, dispatchCommand } from "./services/dispatch-command";
@@ -143,10 +144,19 @@ function handleFrame(ws: BusWebSocket, frame: ReturnType<typeof validateClientFr
     case "heartbeat":
       withRegistered(ws, frame, (conn) => {
         if (conn.agentId) {
-          const runtime = runtimeFromMeta(frame.payload.meta);
-          heartbeat(conn.agentId, { instanceId: conn.instanceId, epoch: conn.epoch }, runtime);
-          if (frame.payload.status) setStatus(conn.agentId, frame.payload.status, { instanceId: conn.instanceId, epoch: conn.epoch });
-          emitAgentStatusEvent(conn.agentId);
+          // #633 — a runner whose provider terminally exited stays alive and keeps sending idle
+          // keepalives (semanticStatus is never "offline"). Those must NOT resurrect an agent the
+          // server already marked offline for that terminal exit, or the death would keep masking
+          // itself. Ignore the keepalive (the row ages out via the normal offline-prune); a genuine
+          // resume sends a `status`/`register` frame, which clears the hold below.
+          const current = getAgent(conn.agentId);
+          const heldOffline = current?.status === "offline" && isRecord(current.meta?.terminalProviderExit);
+          if (!heldOffline) {
+            const runtime = runtimeFromMeta(frame.payload.meta);
+            heartbeat(conn.agentId, { instanceId: conn.instanceId, epoch: conn.epoch }, runtime);
+            if (frame.payload.status) setStatus(conn.agentId, frame.payload.status, { instanceId: conn.instanceId, epoch: conn.epoch });
+            emitAgentStatusEvent(conn.agentId);
+          }
         }
         if (frame.id) send(ws, { type: "ack", payload: { frameId: frame.id } });
       });
@@ -180,6 +190,10 @@ function handleFrame(ws: BusWebSocket, frame: ReturnType<typeof validateClientFr
         emitAgentStatusEvent(conn.agentId);
         // #237 stop-hook → instant ⚪↔🟡 changes detection on the turn-end edge.
         probeWorkspaceOnTurnEnd(before, after);
+        // #633 — silent clean-exit: a runner that stays alive after its provider terminally exited
+        // marks the offline edge with `terminalProviderExit`; the owning service turns that into a
+        // terminal event + #636 diagnosis (and clears the hold on a genuine resume).
+        reconcileTerminalProviderExit(conn.agentId, before, after);
       });
       return;
     case "ack":

package/src/cli/index.ts

@@ -72,7 +72,7 @@ Pair examples:
   agent-relay /message codex "Can you look at that failing action?"
   agent-relay /reply 206 "Sounds good, I'll take a look"
   agent-relay /react 206 👍
-  agent-relay /reply 206 --stdin < result.md
+  agent-relay /reply 206 --body-file result.md
   agent-relay get-message 206 --json
   agent-relay /reply 206 --format markdown "Here is **formatted** text"
   agent-relay /send-claimable tag:backend "Please claim and fix the failing API test"
@@ -166,8 +166,8 @@ Current session
 
 Replying
   agent-relay /reply <messageId> "<response>"
-  agent-relay /reply <messageId> --stdin < response.md
   agent-relay /reply <messageId> --body-file response.md
+  printf '%s\\n' "<response>" | agent-relay /reply <messageId> --stdin
     Reply to the sender of a relay message. Prefer this over /message when
     handling an incoming relay message because the server keeps the routing
     and channel context. Large replies are uploaded as an artifact
@@ -246,7 +246,7 @@ Rules of thumb
     agent-relay get-message 123
 
   If your reply is long, avoid shell quoting and use:
-    agent-relay /reply 123 --stdin < response.md
+    agent-relay /reply 123 --body-file response.md
 
   If you need to know who you are in Relay, run:
     agent-relay /status --json

package/src/cli/reply.ts

@@ -41,6 +41,7 @@ export async function handleReplyCommand(args: string[]): Promise<void> {
   const body = (await resolveBodyInput({ bodyParts, stdinBody, bodyFile })).trim();
   if (!from) throw new Error("Could not detect current Agent Relay ID. Pass --from AGENT_ID or set AGENT_RELAY_ID.");
   if (!body) throw new Error("Reply body required.");
+  if (body === "--stdin") throw new Error('Refusing to send literal "--stdin" as a reply body. Provide real reply text, use --body-file PATH, or pipe non-empty content into --stdin.');
 
   const prepared = await prepareReplyBody({ body, from, replyTo, format, subject });
   const message = await apiRequest("POST", "/api/messages", {
@@ -75,7 +76,11 @@ async function resolveBodyInput(options: {
     options.bodyFile ? "body file" : "",
   ].filter(Boolean);
   if (sources.length > 1) throw new Error("Reply body must come from exactly one source: arguments, --stdin, or --body-file.");
-  if (options.stdinBody) return readStdin();
+  if (options.stdinBody) {
+    const body = await readStdin();
+    if (!body.trim()) throw new Error("--stdin was given but no input was received.");
+    return body;
+  }
   if (options.bodyFile) return readFileSync(resolve(options.bodyFile), "utf8");
   return options.bodyParts.join(" ");
 }

package/src/config.ts

@@ -129,6 +129,10 @@ export function maxArtifactBytesFromEnv(fallback: number): number {
 }
 
 export const OUTBOX_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_OUTBOX_RETENTION_MS", 60 * 60 * 1000);
+// Per-provider quota history retention (#609) — durable burn-rate history for the sparkline/analytics
+// charts (#607/#608). Default 30 days; the quota-history-prune maintenance reaper drops older entries.
+export const QUOTA_HISTORY_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_QUOTA_HISTORY_RETENTION_MS", 30 * DAY_MS);
+export const QUOTA_HISTORY_PRUNE_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_QUOTA_HISTORY_PRUNE_INTERVAL_MS", 6 * 60 * 60 * 1000);
 export const TOKEN_RECORD_RETENTION_SECONDS = envNumberOrDefault("AGENT_RELAY_TOKEN_RECORD_RETENTION_SECONDS", 7 * 24 * 60 * 60);
 export const CONFLICT_SCAN_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_CONFLICT_SCAN_INTERVAL_MS", 10 * 60 * 1000);
 export const WORKSPACE_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_WORKSPACE_RETENTION_MS", DAY_MS);

package/src/db/agents.ts

@@ -331,6 +331,36 @@ export function mergeAgentMeta(id: string, meta: Record<string, unknown>, guard?
   return result.changes > 0;
 }
 
+/** #633/#634 — persist a durable "this child became ready" marker. Death classification
+ *  (`notifyAgentOffline`) keys `agent.exited` vs `agent.spawn_failed` off readiness, but the live
+ *  signal is an in-memory set that a relay restart wipes — so a long-ready child's later clean exit
+ *  would mis-fire `spawn_failed`. This stamp survives the restart in SQLite, so the offline edge
+ *  still classifies as `agent.exited`. Idempotent: only stamped once (first ready). */
+export function markAgentReadyNotified(id: string, at: number = Date.now()): boolean {
+  const agent = getAgent(id);
+  if (!agent) return false;
+  if (typeof agent.meta?.readyNotifiedAt === "number") return false;
+  const merged = { ...(agent.meta ?? {}), readyNotifiedAt: at };
+  return getDb().query("UPDATE agents SET meta = ? WHERE id = ?").run(JSON.stringify(merged), id).changes > 0;
+}
+
+/** Delete specific keys from an agent's meta blob. Returns true if anything changed. Used to clear
+ *  the #633 `terminalProviderExit` hold when a dead provider genuinely resumes. */
+export function clearAgentMetaKeys(id: string, keys: string[]): boolean {
+  const agent = getAgent(id);
+  if (!agent?.meta) return false;
+  const meta = { ...agent.meta };
+  let changed = false;
+  for (const key of keys) {
+    if (key in meta) {
+      delete meta[key];
+      changed = true;
+    }
+  }
+  if (!changed) return false;
+  return getDb().query("UPDATE agents SET meta = ? WHERE id = ?").run(JSON.stringify(meta), id).changes > 0;
+}
+
 export function recordAgentExitDiagnostics(id: string, diagnostics: ManagedSessionExitDiagnostics): AgentCard | null {
   const agent = getAgent(id);
   if (!agent) return null;
@@ -341,9 +371,12 @@ export function recordAgentExitDiagnostics(id: string, diagnostics: ManagedSessi
     lastExit: diagnostics,
     lastExitAt: diagnostics.detectedAt || now,
   };
+  // #636 — do NOT clobber last_seen with `now`: it is the agent's genuine last heartbeat, and the
+  // death diagnosis derives the real heartbeat gap from it (the orchestrator exit path diagnoses
+  // off the row returned here). "When we detected the exit" lives in meta.lastExitAt instead.
   const result = getDb()
-    .query("UPDATE agents SET status = 'offline', ready = 0, meta = ?, last_seen = ? WHERE id = ?")
-    .run(JSON.stringify(merged), now, id);
+    .query("UPDATE agents SET status = 'offline', ready = 0, meta = ? WHERE id = ?")
+    .run(JSON.stringify(merged), id);
   if (result.changes <= 0) return null;
   closeOpenPairsForAgent(id, now);
   return getAgent(id);