agent-relay-server 0.84.0 → 0.85.0

package/src/db/crash-reports.ts

@@ -0,0 +1,115 @@
+// #636 — persisted crash post-mortems. Stored in their own table (no FK to agents) so a death
+// verdict survives `pruneOfflineAgents` / `deleteAgent` and stays retrievable by agentId after the
+// worktree/session reaper has cleaned up. One row per (agentId, createdAt); the latest wins on read.
+import { getDb } from "./connection.ts";
+import { parseJson } from "../utils";
+import type { AgentDeathDiagnosis, CrashReport } from "../types";
+
+interface CrashReportRow {
+  agent_id: string;
+  parent_id: string | null;
+  team_id: string | null;
+  spawn_request_id: string | null;
+  provider: string | null;
+  label: string | null;
+  cause: string;
+  reason: string;
+  diagnosis: string;
+  created_at: number;
+}
+
+export function ensureCrashReportsSchema(): void {
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS crash_reports (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      agent_id TEXT NOT NULL,
+      parent_id TEXT,
+      team_id TEXT,
+      spawn_request_id TEXT,
+      provider TEXT,
+      label TEXT,
+      cause TEXT NOT NULL,
+      reason TEXT NOT NULL,
+      diagnosis TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_crash_reports_agent ON crash_reports(agent_id, created_at DESC)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_crash_reports_parent ON crash_reports(parent_id, created_at DESC)");
+}
+
+function rowToCrashReport(row: CrashReportRow): CrashReport {
+  return {
+    agentId: row.agent_id,
+    ...(row.parent_id ? { parentId: row.parent_id } : {}),
+    ...(row.team_id ? { teamId: row.team_id } : {}),
+    ...(row.spawn_request_id ? { spawnRequestId: row.spawn_request_id } : {}),
+    ...(row.provider ? { provider: row.provider } : {}),
+    ...(row.label ? { label: row.label } : {}),
+    reason: row.reason,
+    diagnosis: parseJson<AgentDeathDiagnosis>(row.diagnosis, {
+      cause: "unknown",
+      confidence: "low",
+      summary: "Cause undetermined",
+      evidence: [],
+      detectedAt: row.created_at,
+    }),
+    createdAt: row.created_at,
+  };
+}
+
+export function recordCrashReport(report: CrashReport): void {
+  getDb()
+    .query(
+      `INSERT INTO crash_reports (agent_id, parent_id, team_id, spawn_request_id, provider, label, cause, reason, diagnosis, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      report.agentId,
+      report.parentId ?? null,
+      report.teamId ?? null,
+      report.spawnRequestId ?? null,
+      report.provider ?? null,
+      report.label ?? null,
+      report.diagnosis.cause,
+      report.reason,
+      JSON.stringify(report.diagnosis),
+      report.createdAt,
+    );
+}
+
+/** The most recent crash report for an agent, or null if none was ever recorded. */
+export function getCrashReport(agentId: string): CrashReport | null {
+  const row = getDb()
+    .query("SELECT * FROM crash_reports WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId) as CrashReportRow | undefined;
+  return row ? rowToCrashReport(row) : null;
+}
+
+export interface ListCrashReportsFilter {
+  /** Restrict to children of this spawn-parent (the parent/self scope for the MCP tool). */
+  parentId?: string;
+  /** Restrict to a single agent. */
+  agentId?: string;
+  limit?: number;
+}
+
+export function listCrashReports(filter: ListCrashReportsFilter = {}): CrashReport[] {
+  const where: string[] = [];
+  const params: (string | number)[] = [];
+  if (filter.parentId) {
+    where.push("parent_id = ?");
+    params.push(filter.parentId);
+  }
+  if (filter.agentId) {
+    where.push("agent_id = ?");
+    params.push(filter.agentId);
+  }
+  const limit = Math.max(1, Math.min(filter.limit ?? 50, 200));
+  const rows = getDb()
+    .query(
+      `SELECT * FROM crash_reports ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY created_at DESC, id DESC LIMIT ?`,
+    )
+    .all(...params, limit) as CrashReportRow[];
+  return rows.map(rowToCrashReport);
+}

package/src/db/index.ts

@@ -27,6 +27,7 @@ export * from "./workspaces.ts";
 export * from "./merge-lease.ts";
 export * from "./continuations.ts";
 export * from "./continuation-archives.ts";
+export * from "./crash-reports.ts";
 export * from "./plans.ts";
 export * from "./teams.ts";
 export * from "./blackboard.ts";

package/src/db/migrations.ts

@@ -20,6 +20,7 @@ import { ensureAuthIdentitiesSchema } from "./auth-identities.ts";
 import { ensureAgentOwnershipSchema, seedDefaultAgentOwnership } from "./agent-ownership.ts";
 import { ensureDriveLeaseSchema } from "./drive-leases.ts";
 import { ensureContinuationArchiveSearchSchema } from "./continuation-archives.ts";
+import { ensureCrashReportsSchema } from "./crash-reports.ts";
 import { normalizeReactionEmoji } from "./mappers.ts";
 import { normalizeProviderQuotaAccountKeys } from "./provider-quotas.ts";
 import { projectRootsMissingIssueRepo, setProjectIssueRepo } from "./projects.ts";
@@ -795,6 +796,10 @@ export function applyMigrations(): void {
     )
   `);
 
+  // #636 — persisted crash post-mortems. Own table, no FK to agents, so a death verdict outlives
+  // the offline-prune/reaper and stays retrievable by agentId.
+  ensureCrashReportsSchema();
+
   // First-class human users (#388 keystone #393): ensure the table (single DDL home in users.ts)
   // and seed the admin-level default user mapped to the existing single human. P0 + idempotent —
   // bare 'user' == user:default resolve to users.id='default' via the SDK resolver; no bytes move.

package/src/maintenance/jobs.ts

@@ -7,6 +7,7 @@ import {
   createActivityEvent,
   evaluatePoolBindings,
   expireQueuedMessages,
+  getAgent,
   pruneOrphanedSharedWorkspaces,
   pruneOfflineAgents,
   pruneOldMessages,
@@ -19,6 +20,7 @@ import {
   sweepArtifacts,
 } from "../db";
 import { notifyAgentOffline } from "../agent-lifecycle-events";
+import { diagnoseAgentDeath } from "../agent-death-diagnosis";
 import { autoMergeCleanFastForwards } from "../workspace-auto-merge";
 import { reapOrphanedWorktrees } from "../workspace-orphans";
 import { getLifecycleManager } from "../lifecycle-manager";
@@ -173,7 +175,10 @@ export const definitions: MaintenanceJobDefinition[] = [
       for (const id of reapedAgentIds) {
         emitAgentStatus(id);
         getLifecycleManager().onAgentDisappeared(id);
-        notifyAgentOffline(id, "heartbeat lost");
+        // #636 — a heartbeat-lost death still gets a best-effort post-mortem from the agent's last
+        // known meta (lastExit/lastError/auth) + heartbeat gap, riding the terminal event.
+        const diagnosis = diagnoseAgentDeath({ agent: getAgent(id), reason: "heartbeat lost" });
+        notifyAgentOffline(id, "heartbeat lost", diagnosis);
         createActivityEvent({
           clientId: "server-agent-" + id + "-heartbeat-lost-" + Date.now(),
           kind: "state",

package/src/maintenance/teams.ts

@@ -2,6 +2,7 @@ import {
   clearTeamZeroMembersSince,
   countLiveTeamMembers,
   createActivityEvent,
+  getCrashReport,
   getTeamBusyState,
   getTeamRoster,
   listActiveTeams,
@@ -13,10 +14,69 @@ import { dissolveTeamById } from "../services/team";
 import { getTeamsConfig } from "../teams-config-store";
 import { teamReapGraceMs } from "./constants";
 import { notifyTarget } from "./events";
-import { teamIdleTracker } from "../team-idle-tracker";
+import { teamIdleTracker, teamMemberWatchTracker, type TeamMemberWatch } from "../team-idle-tracker";
+import type { AgentCard } from "../types";
 
 export function resetTeamIdleWatchdogForTests(): void {
   teamIdleTracker.clear();
+  teamMemberWatchTracker.clear();
+}
+
+/** A member counts as "alive" (worth watching for a future silence) when it's past onboarding and
+ *  heartbeating — i.e. a status that implies it became ready, not a never-started spawn. */
+function memberIsAlive(member: AgentCard, freshCutoff: number): boolean {
+  return (member.status === "idle" || member.status === "busy" || member.status === "online") && member.lastSeen > freshCutoff;
+}
+
+/**
+ * #634 — detect a team member that was ready/busy and then went silent (heartbeat-gap), EVEN IF it
+ * dropped from the roster entirely, and ping the coordinator once. The in-memory watch remembers
+ * each member's last-active time, so a vanished agent (no roster row) is still caught. Re-arms when
+ * a member recovers. Skips the ping when a #633 crash report already exists (the coordinator was
+ * already woken by the terminal event) — this is the backstop for the NO-terminal-event case.
+ */
+function detectSilentMembers(
+  teamId: string,
+  coordinatorId: string | undefined,
+  now: number,
+  thresholdMs: number,
+  out: { pingedMembers: string[]; watchedMembers: number },
+): void {
+  const freshCutoff = now - thresholdMs;
+  const watch = teamMemberWatchTracker.get(teamId) ?? new Map<string, TeamMemberWatch>();
+  teamMemberWatchTracker.set(teamId, watch);
+
+  for (const member of getTeamRoster(teamId)) {
+    if (member.id === coordinatorId) continue; // the coordinator is the recipient, not a watched worker
+    if (!memberIsAlive(member, freshCutoff)) continue;
+    const existing = watch.get(member.id);
+    // Recovery re-arms the ping: a member seen alive again after a silence can re-trigger later.
+    const recovered = existing ? member.lastSeen > existing.lastActiveAt : false;
+    watch.set(member.id, {
+      lastActiveAt: Math.max(member.lastSeen, existing?.lastActiveAt ?? 0),
+      pinged: recovered ? false : existing?.pinged ?? false,
+      ...(member.label ? { label: member.label } : {}),
+    });
+  }
+
+  out.watchedMembers += watch.size;
+  for (const [memberId, entry] of watch) {
+    const gapMs = now - entry.lastActiveAt;
+    if (gapMs < thresholdMs || entry.pinged) continue;
+    entry.pinged = true;
+    if (getCrashReport(memberId)) continue; // terminal event already woke the coordinator (#633)
+    if (!coordinatorId) continue;
+    const minutes = Math.floor(gapMs / 60_000);
+    notifyTarget(
+      "team.member_silent",
+      { teamId, agentId: memberId },
+      coordinatorId,
+      "Team member went silent",
+      `Member \`${entry.label ?? memberId}\` in team \`${teamId}\` has sent no heartbeat for ${minutes} min — it may have died without a terminal event. Check it and restart its unfinished work, or stand it down.`,
+      { kind: "team.member_silent", teamId, coordinatorId, memberId, label: entry.label, silentForMs: gapMs, lastActiveAt: entry.lastActiveAt },
+    );
+    out.pingedMembers.push(memberId);
+  }
 }
 
 function teamIdlePayload(teamId: string, coordinatorId: string, thresholdMinutes: number, idleForMs: number, firstNoBusyAt: number) {
@@ -79,9 +139,13 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
   const pingedTeamIds: string[] = [];
   const alreadyPingedTeamIds: string[] = [];
   const missingCoordinatorTeamIds: string[] = [];
+  const memberSilence = { pingedMembers: [] as string[], watchedMembers: 0 };
 
   for (const team of listActiveTeams()) {
     seen.add(team.id);
+    // #634 — runs every cycle, independent of the team-level idle gate below: a single silent
+    // member must surface even while OTHER members keep the team "busy".
+    detectSilentMembers(team.id, team.coordinatorId, now, thresholdMs, memberSilence);
     const busy = getTeamBusyState(team.id, cutoff);
     if (busy.busyMemberIds.length > 0) {
       teamIdleTracker.delete(team.id);
@@ -112,6 +176,7 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
   }
 
   for (const key of teamIdleTracker.keys()) if (!seen.has(key)) teamIdleTracker.delete(key);
+  for (const key of teamMemberWatchTracker.keys()) if (!seen.has(key)) teamMemberWatchTracker.delete(key);
 
   return {
     enabled: true,
@@ -122,6 +187,8 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
     pingedTeamIds,
     alreadyPingedTeamIds,
     missingCoordinatorTeamIds,
+    silentMemberIds: memberSilence.pingedMembers,
+    watchedMembers: memberSilence.watchedMembers,
   };
 }
 

package/src/mcp/index.ts

@@ -16,7 +16,7 @@ import { errMessage, isRecord } from "agent-relay-sdk";
 import type { JsonRpcId, JsonRpcRequest, McpAuthContext, ToolDefinition } from "./types";
 import { positiveIdOrUndefined, recordField, stringField } from "./validation";
 import { IDENTITY_TOOLS, hasAnyScope, mcpAuthContext, relayCompactAndResume, relayRecall, relayWhoami } from "./identity";
-import { MESSAGING_TOOLS, relayAgentStatus, relayAttachArtifact, relayFindAgents, relayGetMessage, relayGetThread, relayReply, relaySendMessage, relayUploadArtifact } from "./tools-messaging";
+import { MESSAGING_TOOLS, relayAgentCrashReport, relayAgentStatus, relayAttachArtifact, relayFindAgents, relayGetMessage, relayGetThread, relayReply, relaySendMessage, relayUploadArtifact } from "./tools-messaging";
 import { SPAWN_TOOLS, relayShutdownAgent, relaySpawnAgent, relaySpawnTargets } from "./tools-spawn";
 import { WORKSPACE_TOOLS, callerIsolatedWorkspace, relayWorkspaceList, relayWorkspaceMutation, relayWorkspaceStatus } from "./tools-workspace";
 
@@ -122,6 +122,7 @@ async function callTool(auth: McpAuthContext, params: unknown): Promise<Record<s
     else if (name === "relay_upload_artifact") result = await relayUploadArtifact(auth, args);
     else if (name === "relay_attach_artifact") result = relayAttachArtifact(auth, args);
     else if (name === "relay_agent_status") result = relayAgentStatus(args);
+    else if (name === "relay_agent_crash_report") result = relayAgentCrashReport(auth, args);
     else if (name === "relay_find_agents") result = relayFindAgents(auth, args);
     else if (name === "relay_whoami") result = relayWhoami(auth);
     else if (name === "relay_compact_and_resume") result = relayCompactAndResume(auth, args);

package/src/mcp/tools-messaging.ts

@@ -8,7 +8,9 @@ import {
   createArtifact,
   getAgent,
   getArtifact,
+  getCrashReport,
   getMessage,
+  listCrashReports,
   getOrchestrator,
   getTask,
   getThread,
@@ -190,6 +192,19 @@ export const MESSAGING_TOOLS: ToolDefinition[] = [
       additionalProperties: false,
     },
   },
+  {
+    name: "relay_agent_crash_report",
+    description: "Retrieve the post-mortem for a dead agent (#636): a likely-cause label (oom · killed · clean-exit · token-401 · onboarding-gate · provider-crash · transport-loss · unknown) + log tail + heartbeat/active-work context. Pass agentId for one report (yourself or a child you spawned); omit it to list your own spawned children's reports. The report survives the reaper, so it answers 'why did my worker die?' after it's gone.",
+    requiredScopes: ["agents:read", "agent:read"],
+    inputSchema: {
+      type: "object",
+      properties: {
+        agentId: { type: "string", description: "The dead agent's id. Must be yourself or one of your spawned children. Omit to list your own children's crash reports." },
+        limit: { type: "integer", minimum: 1, maximum: 200, description: "Max reports when listing (default 50)." },
+      },
+      additionalProperties: false,
+    },
+  },
 ];
 
 function runMcpSend(auth: McpAuthContext, input: SendMessageInput): Message & { delivery: DeliveryReceipt } {
@@ -450,6 +465,29 @@ export function relayFindAgents(auth: McpAuthContext, args: Record<string, unkno
   return { agents: safe, total, count: safe.length, truncated: true, nextOffset: offset + safe.length };
 }
 
+// #636 — self/parent-scoped crash post-mortem retrieval. A caller may read its OWN report or any of
+// its spawned children's; never an unrelated agent's. Survives the reaper (separate crash_reports table).
+export function relayAgentCrashReport(auth: McpAuthContext, args: Record<string, unknown>): Record<string, unknown> {
+  const caller = callerAgentId(auth);
+  const agentId = optionalString(args.agentId, "agentId", 240);
+  if (agentId) {
+    // Authorize FIRST, regardless of whether a report exists — the self/parent contract must hold
+    // even for an agentId with no report (else any agent:read caller leaks `{report:null}`). Prefer
+    // the live agent row's lineage; fall back to the report's recorded parent once the agent is reaped.
+    const agent = getAgent(agentId);
+    const report = getCrashReport(agentId);
+    const parent = agent?.spawnedBy ?? report?.parentId;
+    const authorized = !!caller && (caller === agentId || (parent !== undefined && caller === parent));
+    if (!authorized) throw new McpAuthError("crash report is scoped to the agent itself or its spawn-parent");
+    return { agentId, report: report ?? null };
+  }
+  // No agentId: list the caller's own spawned children's reports.
+  if (!caller) throw new McpAuthError("relay_agent_crash_report requires a resolvable caller agent to list reports");
+  const limit = optionalPositiveInt(args.limit, "limit");
+  const reports = listCrashReports({ parentId: caller, ...(limit ? { limit } : {}) });
+  return { reports, count: reports.length };
+}
+
 function payloadWithAttachments(
   payload: Record<string, unknown> | undefined,
   attachments: AttachmentRef[] | undefined,

package/src/notification-types.ts

@@ -11,6 +11,7 @@ export type NotificationType =
   | "agent.transient_land_hold"
   | "agent.rate_limit_resume"
   | "team.idle"
+  | "team.member_silent"
   | "plan.bound_agent_exited"
   | "workspace.steward_task"
   | "workspace.conflict"
@@ -62,6 +63,7 @@ export const DEFAULT_TTL_MS: Record<NotificationType, number | null> = {
   "agent.transient_land_hold": null,
   "agent.rate_limit_resume": 15 * MINUTES,
   "team.idle": null,
+  "team.member_silent": null,
   "plan.bound_agent_exited": null,
   "workspace.steward_task": null,
   "workspace.conflict": null,

package/src/routes/agents.ts

@@ -1,6 +1,6 @@
 // Auto-split from routes.ts (#299). Domain: agents.
 import { VALID_AGENT_STATUSES, agentSessionStatus, auditEvent, canViewAgentRoute, error, json, memoryContext, memoryErrorResponse, normalizeAgentSessionGuard, parseBody, parseId, parseQueryInt, visibleAgentIdsForRequest, type Handler } from "./_shared";
-import { ValidationError, deleteAgent, getAgent, getAgentTimeline, heartbeat, listAgentChatHistory, listAgents, listArtifactsForEntity, listContextSnapshots, listPendingReplyObligations, listQuotaSnapshots, markReady, searchAgents, setLabel, setStatus, setTags, validateAgentSession } from "../db";
+import { ValidationError, deleteAgent, getAgent, getAgentTimeline, getCrashReport, heartbeat, listAgentChatHistory, listAgents, listArtifactsForEntity, listContextSnapshots, listPendingReplyObligations, listQuotaSnapshots, markReady, searchAgents, setLabel, setStatus, setTags, validateAgentSession } from "../db";
 import { attachBranchState, attachBranchStates } from "../agent-branch-state";
 import { attachIssueRepo, attachIssueRepos } from "../agent-issue-repo";
 import { cleanStringArray } from "../validation";
@@ -68,6 +68,16 @@ export const getAgentById: Handler = (req, params) => {
   return json(attachIssueRepo(attachBranchState(agent)));
 };
 
+// #636 — "why did it die" data source. The crash report outlives the agent row (separate table),
+// so this resolves even after the reaper removed the agent. Existence-hiding still applies WHILE the
+// agent exists; once reaped, an authenticated operator can read the post-mortem.
+export const getAgentCrashReportRoute: Handler = (req, params) => {
+  const id = params.id!;
+  const agent = getAgent(id);
+  if (agent && !canViewAgentRoute(req, agent.id)) return error("agent not found", 404);
+  return json({ agentId: id, report: getCrashReport(id) });
+};
+
 export const getAgentReplyObligations: Handler = (_req, params) => {
   const agent = getAgent(params.id!);
   return agent ? json(listPendingReplyObligations(agent.id)) : error("agent not found", 404);

package/src/routes/commands.ts

@@ -16,7 +16,7 @@ import { notifyBranchLandFailed, notifyBranchLanded } from "../branch-landed";
 import { seedProviderConfigs } from "../provider-config-store";
 import { notifyAgentSpawnFailed } from "../agent-lifecycle-events";
 import { handleReviewerMergeResult } from "../reviewer-pipeline";
-import { scheduleRepoMerge } from "../workspace-auto-merge";
+import { CLEAR_MERGE_NO_PROGRESS, noteMergeNoProgress, scheduleRepoMerge } from "../workspace-auto-merge";
 import { enqueueContinuationInjectionAfterSelfResume } from "../self-resume";
 import { type Command, type CommandStatus, type CreateCommandInput, type WorkspaceStatus } from "../types";
 
@@ -220,6 +220,11 @@ export const patchCommand: Handler = async (req, params) => {
             mergeResult: command.result,
             mergeCommandId: command.id,
             mergedAt: Date.now(),
+            // A merge that settles `succeeded` made genuine progress (land / recycle /
+            // noop-resolve / PR-opened) — clear any prior no-progress back-off so the lane
+            // is attempted normally next time (#638).
+            ...CLEAR_MERGE_NO_PROGRESS,
+            mergeError: undefined,
             ...(normalizedStatus !== resultStatus ? { mergeStatusNormalizedFrom: resultStatus } : {}),
             // The land consumes the steward's claim (#208 steward contract / vent
             // #62): a successful land must release it, or `steward inspect` keeps
@@ -259,12 +264,25 @@ export const patchCommand: Handler = async (req, params) => {
           }
         }
       } else if (command.status === "failed" && command.correlationId) {
-        // Merge couldn't complete — don't leave it stuck in merge_planned.
+        // Merge couldn't complete — don't leave it stuck in merge_planned. A no-progress
+        // merge now settles `failed` (orchestrator #638) carrying the result; honor the
+        // orchestrator's intended target status when it gave a valid one so a real
+        // `conflict` routes to the steward (a non-landable status the auto-merge job
+        // skips, which itself stops the loop) rather than bouncing back to the landable
+        // `review_requested`. Surface the error and count the no-progress attempt so the
+        // auto-merge job backs off after N instead of busy-looping (#638).
         const current = getWorkspace(command.correlationId);
         if (current && current.status === "merge_planned") {
-          updateWorkspaceStatus(command.correlationId, "review_requested", {
+          const failedResultStatus = isRecord(command.result)
+            ? optionalEnum(command.result.status, "result.status", VALID_WORKSPACE_STATUSES) as WorkspaceStatus | undefined
+            : undefined;
+          const nextStatus: WorkspaceStatus = failedResultStatus && failedResultStatus !== "merge_planned"
+            ? failedResultStatus
+            : "review_requested";
+          updateWorkspaceStatus(command.correlationId, nextStatus, {
             mergeCommandId: command.id,
             mergeError: command.error ?? "merge failed",
+            ...noteMergeNoProgress(current.metadata),
           });
           notifyBranchLandFailed({
             workspace: current,

package/src/routes/index.ts

@@ -1,5 +1,5 @@
 // Auto-split from routes.ts (#299). Router: route table + matchRoute dispatch.
-import { deleteAgentActiveMemories, deleteAgentById, findAgents, getAgentActiveMemories, getAgentById, getAgentChatHistory, getAgentContextSnapshots, getAgentQuotaSnapshots, getAgentReplyObligations, getAgentTimelineRoute, getAgents, getMessageArtifactsRoute, getTaskArtifactsRoute, patchAgentLabel, patchAgentReady, patchAgentStatus, patchAgentTags, postHeartbeat } from "./agents";
+import { deleteAgentActiveMemories, deleteAgentById, findAgents, getAgentActiveMemories, getAgentById, getAgentChatHistory, getAgentContextSnapshots, getAgentCrashReportRoute, getAgentQuotaSnapshots, getAgentReplyObligations, getAgentTimelineRoute, getAgents, getMessageArtifactsRoute, getTaskArtifactsRoute, patchAgentLabel, patchAgentReady, patchAgentStatus, patchAgentTags, postHeartbeat } from "./agents";
 import { deleteAgentProfileRoute, getAgentProfileRoute, getAgentProfilesRoute, putAgentProfileRoute } from "./agent-profiles";
 import { deleteAgentTerminalSession, getAgentDrive, getAgentPresence, postAgentAction, postAgentDrive, postAgentPermissionDecision, postAgentPresence, postAgentPrompt, postAgentTerminalSession } from "./agent-sessions";
 import { deleteMyAvatar, deleteUserAvatarById, getMe, getUserAvatar, getUsers, headUserAvatar, patchMe, patchUserById, postMyAvatar, postUserAvatarById } from "./users";
@@ -121,6 +121,7 @@ const routes: Route[] = [
   route("GET", "/api/agents/:id/timeline", getAgentTimelineRoute),
   route("GET", "/api/agents/:id/active-memories", getAgentActiveMemories),
   route("GET", "/api/agents/:id/reply-obligations", getAgentReplyObligations),
+  route("GET", "/api/agents/:id/crash-report", getAgentCrashReportRoute),
   route("DELETE", "/api/agents/:id/active-memories", deleteAgentActiveMemories),
   route("GET", "/api/agents/:id", getAgentById),
   route("PATCH", "/api/agents/:id/status", patchAgentStatus),

package/src/routes/orchestrator.ts

@@ -2,6 +2,7 @@
 import { APPROVAL_MODES, SPAWN_PROVIDERS, VALID_WORKSPACE_MODES, isRecord } from "agent-relay-sdk";
 import { CONTRACT_VERSIONS, parseRuntimeCapabilities, parseRuntimeContracts, parseRuntimePackage, type RuntimeCapabilities, type RuntimeContracts, type RuntimePackageMetadata } from "../contracts";
 import { notifyAgentOffline } from "../agent-lifecycle-events";
+import { diagnoseAgentDeath } from "../agent-death-diagnosis";
 import { VERSION } from "../config";
 import { ValidationError, deleteOrchestrator, getOrchestrator, listOrchestrators, orchestratorHeartbeat, recordAgentExitDiagnostics, setOrchestratorUpgradeState, updateManagedAgents, upsertOrchestrator } from "../db";
 import { auditEvent, authAuditMetadata, authorizeRoute, cleanJsonArray, cleanSafeNumber, dashboardAttribution, emitCommand, error, isRootCredentialRequest, json, parseBody, spawnRequestId, type Handler } from "./_shared";
@@ -209,7 +210,10 @@ export const patchOrchestratorAgents: Handler = async (req, params) => {
         // (its WHERE excludes status='offline'), guaranteeing a single notifyAgentOffline emission.
         // notifyAgentOffline no-ops for non-spawned agents. Only fire when we actually recorded the
         // exit (agent !== null) so an already-offline row can't double-notify.
-        notifyAgentOffline(exited.agentId, exitReason(exited));
+        // #636 — classify the death from the captured exit diagnostics + agent state and ride the
+        // verdict on the terminal event (also persisted as a crash report inside notifyAgentOffline).
+        const diagnosis = diagnoseAgentDeath({ agent, reason: exitReason(exited), exit: exited });
+        notifyAgentOffline(exited.agentId, exitReason(exited), diagnosis);
       }
       auditEvent({
         clientId: [
@@ -324,6 +328,15 @@ function cleanManagedSessionExitDiagnostics(value: unknown, index: number): Mana
     runnerInfoFile: cleanString(value.runnerInfoFile, `exitedAgents[${index}].runnerInfoFile`, { max: 500 }),
     runnerInfoPresent: typeof value.runnerInfoPresent === "boolean" ? value.runnerInfoPresent : undefined,
     systemd,
+    // #636 — termination signal / exit code / OOM probe captured by the orchestrator.
+    signal: cleanString(value.signal, `exitedAgents[${index}].signal`, { max: 40 }),
+    exitCode: cleanSafeNumber(value.exitCode),
+    oom: isRecord(value.oom)
+      ? {
+        source: cleanString(value.oom.source, `exitedAgents[${index}].oom.source`, { required: true, max: 80 })!,
+        detail: cleanString(value.oom.detail, `exitedAgents[${index}].oom.detail`, { max: 240 }),
+      }
+      : undefined,
     unavailable,
     lastError: cleanString(value.lastError, `exitedAgents[${index}].lastError`, { required: true, max: 1000 })!,
   };

package/src/services/terminal-provider-exit.ts

@@ -0,0 +1,33 @@
+// #633 — owning service for the silent clean-exit terminal event. A runner whose provider
+// terminally exits stays alive (manual-resume hold), so neither the orchestrator tmux-gone path nor
+// the heartbeat reaper fires. The runner marks the offline status edge with a `terminalProviderExit`
+// blob; this service turns that edge into a terminal `agent.exited` event with a #636 diagnosis, and
+// clears the hold when the provider genuinely resumes. Kept out of the bus transport so the
+// transport stays thin (transport-thinness ratchet, #348).
+import { isRecord } from "agent-relay-sdk";
+import { clearAgentMetaKeys } from "../db";
+import { notifyAgentOffline } from "../agent-lifecycle-events";
+import { diagnoseAgentDeath } from "../agent-death-diagnosis";
+import type { AgentCard } from "../types";
+
+/**
+ * React to an agent status transition observed on the bus.
+ * - dead→live with a stale marker: clear the hold so a keepalive can resurrect normally and a future
+ *   death isn't mislabelled off a stale marker.
+ * - live→offline carrying a `terminalProviderExit` marker: fire the terminal event once (edge), with
+ *   a death diagnosis (also persisted as a crash report inside notifyAgentOffline). No-op for
+ *   non-spawned agents (notifyAgentOffline guards on lineage) and graceful shutdowns (no marker).
+ */
+export function reconcileTerminalProviderExit(agentId: string, before: AgentCard | null | undefined, after: AgentCard | null | undefined): void {
+  if (after && after.status !== "offline" && before?.status === "offline" && isRecord(after.meta?.terminalProviderExit)) {
+    clearAgentMetaKeys(agentId, ["terminalProviderExit"]);
+    return;
+  }
+  if (!after || after.status !== "offline") return;
+  if (before && before.status === "offline") return;
+  const marker = after.meta?.terminalProviderExit;
+  if (!isRecord(marker)) return;
+  const reasonText = typeof marker.reason === "string" ? marker.reason : "provider exited";
+  const reason = `provider exited (${reasonText})`;
+  notifyAgentOffline(agentId, reason, diagnoseAgentDeath({ agent: after, reason }));
+}

package/src/spawn-targets.ts

@@ -48,28 +48,39 @@ export function buildSpawnTargets(input: { callerId?: string; quota: number }):
 interface SpawnCatalogish {
   provider: SpawnProvider;
   defaultModel?: string;
-  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string }>;
+  models: Array<{ alias: string; providerModel: string; efforts: readonly string[]; defaultEffort?: string; disabled?: boolean }>;
 }
 
 function providerMatrixFor(o: Orchestrator): Array<Record<string, unknown>> {
   const reported: ProviderCatalogSummary[] = o.providerCatalog ?? [];
   const global = effectiveProviderCatalogList();
+  // #643 — drop disabled models so relay_spawn_targets advertises only spawnable models (parity
+  // with the #485 dashboard pickers' `!model.disabled` filter). The disabled flag is a relay-side
+  // override (provider_model_overrides) that the effective catalog carries but the orchestrator-
+  // REPORTED catalog does not, so cross-reference the effective catalog by alias regardless of
+  // which source supplies the matrix entry — otherwise the reported path keeps advertising a model
+  // the server then rejects with "model is disabled".
+  const disabledAliasesFor = (provider: SpawnProvider): Set<string> =>
+    new Set(global.find((e) => e.provider === provider)?.models.filter((m) => m.disabled).map((m) => m.alias) ?? []);
   return o.providers.map((provider) => {
     const cat: SpawnCatalogish | undefined =
       reported.find((s) => s.provider === provider) ?? global.find((e) => e.provider === provider);
+    const disabled = disabledAliasesFor(provider);
     return {
       provider,
       available: true,
       ...(cat
         ? {
             defaultModel: cat.defaultModel,
-            models: cat.models.map((m) => ({
-              id: m.alias,
-              providerModel: m.providerModel,
-              default: cat.defaultModel === m.alias,
-              effortLevels: m.efforts,
-              ...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
-            })),
+            models: cat.models
+              .filter((m) => !m.disabled && !disabled.has(m.alias))
+              .map((m) => ({
+                id: m.alias,
+                providerModel: m.providerModel,
+                default: cat.defaultModel === m.alias,
+                effortLevels: m.efforts,
+                ...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
+              })),
           }
         : { models: [] }),
     };

package/src/team-idle-tracker.ts

@@ -11,3 +11,18 @@ export const teamIdleTracker = new Map<string, TeamIdleEntry>();
 export function getTeamIdleEntry(teamId: string): TeamIdleEntry | undefined {
   return teamIdleTracker.get(teamId);
 }
+
+// #634 — per-member heartbeat-gap watch. Kept in-memory (not derived from the live roster) ON
+// PURPOSE: a member that was ready/busy and then vanished from the roster entirely escapes any
+// roster-only enumeration, so the watch must remember it independently to detect the silence gap.
+export interface TeamMemberWatch {
+  /** Most recent time the member was seen alive (non-offline heartbeat). */
+  lastActiveAt: number;
+  /** Whether the member's CURRENT silence has already pinged the coordinator (re-armed on recovery). */
+  pinged: boolean;
+  /** Member label for a readable ping. */
+  label?: string;
+}
+
+// teamId → (memberId → watch)
+export const teamMemberWatchTracker = new Map<string, Map<string, TeamMemberWatch>>();

package/src/workspace-actions.ts

@@ -25,7 +25,7 @@ import { isOwnerAlive, requestWorkspaceMerge } from "./workspace-merge";
 export { LAND_STRATEGIES, DEFAULT_MERGE_STRATEGY } from "./workspace-merge";
 import { claimMetadataPatch, workspaceActiveClaim } from "./workspace-claim";
 import { READY_TO_LAND_STATUSES, TERMINAL_WORKSPACE_STATUSES } from "./workspace-phase";
-import { scheduleRepoMerge } from "./workspace-auto-merge";
+import { CLEAR_MERGE_NO_PROGRESS, scheduleRepoMerge } from "./workspace-auto-merge";
 import type { Command, WorkspaceAutoMergePolicy, WorkspaceMergeStrategy, WorkspaceRecord, WorkspaceStatus } from "./types";
 
 // Single source of truth for the action verb set. The route's `optionalEnum` and
@@ -235,6 +235,10 @@ export function applyWorkspaceAction(workspace: WorkspaceRecord, input: ApplyWor
     ...metadata,
     ...(detail ? { detail } : {}),
     ...(agentId ? { updatedByAgentId: agentId } : {}),
+    // #638 — a fresh ready/request-review hand-off means the owner did more work, so a
+    // prior auto-merge no-progress back-off should clear: the new commits may resolve
+    // whatever blocked the land (e.g. a base that's since been synced). Re-arm the lane.
+    ...(READY_TO_LAND_STATUSES.has(nextStatus) ? CLEAR_MERGE_NO_PROGRESS : {}),
     lastWorkspaceAction: action,
     lastWorkspaceActionAt: Date.now(),
   });