agent-relay-server 0.84.0 → 0.86.0

package/src/db/crash-reports.ts

@@ -0,0 +1,115 @@
+// #636 — persisted crash post-mortems. Stored in their own table (no FK to agents) so a death
+// verdict survives `pruneOfflineAgents` / `deleteAgent` and stays retrievable by agentId after the
+// worktree/session reaper has cleaned up. One row per (agentId, createdAt); the latest wins on read.
+import { getDb } from "./connection.ts";
+import { parseJson } from "../utils";
+import type { AgentDeathDiagnosis, CrashReport } from "../types";
+
+interface CrashReportRow {
+  agent_id: string;
+  parent_id: string | null;
+  team_id: string | null;
+  spawn_request_id: string | null;
+  provider: string | null;
+  label: string | null;
+  cause: string;
+  reason: string;
+  diagnosis: string;
+  created_at: number;
+}
+
+export function ensureCrashReportsSchema(): void {
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS crash_reports (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      agent_id TEXT NOT NULL,
+      parent_id TEXT,
+      team_id TEXT,
+      spawn_request_id TEXT,
+      provider TEXT,
+      label TEXT,
+      cause TEXT NOT NULL,
+      reason TEXT NOT NULL,
+      diagnosis TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_crash_reports_agent ON crash_reports(agent_id, created_at DESC)");
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_crash_reports_parent ON crash_reports(parent_id, created_at DESC)");
+}
+
+function rowToCrashReport(row: CrashReportRow): CrashReport {
+  return {
+    agentId: row.agent_id,
+    ...(row.parent_id ? { parentId: row.parent_id } : {}),
+    ...(row.team_id ? { teamId: row.team_id } : {}),
+    ...(row.spawn_request_id ? { spawnRequestId: row.spawn_request_id } : {}),
+    ...(row.provider ? { provider: row.provider } : {}),
+    ...(row.label ? { label: row.label } : {}),
+    reason: row.reason,
+    diagnosis: parseJson<AgentDeathDiagnosis>(row.diagnosis, {
+      cause: "unknown",
+      confidence: "low",
+      summary: "Cause undetermined",
+      evidence: [],
+      detectedAt: row.created_at,
+    }),
+    createdAt: row.created_at,
+  };
+}
+
+export function recordCrashReport(report: CrashReport): void {
+  getDb()
+    .query(
+      `INSERT INTO crash_reports (agent_id, parent_id, team_id, spawn_request_id, provider, label, cause, reason, diagnosis, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      report.agentId,
+      report.parentId ?? null,
+      report.teamId ?? null,
+      report.spawnRequestId ?? null,
+      report.provider ?? null,
+      report.label ?? null,
+      report.diagnosis.cause,
+      report.reason,
+      JSON.stringify(report.diagnosis),
+      report.createdAt,
+    );
+}
+
+/** The most recent crash report for an agent, or null if none was ever recorded. */
+export function getCrashReport(agentId: string): CrashReport | null {
+  const row = getDb()
+    .query("SELECT * FROM crash_reports WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId) as CrashReportRow | undefined;
+  return row ? rowToCrashReport(row) : null;
+}
+
+export interface ListCrashReportsFilter {
+  /** Restrict to children of this spawn-parent (the parent/self scope for the MCP tool). */
+  parentId?: string;
+  /** Restrict to a single agent. */
+  agentId?: string;
+  limit?: number;
+}
+
+export function listCrashReports(filter: ListCrashReportsFilter = {}): CrashReport[] {
+  const where: string[] = [];
+  const params: (string | number)[] = [];
+  if (filter.parentId) {
+    where.push("parent_id = ?");
+    params.push(filter.parentId);
+  }
+  if (filter.agentId) {
+    where.push("agent_id = ?");
+    params.push(filter.agentId);
+  }
+  const limit = Math.max(1, Math.min(filter.limit ?? 50, 200));
+  const rows = getDb()
+    .query(
+      `SELECT * FROM crash_reports ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY created_at DESC, id DESC LIMIT ?`,
+    )
+    .all(...params, limit) as CrashReportRow[];
+  return rows.map(rowToCrashReport);
+}

package/src/db/index.ts

@@ -27,6 +27,8 @@ export * from "./workspaces.ts";
 export * from "./merge-lease.ts";
 export * from "./continuations.ts";
 export * from "./continuation-archives.ts";
+export * from "./crash-reports.ts";
+export * from "./terminal-events.ts";
 export * from "./plans.ts";
 export * from "./teams.ts";
 export * from "./blackboard.ts";

package/src/db/migrations.ts

@@ -20,6 +20,8 @@ import { ensureAuthIdentitiesSchema } from "./auth-identities.ts";
 import { ensureAgentOwnershipSchema, seedDefaultAgentOwnership } from "./agent-ownership.ts";
 import { ensureDriveLeaseSchema } from "./drive-leases.ts";
 import { ensureContinuationArchiveSearchSchema } from "./continuation-archives.ts";
+import { ensureCrashReportsSchema } from "./crash-reports.ts";
+import { ensureTerminalEventsSchema } from "./terminal-events.ts";
 import { normalizeReactionEmoji } from "./mappers.ts";
 import { normalizeProviderQuotaAccountKeys } from "./provider-quotas.ts";
 import { projectRootsMissingIssueRepo, setProjectIssueRepo } from "./projects.ts";
@@ -656,6 +658,17 @@ export function applyMigrations(): void {
     `);
     getDb().run("DROP TABLE quota_snapshots_legacy_agent");
   }
+  // #609 — durable 30-day quota history needs explicit "couldn't collect" markers so gaps are
+  // first-class, not silent absences. An unavailable row carries no quota: `unavailable=1` plus the
+  // skip reason in `last_error` (the NOT NULL windows/source/quota_state columns get inert placeholders
+  // that `rowToSnapshot` never reads when `unavailable=1`).
+  const quotaSnapshotColsNow = (getDb().query("PRAGMA table_info(quota_snapshots)").all() as Array<{ name: string }>).map((col) => col.name);
+  if (!quotaSnapshotColsNow.includes("unavailable")) {
+    getDb().run("ALTER TABLE quota_snapshots ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!quotaSnapshotColsNow.includes("last_error")) {
+    getDb().run("ALTER TABLE quota_snapshots ADD COLUMN last_error TEXT");
+  }
   getDb().run(`
     CREATE TABLE IF NOT EXISTS provider_quotas (
       provider TEXT NOT NULL,
@@ -795,6 +808,15 @@ export function applyMigrations(): void {
     )
   `);
 
+  // #636 — persisted crash post-mortems. Own table, no FK to agents, so a death verdict outlives
+  // the offline-prune/reaper and stays retrievable by agentId.
+  ensureCrashReportsSchema();
+
+  // #645 — clean terminal markers (sibling of crash_reports). Own table, no FK, so an EXPECTED
+  // terminus (transient reap after land, graceful shutdown) outlives the reaper and lets the #634
+  // watchdog tell a benign completion from a silent death. Idempotent CREATE IF NOT EXISTS.
+  ensureTerminalEventsSchema();
+
   // First-class human users (#388 keystone #393): ensure the table (single DDL home in users.ts)
   // and seed the admin-level default user mapped to the existing single human. P0 + idempotent —
   // bare 'user' == user:default resolve to users.id='default' via the SDK resolver; no bytes move.

package/src/db/provider-quotas.ts

@@ -1,4 +1,4 @@
-import { DAY_MS } from "../config";
+import { DAY_MS, QUOTA_HISTORY_RETENTION_MS } from "../config";
 import { parseJson } from "../utils";
 import { getDb } from "./connection.ts";
 import { parseStringArray } from "./shared.ts";
@@ -41,6 +41,8 @@ type ProviderQuotaSnapshotRow = {
   agent_id?: string | null;
   quota_state: string;
   captured_at: number;
+  unavailable?: number | null;
+  last_error?: string | null;
 };
 
 function cleanKey(value: string): string {
@@ -184,11 +186,19 @@ function quotaFingerprint(quota: QuotaState): string {
 }
 
 function rowToSnapshot(row: ProviderQuotaSnapshotRow): ProviderQuotaSnapshot {
-  return {
+  const base = {
     provider: row.provider,
     accountKey: row.account_key,
     capturedAt: row.captured_at,
     ...(row.agent_id ? { sourceAgentId: row.agent_id } : {}),
+  };
+  // #609 — an unavailable marker carries no quota; the placeholder windows/source/quota_state are inert.
+  if (row.unavailable) {
+    const error = parseJson<ProviderQuotaError | undefined>(row.last_error, undefined);
+    return { ...base, unavailable: true, ...(error ? { error } : {}) };
+  }
+  return {
+    ...base,
     quota: parseJson<QuotaState>(row.quota_state, { windows: [], source: "probe", updatedAt: row.captured_at }),
   };
 }
@@ -214,7 +224,7 @@ function rowToRecord(row: ProviderQuotaRow, history: ProviderQuotaSnapshot[], no
 function latestSnapshotFingerprint(provider: string, accountKey: string): string | undefined {
   const row = getDb().query(`
     SELECT quota_state FROM quota_snapshots
-    WHERE provider = ? AND account_key = ?
+    WHERE provider = ? AND account_key = ? AND unavailable = 0
     ORDER BY captured_at DESC, id DESC
     LIMIT 1
   `).get(provider, accountKey) as { quota_state: string } | undefined;
@@ -287,6 +297,11 @@ export function upsertProviderQuota(input: ProviderQuotaUpdateInput, now = Date.
 
   if (input.quota && latestSnapshotFingerprint(provider, accountKey) !== quotaFingerprint(input.quota)) {
     recordProviderQuotaSnapshot({ provider, accountKey, quota: input.quota, sourceAgentId: input.sourceAgentId }, now);
+  } else if (!input.quota && input.lastError?.type === "unavailable" && !latestSnapshotIsUnavailable(provider, accountKey)) {
+    // #609 — the provider couldn't be collected (orchestrator skip, #601). Record one explicit gap
+    // marker on the transition into unavailable (deduped while it stays unavailable) so history shows
+    // an honest gap rather than a silent absence; the next successful poll closes the gap.
+    recordProviderQuotaUnavailable({ provider, accountKey, error: input.lastError, sourceAgentId: input.sourceAgentId }, now);
   }
   if (hasFreshSuccessfulProviderQuota(input, now)) {
     const quotaAdvisoryState = providerQuotaAdvisoryHandler?.({
@@ -310,32 +325,65 @@ export function upsertProviderQuota(input: ProviderQuotaUpdateInput, now = Date.
   return listProviderQuotas({ provider, accountKey, historyLimit: 24, now }).quotas[0]!;
 }
 
+function resolveSnapshotAgentId(sourceAgentId: string | undefined): string | undefined {
+  return sourceAgentId && getDb().query("SELECT 1 FROM agents WHERE id = ?").get(sourceAgentId) ? sourceAgentId : undefined;
+}
+
 function recordProviderQuotaSnapshot(input: { provider: string; accountKey: string; quota: QuotaState; sourceAgentId?: string }, now: number): void {
-  const sourceAgentId = input.sourceAgentId && getDb().query("SELECT 1 FROM agents WHERE id = ?").get(input.sourceAgentId)
-    ? input.sourceAgentId
-    : undefined;
   getDb().query(`
-    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, captured_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, unavailable, last_error, captured_at)
+    VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
   `).run(
     input.provider,
     input.accountKey,
-    sourceAgentId ?? null,
+    resolveSnapshotAgentId(input.sourceAgentId) ?? null,
     JSON.stringify(input.quota.windows),
     input.quota.source,
     JSON.stringify(input.quota),
     now,
   );
-  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - DAY_MS);
+  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - QUOTA_HISTORY_RETENTION_MS);
+}
+
+// #609 — an unavailable marker is a quota-less history row: `unavailable=1` + the skip reason in
+// last_error. windows/source/quota_state are NOT NULL, so they carry inert placeholders that
+// rowToSnapshot never reads for unavailable rows.
+function recordProviderQuotaUnavailable(input: { provider: string; accountKey: string; error: ProviderQuotaError; sourceAgentId?: string }, now: number): void {
+  getDb().query(`
+    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, unavailable, last_error, captured_at)
+    VALUES (?, ?, ?, '[]', 'unavailable', 'null', 1, ?, ?)
+  `).run(
+    input.provider,
+    input.accountKey,
+    resolveSnapshotAgentId(input.sourceAgentId) ?? null,
+    JSON.stringify(input.error),
+    now,
+  );
+  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - QUOTA_HISTORY_RETENTION_MS);
+}
+
+function latestSnapshotIsUnavailable(provider: string, accountKey: string): boolean {
+  const row = getDb().query(`
+    SELECT unavailable FROM quota_snapshots
+    WHERE provider = ? AND account_key = ?
+    ORDER BY captured_at DESC, id DESC
+    LIMIT 1
+  `).get(provider, accountKey) as { unavailable?: number | null } | undefined;
+  return Boolean(row?.unavailable);
 }
 
-function pruneProviderQuotaSnapshots(provider?: string, accountKey?: string, olderThan = Date.now() - DAY_MS): number {
+function pruneProviderQuotaSnapshots(provider?: string, accountKey?: string, olderThan = Date.now() - QUOTA_HISTORY_RETENTION_MS): number {
   const result = provider && accountKey
     ? getDb().query("DELETE FROM quota_snapshots WHERE provider = ? AND account_key = ? AND captured_at < ?").run(provider, accountKey, olderThan)
     : getDb().query("DELETE FROM quota_snapshots WHERE captured_at < ?").run(olderThan);
   return result.changes;
 }
 
+// Maintenance reaper hook (#609): drop quota history beyond the 30-day retention window across all providers.
+export function pruneProviderQuotaHistory(olderThan = Date.now() - QUOTA_HISTORY_RETENTION_MS): number {
+  return pruneProviderQuotaSnapshots(undefined, undefined, olderThan);
+}
+
 function pruneProviderQuotaRecords(provider?: string, accountKey?: string, olderThan = Date.now() - DAY_MS): number {
   const result = provider && accountKey
     ? getDb().query("DELETE FROM provider_quotas WHERE provider = ? AND account_key = ? AND updated_at < ?").run(provider, accountKey, olderThan)
@@ -387,10 +435,13 @@ function suppressRedundantUnavailableRows(rows: ProviderQuotaRow[]): ProviderQuo
   return rows.filter((row) => !(isUnavailableMarkerRow(row) && collectableProviders.has(row.provider)));
 }
 
-function listProviderQuotaSnapshots(provider: string, accountKey: string, options: { since?: number; limit?: number } = {}): ProviderQuotaSnapshot[] {
-  const limit = Math.max(1, Math.min(options.limit ?? 288, 1000));
+const QUOTA_HISTORY_MAX_ROWS = 10_000;
+
+function listProviderQuotaSnapshots(provider: string, accountKey: string, options: { since?: number; limit?: number; includeUnavailable?: boolean } = {}): ProviderQuotaSnapshot[] {
+  const limit = Math.max(1, Math.min(options.limit ?? 288, QUOTA_HISTORY_MAX_ROWS));
   const params: any[] = [normalizeProvider(provider), cleanKey(accountKey)];
-  let sql = "SELECT * FROM quota_snapshots WHERE provider = ? AND account_key = ?";
+  // The live matrix/widget history is quota-only; the 30-day history API opts into gap markers.
+  let sql = `SELECT * FROM quota_snapshots WHERE provider = ? AND account_key = ?${options.includeUnavailable ? "" : " AND unavailable = 0"}`;
   if (options.since !== undefined) {
     sql += " AND captured_at >= ?";
     params.push(options.since);
@@ -400,15 +451,42 @@ function listProviderQuotaSnapshots(provider: string, accountKey: string, option
   return (getDb().query(sql).all(...params) as ProviderQuotaSnapshotRow[]).map(rowToSnapshot);
 }
 
+/**
+ * #609 — durable 30-day per-provider quota history (the read API the #607 sparkline + #608 analytics
+ * chart consume). Returns snapshots oldest-first across all of the provider's canonical accountKeys,
+ * including explicit `unavailable` gap markers (`quota` absent, `unavailable: true`, `error` set).
+ * Group by `accountKey` for a per-account series; chart `quota.windows[].utilization` over `capturedAt`.
+ */
+export function getProviderQuotaHistory(provider: string, options: { sinceMs?: number; accountKey?: string; limit?: number } = {}): ProviderQuotaSnapshot[] {
+  const normalizedProvider = normalizeProvider(provider);
+  if (!normalizedProvider) return [];
+  const limit = Math.max(1, Math.min(options.limit ?? QUOTA_HISTORY_MAX_ROWS, QUOTA_HISTORY_MAX_ROWS));
+  const params: any[] = [normalizedProvider];
+  let sql = "SELECT * FROM quota_snapshots WHERE provider = ?";
+  if (options.accountKey) {
+    sql += " AND account_key = ?";
+    params.push(cleanKey(options.accountKey));
+  }
+  if (options.sinceMs !== undefined) {
+    sql += " AND captured_at >= ?";
+    params.push(options.sinceMs);
+  }
+  sql += " ORDER BY captured_at ASC, id ASC LIMIT ?";
+  params.push(limit);
+  return (getDb().query(sql).all(...params) as ProviderQuotaSnapshotRow[]).map(rowToSnapshot);
+}
+
 export function listAgentQuotaSnapshotsFromProviderStore(agent: AgentCard, options: { since?: number; limit?: number } = {}) {
   const identity = providerQuotaIdentityForAgent(agent);
-  return listProviderQuotaSnapshots(identity.provider, identity.accountKey, options).map((snapshot) => ({
-    agentId: agent.id,
-    capturedAt: snapshot.capturedAt,
-    windows: snapshot.quota.windows,
-    source: snapshot.quota.source,
-    updatedAt: snapshot.quota.updatedAt,
-  }));
+  return listProviderQuotaSnapshots(identity.provider, identity.accountKey, options)
+    .filter((snapshot): snapshot is ProviderQuotaSnapshot & { quota: QuotaState } => Boolean(snapshot.quota))
+    .map((snapshot) => ({
+      agentId: agent.id,
+      capturedAt: snapshot.capturedAt,
+      windows: snapshot.quota.windows,
+      source: snapshot.quota.source,
+      updatedAt: snapshot.quota.updatedAt,
+    }));
 }
 
 export function listProviderQuotas(options: { provider?: string; accountKey?: string; since?: number; historyLimit?: number; now?: number } = {}): ProviderQuotaMatrix {

package/src/db/terminal-events.ts

@@ -0,0 +1,56 @@
+// #645 — persisted CLEAN terminal markers. A sibling of crash_reports: where crash_reports record a
+// DEATH (with a diagnosis), this records an EXPECTED, clean terminus — a transient worker reaped
+// after a land, a graceful shutdown, a spawn that failed to come up. Same survival property: its own
+// table, no FK to agents, so the marker outlives the offline-prune/reaper and stays retrievable by
+// agentId. The #634 heartbeat-gap watchdog consults it to tell a benign completion from a silent
+// death — the watchdog must fire ONLY when an agent went quiet with NO terminal event of any kind.
+import { getDb } from "./connection.ts";
+
+interface TerminalEventRow {
+  agent_id: string;
+  reason: string;
+  kind: string;
+  created_at: number;
+}
+
+export function ensureTerminalEventsSchema(): void {
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS terminal_events (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      agent_id TEXT NOT NULL,
+      reason TEXT NOT NULL,
+      kind TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, created_at DESC)");
+}
+
+export interface TerminalEvent {
+  agentId: string;
+  /** Cause text, e.g. "transient-complete", "shutdown", "heartbeat lost". */
+  reason: string;
+  /** Relay event kind that terminated the agent, e.g. "agent.exited", "agent.spawn_failed". */
+  kind: string;
+  createdAt: number;
+}
+
+export function recordTerminalEvent(event: TerminalEvent): void {
+  getDb()
+    .query("INSERT INTO terminal_events (agent_id, reason, kind, created_at) VALUES (?, ?, ?, ?)")
+    .run(event.agentId, event.reason, event.kind, event.createdAt);
+}
+
+/** The most recent terminal event for an agent, or null if none was ever recorded. */
+export function getTerminalEvent(agentId: string): TerminalEvent | null {
+  const row = getDb()
+    .query("SELECT * FROM terminal_events WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId) as TerminalEventRow | undefined;
+  if (!row) return null;
+  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, createdAt: row.created_at };
+}
+
+/** True when any terminal event (clean exit, spawn-fail, reap) was recorded for the agent. */
+export function hasTerminalEvent(agentId: string): boolean {
+  return getTerminalEvent(agentId) !== null;
+}

package/src/maintenance/constants.ts

@@ -5,6 +5,8 @@ import {
   DB_VACUUM_EVERY,
   ORPHAN_REAPER_INTERVAL_MS,
   OUTBOX_RETENTION_MS,
+  QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+  QUOTA_HISTORY_RETENTION_MS,
   REAP_INTERVAL_MS,
   TOKEN_RECORD_RETENTION_SECONDS,
   WORKSPACE_AUTO_MERGE_INTERVAL_MS,
@@ -38,6 +40,8 @@ export {
   DB_VACUUM_EVERY,
   ORPHAN_REAPER_INTERVAL_MS,
   OUTBOX_RETENTION_MS,
+  QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+  QUOTA_HISTORY_RETENTION_MS,
   TOKEN_RECORD_RETENTION_SECONDS,
   WORKSPACE_AUTO_MERGE_INTERVAL_MS,
   WORKSPACE_GC_INTERVAL_MS,

package/src/maintenance/jobs.ts

@@ -7,9 +7,11 @@ import {
   createActivityEvent,
   evaluatePoolBindings,
   expireQueuedMessages,
+  getAgent,
   pruneOrphanedSharedWorkspaces,
   pruneOfflineAgents,
   pruneOldMessages,
+  pruneProviderQuotaHistory,
   reapStaleAgents,
   reapStaleOrchestrators,
   releaseExpiredClaims,
@@ -19,6 +21,7 @@ import {
   sweepArtifacts,
 } from "../db";
 import { notifyAgentOffline } from "../agent-lifecycle-events";
+import { diagnoseAgentDeath } from "../agent-death-diagnosis";
 import { autoMergeCleanFastForwards } from "../workspace-auto-merge";
 import { reapOrphanedWorktrees } from "../workspace-orphans";
 import { getLifecycleManager } from "../lifecycle-manager";
@@ -43,6 +46,8 @@ import {
   DAY_MS,
   ORPHAN_REAPER_INTERVAL_MS,
   OUTBOX_RETENTION_MS,
+  QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+  QUOTA_HISTORY_RETENTION_MS,
   TOKEN_RECORD_RETENTION_SECONDS,
   WORKSPACE_AUTO_MERGE_INTERVAL_MS,
   WORKSPACE_GC_INTERVAL_MS,
@@ -173,7 +178,10 @@ export const definitions: MaintenanceJobDefinition[] = [
       for (const id of reapedAgentIds) {
         emitAgentStatus(id);
         getLifecycleManager().onAgentDisappeared(id);
-        notifyAgentOffline(id, "heartbeat lost");
+        // #636 — a heartbeat-lost death still gets a best-effort post-mortem from the agent's last
+        // known meta (lastExit/lastError/auth) + heartbeat gap, riding the terminal event.
+        const diagnosis = diagnoseAgentDeath({ agent: getAgent(id), reason: "heartbeat lost" });
+        notifyAgentOffline(id, "heartbeat lost", diagnosis);
         createActivityEvent({
           clientId: "server-agent-" + id + "-heartbeat-lost-" + Date.now(),
           kind: "state",
@@ -275,6 +283,16 @@ export const definitions: MaintenanceJobDefinition[] = [
       return runDbMaintenance({ vacuum });
     },
   },
+  {
+    id: "quota-history-prune",
+    title: "Quota history prune",
+    description: "Delete per-provider quota history snapshots (incl. unavailable markers) beyond the 30-day retention window (#609).",
+    intervalMs: QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+    runOnStart: true,
+    handler() {
+      return { prunedQuotaHistoryRows: pruneProviderQuotaHistory(Date.now() - QUOTA_HISTORY_RETENTION_MS) };
+    },
+  },
   {
     id: "bus-outbox-prune",
     title: "Bus outbox prune",

package/src/maintenance/teams.ts

@@ -2,6 +2,8 @@ import {
   clearTeamZeroMembersSince,
   countLiveTeamMembers,
   createActivityEvent,
+  getCrashReport,
+  hasTerminalEvent,
   getTeamBusyState,
   getTeamRoster,
   listActiveTeams,
@@ -13,10 +15,73 @@ import { dissolveTeamById } from "../services/team";
 import { getTeamsConfig } from "../teams-config-store";
 import { teamReapGraceMs } from "./constants";
 import { notifyTarget } from "./events";
-import { teamIdleTracker } from "../team-idle-tracker";
+import { teamIdleTracker, teamMemberWatchTracker, type TeamMemberWatch } from "../team-idle-tracker";
+import type { AgentCard } from "../types";
 
 export function resetTeamIdleWatchdogForTests(): void {
   teamIdleTracker.clear();
+  teamMemberWatchTracker.clear();
+}
+
+/** A member counts as "alive" (worth watching for a future silence) when it's past onboarding and
+ *  heartbeating — i.e. a status that implies it became ready, not a never-started spawn. */
+function memberIsAlive(member: AgentCard, freshCutoff: number): boolean {
+  return (member.status === "idle" || member.status === "busy" || member.status === "online") && member.lastSeen > freshCutoff;
+}
+
+/**
+ * #634 — detect a team member that was ready/busy and then went silent (heartbeat-gap), EVEN IF it
+ * dropped from the roster entirely, and ping the coordinator once. The in-memory watch remembers
+ * each member's last-active time, so a vanished agent (no roster row) is still caught. Re-arms when
+ * a member recovers. Skips the ping when any terminal event already fired — a #633 crash report (a
+ * DEATH the coordinator was already woken on) OR a #645 clean terminus (a transient worker reaped
+ * after its land, a graceful shutdown). This watchdog must surface ONLY a genuine silent death:
+ * an agent that went quiet with NO terminal event of any kind.
+ */
+function detectSilentMembers(
+  teamId: string,
+  coordinatorId: string | undefined,
+  now: number,
+  thresholdMs: number,
+  out: { pingedMembers: string[]; watchedMembers: number },
+): void {
+  const freshCutoff = now - thresholdMs;
+  const watch = teamMemberWatchTracker.get(teamId) ?? new Map<string, TeamMemberWatch>();
+  teamMemberWatchTracker.set(teamId, watch);
+
+  for (const member of getTeamRoster(teamId)) {
+    if (member.id === coordinatorId) continue; // the coordinator is the recipient, not a watched worker
+    if (!memberIsAlive(member, freshCutoff)) continue;
+    const existing = watch.get(member.id);
+    // Recovery re-arms the ping: a member seen alive again after a silence can re-trigger later.
+    const recovered = existing ? member.lastSeen > existing.lastActiveAt : false;
+    watch.set(member.id, {
+      lastActiveAt: Math.max(member.lastSeen, existing?.lastActiveAt ?? 0),
+      pinged: recovered ? false : existing?.pinged ?? false,
+      ...(member.label ? { label: member.label } : {}),
+    });
+  }
+
+  out.watchedMembers += watch.size;
+  for (const [memberId, entry] of watch) {
+    const gapMs = now - entry.lastActiveAt;
+    if (gapMs < thresholdMs || entry.pinged) continue;
+    entry.pinged = true;
+    // A terminal event already fired — a #633 crash report (death) or a #645 clean terminus
+    // (transient reap after land / graceful shutdown). Either way this is not a silent death.
+    if (getCrashReport(memberId) || hasTerminalEvent(memberId)) continue;
+    if (!coordinatorId) continue;
+    const minutes = Math.floor(gapMs / 60_000);
+    notifyTarget(
+      "team.member_silent",
+      { teamId, agentId: memberId },
+      coordinatorId,
+      "Team member went silent",
+      `Member \`${entry.label ?? memberId}\` in team \`${teamId}\` has sent no heartbeat for ${minutes} min — it may have died without a terminal event. Check it and restart its unfinished work, or stand it down.`,
+      { kind: "team.member_silent", teamId, coordinatorId, memberId, label: entry.label, silentForMs: gapMs, lastActiveAt: entry.lastActiveAt },
+    );
+    out.pingedMembers.push(memberId);
+  }
 }
 
 function teamIdlePayload(teamId: string, coordinatorId: string, thresholdMinutes: number, idleForMs: number, firstNoBusyAt: number) {
@@ -79,9 +144,13 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
   const pingedTeamIds: string[] = [];
   const alreadyPingedTeamIds: string[] = [];
   const missingCoordinatorTeamIds: string[] = [];
+  const memberSilence = { pingedMembers: [] as string[], watchedMembers: 0 };
 
   for (const team of listActiveTeams()) {
     seen.add(team.id);
+    // #634 — runs every cycle, independent of the team-level idle gate below: a single silent
+    // member must surface even while OTHER members keep the team "busy".
+    detectSilentMembers(team.id, team.coordinatorId, now, thresholdMs, memberSilence);
     const busy = getTeamBusyState(team.id, cutoff);
     if (busy.busyMemberIds.length > 0) {
       teamIdleTracker.delete(team.id);
@@ -112,6 +181,7 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
   }
 
   for (const key of teamIdleTracker.keys()) if (!seen.has(key)) teamIdleTracker.delete(key);
+  for (const key of teamMemberWatchTracker.keys()) if (!seen.has(key)) teamMemberWatchTracker.delete(key);
 
   return {
     enabled: true,
@@ -122,6 +192,8 @@ export function runTeamIdleWatchdog(input: { now?: number } = {}): Record<string
     pingedTeamIds,
     alreadyPingedTeamIds,
     missingCoordinatorTeamIds,
+    silentMemberIds: memberSilence.pingedMembers,
+    watchedMembers: memberSilence.watchedMembers,
   };
 }