agent-relay-server 0.85.0 → 0.87.0

package/src/agent-busy-health.ts

@@ -0,0 +1,93 @@
+import {
+  STUCK_BUSY_PROVIDER_TURN_MS,
+  STUCK_BUSY_RECONNECT_MS,
+  STUCK_BUSY_SHUTDOWN_MS,
+  STUCK_BUSY_STARTUP_MS,
+} from "./config";
+import { stringValue } from "agent-relay-sdk";
+import type { AgentCard } from "./types";
+
+type BusyHealthKind = "provider-turn" | "startup" | "reconnect" | "shutdown";
+
+interface ActiveWorkLike {
+  id?: unknown;
+  kind?: unknown;
+  label?: unknown;
+  startedAt?: unknown;
+}
+
+export interface AgentBusyHealth {
+  status: "ok" | "stuck-busy";
+  kind: BusyHealthKind;
+  thresholdMs: number;
+  activeWorkId?: string;
+  activeWorkKind?: string;
+  activeWorkLabel?: string;
+  activeWorkStartedAt: number;
+  activeWorkAgeMs: number;
+  lastProviderEventAt?: number;
+  lastProviderEventAgeMs?: number;
+  staleForMs: number;
+}
+
+function numberValue(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function activeWork(agent: AgentCard): ActiveWorkLike[] {
+  return Array.isArray(agent.meta?.activeWork) ? agent.meta.activeWork as ActiveWorkLike[] : [];
+}
+
+function thresholdFor(kind: BusyHealthKind): number {
+  if (kind === "startup") return STUCK_BUSY_STARTUP_MS;
+  if (kind === "reconnect") return STUCK_BUSY_RECONNECT_MS;
+  if (kind === "shutdown") return STUCK_BUSY_SHUTDOWN_MS;
+  return STUCK_BUSY_PROVIDER_TURN_MS;
+}
+
+function classifyBusy(agent: AgentCard, now: number): { kind: BusyHealthKind; work?: ActiveWorkLike; startedAt: number } | null {
+  if (agent.status !== "busy") return null;
+  const works = activeWork(agent);
+  const providerTurn = works.find((item) => item.kind === "provider-turn");
+  if (providerTurn) {
+    return { kind: "provider-turn", work: providerTurn, startedAt: numberValue(providerTurn.startedAt) ?? agent.lastSeen };
+  }
+
+  const lifecycleAction = stringValue(agent.meta?.lifecycleAction) ?? "";
+  const lifecycleActionAt = numberValue(agent.meta?.lifecycleActionAt);
+  if (lifecycleAction.includes("shutdown") || lifecycleAction === "killing") {
+    return { kind: "shutdown", work: works[0], startedAt: lifecycleActionAt ?? numberValue(works[0]?.startedAt) ?? agent.lastSeen };
+  }
+  if (lifecycleAction.includes("reconnect") || lifecycleAction === "resuming") {
+    return { kind: "reconnect", work: works[0], startedAt: lifecycleActionAt ?? numberValue(works[0]?.startedAt) ?? agent.lastSeen };
+  }
+  if (!agent.ready) {
+    return { kind: "startup", work: works[0], startedAt: numberValue(agent.meta?.startedAt) ?? agent.createdAt };
+  }
+  return works[0] ? { kind: "provider-turn", work: works[0], startedAt: numberValue(works[0].startedAt) ?? agent.lastSeen } : null;
+}
+
+export function deriveAgentBusyHealth(agent: AgentCard, now: number = Date.now()): AgentBusyHealth | undefined {
+  const classified = classifyBusy(agent, now);
+  if (!classified) return undefined;
+  const thresholdMs = thresholdFor(classified.kind);
+  const activeWorkStartedAt = Math.max(0, classified.startedAt);
+  const activeWorkAgeMs = Math.max(0, now - activeWorkStartedAt);
+  const lastProviderEventAt = numberValue(agent.meta?.lastProviderEventAt);
+  const lastProgressAt = lastProviderEventAt ?? activeWorkStartedAt;
+  const staleForMs = Math.max(0, now - lastProgressAt);
+  const stuck = activeWorkAgeMs >= thresholdMs && staleForMs >= thresholdMs;
+  const work = classified.work;
+  return {
+    status: stuck ? "stuck-busy" : "ok",
+    kind: classified.kind,
+    thresholdMs,
+    ...(stringValue(work?.id) ? { activeWorkId: stringValue(work?.id) } : {}),
+    ...(stringValue(work?.kind) ? { activeWorkKind: stringValue(work?.kind) } : {}),
+    ...(stringValue(work?.label) ? { activeWorkLabel: stringValue(work?.label) } : {}),
+    activeWorkStartedAt,
+    activeWorkAgeMs,
+    ...(lastProviderEventAt !== undefined ? { lastProviderEventAt, lastProviderEventAgeMs: Math.max(0, now - lastProviderEventAt) } : {}),
+    staleForMs,
+  };
+}

package/src/agent-card-projection.ts

@@ -10,12 +10,15 @@
 // surface; a new emitter can't reintroduce the drift without going around this function.
 
 import { attachBranchState } from "./agent-branch-state";
+import { deriveAgentBusyHealth } from "./agent-busy-health";
 import { attachIssueRepo } from "./agent-issue-repo";
 import type { AgentCard } from "./types";
 
 /** Enrich an AgentCard with the derived fields the dashboard expects on every surface. */
 export function projectAgentCard(agent: AgentCard): AgentCard {
-  return attachIssueRepo(attachBranchState(agent));
+  const projected = attachIssueRepo(attachBranchState(agent));
+  const busyHealth = deriveAgentBusyHealth(projected);
+  return busyHealth ? { ...projected, meta: { ...(projected.meta ?? {}), busyHealth } } : projected;
 }
 
 /** The `agent.status` SSE payload: the enriched card when the agent is known, else the

package/src/agent-lifecycle-events.ts

@@ -1,7 +1,7 @@
 import { emitRelayEvent } from "./events";
 import { getNotificationsConfig } from "./config-store";
 import { routeNotification } from "./notification-router";
-import { getAgent, markAgentReadyNotified, recordCrashReport } from "./db";
+import { getAgent, markAgentReadyNotified, recordCrashReport, recordTerminalEvent } from "./db";
 import { crashReportFor } from "./agent-death-diagnosis";
 import type { AgentCard, AgentDeathDiagnosis } from "./types";
 
@@ -105,6 +105,11 @@ export function notifyAgentOffline(childId: string, reason: string, diagnosis?:
   // whether the parent push is enabled. Recorded once per terminal-event emission.
   if (diagnosis) recordCrashReport(crashReportFor(child, reason, diagnosis));
 
+  // #645 — persist a terminal marker for ANY offline terminus (clean exit or spawn-fail) keyed by
+  // agentId so it outlives the reaper. The #634 watchdog reads this to skip a benign shutdown; it
+  // must fire ONLY on a genuine silent death that emitted no terminal event at all.
+  recordTerminalEvent({ agentId: childId, reason, kind: wasReady ? "agent.exited" : "agent.spawn_failed", createdAt: Date.now() });
+
   if (wasReady) {
     emitRelayEvent({
       type: "agent.exited",

package/src/branch-landed.ts

@@ -257,6 +257,87 @@ export function notifyBranchLandFailed(input: {
   });
 }
 
+/**
+ * Steward-handoff awareness wake (#641/#642). When the relay can't auto-merge a branch and
+ * hands the workspace to a steward, the two ends used to be mutually blind: the branch agent
+ * was told only that land failed (never which steward holds it), and its coordinator — the
+ * agent responsible for the merge landing — was never looped in at all. This fires BOTH
+ * notices off one handoff event:
+ *   - the OWNER branch agent learns which steward took over (#641), and
+ *   - the branch agent's COORDINATOR (owner's `spawnedBy`, resolved in steward-identity.ts)
+ *     is proactively notified so it keeps the big picture (#642).
+ * The steward id is the managed steward's policy name (reachable via `policy:<name>`) or an
+ * elected steward's agent id, whichever the caller has in scope.
+ */
+export function notifyStewardHandoff(input: {
+  workspace: Pick<WorkspaceRecord, "id" | "repoRoot" | "branch" | "baseRef" | "ownerAgentId">;
+  /** The steward now holding the merge — managed steward policy name or elected agent id. */
+  stewardId: string;
+  /** The branch agent's coordinator (owner's `spawnedBy` lineage), or null when none resolves. */
+  coordinatorAgentId?: string | null;
+  /** Why it was handed over ("conflict", "merge state unknown", auto-merge back-off, …). */
+  reason?: string;
+}): void {
+  const { workspace } = input;
+  const base = workspace.baseRef ?? "base";
+  const owner = workspace.ownerAgentId;
+  const coordinator = input.coordinatorAgentId?.trim() || undefined;
+  const branchLabel = workspace.branch ? `\`${workspace.branch}\`` : "Your branch";
+  const reasonLabel = input.reason ? ` (${input.reason})` : "";
+  const payload = {
+    kind: "workspace.steward-handoff",
+    workspaceId: workspace.id,
+    repoRoot: workspace.repoRoot,
+    branch: workspace.branch,
+    base,
+    author: owner,
+    stewardId: input.stewardId,
+    coordinatorAgentId: coordinator,
+    reason: input.reason,
+  };
+
+  emitRelayEvent({ type: "workspace.steward_handoff", source: "server", subject: workspace.id, data: payload });
+
+  const config = getNotificationsConfig();
+  if (!config.enabled) return;
+
+  // #641 — tell the owner branch agent which steward now holds its merge, so it stops
+  // thinking the branch silently failed. Durable (ttl null): the owner must learn this
+  // even if it went idle. Mirror the "don't touch main" guardrail of the landed-failure wake.
+  if (owner) {
+    routeNotification({
+      type: "workspace.steward_handoff",
+      scope: { author: owner, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
+      recipients: [owner],
+      ttlMs: null,
+      message: {
+        subject: "Your merge is now with a steward",
+        body: `🛟 ${branchLabel} couldn't be auto-merged to \`${base}\` and is now being handled by steward agent \`${input.stewardId}\`${reasonLabel}. The steward will rebase/resolve and land it. Do NOT push, rebase, merge, or touch the main checkout yourself; stay idle until you're woken with the result.`,
+        payload: { ...payload, notification: "steward-handoff" },
+        replyExpected: false,
+      },
+    });
+  }
+
+  // #642 — proactively loop in the branch agent's coordinator so it keeps the big picture
+  // (it spawned the branch agent and owns the merge landing). Skip when the coordinator IS
+  // the owner (no real lineage) to avoid a duplicate ping.
+  if (coordinator && coordinator !== owner) {
+    routeNotification({
+      type: "workspace.steward_handoff",
+      scope: { author: owner, parent: coordinator, repoRoot: workspace.repoRoot, workspaceId: workspace.id },
+      recipients: [coordinator],
+      ttlMs: null,
+      message: {
+        subject: "A branch agent's merge is now with a steward",
+        body: `🛟 Your branch agent \`${owner ?? "?"}\`'s merge (${branchLabel} → \`${base}\`) couldn't be auto-merged and is now with steward agent \`${input.stewardId}\`${reasonLabel}. The steward will land it or escalate to you if it can't.`,
+        payload: { ...payload, notification: "steward-handoff", audience: "coordinator" },
+        replyExpected: false,
+      },
+    });
+  }
+}
+
 /**
  * Finalize a workspace whose work landed in base out-of-band — a pr-strategy land whose
  * PR merged (by anyone, incl. `gh pr merge`), or a squash/cherry-pick already in base

package/src/cli/index.ts

@@ -72,7 +72,7 @@ Pair examples:
   agent-relay /message codex "Can you look at that failing action?"
   agent-relay /reply 206 "Sounds good, I'll take a look"
   agent-relay /react 206 👍
-  agent-relay /reply 206 --stdin < result.md
+  agent-relay /reply 206 --body-file result.md
   agent-relay get-message 206 --json
   agent-relay /reply 206 --format markdown "Here is **formatted** text"
   agent-relay /send-claimable tag:backend "Please claim and fix the failing API test"
@@ -166,8 +166,8 @@ Current session
 
 Replying
   agent-relay /reply <messageId> "<response>"
-  agent-relay /reply <messageId> --stdin < response.md
   agent-relay /reply <messageId> --body-file response.md
+  printf '%s\\n' "<response>" | agent-relay /reply <messageId> --stdin
     Reply to the sender of a relay message. Prefer this over /message when
     handling an incoming relay message because the server keeps the routing
     and channel context. Large replies are uploaded as an artifact
@@ -246,7 +246,7 @@ Rules of thumb
     agent-relay get-message 123
 
   If your reply is long, avoid shell quoting and use:
-    agent-relay /reply 123 --stdin < response.md
+    agent-relay /reply 123 --body-file response.md
 
   If you need to know who you are in Relay, run:
     agent-relay /status --json

package/src/cli/reply.ts

@@ -41,6 +41,7 @@ export async function handleReplyCommand(args: string[]): Promise<void> {
   const body = (await resolveBodyInput({ bodyParts, stdinBody, bodyFile })).trim();
   if (!from) throw new Error("Could not detect current Agent Relay ID. Pass --from AGENT_ID or set AGENT_RELAY_ID.");
   if (!body) throw new Error("Reply body required.");
+  if (body === "--stdin") throw new Error('Refusing to send literal "--stdin" as a reply body. Provide real reply text, use --body-file PATH, or pipe non-empty content into --stdin.');
 
   const prepared = await prepareReplyBody({ body, from, replyTo, format, subject });
   const message = await apiRequest("POST", "/api/messages", {
@@ -75,7 +76,11 @@ async function resolveBodyInput(options: {
     options.bodyFile ? "body file" : "",
   ].filter(Boolean);
   if (sources.length > 1) throw new Error("Reply body must come from exactly one source: arguments, --stdin, or --body-file.");
-  if (options.stdinBody) return readStdin();
+  if (options.stdinBody) {
+    const body = await readStdin();
+    if (!body.trim()) throw new Error("--stdin was given but no input was received.");
+    return body;
+  }
   if (options.bodyFile) return readFileSync(resolve(options.bodyFile), "utf8");
   return options.bodyParts.join(" ");
 }

package/src/config.ts

@@ -59,6 +59,11 @@ export const SPAWN_REGISTRATION_NOTIFY_MAX_AGE_MS = envPositiveInt("AGENT_RELAY_
 export const DEFAULT_FANOUT_TTL_SECONDS = envPositiveInt("AGENT_RELAY_FANOUT_TTL_SECONDS", 2 * 60 * 60); // 2h
 export const OFFLINE_PRUNE_MS = envPositiveInt("OFFLINE_PRUNE_MS", DAY_MS); // 24h offline → delete
 export const REAP_INTERVAL_MS = envPositiveInt("REAP_INTERVAL_MS", 60_000); // reaper cadence
+export const STUCK_BUSY_THRESHOLD_MS = envPositiveInt("AGENT_RELAY_STUCK_BUSY_THRESHOLD_MS", 15 * 60_000);
+export const STUCK_BUSY_PROVIDER_TURN_MS = envPositiveInt("AGENT_RELAY_STUCK_BUSY_PROVIDER_TURN_MS", STUCK_BUSY_THRESHOLD_MS);
+export const STUCK_BUSY_STARTUP_MS = envPositiveInt("AGENT_RELAY_STUCK_BUSY_STARTUP_MS", STUCK_BUSY_THRESHOLD_MS);
+export const STUCK_BUSY_RECONNECT_MS = envPositiveInt("AGENT_RELAY_STUCK_BUSY_RECONNECT_MS", STUCK_BUSY_THRESHOLD_MS);
+export const STUCK_BUSY_SHUTDOWN_MS = envPositiveInt("AGENT_RELAY_STUCK_BUSY_SHUTDOWN_MS", STUCK_BUSY_THRESHOLD_MS);
 export const CLAIM_LEASE_MS = envPositiveInt("AGENT_RELAY_CLAIM_LEASE_MS", 1_800_000); // 30min claim lease
 export const POOL_CLAIM_LEASE_MS = envPositiveInt("AGENT_RELAY_POOL_CLAIM_LEASE_MS", STALE_TTL_MS * 3); // pool binding lease
 // Per-repo merge serialization lease — only one base merge may run at a time per
@@ -129,6 +134,10 @@ export function maxArtifactBytesFromEnv(fallback: number): number {
 }
 
 export const OUTBOX_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_OUTBOX_RETENTION_MS", 60 * 60 * 1000);
+// Per-provider quota history retention (#609) — durable burn-rate history for the sparkline/analytics
+// charts (#607/#608). Default 30 days; the quota-history-prune maintenance reaper drops older entries.
+export const QUOTA_HISTORY_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_QUOTA_HISTORY_RETENTION_MS", 30 * DAY_MS);
+export const QUOTA_HISTORY_PRUNE_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_QUOTA_HISTORY_PRUNE_INTERVAL_MS", 6 * 60 * 60 * 1000);
 export const TOKEN_RECORD_RETENTION_SECONDS = envNumberOrDefault("AGENT_RELAY_TOKEN_RECORD_RETENTION_SECONDS", 7 * 24 * 60 * 60);
 export const CONFLICT_SCAN_INTERVAL_MS = envNumberOrDefault("AGENT_RELAY_CONFLICT_SCAN_INTERVAL_MS", 10 * 60 * 1000);
 export const WORKSPACE_RETENTION_MS = envNumberOrDefault("AGENT_RELAY_WORKSPACE_RETENTION_MS", DAY_MS);

package/src/db/index.ts

@@ -28,6 +28,7 @@ export * from "./merge-lease.ts";
 export * from "./continuations.ts";
 export * from "./continuation-archives.ts";
 export * from "./crash-reports.ts";
+export * from "./terminal-events.ts";
 export * from "./plans.ts";
 export * from "./teams.ts";
 export * from "./blackboard.ts";

package/src/db/migrations.ts

@@ -21,6 +21,7 @@ import { ensureAgentOwnershipSchema, seedDefaultAgentOwnership } from "./agent-o
 import { ensureDriveLeaseSchema } from "./drive-leases.ts";
 import { ensureContinuationArchiveSearchSchema } from "./continuation-archives.ts";
 import { ensureCrashReportsSchema } from "./crash-reports.ts";
+import { ensureTerminalEventsSchema } from "./terminal-events.ts";
 import { normalizeReactionEmoji } from "./mappers.ts";
 import { normalizeProviderQuotaAccountKeys } from "./provider-quotas.ts";
 import { projectRootsMissingIssueRepo, setProjectIssueRepo } from "./projects.ts";
@@ -657,6 +658,17 @@ export function applyMigrations(): void {
     `);
     getDb().run("DROP TABLE quota_snapshots_legacy_agent");
   }
+  // #609 — durable 30-day quota history needs explicit "couldn't collect" markers so gaps are
+  // first-class, not silent absences. An unavailable row carries no quota: `unavailable=1` plus the
+  // skip reason in `last_error` (the NOT NULL windows/source/quota_state columns get inert placeholders
+  // that `rowToSnapshot` never reads when `unavailable=1`).
+  const quotaSnapshotColsNow = (getDb().query("PRAGMA table_info(quota_snapshots)").all() as Array<{ name: string }>).map((col) => col.name);
+  if (!quotaSnapshotColsNow.includes("unavailable")) {
+    getDb().run("ALTER TABLE quota_snapshots ADD COLUMN unavailable INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!quotaSnapshotColsNow.includes("last_error")) {
+    getDb().run("ALTER TABLE quota_snapshots ADD COLUMN last_error TEXT");
+  }
   getDb().run(`
     CREATE TABLE IF NOT EXISTS provider_quotas (
       provider TEXT NOT NULL,
@@ -800,6 +812,11 @@ export function applyMigrations(): void {
   // the offline-prune/reaper and stays retrievable by agentId.
   ensureCrashReportsSchema();
 
+  // #645 — clean terminal markers (sibling of crash_reports). Own table, no FK, so an EXPECTED
+  // terminus (transient reap after land, graceful shutdown) outlives the reaper and lets the #634
+  // watchdog tell a benign completion from a silent death. Idempotent CREATE IF NOT EXISTS.
+  ensureTerminalEventsSchema();
+
   // First-class human users (#388 keystone #393): ensure the table (single DDL home in users.ts)
   // and seed the admin-level default user mapped to the existing single human. P0 + idempotent —
   // bare 'user' == user:default resolve to users.id='default' via the SDK resolver; no bytes move.

package/src/db/provider-quotas.ts

@@ -1,4 +1,4 @@
-import { DAY_MS } from "../config";
+import { DAY_MS, QUOTA_HISTORY_RETENTION_MS } from "../config";
 import { parseJson } from "../utils";
 import { getDb } from "./connection.ts";
 import { parseStringArray } from "./shared.ts";
@@ -41,6 +41,8 @@ type ProviderQuotaSnapshotRow = {
   agent_id?: string | null;
   quota_state: string;
   captured_at: number;
+  unavailable?: number | null;
+  last_error?: string | null;
 };
 
 function cleanKey(value: string): string {
@@ -184,11 +186,19 @@ function quotaFingerprint(quota: QuotaState): string {
 }
 
 function rowToSnapshot(row: ProviderQuotaSnapshotRow): ProviderQuotaSnapshot {
-  return {
+  const base = {
     provider: row.provider,
     accountKey: row.account_key,
     capturedAt: row.captured_at,
     ...(row.agent_id ? { sourceAgentId: row.agent_id } : {}),
+  };
+  // #609 — an unavailable marker carries no quota; the placeholder windows/source/quota_state are inert.
+  if (row.unavailable) {
+    const error = parseJson<ProviderQuotaError | undefined>(row.last_error, undefined);
+    return { ...base, unavailable: true, ...(error ? { error } : {}) };
+  }
+  return {
+    ...base,
     quota: parseJson<QuotaState>(row.quota_state, { windows: [], source: "probe", updatedAt: row.captured_at }),
   };
 }
@@ -214,7 +224,7 @@ function rowToRecord(row: ProviderQuotaRow, history: ProviderQuotaSnapshot[], no
 function latestSnapshotFingerprint(provider: string, accountKey: string): string | undefined {
   const row = getDb().query(`
     SELECT quota_state FROM quota_snapshots
-    WHERE provider = ? AND account_key = ?
+    WHERE provider = ? AND account_key = ? AND unavailable = 0
     ORDER BY captured_at DESC, id DESC
     LIMIT 1
   `).get(provider, accountKey) as { quota_state: string } | undefined;
@@ -287,6 +297,11 @@ export function upsertProviderQuota(input: ProviderQuotaUpdateInput, now = Date.
 
   if (input.quota && latestSnapshotFingerprint(provider, accountKey) !== quotaFingerprint(input.quota)) {
     recordProviderQuotaSnapshot({ provider, accountKey, quota: input.quota, sourceAgentId: input.sourceAgentId }, now);
+  } else if (!input.quota && input.lastError?.type === "unavailable" && !latestSnapshotIsUnavailable(provider, accountKey)) {
+    // #609 — the provider couldn't be collected (orchestrator skip, #601). Record one explicit gap
+    // marker on the transition into unavailable (deduped while it stays unavailable) so history shows
+    // an honest gap rather than a silent absence; the next successful poll closes the gap.
+    recordProviderQuotaUnavailable({ provider, accountKey, error: input.lastError, sourceAgentId: input.sourceAgentId }, now);
   }
   if (hasFreshSuccessfulProviderQuota(input, now)) {
     const quotaAdvisoryState = providerQuotaAdvisoryHandler?.({
@@ -310,32 +325,65 @@ export function upsertProviderQuota(input: ProviderQuotaUpdateInput, now = Date.
   return listProviderQuotas({ provider, accountKey, historyLimit: 24, now }).quotas[0]!;
 }
 
+function resolveSnapshotAgentId(sourceAgentId: string | undefined): string | undefined {
+  return sourceAgentId && getDb().query("SELECT 1 FROM agents WHERE id = ?").get(sourceAgentId) ? sourceAgentId : undefined;
+}
+
 function recordProviderQuotaSnapshot(input: { provider: string; accountKey: string; quota: QuotaState; sourceAgentId?: string }, now: number): void {
-  const sourceAgentId = input.sourceAgentId && getDb().query("SELECT 1 FROM agents WHERE id = ?").get(input.sourceAgentId)
-    ? input.sourceAgentId
-    : undefined;
   getDb().query(`
-    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, captured_at)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, unavailable, last_error, captured_at)
+    VALUES (?, ?, ?, ?, ?, ?, 0, NULL, ?)
   `).run(
     input.provider,
     input.accountKey,
-    sourceAgentId ?? null,
+    resolveSnapshotAgentId(input.sourceAgentId) ?? null,
     JSON.stringify(input.quota.windows),
     input.quota.source,
     JSON.stringify(input.quota),
     now,
   );
-  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - DAY_MS);
+  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - QUOTA_HISTORY_RETENTION_MS);
+}
+
+// #609 — an unavailable marker is a quota-less history row: `unavailable=1` + the skip reason in
+// last_error. windows/source/quota_state are NOT NULL, so they carry inert placeholders that
+// rowToSnapshot never reads for unavailable rows.
+function recordProviderQuotaUnavailable(input: { provider: string; accountKey: string; error: ProviderQuotaError; sourceAgentId?: string }, now: number): void {
+  getDb().query(`
+    INSERT INTO quota_snapshots (provider, account_key, agent_id, windows, source, quota_state, unavailable, last_error, captured_at)
+    VALUES (?, ?, ?, '[]', 'unavailable', 'null', 1, ?, ?)
+  `).run(
+    input.provider,
+    input.accountKey,
+    resolveSnapshotAgentId(input.sourceAgentId) ?? null,
+    JSON.stringify(input.error),
+    now,
+  );
+  pruneProviderQuotaSnapshots(input.provider, input.accountKey, now - QUOTA_HISTORY_RETENTION_MS);
+}
+
+function latestSnapshotIsUnavailable(provider: string, accountKey: string): boolean {
+  const row = getDb().query(`
+    SELECT unavailable FROM quota_snapshots
+    WHERE provider = ? AND account_key = ?
+    ORDER BY captured_at DESC, id DESC
+    LIMIT 1
+  `).get(provider, accountKey) as { unavailable?: number | null } | undefined;
+  return Boolean(row?.unavailable);
 }
 
-function pruneProviderQuotaSnapshots(provider?: string, accountKey?: string, olderThan = Date.now() - DAY_MS): number {
+function pruneProviderQuotaSnapshots(provider?: string, accountKey?: string, olderThan = Date.now() - QUOTA_HISTORY_RETENTION_MS): number {
   const result = provider && accountKey
     ? getDb().query("DELETE FROM quota_snapshots WHERE provider = ? AND account_key = ? AND captured_at < ?").run(provider, accountKey, olderThan)
     : getDb().query("DELETE FROM quota_snapshots WHERE captured_at < ?").run(olderThan);
   return result.changes;
 }
 
+// Maintenance reaper hook (#609): drop quota history beyond the 30-day retention window across all providers.
+export function pruneProviderQuotaHistory(olderThan = Date.now() - QUOTA_HISTORY_RETENTION_MS): number {
+  return pruneProviderQuotaSnapshots(undefined, undefined, olderThan);
+}
+
 function pruneProviderQuotaRecords(provider?: string, accountKey?: string, olderThan = Date.now() - DAY_MS): number {
   const result = provider && accountKey
     ? getDb().query("DELETE FROM provider_quotas WHERE provider = ? AND account_key = ? AND updated_at < ?").run(provider, accountKey, olderThan)
@@ -387,10 +435,13 @@ function suppressRedundantUnavailableRows(rows: ProviderQuotaRow[]): ProviderQuo
   return rows.filter((row) => !(isUnavailableMarkerRow(row) && collectableProviders.has(row.provider)));
 }
 
-function listProviderQuotaSnapshots(provider: string, accountKey: string, options: { since?: number; limit?: number } = {}): ProviderQuotaSnapshot[] {
-  const limit = Math.max(1, Math.min(options.limit ?? 288, 1000));
+const QUOTA_HISTORY_MAX_ROWS = 10_000;
+
+function listProviderQuotaSnapshots(provider: string, accountKey: string, options: { since?: number; limit?: number; includeUnavailable?: boolean } = {}): ProviderQuotaSnapshot[] {
+  const limit = Math.max(1, Math.min(options.limit ?? 288, QUOTA_HISTORY_MAX_ROWS));
   const params: any[] = [normalizeProvider(provider), cleanKey(accountKey)];
-  let sql = "SELECT * FROM quota_snapshots WHERE provider = ? AND account_key = ?";
+  // The live matrix/widget history is quota-only; the 30-day history API opts into gap markers.
+  let sql = `SELECT * FROM quota_snapshots WHERE provider = ? AND account_key = ?${options.includeUnavailable ? "" : " AND unavailable = 0"}`;
   if (options.since !== undefined) {
     sql += " AND captured_at >= ?";
     params.push(options.since);
@@ -400,15 +451,42 @@ function listProviderQuotaSnapshots(provider: string, accountKey: string, option
   return (getDb().query(sql).all(...params) as ProviderQuotaSnapshotRow[]).map(rowToSnapshot);
 }
 
+/**
+ * #609 — durable 30-day per-provider quota history (the read API the #607 sparkline + #608 analytics
+ * chart consume). Returns snapshots oldest-first across all of the provider's canonical accountKeys,
+ * including explicit `unavailable` gap markers (`quota` absent, `unavailable: true`, `error` set).
+ * Group by `accountKey` for a per-account series; chart `quota.windows[].utilization` over `capturedAt`.
+ */
+export function getProviderQuotaHistory(provider: string, options: { sinceMs?: number; accountKey?: string; limit?: number } = {}): ProviderQuotaSnapshot[] {
+  const normalizedProvider = normalizeProvider(provider);
+  if (!normalizedProvider) return [];
+  const limit = Math.max(1, Math.min(options.limit ?? QUOTA_HISTORY_MAX_ROWS, QUOTA_HISTORY_MAX_ROWS));
+  const params: any[] = [normalizedProvider];
+  let sql = "SELECT * FROM quota_snapshots WHERE provider = ?";
+  if (options.accountKey) {
+    sql += " AND account_key = ?";
+    params.push(cleanKey(options.accountKey));
+  }
+  if (options.sinceMs !== undefined) {
+    sql += " AND captured_at >= ?";
+    params.push(options.sinceMs);
+  }
+  sql += " ORDER BY captured_at ASC, id ASC LIMIT ?";
+  params.push(limit);
+  return (getDb().query(sql).all(...params) as ProviderQuotaSnapshotRow[]).map(rowToSnapshot);
+}
+
 export function listAgentQuotaSnapshotsFromProviderStore(agent: AgentCard, options: { since?: number; limit?: number } = {}) {
   const identity = providerQuotaIdentityForAgent(agent);
-  return listProviderQuotaSnapshots(identity.provider, identity.accountKey, options).map((snapshot) => ({
-    agentId: agent.id,
-    capturedAt: snapshot.capturedAt,
-    windows: snapshot.quota.windows,
-    source: snapshot.quota.source,
-    updatedAt: snapshot.quota.updatedAt,
-  }));
+  return listProviderQuotaSnapshots(identity.provider, identity.accountKey, options)
+    .filter((snapshot): snapshot is ProviderQuotaSnapshot & { quota: QuotaState } => Boolean(snapshot.quota))
+    .map((snapshot) => ({
+      agentId: agent.id,
+      capturedAt: snapshot.capturedAt,
+      windows: snapshot.quota.windows,
+      source: snapshot.quota.source,
+      updatedAt: snapshot.quota.updatedAt,
+    }));
 }
 
 export function listProviderQuotas(options: { provider?: string; accountKey?: string; since?: number; historyLimit?: number; now?: number } = {}): ProviderQuotaMatrix {

package/src/db/terminal-events.ts

@@ -0,0 +1,56 @@
+// #645 — persisted CLEAN terminal markers. A sibling of crash_reports: where crash_reports record a
+// DEATH (with a diagnosis), this records an EXPECTED, clean terminus — a transient worker reaped
+// after a land, a graceful shutdown, a spawn that failed to come up. Same survival property: its own
+// table, no FK to agents, so the marker outlives the offline-prune/reaper and stays retrievable by
+// agentId. The #634 heartbeat-gap watchdog consults it to tell a benign completion from a silent
+// death — the watchdog must fire ONLY when an agent went quiet with NO terminal event of any kind.
+import { getDb } from "./connection.ts";
+
+interface TerminalEventRow {
+  agent_id: string;
+  reason: string;
+  kind: string;
+  created_at: number;
+}
+
+export function ensureTerminalEventsSchema(): void {
+  getDb().run(`
+    CREATE TABLE IF NOT EXISTS terminal_events (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      agent_id TEXT NOT NULL,
+      reason TEXT NOT NULL,
+      kind TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  getDb().run("CREATE INDEX IF NOT EXISTS idx_terminal_events_agent ON terminal_events(agent_id, created_at DESC)");
+}
+
+export interface TerminalEvent {
+  agentId: string;
+  /** Cause text, e.g. "transient-complete", "shutdown", "heartbeat lost". */
+  reason: string;
+  /** Relay event kind that terminated the agent, e.g. "agent.exited", "agent.spawn_failed". */
+  kind: string;
+  createdAt: number;
+}
+
+export function recordTerminalEvent(event: TerminalEvent): void {
+  getDb()
+    .query("INSERT INTO terminal_events (agent_id, reason, kind, created_at) VALUES (?, ?, ?, ?)")
+    .run(event.agentId, event.reason, event.kind, event.createdAt);
+}
+
+/** The most recent terminal event for an agent, or null if none was ever recorded. */
+export function getTerminalEvent(agentId: string): TerminalEvent | null {
+  const row = getDb()
+    .query("SELECT * FROM terminal_events WHERE agent_id = ? ORDER BY created_at DESC, id DESC LIMIT 1")
+    .get(agentId) as TerminalEventRow | undefined;
+  if (!row) return null;
+  return { agentId: row.agent_id, reason: row.reason, kind: row.kind, createdAt: row.created_at };
+}
+
+/** True when any terminal event (clean exit, spawn-fail, reap) was recorded for the agent. */
+export function hasTerminalEvent(agentId: string): boolean {
+  return getTerminalEvent(agentId) !== null;
+}

package/src/maintenance/constants.ts

@@ -5,6 +5,8 @@ import {
   DB_VACUUM_EVERY,
   ORPHAN_REAPER_INTERVAL_MS,
   OUTBOX_RETENTION_MS,
+  QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+  QUOTA_HISTORY_RETENTION_MS,
   REAP_INTERVAL_MS,
   TOKEN_RECORD_RETENTION_SECONDS,
   WORKSPACE_AUTO_MERGE_INTERVAL_MS,
@@ -38,6 +40,8 @@ export {
   DB_VACUUM_EVERY,
   ORPHAN_REAPER_INTERVAL_MS,
   OUTBOX_RETENTION_MS,
+  QUOTA_HISTORY_PRUNE_INTERVAL_MS,
+  QUOTA_HISTORY_RETENTION_MS,
   TOKEN_RECORD_RETENTION_SECONDS,
   WORKSPACE_AUTO_MERGE_INTERVAL_MS,
   WORKSPACE_GC_INTERVAL_MS,