agent-relay-runner 0.16.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-runner",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Unified provider lifecycle runner for Agent Relay",
   "type": "module",
   "bin": {
@@ -20,7 +20,7 @@
     "directory": "runner"
   },
   "dependencies": {
-    "agent-relay-sdk": "0.2.7"
+    "agent-relay-sdk": "0.2.8"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/plugins/claude/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "agent-relay-runner",
   "description": "Thin Agent Relay runner bridge for Claude Code",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "agentRelayContracts": {
     "providerPluginProtocol": 1
   }

package/src/outbox.ts

@@ -0,0 +1,303 @@
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { tmpdir } from "node:os";
+import { logger } from "./logger";
+
+// Phase 2 (#196) — the "nothing is ever lost" half. Runner→server events that used to be
+// fire-and-forget over HTTP (session turns, reasoning/tool traces, prompt echoes, insights,
+// hook-fatal reports) were silently dropped whenever the server was momentarily down. This
+// is a durable, FIFO, disk-backed queue that:
+//   - survives Runner/server restart (bun:sqlite file in the runtime dir),
+//   - stamps true event time (`occurredAt`) once at enqueue and preserves it through retries,
+//   - retries with capped exponential backoff, strictly in order (an append log must not
+//     reorder turns),
+//   - poisons a permanently-failing head after maxAttempts so it can't block the queue,
+//   - is bounded with a logged drop policy (never silently truncates).
+//
+// Status deliberately does NOT go through here: it rides the WebSocket bus, which is
+// last-wins and self-heals on reconnect (so it already satisfies "coalesce, don't replay
+// stale busyes"). The coalesce mode below exists so a future state event could migrate here.
+
+export type OutboxMode = "append" | "coalesce";
+
+export interface OutboxEventInput {
+  kind: string;
+  payload: unknown;
+  mode?: OutboxMode;
+  // Required for coalesce mode: prior un-poisoned rows with the same dedupeKey are replaced.
+  dedupeKey?: string;
+  // Defaults to now. Set explicitly only to backdate (e.g. replaying a captured timestamp).
+  occurredAt?: number;
+  // Defaults to a stable derived key so server-side dedup makes retries exactly-once.
+  idempotencyKey?: string;
+}
+
+export interface OutboxRecord {
+  seq: number;
+  kind: string;
+  mode: OutboxMode;
+  occurredAt: number;
+  idempotencyKey: string;
+  payload: unknown;
+  attempts: number;
+}
+
+// The transport. Resolve = delivered (row deleted). Reject = failed (retried with backoff).
+export type OutboxSend = (record: OutboxRecord) => Promise<void>;
+
+export interface OutboxOptions {
+  agentId: string;
+  send: OutboxSend;
+  // Storage directory. Defaults to AGENT_RELAY_RUNNER_OUTBOX_DIR, else a per-host temp dir.
+  dir?: string;
+  maxRows?: number;
+  maxAttempts?: number;
+  baseBackoffMs?: number;
+  maxBackoffMs?: number;
+  pollMs?: number;
+}
+
+const DEFAULTS = {
+  maxRows: 5000,
+  maxAttempts: 12,
+  baseBackoffMs: 1_000,
+  maxBackoffMs: 60_000,
+  pollMs: 5_000,
+};
+
+interface Row {
+  seq: number;
+  kind: string;
+  mode: string;
+  occurred_at: number;
+  idempotency_key: string;
+  payload: string;
+  attempts: number;
+  next_attempt_at: number;
+  poisoned: number;
+}
+
+export class Outbox {
+  private readonly db: Database;
+  private readonly agentId: string;
+  private readonly send: OutboxSend;
+  private readonly maxRows: number;
+  private readonly maxAttempts: number;
+  private readonly baseBackoffMs: number;
+  private readonly maxBackoffMs: number;
+  private readonly pollMs: number;
+  readonly path: string;
+
+  private draining = false;
+  private rerun = false;
+  private pollTimer?: ReturnType<typeof setInterval>;
+  private dueTimer?: ReturnType<typeof setTimeout>;
+  private stopped = false;
+
+  constructor(options: OutboxOptions) {
+    this.agentId = options.agentId;
+    this.send = options.send;
+    this.maxRows = options.maxRows ?? DEFAULTS.maxRows;
+    this.maxAttempts = options.maxAttempts ?? DEFAULTS.maxAttempts;
+    this.baseBackoffMs = options.baseBackoffMs ?? DEFAULTS.baseBackoffMs;
+    this.maxBackoffMs = options.maxBackoffMs ?? DEFAULTS.maxBackoffMs;
+    this.pollMs = options.pollMs ?? DEFAULTS.pollMs;
+
+    const dir = options.dir ?? process.env.AGENT_RELAY_RUNNER_OUTBOX_DIR ?? join(tmpdir(), "agent-relay-outbox");
+    this.path = options.dir === ":memory:" ? ":memory:" : join(dir, `outbox-${safeName(this.agentId)}.sqlite`);
+    if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
+
+    this.db = new Database(this.path, { create: true });
+    this.db.exec("PRAGMA journal_mode = WAL");
+    this.db.exec("PRAGMA busy_timeout = 2000");
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS outbox (
+        seq INTEGER PRIMARY KEY AUTOINCREMENT,
+        kind TEXT NOT NULL,
+        mode TEXT NOT NULL DEFAULT 'append',
+        dedupe_key TEXT,
+        occurred_at INTEGER NOT NULL,
+        idempotency_key TEXT NOT NULL,
+        payload TEXT NOT NULL,
+        attempts INTEGER NOT NULL DEFAULT 0,
+        next_attempt_at INTEGER NOT NULL DEFAULT 0,
+        poisoned INTEGER NOT NULL DEFAULT 0,
+        created_at INTEGER NOT NULL
+      )
+    `);
+    // A restart is a fresh start: clear any backoff timers left by the prior process so
+    // pending events get an immediate retry (the down server may now be back). `attempts`
+    // is kept so the poison threshold still counts cumulative failures.
+    this.db.exec("UPDATE outbox SET next_attempt_at = 0 WHERE next_attempt_at > 0");
+  }
+
+  // Persist an event. Returns the assigned seq. Triggers a drain.
+  enqueue(input: OutboxEventInput): number {
+    if (this.stopped) throw new Error("outbox is stopped");
+    const mode: OutboxMode = input.mode ?? "append";
+    const occurredAt = input.occurredAt ?? Date.now();
+    const payloadJson = JSON.stringify(input.payload ?? null);
+    const idempotencyKey = input.idempotencyKey ?? `${this.agentId}:${input.kind}:${occurredAt}:${shortHash(payloadJson)}`;
+
+    if (mode === "coalesce") {
+      if (!input.dedupeKey) throw new Error("coalesce mode requires a dedupeKey");
+      this.db.query("DELETE FROM outbox WHERE dedupe_key = ? AND poisoned = 0").run(input.dedupeKey);
+    }
+
+    const info = this.db
+      .query(`INSERT INTO outbox (kind, mode, dedupe_key, occurred_at, idempotency_key, payload, created_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?)`)
+      .run(input.kind, mode, input.dedupeKey ?? null, occurredAt, idempotencyKey, payloadJson, Date.now());
+    const seq = Number(info.lastInsertRowid);
+
+    this.enforceBound();
+    // Defer the drain to a microtask so a synchronous burst of enqueues (e.g. several
+    // coalesce updates) all land — and coalesce — before the pump pulls the head.
+    queueMicrotask(() => { void this.drain(); });
+    return seq;
+  }
+
+  // Bounded ring buffer: if over capacity, drop the oldest rows (defined overflow policy).
+  // Logged, never silent. Prefers dropping already-poisoned rows first, then oldest by seq.
+  private enforceBound(): void {
+    const { n } = this.db.query("SELECT count(*) AS n FROM outbox").get() as { n: number };
+    if (n <= this.maxRows) return;
+    const overflow = n - this.maxRows;
+    // Oldest poisoned first, then oldest live — both by seq.
+    const victims = this.db
+      .query("SELECT seq FROM outbox ORDER BY poisoned DESC, seq ASC LIMIT ?")
+      .all(overflow) as Array<{ seq: number }>;
+    const ids = victims.map((v) => v.seq);
+    if (ids.length === 0) return;
+    const placeholders = ids.map(() => "?").join(",");
+    this.db.query(`DELETE FROM outbox WHERE seq IN (${placeholders})`).run(...ids);
+    logger.warn("outbox", `bound exceeded (${n}/${this.maxRows}) — dropped ${ids.length} oldest event(s)`);
+  }
+
+  // Begin the background pump: an initial drain plus a poll timer as a backstop.
+  start(): void {
+    if (this.pollTimer || this.stopped) return;
+    void this.drain();
+    this.pollTimer = setInterval(() => { void this.drain(); }, this.pollMs);
+    this.pollTimer.unref?.();
+  }
+
+  // Process the queue strictly oldest-first. Coalesces concurrent calls; if a drain is
+  // requested while one is running, it re-runs once at the end (so an enqueue during a
+  // send isn't missed).
+  async drain(): Promise<void> {
+    if (this.stopped) return;
+    if (this.draining) { this.rerun = true; return; }
+    this.draining = true;
+    try {
+      do {
+        this.rerun = false;
+        await this.drainOnce();
+      } while (this.rerun && !this.stopped);
+    } finally {
+      this.draining = false;
+    }
+  }
+
+  private async drainOnce(): Promise<void> {
+    for (;;) {
+      if (this.stopped) return;
+      const row = this.db
+        .query("SELECT * FROM outbox WHERE poisoned = 0 ORDER BY seq ASC LIMIT 1")
+        .get() as Row | null;
+      if (!row) return;
+
+      const now = Date.now();
+      if (row.next_attempt_at > now) {
+        // Head isn't due yet. Don't reorder past it (FIFO) — schedule a wake-up and stop.
+        this.scheduleDue(row.next_attempt_at - now);
+        return;
+      }
+
+      const record: OutboxRecord = {
+        seq: row.seq,
+        kind: row.kind,
+        mode: row.mode as OutboxMode,
+        occurredAt: row.occurred_at,
+        idempotencyKey: row.idempotency_key,
+        payload: safeParse(row.payload),
+        attempts: row.attempts,
+      };
+
+      try {
+        await this.send(record);
+        this.db.query("DELETE FROM outbox WHERE seq = ?").run(row.seq);
+      } catch (error) {
+        const attempts = row.attempts + 1;
+        const reason = error instanceof Error ? error.message : String(error);
+        if (attempts >= this.maxAttempts) {
+          this.db.query("UPDATE outbox SET attempts = ?, poisoned = 1 WHERE seq = ?").run(attempts, row.seq);
+          logger.fatal("outbox", `event seq=${row.seq} kind=${row.kind} poisoned after ${attempts} attempts: ${reason}`);
+          // Move on — the next iteration picks the new head (poison no longer blocks).
+          continue;
+        }
+        const delay = this.backoff(attempts);
+        this.db.query("UPDATE outbox SET attempts = ?, next_attempt_at = ? WHERE seq = ?").run(attempts, now + delay, row.seq);
+        logger.debug("outbox", `event seq=${row.seq} kind=${row.kind} retry ${attempts}/${this.maxAttempts} in ${delay}ms: ${reason}`);
+        this.scheduleDue(delay);
+        return; // head is now scheduled; stop until it's due (preserve order)
+      }
+    }
+  }
+
+  private backoff(attempts: number): number {
+    const exp = Math.min(this.maxBackoffMs, this.baseBackoffMs * 2 ** (attempts - 1));
+    return Math.round(exp / 2 + Math.random() * (exp / 2)); // full-ish jitter, never below half
+  }
+
+  private scheduleDue(delayMs: number): void {
+    if (this.stopped || this.dueTimer) return;
+    this.dueTimer = setTimeout(() => {
+      this.dueTimer = undefined;
+      void this.drain();
+    }, Math.max(0, delayMs));
+    this.dueTimer.unref?.();
+  }
+
+  // Observability / tests.
+  pendingCount(): number {
+    return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 0").get() as { n: number }).n;
+  }
+
+  poisonedCount(): number {
+    return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 1").get() as { n: number }).n;
+  }
+
+  stop(): void {
+    this.stopped = true;
+    if (this.pollTimer) clearInterval(this.pollTimer);
+    this.pollTimer = undefined;
+    if (this.dueTimer) clearTimeout(this.dueTimer);
+    this.dueTimer = undefined;
+  }
+
+  close(): void {
+    this.stop();
+    try { this.db.close(); } catch { /* already closed */ }
+  }
+}
+
+function safeName(value: string): string {
+  return value.replace(/[^a-zA-Z0-9_.-]+/g, "_").slice(0, 180) || "agent";
+}
+
+function safeParse(json: string): unknown {
+  try { return JSON.parse(json); } catch { return null; }
+}
+
+// Small, fast, stable string hash (FNV-1a, 32-bit) — enough to disambiguate identical
+// kind+timestamp payloads in the idempotency key. Not security-sensitive.
+function shortHash(value: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < value.length; i++) {
+    h ^= value.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(36);
+}

package/src/relay-instructions.ts

@@ -45,15 +45,29 @@ export function workspaceDepsNote(input: { mode?: string | null; depsMode?: stri
   }
 }
 
-/** Resolve the workspace deps caveat from the runner/monitor environment.
+/**
+ * Caveat for untracked paths symlinked from main into an isolated worktree
+ * (WorkspaceConfig.symlinkPaths, e.g. AGENTS.md, .claude-rig). Edits to these
+ * write THROUGH to the main checkout — the agent must know so it doesn't mutate
+ * shared config thinking it's worktree-local. Returns "" when nothing was linked.
+ */
+export function workspaceSymlinksNote(linked: string[]): string {
+  if (!linked.length) return "";
+  return `[agent-relay] Isolated workspace: these untracked paths are SYMLINKED from the main checkout: ${linked.join(", ")}. They resolve to the real files in main, so editing or deleting them writes THROUGH to main — treat them as read-only unless you intend to change main.`;
+}
+
+/** Resolve the workspace caveats from the runner/monitor environment.
  * AGENT_RELAY_WORKSPACE_JSON carries the resolved workspace metadata (mode +
- * deps) and is the authoritative source. Best-effort: never throws. */
+ * deps + symlinks) and is the authoritative source. Best-effort: never throws. */
 export function workspaceDepsNoteFromEnv(env: Record<string, string | undefined> = process.env): string {
   const json = env.AGENT_RELAY_WORKSPACE_JSON;
   if (!json) return "";
   try {
-    const parsed = JSON.parse(json) as { mode?: string; deps?: { mode?: string } };
-    return workspaceDepsNote({ mode: parsed.mode ?? null, depsMode: parsed.deps?.mode ?? null });
+    const parsed = JSON.parse(json) as { mode?: string; deps?: { mode?: string }; symlinks?: { linked?: string[] } };
+    return [
+      workspaceDepsNote({ mode: parsed.mode ?? null, depsMode: parsed.deps?.mode ?? null }),
+      parsed.mode === "isolated" ? workspaceSymlinksNote(parsed.symlinks?.linked ?? []) : "",
+    ].filter(Boolean).join("\n\n");
   } catch {
     return "";
   }

package/src/reply-obligation-cache.ts

@@ -0,0 +1,109 @@
+import type { ReplyObligation } from "agent-relay-sdk";
+import { logger } from "./logger";
+
+// Phase 2 (#196) — the crux. The Claude Stop hook used to ask the server, synchronously
+// and in the hot path, "does this agent owe a reply?" before clearing the turn. A slow
+// server answer (the unindexed reply_to scan, #199) blew the hook's timeout and wedged the
+// agent in `busy` forever. The fix: the hook asks the Runner, the Runner answers instantly
+// from this local snapshot, and the snapshot is refreshed from the server only in the
+// background — never on the path that ends a turn.
+//
+// Design rules:
+// - `get()` is synchronous, never throws, never touches the network.
+// - `refresh()` is the only thing that talks to the server; it coalesces concurrent calls
+//   and, on failure, keeps the last-known snapshot (stale-but-serving beats blocking).
+// - A background interval keeps the snapshot warm; `markDirty()` requests an extra,
+//   debounced refresh when state likely just changed (a message arrived, a turn ended).
+
+export type ReplyObligationFetch = () => Promise<ReplyObligation[]>;
+
+export interface ReplyObligationCacheOptions {
+  fetch: ReplyObligationFetch;
+  // Background freshness backstop. Default 10s — well under any turn cadence, cheap.
+  intervalMs?: number;
+  // Debounce window for markDirty()-triggered refreshes so a burst of events
+  // (e.g. a fan-out of messages) collapses into one server round-trip.
+  dirtyDebounceMs?: number;
+}
+
+const DEFAULT_INTERVAL_MS = 10_000;
+const DEFAULT_DIRTY_DEBOUNCE_MS = 400;
+
+export class ReplyObligationCache {
+  private readonly fetch: ReplyObligationFetch;
+  private readonly intervalMs: number;
+  private readonly dirtyDebounceMs: number;
+
+  private snapshot: ReplyObligation[] = [];
+  private lastRefreshedAt = 0;
+  private inFlight: Promise<void> | null = null;
+  private intervalTimer?: ReturnType<typeof setInterval>;
+  private dirtyTimer?: ReturnType<typeof setTimeout>;
+  private stopped = false;
+
+  constructor(options: ReplyObligationCacheOptions) {
+    this.fetch = options.fetch;
+    this.intervalMs = options.intervalMs ?? DEFAULT_INTERVAL_MS;
+    this.dirtyDebounceMs = options.dirtyDebounceMs ?? DEFAULT_DIRTY_DEBOUNCE_MS;
+  }
+
+  // Synchronous, hot-path-safe read. Returns a copy so callers can't mutate the snapshot.
+  get(): ReplyObligation[] {
+    return this.snapshot.slice();
+  }
+
+  getLastRefreshedAt(): number {
+    return this.lastRefreshedAt;
+  }
+
+  // Begin the background freshness loop and prime the first snapshot immediately.
+  start(): void {
+    if (this.intervalTimer || this.stopped) return;
+    void this.refresh();
+    this.intervalTimer = setInterval(() => { void this.refresh(); }, this.intervalMs);
+    // Don't keep the process alive solely for cache refreshes.
+    this.intervalTimer.unref?.();
+  }
+
+  stop(): void {
+    this.stopped = true;
+    if (this.intervalTimer) clearInterval(this.intervalTimer);
+    this.intervalTimer = undefined;
+    if (this.dirtyTimer) clearTimeout(this.dirtyTimer);
+    this.dirtyTimer = undefined;
+  }
+
+  // Request a refresh because state likely changed (message arrived / turn ended).
+  // Debounced so a burst collapses into a single server round-trip.
+  markDirty(): void {
+    if (this.stopped || this.dirtyTimer) return;
+    this.dirtyTimer = setTimeout(() => {
+      this.dirtyTimer = undefined;
+      void this.refresh();
+    }, this.dirtyDebounceMs);
+    this.dirtyTimer.unref?.();
+  }
+
+  // Fetch from the server and replace the snapshot. Coalesces concurrent callers onto a
+  // single in-flight request. Never rejects — a failed fetch leaves the prior snapshot in
+  // place (the hook keeps getting an answer even while the server is down).
+  refresh(): Promise<void> {
+    if (this.stopped) return Promise.resolve();
+    if (this.inFlight) return this.inFlight;
+    this.inFlight = this.doRefresh().finally(() => { this.inFlight = null; });
+    return this.inFlight;
+  }
+
+  private async doRefresh(): Promise<void> {
+    try {
+      const obligations = await this.fetch();
+      if (this.stopped) return;
+      this.snapshot = Array.isArray(obligations) ? obligations : [];
+      this.lastRefreshedAt = Date.now();
+    } catch (error) {
+      // Server-down is a non-event: keep serving the last snapshot. Debug, not error —
+      // this is expected during outages and must not spam the log.
+      logger.debug("obligation-cache", `refresh failed, serving cached snapshot (${this.snapshot.length}): ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+}

package/src/runner.ts

@@ -2,13 +2,15 @@ import { hostname } from "node:os";
 import { closeSync, mkdirSync, openSync, readSync, statSync, writeFileSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
-import type { AgentProfile, ContextState, Message, MessageSessionMeta, ProviderCapabilities, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
+import type { AgentProfile, ContextState, Message, MessageSessionMeta, ProviderCapabilities, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
 import { RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
 import { contextStateFromProbeMetrics, readContextProbeState } from "agent-relay-sdk/context-probe";
 import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
 import { messagesWithCachedAttachments } from "./attachment-cache";
 import { ClaimTracker } from "./claim-tracker";
 import { startControlServer, type ControlServer } from "./control-server";
+import { ReplyObligationCache } from "./reply-obligation-cache";
+import { Outbox, type OutboxRecord } from "./outbox";
 import { extractLastAssistantTurn, extractFinalAssistantMessage, extractHookAssistantMessage, extractLatestTurnSteps, transcriptLooksComplete, analyzeSession } from "./adapters/claude-transcript";
 import { agentProfileProjectionReport } from "./profile-projection";
 import { profileUsesHostProviderGlobals } from "./profile-home";
@@ -116,6 +118,13 @@ export class AgentRunner {
   private readonly claims = new ClaimTracker();
   private readonly http: RelayHttpClient;
   private readonly bus: RelayBusClient;
+  // Phase 2 (#196): the Stop hook reads reply obligations from this local snapshot, never
+  // from the server — so a slow server can no longer wedge a turn (the crux fix).
+  private readonly obligationCache: ReplyObligationCache;
+  // Phase 2 (#196): Runner→server append-log events (session turns, reasoning, prompts,
+  // insights, hook-fatal) go through this durable, disk-backed, timestamped queue instead of
+  // direct fire-and-forget HTTP — so nothing is lost across a server/Runner restart.
+  private readonly outbox: Outbox;
   private currentToken?: string;
   private currentTokenJti?: string;
   private currentTokenProfileId?: string;
@@ -192,6 +201,12 @@ export class AgentRunner {
     this.currentTokenExpiresAt = options.tokenExpiresAt;
     const runtime = runtimeMetadata(options.provider);
     this.http = new RelayHttpClient({ baseUrl: options.relayUrl, token: this.currentToken });
+    this.obligationCache = new ReplyObligationCache({ fetch: () => this.http.listReplyObligations(this.agentId) });
+    // Co-locate the durable outbox with the runner's runtime state (survives reboot) when the
+    // orchestrator told us where that is; otherwise the Outbox falls back to a temp dir.
+    const outboxDir = process.env.AGENT_RELAY_RUNNER_OUTBOX_DIR
+      ?? (process.env.AGENT_RELAY_RUNNER_INFO_FILE ? join(dirname(process.env.AGENT_RELAY_RUNNER_INFO_FILE), "outbox") : undefined);
+    this.outbox = new Outbox({ agentId: this.agentId, dir: outboxDir, send: (record) => this.deliverOutboxEvent(record) });
     this.bus = new RelayBusClient({
       url: relayBusUrl(options.relayUrl),
       role: "provider",
@@ -260,10 +275,13 @@ export class AgentRunner {
     this.control = startControlServer({
       onStatus: (status) => this.setProviderStatus(status),
       onTerminalAttachSpec: () => this.terminalAttachSpec(),
-      onReplyObligations: () => this.http.listReplyObligations(this.agentId),
+      // Hot-path-safe: answered instantly from the local snapshot, never a server
+      // round-trip. The snapshot is kept warm by the background refresh below (#196).
+      onReplyObligations: () => Promise.resolve(this.obligationCache.get()),
       onSessionTurn: (input) => this.publishSessionTurn(input),
       onUserPrompt: (input) => this.handleUserPrompt(input),
       onSessionEnd: (input) => this.handleSessionEnd(input),
+      onHookFatal: (report) => this.reportHookFatal(report),
     });
     this.writeRunnerInfoFile();
     this.options.adapter.onStatusChange((status) => {
@@ -277,12 +295,19 @@ export class AgentRunner {
       if (runnerShouldResolveProviderExit(semanticStatus, this.exitCommandInProgress)) this.options.onProviderExit?.(semanticStatus === "offline" ? 0 : 1);
     });
     this.options.adapter.onSessionEvent?.((event) => { void this.publishProviderSessionEvent(event); });
-    this.bus.on("message.new", (message) => this.enqueueMessage(message as Message));
+    this.bus.on("message.new", (message) => {
+      // A delivered message may create a new reply obligation — warm the snapshot so the
+      // next turn-end sees it without a hot-path server read.
+      this.obligationCache.markDirty();
+      this.enqueueMessage(message as Message);
+    });
     this.bus.on("command", (type, params, commandId, command) => {
       void this.handleCommand(type, params, commandId, command);
     });
     this.bus.on("error", (code, message) => this.handleBusError(String(code), String(message)));
     await this.bus.connect();
+    this.obligationCache.start();
+    this.outbox.start();
     this.ensureScratch();
     void this.sweepStaleScratch();
     this.process = await this.spawnProvider();
@@ -322,6 +347,8 @@ export class AgentRunner {
     this.tokenRenewTimer = undefined;
     this.disarmBusyReconciler();
     this.stopReasoningTail();
+    this.obligationCache.stop();
+    this.outbox.close();
     this.control?.stop();
     await this.bus.close();
   }
@@ -927,13 +954,10 @@ export class AgentRunner {
       replyToMessageId = pendingPrompt;
       this.pendingPromptMessageId = undefined;
     } else {
-      try {
-        const obligations = await this.http.listReplyObligations(this.agentId);
-        const obligation = [...obligations].reverse().find((o) => o.from === "user");
-        replyToMessageId = obligation?.messageId;
-      } catch {
-        // fall through and capture without correlation
-      }
+      // Correlation-only (threading + obligation clearing) — the local snapshot is fresh
+      // enough and never blocks the response-capture path (#196).
+      const obligation = [...this.obligationCache.get()].reverse().find((o) => o.from === "user");
+      replyToMessageId = obligation?.messageId;
     }
 
     // The Stop hook can fire before the final assistant entry is flushed to disk.
@@ -975,31 +999,86 @@ export class AgentRunner {
       ...(replyToMessageId ? { replyTo: replyToMessageId } : {}),
       session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) },
     });
+    // The agent's reply may have cleared an obligation — refresh the snapshot so the next
+    // turn-end doesn't re-prompt for a message already answered (#196).
+    if (replyToMessageId) this.obligationCache.markDirty();
   }
 
   // Post one session-mirror event (prompt echo, assistant response, reasoning or
   // tool step) as a `kind: "session"` relay message tagged with payload.session so
   // the dashboard can render the live provider session faithfully. Display-only:
   // session messages are never delivered back into a provider.
-  private async publishSessionEvent(input: {
+  private publishSessionEvent(input: {
     from: string;
     to: string;
     body: string;
     session: MessageSessionMeta;
     replyTo?: number;
-  }): Promise<void> {
-    try {
-      await this.http.sendMessage({
+  }): void {
+    // Durable, ordered, timestamped (#196): the actual POST happens in deliverOutboxEvent,
+    // retried until it lands. occurredAt is stamped now so a queued event reports when it
+    // truly happened, not when the server finally accepted it.
+    this.outbox.enqueue({
+      kind: "session-message",
+      payload: {
         from: input.from,
         to: input.to,
         ...(input.replyTo ? { replyTo: input.replyTo } : {}),
         kind: "session",
         body: input.body,
         payload: { session: { provider: this.options.provider, ...input.session } },
+      } satisfies SendMessageInput,
+    });
+  }
+
+  // The outbox transport: map a queued record to its HTTP call. Throw to retry, return to
+  // ack (delete). occurredAt + idempotencyKey are injected from the record so retries are
+  // exactly-once server-side and carry true event time.
+  private async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
+    try {
+      if (record.kind === "session-message") {
+        await this.http.sendMessage({
+          ...(record.payload as SendMessageInput),
+          occurredAt: record.occurredAt,
+          idempotencyKey: record.idempotencyKey,
+        });
+        return;
+      }
+      if (record.kind === "insight") {
+        await this.http.recordInsightObservation({
+          ...(record.payload as Parameters<RelayHttpClient["recordInsightObservation"]>[0]),
+          occurredAt: record.occurredAt,
+        });
+        return;
+      }
+      logger.warn("outbox", `dropping event with unknown kind: ${record.kind}`);
+    } catch (error) {
+      // 409 = the server intentionally rejected it (e.g. Insights/feature toggled off). That
+      // is a permanent "don't want this", not a transient failure — ack so it doesn't retry.
+      if (isHttpStatusError(error, 409)) return;
+      if (isHttpAuthError(error)) this.recoverRuntimeTokenAfterAuthFailure("outbox");
+      throw error; // transient (or auth, post-recovery) → let the outbox retry with backoff
+    }
+  }
+
+  // A hook reported an unhandled failure (#198 seam). Already logged FATAL by the control
+  // server; here we additionally surface it durably to the server as a generic insight so
+  // it shows up in observability rather than only in the per-agent log (#196).
+  private reportHookFatal(report: { hook: string; error: string }): void {
+    try {
+      this.outbox.enqueue({
+        kind: "insight",
+        payload: {
+          sessionId: this.providerSessionId,
+          project: this.options.cwd,
+          agentId: this.agentId,
+          signal: "hook_fatal",
+          value: { hook: report.hook, error: report.error },
+          source: "server",
+        },
       });
     } catch (error) {
-      this.logRunnerDiagnostic(`session ${input.session.type} capture failed: ${error instanceof Error ? error.message : String(error)}`);
-      if (isHttpAuthError(error)) this.recoverRuntimeTokenAfterAuthFailure("session-capture");
+      logger.error("outbox", `failed to queue hook-fatal report: ${error instanceof Error ? error.message : String(error)}`);
     }
   }
 
@@ -1043,8 +1122,11 @@ export class AgentRunner {
     }
     const analysis = analyzeSession(jsonl);
     if (!analysis) return; // no tool calls = nothing substantive to measure
-    try {
-      await this.http.recordInsightObservation({
+    // Durable + non-blocking (#196): queue it. SessionEnd can race provider shutdown, so a
+    // direct POST risked being dropped if the server hiccuped; the outbox survives that.
+    this.outbox.enqueue({
+      kind: "insight",
+      payload: {
         sessionId: this.providerSessionId,
         project: this.options.cwd,
         agentId: this.agentId,
@@ -1052,13 +1134,9 @@ export class AgentRunner {
         value: { ...analysis.metric, ...(input.reason ? { endReason: input.reason } : {}) },
         outcome: { ...analysis.outcome },
         source: "server",
-      });
-      this.sessionLog(`insights: context_ratio ${analysis.metric.ratio.toFixed(2)} (${analysis.metric.gatheringCalls}/${analysis.metric.totalToolCalls} gathering)`);
-    } catch (error) {
-      // 409 = Insights/feature toggled off; anything else is best-effort too.
-      this.sessionDebug(`insights context_ratio skipped: ${error instanceof Error ? error.message : String(error)}`);
-      if (isHttpAuthError(error)) this.recoverRuntimeTokenAfterAuthFailure("insights");
-    }
+      },
+    });
+    this.sessionLog(`insights: context_ratio ${analysis.metric.ratio.toFixed(2)} (${analysis.metric.gatheringCalls}/${analysis.metric.totalToolCalls} gathering) queued`);
   }
 
   // Route a provider-emitted session event (Codex app-server) into the chat mirror.
@@ -1087,13 +1165,9 @@ export class AgentRunner {
       if (pendingPrompt) {
         replyToMessageId = pendingPrompt;
         this.pendingPromptMessageId = undefined;
-      } else {
-        try {
-          const obligations = await this.http.listReplyObligations(this.agentId);
-          if (obligations.some((o) => o.from === "user")) return;
-        } catch {
-          // capture anyway on lookup failure
-        }
+      } else if (this.obligationCache.get().some((o) => o.from === "user")) {
+        // The agent will answer the relay obligation itself — don't double-post (#196).
+        return;
       }
       await this.publishSessionEvent({
         from: this.agentId,
@@ -1953,6 +2027,11 @@ function isHttpAuthError(error: unknown): boolean {
   return status === 401 || status === 403;
 }
 
+function isHttpStatusError(error: unknown, code: number): boolean {
+  const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
+  return status === code;
+}
+
 function httpErrorKey(error: unknown): string {
   const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
   if (typeof status === "number") return `status:${status}`;