agent-relay-server 0.121.5 → 0.122.1

package/runner/src/mcp-outbox.ts

@@ -0,0 +1,120 @@
+import type { SendMessageInput } from "agent-relay-sdk";
+import type { RelayHttpClient } from "agent-relay-sdk";
+import { errMessage } from "agent-relay-sdk";
+import { relayMcpEndpoint } from "./relay-mcp";
+import type { OutboxRecord } from "./outbox";
+import { deliverContinuationArchiveRecord } from "./continuation-archive";
+import { logger } from "./logger";
+import { isHttpAuthError, isHttpStatusError } from "./runner-helpers";
+
+interface RunnerOutboxDelivery {
+  record: OutboxRecord;
+  http: RelayHttpClient;
+  relayUrl: string;
+  token?: string;
+  updatePayload(seq: number, payload: unknown): void;
+  sessionLog(message: string): void;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+}
+
+export async function deliverRunnerOutboxEvent(input: RunnerOutboxDelivery): Promise<void> {
+  const { record, http } = input;
+  try {
+    if (record.kind === "session-message") {
+      await http.sendMessage({
+        ...(record.payload as SendMessageInput),
+        occurredAt: record.occurredAt,
+        idempotencyKey: record.idempotencyKey,
+      });
+      return;
+    }
+    if (record.kind === "session-message-batch") {
+      const messages = Array.isArray((record.payload as { messages?: unknown }).messages)
+        ? (record.payload as { messages: SendMessageInput[] }).messages
+        : [];
+      const batch = messages.map((message, index) => ({
+        ...message,
+        occurredAt: typeof message.occurredAt === "number" ? message.occurredAt : record.occurredAt,
+        idempotencyKey: message.idempotencyKey ?? `${record.idempotencyKey}:${index}`,
+      }));
+      try {
+        await http.sendMessages(batch);
+      } catch (error) {
+        if (!isDeterministicValidationError(error)) throw error;
+        await deliverSessionBatchItems(input, batch);
+      }
+      return;
+    }
+    if (record.kind === "insight") {
+      await http.recordInsightObservation({
+        ...(record.payload as Parameters<RelayHttpClient["recordInsightObservation"]>[0]),
+        occurredAt: record.occurredAt,
+      });
+      return;
+    }
+    if (record.kind === "continuation-archive") {
+      await deliverContinuationArchiveRecord({
+        record,
+        http,
+        updatePayload: input.updatePayload,
+        sessionLog: input.sessionLog,
+      });
+      return;
+    }
+    if (record.kind === "mcp-tool-call") {
+      await deliverBufferedMcpCall(input);
+      return;
+    }
+    logger.warn("outbox", `dropping event with unknown kind: ${record.kind}`);
+  } catch (error) {
+    if (isHttpStatusError(error, 409)) return;
+    if (isHttpAuthError(error)) input.recoverRuntimeTokenAfterAuthFailure("outbox");
+    throw error;
+  }
+}
+
+async function deliverSessionBatchItems(input: RunnerOutboxDelivery, messages: SendMessageInput[]): Promise<void> {
+  for (const message of messages) {
+    try {
+      await input.http.sendMessage(message);
+    } catch (error) {
+      if (!isDeterministicValidationError(error)) throw error;
+      logger.warn("outbox", `dropping invalid session batch item: ${errMessage(error)}`);
+    }
+  }
+}
+
+function isDeterministicValidationError(error: unknown): boolean {
+  const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
+  return status === 400 || status === 413 || status === 422;
+}
+
+export async function deliverBufferedMcpCall(input: {
+  record: OutboxRecord;
+  relayUrl: string;
+  token?: string;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+}): Promise<void> {
+  const payload = input.record.payload as { tool: string; arguments: Record<string, unknown> };
+  const headers: Record<string, string> = { "content-type": "application/json" };
+  if (input.token) headers.authorization = `Bearer ${input.token}`;
+  const response = await fetch(relayMcpEndpoint(input.relayUrl), {
+    method: "POST",
+    headers,
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: payload.tool, arguments: payload.arguments } }),
+  });
+  if (response.status === 401 || response.status === 403) {
+    input.recoverRuntimeTokenAfterAuthFailure("mcp-outbox");
+    throw new Error(`relay rejected buffered ${payload.tool} with ${response.status}`);
+  }
+  if (response.status >= 500) throw new Error(`relay ${response.status} on buffered ${payload.tool}`);
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    logger.warn("mcp-outbox", `buffered ${payload.tool} permanently rejected (${response.status}); dropping: ${body.slice(0, 200)}`);
+    return;
+  }
+  const json = await response.json().catch(() => null) as { error?: { message?: string } } | null;
+  if (json?.error) {
+    logger.warn("mcp-outbox", `buffered ${payload.tool} returned a tool error; dropping: ${json.error.message ?? "(no detail)"}`);
+  }
+}

package/runner/src/message-body-config/config.ts

@@ -0,0 +1,3 @@
+export function messageBodyMaxCharsFromEnv(): string | undefined {
+  return process.env.AGENT_RELAY_MESSAGE_BODY_MAX_CHARS;
+}

package/runner/src/message-body-config.ts

@@ -0,0 +1 @@
+export { messageBodyMaxCharsFromEnv } from "./message-body-config/config";

package/runner/src/native-self-resume.ts

@@ -0,0 +1,202 @@
+import { errMessage } from "agent-relay-sdk";
+import type { ProviderStatusEvent, SemanticStatus } from "./adapter";
+import { nativeSelfResumeTimeoutMsFromEnv } from "./config";
+
+interface RunnerTimelineEvent {
+  status: string;
+  id?: string;
+  timestamp: number;
+  title?: string;
+  body?: string;
+  icon?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface NativeSelfResumeHandoff {
+  commandId: string;
+  generation?: number;
+  compactedAt?: number;
+  timer: ReturnType<typeof setTimeout>;
+  resolve: (result: Record<string, unknown>) => void;
+  reject: (error: Error) => void;
+  promise: Promise<Record<string, unknown>>;
+}
+
+export function isNativeSelfResumeCompact(type: string, params: Record<string, unknown>): boolean {
+  if (type !== "agent.compact") return false;
+  const selfResume = params.selfResume;
+  return Boolean(selfResume && typeof selfResume === "object" && !Array.isArray(selfResume) && (selfResume as Record<string, unknown>).mode === "native");
+}
+
+export class NativeSelfResumeTracker {
+  private current?: NativeSelfResumeHandoff;
+
+  get active(): NativeSelfResumeHandoff | undefined {
+    return this.current;
+  }
+
+  begin(commandId: string, params: Record<string, unknown>): NativeSelfResumeHandoff {
+    this.finish(this.current);
+    let resolve!: (result: Record<string, unknown>) => void;
+    let reject!: (error: Error) => void;
+    const promise = new Promise<Record<string, unknown>>((res, rej) => {
+      resolve = res;
+      reject = rej;
+    });
+    const selfResume = params.selfResume && typeof params.selfResume === "object" && !Array.isArray(params.selfResume)
+      ? params.selfResume as Record<string, unknown>
+      : undefined;
+    const generation = typeof selfResume?.generation === "number" ? selfResume.generation : undefined;
+    const timeoutMs = nativeSelfResumeTimeoutMsFromEnv();
+    const handoff: NativeSelfResumeHandoff = {
+      commandId,
+      generation,
+      promise,
+      resolve,
+      reject,
+      timer: setTimeout(() => {
+        if (this.current !== handoff) return;
+        this.current = undefined;
+        reject(new Error(`native self-resume timed out after ${Math.round(timeoutMs / 1000)}s`));
+      }, timeoutMs),
+    };
+    this.current = handoff;
+    return handoff;
+  }
+
+  finish(handoff?: NativeSelfResumeHandoff): void {
+    if (!handoff) return;
+    clearTimeout(handoff.timer);
+    if (this.current === handoff) this.current = undefined;
+  }
+
+  resolve(handoff: NativeSelfResumeHandoff, result: Record<string, unknown>): void {
+    this.finish(handoff);
+    handoff.resolve(result);
+  }
+
+  reject(handoff: NativeSelfResumeHandoff, error: Error): void {
+    this.finish(handoff);
+    handoff.reject(error);
+  }
+
+  noteCompacted(timestamp: number = Date.now()): void {
+    if (!this.current) return;
+    this.current.compactedAt = timestamp;
+  }
+}
+
+interface NativeSelfResumeRecoveryOptions {
+  handoff: NativeSelfResumeHandoff;
+  tracker: NativeSelfResumeTracker;
+  providerSessionId: string;
+  status: SemanticStatus;
+  diagnostics: Record<string, unknown>;
+  now: number;
+  publishTimeline(event: RunnerTimelineEvent): void;
+  setTerminalProviderExit(marker: Record<string, unknown>): void;
+  setProviderStatus(update: ProviderStatusEvent): void;
+  restartProvider(resumeId?: string): Promise<void>;
+  publishStatus(): void;
+  scheduleDrain(): void;
+}
+
+export async function recoverNativeSelfResumeExit(opts: NativeSelfResumeRecoveryOptions): Promise<void> {
+  const resumeId = typeof opts.diagnostics.claudeResumeId === "string" && opts.diagnostics.claudeResumeId.length > 0
+    ? opts.diagnostics.claudeResumeId
+    : undefined;
+  if (opts.handoff.compactedAt && resumeId) {
+    await recoverWithClaudeResume(opts, resumeId);
+    return;
+  }
+  await recoverWithFreshLaunch(opts, resumeId);
+}
+
+async function recoverWithClaudeResume(opts: NativeSelfResumeRecoveryOptions, resumeId: string): Promise<void> {
+  opts.publishTimeline({
+    status: "provider.restart_decision",
+    id: `provider-self-resume-relaunch-${opts.providerSessionId}-${opts.now}`,
+    timestamp: Date.now(),
+    title: "Self-resume relaunching",
+    body: `Claude exited during native self-resume; runner is relaunching with claude --resume ${resumeId}.`,
+    icon: "ti-refresh",
+    metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      decision: "resume-native-self-resume",
+      reason: "self-resume-exit-after-compact",
+      claudeResumeId: resumeId,
+      compactedAt: opts.handoff.compactedAt,
+    }),
+  });
+  try {
+    await opts.restartProvider(resumeId);
+    opts.publishStatus();
+    opts.scheduleDrain();
+    opts.tracker.resolve(opts.handoff, { recovered: true, via: "claude-resume", claudeResumeId: resumeId });
+  } catch (error) {
+    const message = errMessage(error);
+    opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      reason: "self-resume-relaunch-failed",
+      claudeResumeId: resumeId,
+      compactedAt: opts.handoff.compactedAt,
+      lastError: message,
+    }));
+    opts.setProviderStatus({
+      status: "offline",
+      reason: "provider-turn",
+      id: `provider-self-resume-relaunch-failed-${opts.providerSessionId}`,
+      clear: ["provider-turn", "subagent", "background-script"],
+    });
+    opts.tracker.reject(opts.handoff, new Error(`self-resume relaunch failed: ${message}`));
+  }
+}
+
+async function recoverWithFreshLaunch(opts: NativeSelfResumeRecoveryOptions, resumeId?: string): Promise<void> {
+  opts.publishTimeline({
+    status: "provider.restart_decision",
+    id: `provider-self-resume-fresh-launch-${opts.providerSessionId}-${opts.now}`,
+    timestamp: Date.now(),
+    title: "Self-resume fresh-launching",
+    body: "Claude exited during native self-resume before a compacted marker; runner is fresh-launching so Relay can inject the persisted continuation.",
+    icon: "ti-refresh",
+    metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      decision: "fresh-launch-native-self-resume",
+      reason: "self-resume-exit-before-compact",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+    }),
+  });
+  try {
+    await opts.restartProvider();
+    opts.publishStatus();
+    opts.scheduleDrain();
+    opts.tracker.resolve(opts.handoff, {
+      recovered: true,
+      via: "fresh-launch",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+    });
+  } catch (error) {
+    const message = errMessage(error);
+    opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      reason: "self-resume-fresh-launch-failed",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+      lastError: message,
+    }));
+    opts.setProviderStatus({
+      status: "offline",
+      reason: "provider-turn",
+      id: `provider-self-resume-fresh-launch-failed-${opts.providerSessionId}`,
+      clear: ["provider-turn", "subagent", "background-script"],
+    });
+    opts.tracker.reject(opts.handoff, new Error(`self-resume fresh launch failed: ${message}`));
+  }
+}
+
+function selfResumeMetadata(handoff: NativeSelfResumeHandoff, diagnostics: Record<string, unknown>, extra: Record<string, unknown>): Record<string, unknown> {
+  return {
+    eventType: "provider.restart_decision",
+    selfResume: true,
+    selfResumeCommandId: handoff.commandId,
+    selfResumeGeneration: handoff.generation ?? null,
+    ...extra,
+    ...diagnostics,
+  };
+}

package/runner/src/outbox.ts

@@ -0,0 +1,342 @@
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { tmpdir } from "node:os";
+import { sanitizeFsName } from "agent-relay-sdk/fs-name";
+import { errMessage } from "agent-relay-sdk";
+import { logger } from "./logger";
+import { runnerOutboxDirFromEnv } from "./config";
+
+// Phase 2 (#196) — the "nothing is ever lost" half. Runner→server events that used to be
+// fire-and-forget over HTTP (session turns, reasoning/tool traces, prompt echoes, insights,
+// hook-fatal reports) were silently dropped whenever the server was momentarily down. This
+// is a durable, FIFO, disk-backed queue that:
+//   - survives Runner/server restart (bun:sqlite file in the runtime dir),
+//   - stamps true event time (`occurredAt`) once at enqueue and preserves it through retries,
+//   - retries with capped exponential backoff, strictly in order (an append log must not
+//     reorder turns),
+//   - poisons a permanently-failing head after maxAttempts so it can't block the queue,
+//   - is bounded with a logged drop policy (never silently truncates).
+//
+// Status deliberately does NOT go through here: it rides the WebSocket bus, which is
+// last-wins and self-heals on reconnect (so it already satisfies "coalesce, don't replay
+// stale busyes"). The coalesce mode below exists so a future state event could migrate here.
+
+type OutboxMode = "append" | "coalesce";
+
+interface OutboxEventInput {
+  kind: string;
+  payload: unknown;
+  mode?: OutboxMode;
+  // Required for coalesce mode: prior un-poisoned rows with the same dedupeKey are replaced.
+  dedupeKey?: string;
+  // Defaults to now. Set explicitly only to backdate (e.g. replaying a captured timestamp).
+  occurredAt?: number;
+  // Defaults to a stable derived key so server-side dedup makes retries exactly-once.
+  idempotencyKey?: string;
+}
+
+export interface OutboxRecord {
+  seq: number;
+  kind: string;
+  mode: OutboxMode;
+  occurredAt: number;
+  idempotencyKey: string;
+  payload: unknown;
+  attempts: number;
+}
+
+// The transport. Resolve = delivered (row deleted). Reject = failed (retried with backoff).
+type OutboxSend = (record: OutboxRecord) => Promise<void>;
+
+interface OutboxOptions {
+  agentId: string;
+  send: OutboxSend;
+  // Storage directory. Defaults to AGENT_RELAY_RUNNER_OUTBOX_DIR, else a per-host temp dir.
+  dir?: string;
+  maxRows?: number;
+  maxAttempts?: number;
+  baseBackoffMs?: number;
+  maxBackoffMs?: number;
+  pollMs?: number;
+}
+
+const DEFAULTS = {
+  maxRows: 5000,
+  maxAttempts: 12,
+  baseBackoffMs: 1_000,
+  maxBackoffMs: 60_000,
+  pollMs: 5_000,
+};
+
+// Pause between forced delivery attempts during flush() — keeps a brief server outage
+// from hot-looping while still retrying fast enough to land within the flush deadline.
+const FLUSH_RETRY_PAUSE_MS = 150;
+
+interface Row {
+  seq: number;
+  kind: string;
+  mode: string;
+  occurred_at: number;
+  idempotency_key: string;
+  payload: string;
+  attempts: number;
+  next_attempt_at: number;
+  poisoned: number;
+}
+
+export class Outbox {
+  private readonly db: Database;
+  private readonly agentId: string;
+  private readonly send: OutboxSend;
+  private readonly maxRows: number;
+  private readonly maxAttempts: number;
+  private readonly baseBackoffMs: number;
+  private readonly maxBackoffMs: number;
+  private readonly pollMs: number;
+  readonly path: string;
+
+  private draining = false;
+  private rerun = false;
+  // Set by flush() to make the next drain pass ignore per-row backoff (deliver NOW).
+  private forceDue = false;
+  private pollTimer?: ReturnType<typeof setInterval>;
+  private dueTimer?: ReturnType<typeof setTimeout>;
+  private stopped = false;
+
+  constructor(options: OutboxOptions) {
+    this.agentId = options.agentId;
+    this.send = options.send;
+    this.maxRows = options.maxRows ?? DEFAULTS.maxRows;
+    this.maxAttempts = options.maxAttempts ?? DEFAULTS.maxAttempts;
+    this.baseBackoffMs = options.baseBackoffMs ?? DEFAULTS.baseBackoffMs;
+    this.maxBackoffMs = options.maxBackoffMs ?? DEFAULTS.maxBackoffMs;
+    this.pollMs = options.pollMs ?? DEFAULTS.pollMs;
+
+    const dir = options.dir ?? runnerOutboxDirFromEnv() ?? join(tmpdir(), "agent-relay-outbox");
+    this.path = options.dir === ":memory:" ? ":memory:" : join(dir, `outbox-${safeName(this.agentId)}.sqlite`);
+    if (this.path !== ":memory:") mkdirSync(dirname(this.path), { recursive: true });
+
+    this.db = new Database(this.path, { create: true });
+    this.db.exec("PRAGMA journal_mode = WAL");
+    this.db.exec("PRAGMA busy_timeout = 2000");
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS outbox (
+        seq INTEGER PRIMARY KEY AUTOINCREMENT,
+        kind TEXT NOT NULL,
+        mode TEXT NOT NULL DEFAULT 'append',
+        dedupe_key TEXT,
+        occurred_at INTEGER NOT NULL,
+        idempotency_key TEXT NOT NULL,
+        payload TEXT NOT NULL,
+        attempts INTEGER NOT NULL DEFAULT 0,
+        next_attempt_at INTEGER NOT NULL DEFAULT 0,
+        poisoned INTEGER NOT NULL DEFAULT 0,
+        created_at INTEGER NOT NULL
+      )
+    `);
+    // A restart is a fresh start: clear any backoff timers left by the prior process so
+    // pending events get an immediate retry (the down server may now be back). `attempts`
+    // is kept so the poison threshold still counts cumulative failures.
+    this.db.exec("UPDATE outbox SET next_attempt_at = 0 WHERE next_attempt_at > 0");
+  }
+
+  // Persist an event. Returns the assigned seq. Triggers a drain.
+  enqueue(input: OutboxEventInput): number {
+    if (this.stopped) throw new Error("outbox is stopped");
+    const mode: OutboxMode = input.mode ?? "append";
+    const occurredAt = input.occurredAt ?? Date.now();
+    const payloadJson = JSON.stringify(input.payload ?? null);
+    const idempotencyKey = input.idempotencyKey ?? `${this.agentId}:${input.kind}:${occurredAt}:${shortHash(payloadJson)}`;
+
+    if (mode === "coalesce") {
+      if (!input.dedupeKey) throw new Error("coalesce mode requires a dedupeKey");
+      this.db.query("DELETE FROM outbox WHERE dedupe_key = ? AND poisoned = 0").run(input.dedupeKey);
+    }
+
+    const info = this.db
+      .query(`INSERT INTO outbox (kind, mode, dedupe_key, occurred_at, idempotency_key, payload, created_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?)`)
+      .run(input.kind, mode, input.dedupeKey ?? null, occurredAt, idempotencyKey, payloadJson, Date.now());
+    const seq = Number(info.lastInsertRowid);
+
+    this.enforceBound();
+    // Defer the drain to a microtask so a synchronous burst of enqueues (e.g. several
+    // coalesce updates) all land — and coalesce — before the pump pulls the head.
+    queueMicrotask(() => { void this.drain(); });
+    return seq;
+  }
+
+  // Bounded ring buffer: if over capacity, drop the oldest rows (defined overflow policy).
+  // Logged, never silent. Prefers dropping already-poisoned rows first, then oldest by seq.
+  private enforceBound(): void {
+    const { n } = this.db.query("SELECT count(*) AS n FROM outbox").get() as { n: number };
+    if (n <= this.maxRows) return;
+    const overflow = n - this.maxRows;
+    // Oldest poisoned first, then oldest live — both by seq.
+    const victims = this.db
+      .query("SELECT seq FROM outbox ORDER BY poisoned DESC, seq ASC LIMIT ?")
+      .all(overflow) as Array<{ seq: number }>;
+    const ids = victims.map((v) => v.seq);
+    if (ids.length === 0) return;
+    const placeholders = ids.map(() => "?").join(",");
+    this.db.query(`DELETE FROM outbox WHERE seq IN (${placeholders})`).run(...ids);
+    logger.warn("outbox", `bound exceeded (${n}/${this.maxRows}) — dropped ${ids.length} oldest event(s)`);
+  }
+
+  // Begin the background pump: an initial drain plus a poll timer as a backstop.
+  start(): void {
+    if (this.pollTimer || this.stopped) return;
+    void this.drain();
+    this.pollTimer = setInterval(() => { void this.drain(); }, this.pollMs);
+    this.pollTimer.unref?.();
+  }
+
+  // Drain everything we can before the process exits, ignoring per-row backoff. Used on
+  // shutdown/kill/crash (#183): the capture seam durably enqueues the end-of-session
+  // Insights datapoint, but the per-agent outbox is never reopened once the agent is gone,
+  // so a row that hasn't been POSTed yet is stranded forever. flush() blocks (bounded) on
+  // delivery — the opposite intent of the polite, backoff-respecting steady-state drain.
+  // Returns true if the queue fully drained within the deadline.
+  async flush(timeoutMs: number): Promise<boolean> {
+    const deadline = Date.now() + Math.max(0, timeoutMs);
+    for (;;) {
+      if (this.stopped) return this.pendingCount() === 0;
+      if (this.pendingCount() === 0) return true;
+      await this.drain(true);
+      if (this.pendingCount() === 0) return true;
+      if (Date.now() >= deadline) return false;
+      await new Promise<void>((resolve) => setTimeout(resolve, FLUSH_RETRY_PAUSE_MS));
+    }
+  }
+
+  // Process the queue strictly oldest-first. Coalesces concurrent calls; if a drain is
+  // requested while one is running, it re-runs once at the end (so an enqueue during a
+  // send isn't missed). `force` (from flush) makes this pass ignore per-row backoff.
+  async drain(force = false): Promise<void> {
+    if (this.stopped) return;
+    if (force) this.forceDue = true;
+    if (this.draining) { this.rerun = true; return; }
+    this.draining = true;
+    try {
+      do {
+        this.rerun = false;
+        const due = this.forceDue;
+        this.forceDue = false;
+        await this.drainOnce(due);
+      } while ((this.rerun || this.forceDue) && !this.stopped);
+    } finally {
+      this.draining = false;
+    }
+  }
+
+  private async drainOnce(force = false): Promise<void> {
+    for (;;) {
+      if (this.stopped) return;
+      const row = this.db
+        .query("SELECT * FROM outbox WHERE poisoned = 0 ORDER BY seq ASC LIMIT 1")
+        .get() as Row | null;
+      if (!row) return;
+
+      const now = Date.now();
+      if (!force && row.next_attempt_at > now) {
+        // Head isn't due yet. Don't reorder past it (FIFO) — schedule a wake-up and stop.
+        this.scheduleDue(row.next_attempt_at - now);
+        return;
+      }
+
+      const record: OutboxRecord = {
+        seq: row.seq,
+        kind: row.kind,
+        mode: row.mode as OutboxMode,
+        occurredAt: row.occurred_at,
+        idempotencyKey: row.idempotency_key,
+        payload: safeParse(row.payload),
+        attempts: row.attempts,
+      };
+
+      try {
+        await this.send(record);
+        this.db.query("DELETE FROM outbox WHERE seq = ?").run(row.seq);
+      } catch (error) {
+        const attempts = row.attempts + 1;
+        const reason = errMessage(error);
+        if (attempts >= this.maxAttempts) {
+          this.db.query("UPDATE outbox SET attempts = ?, poisoned = 1 WHERE seq = ?").run(attempts, row.seq);
+          logger.fatal("outbox", `event seq=${row.seq} kind=${row.kind} poisoned after ${attempts} attempts: ${reason}`);
+          // Move on — the next iteration picks the new head (poison no longer blocks).
+          continue;
+        }
+        const delay = this.backoff(attempts);
+        this.db.query("UPDATE outbox SET attempts = ?, next_attempt_at = ? WHERE seq = ?").run(attempts, now + delay, row.seq);
+        logger.debug("outbox", `event seq=${row.seq} kind=${row.kind} retry ${attempts}/${this.maxAttempts} in ${delay}ms: ${reason}`);
+        this.scheduleDue(delay);
+        return; // head is now scheduled; stop until it's due (preserve order)
+      }
+    }
+  }
+
+  // Pure per-row delay calc: the attempt count lives in SQLite (one outbox row
+  // at a time may be due), so this stays a stateless function rather than the
+  // stateful SDK ReconnectionManager (single in-memory counter) used for the
+  // orchestrator/voice reconnect loops — different model, deliberately not shared.
+  private backoff(attempts: number): number {
+    const exp = Math.min(this.maxBackoffMs, this.baseBackoffMs * 2 ** (attempts - 1));
+    return Math.round(exp / 2 + Math.random() * (exp / 2)); // full-ish jitter, never below half
+  }
+
+  private scheduleDue(delayMs: number): void {
+    if (this.stopped || this.dueTimer) return;
+    this.dueTimer = setTimeout(() => {
+      this.dueTimer = undefined;
+      void this.drain();
+    }, Math.max(0, delayMs));
+    this.dueTimer.unref?.();
+  }
+
+  // Observability / tests.
+  pendingCount(): number {
+    return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 0").get() as { n: number }).n;
+  }
+
+  updatePayload(seq: number, payload: unknown): void {
+    const payloadJson = JSON.stringify(payload ?? null);
+    this.db.query("UPDATE outbox SET payload = ? WHERE seq = ? AND poisoned = 0").run(payloadJson, seq);
+  }
+
+  poisonedCount(): number {
+    return (this.db.query("SELECT count(*) AS n FROM outbox WHERE poisoned = 1").get() as { n: number }).n;
+  }
+
+  stop(): void {
+    this.stopped = true;
+    if (this.pollTimer) clearInterval(this.pollTimer);
+    this.pollTimer = undefined;
+    if (this.dueTimer) clearTimeout(this.dueTimer);
+    this.dueTimer = undefined;
+  }
+
+  close(): void {
+    this.stop();
+    try { this.db.close(); } catch { /* already closed */ }
+  }
+}
+
+function safeName(value: string): string {
+  return sanitizeFsName(value, { replacement: "_", maxLen: 180, fallback: "agent" });
+}
+
+function safeParse(json: string): unknown {
+  try { return JSON.parse(json); } catch { return null; }
+}
+
+// Small, fast, stable string hash (FNV-1a, 32-bit) — enough to disambiguate identical
+// kind+timestamp payloads in the idempotency key. Not security-sensitive.
+function shortHash(value: string): string {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < value.length; i++) {
+    h ^= value.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(36);
+}

package/runner/src/process-meta.ts

@@ -0,0 +1,9 @@
+import type { ManagedProcess } from "./adapter";
+
+export function providerTerminalSession(process?: ManagedProcess): string | undefined {
+  return typeof process?.meta?.tmuxSession === "string" ? process.meta.tmuxSession : undefined;
+}
+
+export function providerTerminalSocket(process?: ManagedProcess): string | undefined {
+  return typeof process?.meta?.tmuxSocket === "string" ? process.meta.tmuxSocket : undefined;
+}