@bitkyc08/opencodex 2.91.0 → 2.92.0

package/src/lib/decision-quota-sink.ts

@@ -18,6 +18,10 @@ import type { CodexAccountCredentialRecord } from "../types";
 
 export type DecisionPoolRead = readonly { id: string; usable: boolean; windows?: readonly DecisionQuotaWindow[] }[];
 export interface MainDecisionRow { generation: number; usable: boolean }
+/** Opaque, process-local producer receipt; only the active snapshot can validate it. */
+export interface AntigravityQuotaReceipt { readonly kind: "antigravity" }
+export interface AntigravityQuotaObservation { name: string; provider: OcxProviderConfig; receipt: AntigravityQuotaReceipt; windows: readonly DecisionQuotaWindow[] }
+export interface DecisionKeyObservation { name: string; provider: OcxProviderConfig; windows: readonly DecisionQuotaWindow[] }
 
 export interface DecisionQuotaSink {
   observeKeyCredential(reference: string, resolved: string | undefined): void;
@@ -32,6 +36,8 @@ export interface DecisionQuotaSink {
   mainRosterChanged(row: MainDecisionRow | undefined): void;
   readKeyQuota(name: string, provider: OcxProviderConfig): readonly DecisionQuotaWindow[] | undefined;
   readPool(provider: string): DecisionPoolRead | undefined;
+  captureAntigravityReceipt(name: string, provider: OcxProviderConfig, id: string, generation: string): AntigravityQuotaReceipt | undefined;
+  replaceCollectedQuotas(keys: readonly DecisionKeyObservation[], oauth: readonly AntigravityQuotaObservation[]): void;
 }
 
 let sink: DecisionQuotaSink | undefined;
@@ -71,6 +77,10 @@ export function notifyCodexStoreLoaded(store: Readonly<Record<string, CodexAccou
 export function notifyMainRosterChanged(row: () => MainDecisionRow | undefined): void { forward(s => s.mainRosterChanged(row())); }
 export function readLoadedDecisionKeyQuota(name: string, provider: OcxProviderConfig): readonly DecisionQuotaWindow[] | undefined { return read(s => s.readKeyQuota(name, provider)); }
 export function readLoadedDecisionQuotaPool(provider: string): DecisionPoolRead | undefined { return read(s => s.readPool(provider)); }
+/** Capture from the producer's already-loaded credential, never from a decision read. */
+export function captureAntigravityQuotaReceipt(name: string, provider: OcxProviderConfig, id: string, generation: string): AntigravityQuotaReceipt | undefined { return read(s => s.captureAntigravityReceipt(name, provider, id, generation)); }
+/** Accepted collector atomically replaces its key/instance evidence set. */
+export function replaceCollectedDecisionQuotas(keys: readonly DecisionKeyObservation[], oauth: readonly AntigravityQuotaObservation[]): void { forward(s => s.replaceCollectedQuotas(keys, oauth)); }
 
 /** Test-only: drop any registration a previous test left behind. */
 export function resetDecisionQuotaSinkForTests(): void { sink = undefined; }

package/src/lib/translator-budget.ts

@@ -1,4 +1,4 @@
-import { isNonReplayableResponse, markResponseNonReplayable } from "./upstream-retry";
+import { carryReplayRefusal, isNonReplayableResponse, markResponseNonReplayable } from "./upstream-retry";
 
 export const TRANSLATOR_MAX_CALL_ARGUMENT_BYTES = 2 * 1024 * 1024;
 export const TRANSLATOR_MAX_TURN_BYTES = 32 * 1024 * 1024;
@@ -413,8 +413,8 @@ export function finalizeTranslatorBudgetResponse(response: Response, budget: Tra
   if (signal?.aborted) finalize();
   else signal?.addEventListener("abort", finalize, { once: true });
   const finalizedResponse = new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers });
-  // The rebuild above drops identity markers. A refusal that must not be replayed
-  // stays non-replayable through accounting finalization.
+  // Preserve both replay suppression and refusal provenance across the rebuild
+  // so deferred logging can distinguish synthetic refusals from upstream 429s.
   if (isNonReplayableResponse(response)) markResponseNonReplayable(finalizedResponse);
-  return finalizedResponse;
+  return carryReplayRefusal(response, finalizedResponse);
 }

package/src/lib/upstream-retry.ts

@@ -770,7 +770,10 @@ export async function fetchWithTransientRetry(
   for (let attempt = 0; sent < budget; attempt++) {
     // A non-replayable gateway status was settled after the request body had already left
     // for the origin; retrying it here is the automatic resend the marker exists to forbid.
-    if (res.ok || !isTransientUpstreamStatus(res.status) || isNonReplayableResponse(res)) return res;
+    // An upstream that answers with the exact `x-should-retry: false` verdict has said the same
+    // thing about its own failure, so the ladder stops and hands the response back untouched.
+    if (res.ok || !isTransientUpstreamStatus(res.status) || isNonReplayableResponse(res)
+      || res.headers.get(REPLAY_REFUSAL_NO_RETRY_HEADER) === REPLAY_REFUSAL_NO_RETRY_VALUE) return res;
     // Checked before cancelResponseBodyBestEffort so an already-aborted caller never receives
     // a response whose body we just cancelled.
     if (opts.abortSignal?.aborted) return res;

package/src/link/diagnostics.ts

@@ -0,0 +1,193 @@
+import { constants } from "node:fs";
+import { chmod, lstat, mkdir, open, rename, unlink } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { assertNotRealHomeUnderTest } from "../lib/test-home-guard";
+import { hardenSecretDirAsync, hardenSecretPathAsync, reattributeHardenedSecretPath, windowsSecretAclApplies } from "../lib/windows-secret-acl";
+import { linkDir } from "./paths";
+import type { TunnelState } from "./tunnel-state";
+
+export type LinkDiagnosticRecord = {
+  event: "owner_lookup" | "relay_retry" | "relay_failure" | "relay_recovered" | "tunnel_transition";
+  outcome: "match" | "mismatch" | "lookup_error" | "deadline" | "stale" | "unavailable"
+    | "identity_unknown" | "identity_mismatch" | "auth_timeout" | "auth_failed"
+    | "transport_error" | "client_abort" | "header_timeout" | "recovered"
+    // tunnel_transition: the Home's -R tunnel changed state, and why (src/link/tunnel-history.ts).
+    | "connected" | "spawned" | "stopped" | "auth" | "hostkey" | "forward" | "remote_forward_conflict"
+    | "timeout" | "network" | "unknown" | "spawn_failed" | "listener_unavailable";
+  requestId?: string;
+  checkId?: string;
+  generation?: number;
+  state?: string;
+  previousState?: string;
+  elapsedMs?: number;
+  attempt?: number;
+  tunnelPort?: number;
+  retryAfterMs?: number;
+};
+export type LinkDiagnosticSink = (record: LinkDiagnosticRecord) => void;
+
+const EVENTS = new Set(["owner_lookup", "relay_retry", "relay_failure", "relay_recovered", "tunnel_transition"]);
+const OUTCOMES = new Set([
+  "match", "mismatch", "lookup_error", "deadline", "stale", "unavailable", "identity_unknown",
+  "identity_mismatch", "auth_timeout", "auth_failed", "transport_error", "client_abort", "header_timeout", "recovered",
+  "connected", "spawned", "stopped", "auth", "hostkey", "forward", "remote_forward_conflict", "timeout", "network",
+  "unknown", "spawn_failed", "listener_unavailable",
+]);
+const STATES = new Set<TunnelState["kind"]>(["idle", "connecting", "connected", "reconnecting", "failed"]);
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const MAX_BYTES = 256 * 1024;
+
+/** Copy only bounded enum, UUID and numeric fields; unknown fields never reach disk. */
+function metadata(record: LinkDiagnosticRecord): LinkDiagnosticRecord | null {
+  if (!record) return null;
+  const { event, outcome, state, previousState } = record;
+  if (!EVENTS.has(event) || !OUTCOMES.has(outcome)) return null;
+  const result: LinkDiagnosticRecord = { event, outcome };
+  for (const key of ["requestId", "checkId"] as const) {
+    const value = record[key];
+    if (typeof value === "string" && value.length === 36 && UUID.test(value)) result[key] = value;
+  }
+  if (typeof state === "string" && STATES.has(state as TunnelState["kind"])) result.state = state;
+  if (typeof previousState === "string" && STATES.has(previousState as TunnelState["kind"])) result.previousState = previousState;
+  for (const key of ["generation", "elapsedMs", "attempt", "retryAfterMs"] as const) {
+    const value = record[key];
+    if (typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER
+      && (key === "elapsedMs" || key === "retryAfterMs" || Number.isSafeInteger(value))) result[key] = value;
+  }
+  const { tunnelPort } = record;
+  if (typeof tunnelPort === "number" && Number.isInteger(tunnelPort) && tunnelPort >= 1 && tunnelPort <= 65_535) result.tunnelPort = tunnelPort;
+  return result;
+}
+
+/** Append through a private no-follow handle, keeping at most one 256 KiB rotated backup. */
+async function append(dir: string, line: string): Promise<void> {
+  assertNotRealHomeUnderTest(dirname(dir));
+  await mkdir(dir, { recursive: true, mode: 0o700 });
+  const directory = await lstat(dir);
+  if (!directory.isDirectory() || directory.isSymbolicLink()) return;
+  const windows = windowsSecretAclApplies();
+  if (windows) await hardenSecretDirAsync(dir, { required: true });
+  else await chmod(dir, 0o700);
+  const path = join(dir, "diagnostics.jsonl");
+  const backup = `${path}.1`;
+  let file: Awaited<ReturnType<typeof open>> | undefined;
+  try {
+    // No read of log contents, and no following a file symlink or writing to a pipe.
+    const flags = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK;
+    file = await open(path, flags, 0o600);
+    const current = await file.stat();
+    if (!current.isFile() || current.nlink !== 1) return;
+    if (windows) await hardenSecretPathAsync(path, { required: true });
+    else await file.chmod(0o600);
+    const previous = await lstat(backup).catch(error => {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      return null;
+    });
+    if (previous) {
+      if (!previous.isFile() || previous.isSymbolicLink() || previous.nlink !== 1) return;
+      if (previous.size > MAX_BYTES) await unlink(backup);
+      else if (windows) await hardenSecretPathAsync(backup, { required: true });
+      else await chmod(backup, 0o600);
+    }
+    if (current.size > MAX_BYTES) await file.truncate(0);
+    else if (current.size + Buffer.byteLength(line) > MAX_BYTES) {
+      await file.close();
+      file = undefined;
+      await rename(path, backup);
+      file = await open(path, flags, 0o600);
+      const fresh = await file.stat();
+      if (!fresh.isFile() || fresh.nlink !== 1 || fresh.size + Buffer.byteLength(line) > MAX_BYTES) return;
+      if (windows) await hardenSecretPathAsync(path, { required: true });
+      else await file.chmod(0o600);
+    }
+    await file.write(line);
+    if (windows) {
+      const written = await file.stat({ bigint: true });
+      const named = await lstat(path, { bigint: true });
+      if (!named.isSymbolicLink() && named.dev === written.dev && named.ino === written.ino) {
+        // Windows publishes content-write ChangeTime on close; retain only the same object's ACL memo.
+        await file.close();
+        file = undefined;
+        reattributeHardenedSecretPath(path);
+      }
+    }
+  } finally {
+    await file?.close();
+  }
+}
+
+/**
+ * Create a best-effort metadata sink with one-second event/outcome buckets and a bounded queue.
+ * File and ACL work starts on the first eligible record and never blocks admission. flush() waits
+ * at most one second; its timeout does not cancel the writer or guarantee durable storage.
+ */
+export function createLinkDiagnosticSink(
+  { configDir, now = Date.now }: { configDir?: string; now?: () => number } = {},
+): LinkDiagnosticSink & { flush: (timeoutMs?: number) => Promise<void> } {
+  // At most 56 buckets; correlation IDs never become keys or retained memory.
+  const buckets = new Map<string, { at: number; suppressed: number }>();
+  const queue: Array<{ dir: string; line: string }> = [];
+  let flight: Promise<void> | undefined;
+  const drain = async () => {
+    while (queue.length) {
+      const record = queue.shift()!;
+      try { await append(record.dir, record.line); } catch { /* Best effort; never publish raw errors. */ }
+    }
+  };
+  const startWriter = () => {
+    if (flight) return;
+    flight = drain().finally(() => {
+      flight = undefined;
+      if (queue.length) startWriter();
+    });
+  };
+  const sink: LinkDiagnosticSink = record => {
+    try {
+      const safe = metadata(record);
+      if (!safe) return;
+      const at = now();
+      if (!Number.isSafeInteger(at) || at < 0) return;
+      const key = `${safe.event}:${safe.outcome}`;
+      const bucket = buckets.get(key);
+      // Wall-clock rollback starts a new window, even when the writer is backed up.
+      if (bucket && at >= bucket.at && at - bucket.at < 1_000) {
+        bucket.suppressed = Math.min(Number.MAX_SAFE_INTEGER, bucket.suppressed + 1);
+        return;
+      }
+      if (queue.length >= 56) {
+        buckets.set(key, { at: Math.min(bucket?.at ?? at, at),
+          suppressed: Math.min(Number.MAX_SAFE_INTEGER, (bucket?.suppressed ?? 0) + 1) });
+        return;
+      }
+      const suppressed = bucket?.suppressed ?? 0;
+      // Failed writes are rate-limited too; diagnostics cannot hammer an unavailable disk.
+      buckets.set(key, { at, suppressed: 0 });
+      queue.push({ dir: linkDir(configDir), line: `${JSON.stringify({ ...safe, timestamp: at, ...(suppressed ? { suppressed } : {}) })}\n` });
+      startWriter();
+    } catch { /* Diagnostics must never change request admission or expose raw errors. */ }
+  };
+  return Object.assign(sink, { flush: async (timeoutMs = 1_000) => {
+    if (!flight) return;
+    // Keep shutdown bounded even if an asynchronous filesystem or ACL operation stalls.
+    // Timeout leaves the existing writer alone; it may still finish before process exit.
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    try {
+      await Promise.race([
+        (async () => { while (flight) await flight; })(),
+        new Promise<void>(resolve => { timer = setTimeout(resolve, Math.min(1_000, Math.max(0, timeoutMs))); }),
+      ]);
+    } catch { /* Best effort; never publish raw errors. */ }
+    finally { clearTimeout(timer); }
+  } });
+}
+
+let defaultSink: ReturnType<typeof createLinkDiagnosticSink> | undefined;
+/** Lazily enqueue sanitized evidence in the default sink without propagating diagnostic failures. */
+export function recordLinkDiagnostic(record: LinkDiagnosticRecord): void {
+  try { (defaultSink ??= createLinkDiagnosticSink())(record); } catch { /* Best effort. */ }
+}
+
+/** Drain existing diagnostics for at most one second without activating an unused sink. */
+export async function flushLinkDiagnostics(): Promise<void> {
+  await defaultSink?.flush();
+}

package/src/link/relay-auth.ts

@@ -2,6 +2,12 @@ import { createHmac, randomBytes, timingSafeEqual } from "node:crypto";
 
 export const LINK_RELAY_AUTH_PATH = "/.well-known/opencodex/link-relay-auth";
 export const LINK_RELAY_SESSION_HEADER = "x-opencodex-link-session";
+/**
+ * Marks a challenge that only asks whether the Home answers (the Child's readiness preflight): the
+ * Home proves itself without reserving a session, so no data request can follow it and it holds
+ * neither session capacity nor a drain lease. An older Home ignores it and reserves as before.
+ */
+export const LINK_RELAY_PROBE_HEADER = "x-opencodex-link-probe";
 export const LINK_RELAY_AUTH_VERSION = "2";
 export const LINK_RELAY_AUTH_TIMEOUT_MS = 5_000;
 

package/src/link/status-projection.ts

@@ -22,6 +22,8 @@ export interface LinkStatusDto {
     tunnelPort: number;
     /** Home only: what the link carried from this Child, and the compatibility verdict. */
     peer?: LinkPeerStatus;
+    /** Home-initiated links on request (`?diagnostics=1`): retry progress and the last failure. */
+    tunnel?: LinkTunnelDiagnostics;
   }>;
   child: null | {
     alias: string;
@@ -33,6 +35,43 @@ export interface LinkStatusDto {
   local?: LinkLocalPeer;
 }
 
+/**
+ * The Home's account of one `-R` tunnel: ssh attempts since it was last connected, when the next
+ * one is due (null while one runs or none is scheduled), the last state change and the last
+ * failure. `cause` is a fixed word from `tunnel-history.ts`, never ssh output.
+ */
+export interface LinkTunnelDiagnostics {
+  attempt: number;
+  nextRetryAt: string | null;
+  lastTransition: { from: LinkWireState; to: LinkWireState; at: string; cause: string | null } | null;
+  lastFailure: { cause: string; at: string } | null;
+}
+
+function iso(at: number): string {
+  return new Date(at).toISOString();
+}
+
+/** Add `tunnel` to each Home-initiated link the supervisor reports. */
+export function attachLinkTunnelDiagnostics(dto: LinkStatusDto, supervisorStates: readonly LinkTunnelStatus[]): LinkStatusDto {
+  const byId = new Map(supervisorStates.map(status => [status.linkId, status]));
+  for (const link of dto.links) {
+    const status = byId.get(link.id);
+    if (!status || status.direction !== "hub-initiated") continue;
+    const { state, history } = status;
+    const due = (state.kind === "failed" || state.kind === "reconnecting") && state.retryAt !== undefined && state.inFlight !== true
+      ? state.retryAt : null;
+    link.tunnel = {
+      attempt: history?.attempt ?? 0,
+      nextRetryAt: due === null ? null : iso(due),
+      lastTransition: history?.lastTransition
+        ? { from: history.lastTransition.from, to: history.lastTransition.to, at: iso(history.lastTransition.at), cause: history.lastTransition.cause }
+        : null,
+      lastFailure: history?.lastFailure ? { cause: history.lastFailure.cause, at: iso(history.lastFailure.at) } : null,
+    };
+  }
+  return dto;
+}
+
 /** Add this build's identity and, on a Home, each Child's observed peer status. */
 export function attachLinkPeers(dto: LinkStatusDto, local: LinkLocalPeer, observation: (linkId: string) => LinkPeerEntry): LinkStatusDto {
   if (dto.role === "home") {

package/src/link/supervisor.ts

@@ -4,13 +4,16 @@ import { atomicWriteFile } from "../config/atomic-write";
 import { linkDir, linkKnownHostsPath, linkStorePath } from "./paths";
 import { buildTunnelArgv } from "./ssh-argv";
 import { createSshRunner, type SshChild, type SshRunner } from "./ssh-runner";
+import type { LinkDiagnosticSink } from "./diagnostics";
+import { createTunnelHistoryBook, exitCause, type TunnelCause, type TunnelHistory } from "./tunnel-history";
 import {
   classifySshStderr,
   dueForSpawn,
-  failedTunnel,
+  failedSpawnTunnel,
   HOME_TUNNEL_RETRY_POLICY,
   IDLE,
   reduceTunnel,
+  type TunnelFailure,
   type TunnelState,
 } from "./tunnel-state";
 import { emptyLinkStore, readLinkStore, type LinkRecord, type LinkStore } from "./store";
@@ -22,6 +25,8 @@ export type LinkTunnelStatus =
     state: TunnelState;
     pid: number | null;
     orphan?: "orphan-unverified";
+    /** Attempts, last change and last failure cause of this tunnel (`tunnel-history.ts`). */
+    history?: TunnelHistory;
   }
   | {
     linkId: string;
@@ -72,6 +77,10 @@ export interface LinkSupervisorDeps {
   apiKeys?: () => readonly { id: string; name: string }[];
   revokeApiKey?: (id: string) => boolean;
   warn?: (message: string) => void;
+  /** Where tunnel state changes are recorded; production passes the bounded diagnostics file. */
+  diagnostic?: LinkDiagnosticSink;
+  /** Bounded drain of `diagnostic` at stop. */
+  flushDiagnostics?: () => Promise<void>;
 }
 
 const TIMER_MS = 1_000;
@@ -166,6 +175,10 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
   /** The generation of the child whose exit is still being classified, per link. */
   const exiting = new Map<string, number>();
   const recordInstances = new Map<string, string>();
+  const history = createTunnelHistoryBook({
+    diagnostic: deps.diagnostic,
+    tunnelPort: linkId => store.links.find(link => link.id === linkId)?.tunnelPort,
+  });
 
   const reconcileApiKeys = (): void => {
     const keys = deps.apiKeys?.() ?? [];
@@ -218,12 +231,22 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
     }
   };
 
-  const setEvent = (linkId: string, event: Parameters<typeof reduceTunnel>[1]): TunnelState => {
-    const next = reduceTunnel(states.get(linkId) ?? IDLE, event, deps.random, HOME_TUNNEL_RETRY_POLICY);
+  /** Every state change goes through here, so the history sees each transition and its cause. */
+  const put = (linkId: string, next: TunnelState, cause: TunnelCause | null = null): TunnelState => {
+    const before = states.get(linkId) ?? IDLE;
     states.set(linkId, next);
+    history.observe(linkId, before, next, cause, now());
     return next;
   };
 
+  const setEvent = (linkId: string, event: Parameters<typeof reduceTunnel>[1], cause: TunnelCause | null = null): TunnelState =>
+    put(linkId, reduceTunnel(states.get(linkId) ?? IDLE, event, deps.random, HOME_TUNNEL_RETRY_POLICY), cause);
+
+  /** Fail a link whose attempt could not start; a due retry that fails this way counts as one. */
+  const failAttempt = (linkId: string, prior: TunnelState | undefined, reason: TunnelFailure, cause: TunnelCause): void => {
+    put(linkId, failedSpawnTunnel(prior, reason, now(), HOME_TUNNEL_RETRY_POLICY), cause);
+  };
+
   const spawnFor = (record: LinkRecord): void => {
     if (stopping || holds.has(record.id) || record.direction !== "hub-initiated" || children.has(record.id)) return;
     // Only a fresh link or a due attempt spawns. A reload must neither skip the backoff nor start
@@ -231,7 +254,8 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
     const prior = states.get(record.id);
     if (prior && prior.kind !== "idle" && !dueForSpawn(prior, now())) return;
     if (store.listenerPort === null) {
-      states.set(record.id, { kind: "failed", since: now(), reason: "forward" });
+      // Retried like any forward failure; a reload that brings the listener back starts it fresh.
+      failAttempt(record.id, prior, "forward", "listener_unavailable");
       return;
     }
     let argv: string[];
@@ -250,6 +274,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
       exiting.delete(record.id);
       children.set(record.id, { child, argv, spawnedAt: now() });
       orphanUnverified.delete(record.id);
+      history.spawned(record.id);
       setEvent(record.id, { type: "spawn", now: now() });
       writePidfile(pidfilePath(record.id), { version: 1, linkId: record.id, pid: child.pid, argv });
       void child.exited.then(async () => {
@@ -264,7 +289,8 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
           try { stderr = child.stderr ? await child.stderr : ""; } catch { /* classified as unknown */ }
           // A reconnect may have started a fresh child meanwhile; its own lifecycle owns the state.
           if (generations.get(record.id) !== generation) return;
-          setEvent(record.id, { type: "exit", now: now(), stderrClass: classifySshStderr(stderr) });
+          const stderrClass = classifySshStderr(stderr);
+          setEvent(record.id, { type: "exit", now: now(), stderrClass }, exitCause(stderrClass));
         } finally {
           if (exiting.get(record.id) === generation) exiting.delete(record.id);
         }
@@ -272,11 +298,10 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
         if (children.get(record.id)?.child !== child) return;
         children.delete(record.id);
         conditionalRemovePidfile(record.id, child.pid);
-        if (!stopping) setEvent(record.id, { type: "exit", now: now(), stderrClass: "network" });
+        if (!stopping) setEvent(record.id, { type: "exit", now: now(), stderrClass: "network" }, "network");
       });
     } catch {
-      const at = now();
-      states.set(record.id, failedTunnel("forward", at, HOME_TUNNEL_RETRY_POLICY, prior?.kind === "failed" ? prior.since : at));
+      failAttempt(record.id, prior, "forward", "spawn_failed");
     }
   };
 
@@ -300,7 +325,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
         setEvent(record.id, { type: "ready", now: current });
         continue;
       }
-      const next = setEvent(record.id, { type: "tick", now: current });
+      const next = setEvent(record.id, { type: "tick", now: current }, "timeout");
       if (next.kind === "failed" && !next.inFlight && child && retryInFlight(before)) {
         // A retry started just before the outage limit has not had its grace yet. It carries on as
         // the failed state's retry instead of being killed with a link that may already be back.
@@ -349,7 +374,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
   const stopChild = async (linkId: string): Promise<void> => {
     const current = children.get(linkId);
     if (!current) {
-      states.set(linkId, IDLE);
+      put(linkId, IDLE);
       return;
     }
     const generation = generations.get(linkId);
@@ -358,7 +383,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
     await current.child.exited;
     conditionalRemovePidfile(linkId, current.child.pid);
     // A stop or removal reservation made while this child exited owns the next state.
-    if (generations.get(linkId) === generation) states.set(linkId, IDLE);
+    if (generations.get(linkId) === generation) put(linkId, IDLE);
   };
 
   const invalidateLink = (linkId: string): void => {
@@ -407,6 +432,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
           if (nextInstances.has(linkId) && !changed.has(linkId)) continue;
           await stopChild(linkId);
           states.delete(linkId);
+          history.forget(linkId);
           orphanUnverified.delete(linkId);
           recordInstances.delete(linkId);
           if (stopping) return;
@@ -414,9 +440,13 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
         for (const record of nextStore.links) {
           if (changed.has(record.id)) {
             states.delete(record.id);
+            history.forget(record.id);
             orphanUnverified.delete(record.id);
           }
         }
+        for (const linkId of [...states.keys()]) {
+          if (!nextInstances.has(linkId)) history.forget(linkId);
+        }
         recordInstances.clear();
         for (const [linkId, instance] of nextInstances) recordInstances.set(linkId, instance);
         for (const record of nextStore.links) {
@@ -471,12 +501,14 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
           return { linkId: record.id, direction: record.direction, state: "client-owned", pid: null };
         }
         const child = children.get(record.id);
+        const recorded = history.get(record.id);
         return {
           linkId: record.id,
           direction: record.direction,
           state: states.get(record.id) ?? IDLE,
           pid: child?.child.pid ?? null,
           ...(orphanUnverified.has(record.id) ? { orphan: "orphan-unverified" as const } : {}),
+          ...(recorded ? { history: recorded } : {}),
         };
       });
     },
@@ -489,9 +521,10 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
       await runLifecycle(async () => {
         await Promise.all([...children.keys()].map(stopChild));
         for (const record of store.links) {
-          if (record.direction === "hub-initiated") states.set(record.id, IDLE);
+          if (record.direction === "hub-initiated") put(record.id, IDLE);
         }
       });
+      try { await deps.flushDiagnostics?.(); } catch { /* best effort */ }
     },
   };
 }

package/src/link/tunnel-history.ts

@@ -0,0 +1,109 @@
+/**
+ * What the Home remembers about one `-R` tunnel besides its current state: how many ssh attempts
+ * the current outage has cost, the last state change and the last failure with its cause. It lives
+ * in memory next to the supervisor's state, appears in `GET /api/link/status` as `tunnel`, and
+ * each change it records also goes to the bounded diagnostics file (`diagnostics.ts`), so a link
+ * that failed for an hour leaves evidence of why after the fact.
+ *
+ * Nothing here carries ssh output, argv, host names or keys: a cause is one of a fixed set of
+ * words, and times are numbers.
+ */
+import type { LinkDiagnosticSink } from "./diagnostics";
+import type { StderrClass, TunnelState } from "./tunnel-state";
+
+export type TunnelCause =
+  | "auth" | "hostkey" | "forward" | "remote_forward_conflict" | "timeout" | "network" | "unknown"
+  | "spawn_failed" | "listener_unavailable";
+
+export const TUNNEL_CAUSES: readonly TunnelCause[] = [
+  "auth", "hostkey", "forward", "remote_forward_conflict", "timeout", "network", "unknown",
+  "spawn_failed", "listener_unavailable",
+];
+
+export interface TunnelTransitionRecord {
+  from: TunnelState["kind"];
+  to: TunnelState["kind"];
+  at: number;
+  cause: TunnelCause | null;
+}
+
+export interface TunnelHistory {
+  /** ssh attempts started since the link was last connected (0 while connected). */
+  attempt: number;
+  lastTransition: TunnelTransitionRecord | null;
+  lastFailure: { cause: TunnelCause; at: number } | null;
+}
+
+export function exitCause(stderrClass: StderrClass): TunnelCause {
+  return stderrClass === "remote_forward" ? "remote_forward_conflict" : stderrClass;
+}
+
+/**
+ * A retry of a failed link ended without connecting: one that ran while the state read failed, or
+ * one that could not even start (the retry count still grew).
+ */
+function retryFailedAgain(before: TunnelState, after: TunnelState): boolean {
+  if (before.kind !== "failed" || after.kind !== "failed" || after.inFlight === true) return false;
+  return before.inFlight === true || (after.retries ?? 0) > (before.retries ?? 0);
+}
+
+export interface TunnelHistoryBook {
+  /** Count one ssh attempt for the link. */
+  spawned(linkId: string): void;
+  /** Record a state change; `cause` names why the link left a working state, when known. */
+  observe(linkId: string, before: TunnelState, after: TunnelState, cause: TunnelCause | null, at: number): void;
+  get(linkId: string): TunnelHistory | undefined;
+  forget(linkId: string): void;
+}
+
+export function createTunnelHistoryBook(
+  options: { diagnostic?: LinkDiagnosticSink; tunnelPort?: (linkId: string) => number | undefined } = {},
+): TunnelHistoryBook {
+  const histories = new Map<string, TunnelHistory>();
+  const entry = (linkId: string): TunnelHistory => {
+    let history = histories.get(linkId);
+    if (!history) {
+      history = { attempt: 0, lastTransition: null, lastFailure: null };
+      histories.set(linkId, history);
+    }
+    return history;
+  };
+  return {
+    spawned(linkId) {
+      entry(linkId).attempt += 1;
+    },
+    observe(linkId, before, after, cause, at) {
+      const refailed = retryFailedAgain(before, after);
+      if (before.kind === after.kind && !refailed) return;
+      const history = entry(linkId);
+      if (after.kind === "connected" || after.kind === "idle") history.attempt = 0;
+      if (cause && (after.kind === "failed" || after.kind === "reconnecting")) history.lastFailure = { cause, at };
+      history.lastTransition = { from: before.kind, to: after.kind, at, cause };
+      const retryAt = after.kind === "failed" || after.kind === "reconnecting" ? after.retryAt : undefined;
+      const tunnelPort = options.tunnelPort?.(linkId);
+      try {
+        options.diagnostic?.({
+          event: "tunnel_transition",
+          outcome: cause ?? (after.kind === "connected" ? "connected" : after.kind === "idle" ? "stopped" : "spawned"),
+          state: after.kind,
+          previousState: before.kind,
+          attempt: history.attempt,
+          ...(tunnelPort !== undefined ? { tunnelPort } : {}),
+          ...(retryAt !== undefined ? { retryAfterMs: Math.max(0, retryAt - at) } : {}),
+        });
+      } catch { /* Diagnostics never change the tunnel lifecycle. */ }
+    },
+    get(linkId) {
+      const history = histories.get(linkId);
+      if (!history) return undefined;
+      return {
+        attempt: history.attempt,
+        lastTransition: history.lastTransition && { ...history.lastTransition },
+        lastFailure: history.lastFailure && { ...history.lastFailure },
+      };
+    },
+    forget(linkId) {
+      histories.delete(linkId);
+    },
+  };
+}