@addai/node 0.4.0

package/dist/prompt-keystrokes.js

@@ -0,0 +1,56 @@
+"use strict";
+// Encodes a user prompt into the raw keystroke byte-stream we type into
+// the Claude TUI's PTY before pressing Enter to submit.
+//
+// THE BUG THIS FIXES — "half the prompt gets cut off in TUI mode":
+//   The TUI path types the prompt into the PTY as raw keystrokes. The
+//   terminal/Ink input parser interprets terminal control bytes, so any
+//   literal ESC / C0 control character *inside the prompt content* (e.g.
+//   ANSI colour codes like \x1b[31m that get captured into a for/loop
+//   node's item data) is read as the start of an escape sequence and
+//   SWALLOWS the surrounding text. An ESC before a bracketed run
+//   (\x1b[0,1,...,199]) eats everything up to the terminating "]" — which
+//   is how a single stray byte truncates ~half a prompt. It is
+//   content-dependent, hence "sometimes".
+//
+//   Print mode (`claude -p <prompt>`) is immune because the prompt is
+//   passed as an argv string and never parsed as terminal input.
+//
+//   Bracketed paste (\x1b[200~ … \x1b[201~) does NOT help — the terminal
+//   strips/interprets the embedded ESC regardless of the paste framing.
+//
+// THE FIX: strip terminal control sequences/characters OUT of the content
+//   first, THEN turn real newlines into the in-message line-break chord.
+//
+// Newlines must become ESC+CR (Alt/Shift+Enter inside claude) — a bare \r
+// is Enter=submit and would fire the message at the first newline. The
+// ESC+CR pairs are added AFTER sanitisation so they aren't stripped.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TUI_NEWLINE = void 0;
+exports.stripTerminalControls = stripTerminalControls;
+exports.encodeTuiPromptKeystrokes = encodeTuiPromptKeystrokes;
+// The in-message line-break chord (Alt/Shift+Enter) as the TUI sees it.
+exports.TUI_NEWLINE = '\x1b\r';
+// Strip ANSI/VT escape sequences and stray C0 control bytes from prompt
+// content. Tabs (\t) and newlines (\n) are preserved; \r is normalized to
+// \n separately by the caller path below.
+function stripTerminalControls(text) {
+    return text
+        // eslint-disable-next-line no-control-regex
+        .replace(/\x1b\[[0-9;:<=>?]*[ -\/]*[@-~]/g, '') // CSI sequences (colours, cursor moves, …)
+        // eslint-disable-next-line no-control-regex
+        .replace(/\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g, '') // OSC sequences (titles, hyperlinks, …)
+        // eslint-disable-next-line no-control-regex
+        .replace(/\x1b[@-Z\\-_]/g, '') // two-character ESC sequences
+        // eslint-disable-next-line no-control-regex
+        .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); // residual C0 controls (keep \t \n)
+}
+function encodeTuiPromptKeystrokes(prompt) {
+    // 1. Remove terminal control bytes so nothing in the content is parsed
+    //    as an escape sequence that eats surrounding text.
+    const safe = stripTerminalControls(prompt);
+    // 2. Normalize CRLF / lone CR to LF so the rewrite is uniform.
+    const normalized = safe.replace(/\r\n?/g, '\n');
+    // 3. Turn real newlines into the in-message line-break chord.
+    return normalized.replace(/\n/g, exports.TUI_NEWLINE);
+}

package/dist/pty-helper.d.ts

@@ -0,0 +1,26 @@
+export interface PtyHelperRepair {
+    /** Helpers that were not executable and now are. */
+    repaired: string[];
+    failed: Array<{
+        path: string;
+        error: string;
+    }>;
+}
+/** node-pty's package root, resolved the way Node itself would — so a
+ *  hoisted copy is found just as reliably as a nested one. `fromDir`
+ *  overrides the resolution base (this module's own directory by default).
+ *  Returns null when node-pty isn't installed. */
+export declare function findNodePtyDir(fromDir?: string): string | null;
+/** chmod +x every spawn-helper under `nodePtyDir` that lacks it. Only
+ *  spawn-helper: the sibling pty.node is dlopen'd, never executed. Never
+ *  throws — a runtime that can't repair itself must still boot. */
+export declare function repairPtyHelpers(nodePtyDir: string | null): PtyHelperRepair;
+/** Resolve node-pty and restore its helper's +x bit. Safe to call on every
+ *  boot and before any PTY spawn: a healthy install costs a few stats. */
+export declare function ensurePtyHelperExecutable(fromDir?: string): PtyHelperRepair & {
+    dir: string | null;
+};
+/** One-line summary for logs, or null when there was nothing to say. */
+export declare function describeRepair(r: PtyHelperRepair & {
+    dir: string | null;
+}): string | null;

package/dist/pty-helper.js

@@ -0,0 +1,156 @@
+"use strict";
+// Keeps node-pty's `spawn-helper` executable.
+//
+// On macOS node-pty does not fork/exec the target itself: it posix_spawn()s
+// a small `spawn-helper` binary as argv[0] (cwd, then the real command,
+// follow as argv[1..]). That helper ships inside node-pty's prebuilds, and
+// npm's tarball extraction sometimes lands it as 0644 — reproducibly so for
+// installs made through `npx`, which is how AiNode daemons run. posix_spawn
+// then fails with EACCES and node-pty throws the opaque
+//
+//     posix_spawnp failed.
+//
+// which is all the operator ever sees. Every PTY path on that machine is
+// dead: remote harness logins (codex --device-auth, claude/gemini OAuth)
+// and claude TUI runs alike.
+//
+// A postinstall script has always tried to restore the bit, but it looked
+// only in <our-package>/node_modules/node-pty — a path that exists only when
+// node-pty is NESTED. npm hoists shared deps to the tree root, so under npx
+// node-pty sits beside us, not beneath us, and the fix silently no-op'd on
+// exactly the installs that needed it (live incident 2026-07-29,
+// Lennys-Mac-mini). Resolve the package instead of guessing its path, and
+// re-check at daemon start so an already-broken install heals on restart
+// rather than waiting for a reinstall.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findNodePtyDir = findNodePtyDir;
+exports.repairPtyHelpers = repairPtyHelpers;
+exports.ensurePtyHelperExecutable = ensurePtyHelperExecutable;
+exports.describeRepair = describeRepair;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/** node-pty's package root, resolved the way Node itself would — so a
+ *  hoisted copy is found just as reliably as a nested one. `fromDir`
+ *  overrides the resolution base (this module's own directory by default).
+ *  Returns null when node-pty isn't installed. */
+function findNodePtyDir(fromDir) {
+    const paths = [fromDir ?? __dirname];
+    let entry;
+    try {
+        // Resolve the package's main entry rather than a subpath: an "exports"
+        // map can refuse `node-pty/package.json` outright.
+        entry = require.resolve('node-pty', { paths });
+    }
+    catch {
+        return null;
+    }
+    // Walk up from lib/index.js to the directory holding node-pty's manifest.
+    let dir = path.dirname(entry);
+    for (let i = 0; i < 5; i++) {
+        try {
+            const pkg = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8'));
+            if (pkg.name === 'node-pty')
+                return dir;
+        }
+        catch { /* keep climbing */ }
+        const up = path.dirname(dir);
+        if (up === dir)
+            break;
+        dir = up;
+    }
+    return null;
+}
+/** Every place node-pty may keep a `spawn-helper`: a locally compiled build
+ *  and each shipped prebuild arch. */
+function helperCandidates(nodePtyDir) {
+    const out = [
+        path.join(nodePtyDir, 'build', 'Release', 'spawn-helper'),
+        path.join(nodePtyDir, 'build', 'Debug', 'spawn-helper'),
+    ];
+    const prebuilds = path.join(nodePtyDir, 'prebuilds');
+    try {
+        for (const arch of fs.readdirSync(prebuilds))
+            out.push(path.join(prebuilds, arch, 'spawn-helper'));
+    }
+    catch { /* no prebuilds dir — build/ candidates still stand */ }
+    return out;
+}
+/** chmod +x every spawn-helper under `nodePtyDir` that lacks it. Only
+ *  spawn-helper: the sibling pty.node is dlopen'd, never executed. Never
+ *  throws — a runtime that can't repair itself must still boot. */
+function repairPtyHelpers(nodePtyDir) {
+    const res = { repaired: [], failed: [] };
+    if (!nodePtyDir)
+        return res;
+    for (const helper of helperCandidates(nodePtyDir)) {
+        let mode;
+        try {
+            const st = fs.statSync(helper);
+            if (!st.isFile())
+                continue;
+            mode = st.mode;
+        }
+        catch {
+            continue;
+        } // not present for this arch/build
+        if ((mode & 0o111) !== 0)
+            continue; // already executable
+        try {
+            fs.chmodSync(helper, (mode & 0o777) | 0o755);
+            res.repaired.push(helper);
+        }
+        catch (err) {
+            res.failed.push({ path: helper, error: err.message });
+        }
+    }
+    return res;
+}
+/** Resolve node-pty and restore its helper's +x bit. Safe to call on every
+ *  boot and before any PTY spawn: a healthy install costs a few stats. */
+function ensurePtyHelperExecutable(fromDir) {
+    const dir = findNodePtyDir(fromDir);
+    return { dir, ...repairPtyHelpers(dir) };
+}
+/** One-line summary for logs, or null when there was nothing to say. */
+function describeRepair(r) {
+    if (r.repaired.length > 0) {
+        return `[pty] restored +x on node-pty spawn-helper (${r.repaired.length}): ${r.repaired.join(', ')}`;
+    }
+    if (r.failed.length > 0) {
+        return `[pty] spawn-helper is not executable and could not be fixed: ${r.failed.map(f => `${f.path} (${f.error})`).join(', ')}`;
+    }
+    return null;
+}

package/dist/request-pump.d.ts

@@ -0,0 +1,12 @@
+export declare function inflightCount(): number;
+export declare function start(): void;
+export declare function stop(): void;
+/**
+ * Wait for all in-flight requests to finish, with a hard ceiling so a
+ * hung spawn can't block daemon shutdown forever. Resolves when either
+ * all promises settle OR the timeout elapses.
+ */
+export declare function drain(timeoutMs: number): Promise<{
+    drained: number;
+    timedOut: number;
+}>;

package/dist/request-pump.js

@@ -0,0 +1,249 @@
+"use strict";
+// Polls Supabase for pending requests targeted at this runtime and runs
+// them via session-runner. Bounded concurrency keeps a single runtime
+// from spawning unlimited Claude processes when a burst of requests
+// arrives.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inflightCount = inflightCount;
+exports.start = start;
+exports.stop = stop;
+exports.drain = drain;
+const supabase_client_1 = require("./supabase-client");
+const store_1 = require("./store");
+const session_runner_1 = require("./session-runner");
+const node_health_1 = require("./node-health");
+const diskguard_1 = require("./diskguard");
+// Poll cadence for new pending requests. We previously polled every
+// 250ms (4 RPS per daemon) which sounded harmless but under load —
+// each call could hold a Supabase pooler connection for the full RPC
+// timeout — three or four paired daemons would saturate the project's
+// connection pool, every request 522'd, and the daemon's own RPCs
+// timed out too, producing a death spiral. 2s is a fine UX trade
+// (chat_send lands within 2s vs 250ms; users won't notice) and is
+// ~16x cheaper at steady state. Successful pickups still re-tick
+// immediately so bursts drain fast.
+const POLL_INTERVAL_MS = 2_000;
+const MAX_CONCURRENT = 4;
+// Anti-leak backstop ONLY — how long the pump will hold a concurrency slot
+// for a run whose promise never resolves (e.g. a spawn whose onExit is
+// lost). This does NOT kill the agent and is NOT an execution limit: it
+// only frees the slot so a leak can't wedge the pump; the run keeps going
+// and keeps heartbeating if it's still alive. Real time limits come from
+// the node's timeout (enforced in session-runner), never from here. For a
+// request WITH a node timeout we pad it; with none (unlimited) we fall
+// back to this large default.
+const SLOT_LEAK_BACKSTOP_MS = 12 * 60 * 60 * 1000; // 12 hours
+const SLOT_TIMEOUT_GRACE_MS = 5 * 60 * 1000; // pad over a node timeout
+let timer = null;
+let stopped = false;
+let inflight = 0;
+// Request ids with a runRequest currently in-flight on THIS daemon.
+// Guards against launching a second run for the same request — e.g. if a
+// reclaim ever requeues one whose spawn is still alive — which is exactly
+// the duplicate-session / wrong-attribution bug we're fixing.
+const activeRequestIds = new Set();
+// Failure backoff state: when Supabase goes unreachable we used to log
+// `pick failed: fetch failed` every 250ms (= 14k/hour) and keep hammering
+// the endpoint. Now we exponentially back off and dedupe identical
+// errors: only the first occurrence + once per minute thereafter is
+// logged, and the next-tick delay is `min(60s, base * 2^n)` so a 10-min
+// outage produces ~10 log lines instead of 240,000.
+let consecutivePickFailures = 0;
+let lastLoggedErrorMessage = '';
+let lastLoggedAt = 0;
+let nextPickAllowedAt = 0;
+// Health-gate log dedupe: when we refuse pickup (e.g. disk full) we don't
+// want a log line every 2s. Log the first occurrence + once/minute.
+let lastHealthBlockLogged = '';
+let lastHealthBlockAt = 0;
+// Exported so /stats can surface "pump stuck at N inflight" without
+// having to guess.
+function inflightCount() { return inflight; }
+async function tick() {
+    if (stopped)
+        return;
+    if (inflight >= MAX_CONCURRENT)
+        return;
+    // Honour the backoff window — skip this tick if we recently failed.
+    if (Date.now() < nextPickAllowedAt)
+        return;
+    // Honour the shared circuit breaker — when Supabase is in distress
+    // every poller in the daemon pauses, not just this one.
+    if ((0, supabase_client_1.rpcShouldSkip)())
+        return;
+    const pairing = (0, store_1.readPairing)();
+    if (!pairing)
+        return;
+    // Health gate: don't ACCEPT work this node physically can't finish. A full
+    // scratch disk makes every spawn die with ENOSPC — by refusing the pickup
+    // we leave the request `pending` so the server-side failover hands it to a
+    // healthy backup, instead of us picking it up only to fail it immediately.
+    const health = (0, node_health_1.evaluateNodeHealth)({ diskUsedPct: (0, node_health_1.readDiskUsedPct)((0, diskguard_1.getDiskGuard)().config.tempRoot) });
+    if (!health.healthy) {
+        const msg = health.blockers.map(b => b.message).join('; ');
+        const now = Date.now();
+        if (msg !== lastHealthBlockLogged || (now - lastHealthBlockAt) > 60_000) {
+            console.warn(`[request-pump] refusing pickup — node unhealthy: ${msg}`);
+            lastHealthBlockLogged = msg;
+            lastHealthBlockAt = now;
+        }
+        // Kick a disk sweep so we recover and resume accepting work.
+        void (0, diskguard_1.getDiskGuard)().sweep('health-gate').catch(() => { });
+        return;
+    }
+    let row = null;
+    try {
+        // runtime_pick_next_request returns a row record. PostgREST wraps
+        // it as an object or null depending on the function shape.
+        const result = await (0, supabase_client_1.rpc)('runtime_pick_next_request', {
+            p_token: pairing.daemonToken,
+        });
+        row = (result && result.id) ? result : null;
+        // Success — reset backoff state.
+        if (consecutivePickFailures > 0) {
+            console.log(`[request-pump] recovered after ${consecutivePickFailures} consecutive failures`);
+            consecutivePickFailures = 0;
+            lastLoggedErrorMessage = '';
+        }
+    }
+    catch (err) {
+        if (err instanceof supabase_client_1.RpcError && err.status === 401) {
+            console.error('[request-pump] unauthorized; will retry on next heartbeat.');
+            return;
+        }
+        consecutivePickFailures++;
+        // Exponential backoff: 250ms × 2^n, capped at 60s. After 8 failures
+        // we're at the cap so a sustained outage doesn't snowball.
+        const backoffMs = Math.min(60_000, POLL_INTERVAL_MS * Math.pow(2, Math.min(consecutivePickFailures, 8)));
+        nextPickAllowedAt = Date.now() + backoffMs;
+        const msg = err.message;
+        const now = Date.now();
+        // Log first occurrence + every 60s of a sustained failure stream,
+        // plus on message change.
+        if (msg !== lastLoggedErrorMessage || (now - lastLoggedAt) > 60_000) {
+            console.error(`[request-pump] pick failed (#${consecutivePickFailures}, backing off ${backoffMs}ms):`, msg);
+            lastLoggedErrorMessage = msg;
+            lastLoggedAt = now;
+        }
+        return;
+    }
+    if (!row)
+        return;
+    // Defense-in-depth: never run the same request twice concurrently. The
+    // server-side reclaim now keys off real activity so it shouldn't requeue
+    // a live request, but if one ever slips through, running it again is the
+    // double-session bug. Leave the row as the pick RPC set it; the original
+    // in-flight run will carry it to a terminal state.
+    if (activeRequestIds.has(row.id)) {
+        console.warn(`[request-pump] skipping re-pick of in-flight request [reqId=${row.id}] — already running`);
+        return;
+    }
+    inflight++;
+    const reqId = row.id;
+    activeRequestIds.add(reqId);
+    // Liveness heartbeat: refresh last_activity_at every 60s while this
+    // request runs, so the server-side reclaim (which keys off real activity,
+    // not the one-shot picked_up_at) never requeues a healthy long-running
+    // agent out from under us and re-spawns it as a new session. Bounded by
+    // the hard-timeout race below — when that settles, the interval clears.
+    const heartbeat = setInterval(() => {
+        void (0, supabase_client_1.rpc)('runtime_request_touch', {
+            p_token: pairing.daemonToken,
+            p_request_id: reqId,
+        }).catch(() => { });
+    }, 60_000);
+    heartbeat.unref?.();
+    // Track for graceful drain on SIGTERM/SIGINT.
+    let resolveInflight;
+    const inflightPromise = new Promise((r) => { resolveInflight = r; });
+    inflightPromises.add(inflightPromise);
+    // Race the real promise against a hard timeout. Whichever settles
+    // first wins; the timeout's resolution is enough to free the slot —
+    // server-side reclaim handles the database row.
+    let timeoutFired = false;
+    // Slot backstop = node timeout + grace if the node set one, else the
+    // large leak-only default. Frees the slot; never kills the run.
+    const slotBackstopMs = (row.timeout_seconds && row.timeout_seconds > 0)
+        ? row.timeout_seconds * 1000 + SLOT_TIMEOUT_GRACE_MS
+        : SLOT_LEAK_BACKSTOP_MS;
+    const timeoutPromise = new Promise((resolve) => {
+        const t = setTimeout(() => {
+            timeoutFired = true;
+            console.error(`[request-pump] runRequest exceeded slot backstop (${Math.round(slotBackstopMs / 60000)}m) [reqId=${reqId}] — freeing pump slot. The run continues; server-side reclaim handles the DB row only if it's actually dead.`);
+            resolve();
+        }, slotBackstopMs);
+        t.unref?.();
+    });
+    const runPromise = (0, session_runner_1.runRequest)(row).catch(err => console.error(`[request-pump] runRequest threw [reqId=${reqId}]: ${err.message}`));
+    // Heartbeat + in-flight registration live for the REAL lifetime of the
+    // run (until the spawn exits), NOT just until the slot's hard timeout.
+    // A task that runs longer than REQUEST_HARD_TIMEOUT_MS must keep
+    // refreshing last_activity_at, or the server-side reclaim would requeue
+    // and re-spawn it mid-run — the exact double-session bug we're fixing.
+    void runPromise.finally(() => {
+        clearInterval(heartbeat);
+        activeRequestIds.delete(reqId);
+        // Drain tracking follows the REAL run, not the slot. These used to be
+        // released together on the race below, which meant a run that outlived
+        // its slot backstop vanished from `inflightPromises` while its child was
+        // still working — so drain() had nothing to wait for and a shutdown or a
+        // remote roll would hand over mid-turn. The backstop exists to free a
+        // concurrency slot, not to declare the run finished.
+        inflightPromises.delete(inflightPromise);
+        resolveInflight();
+    });
+    // Slot accounting is separate: free the concurrency slot when the run
+    // finishes OR after the hard timeout (whichever is first), so a run that
+    // never resolves (onExit lost) can't leak a slot forever. The run keeps
+    // going + keeps heartbeating past this point if it's still alive.
+    Promise.race([runPromise, timeoutPromise]).finally(() => {
+        if (!timeoutFired)
+            inflight--;
+        else
+            inflight = Math.max(0, inflight - 1);
+    });
+    // Re-tick immediately so back-to-back requests don't wait for the
+    // next interval. If there's nothing to pick up the next call is a
+    // cheap no-op.
+    if (!stopped && inflight < MAX_CONCURRENT) {
+        queueMicrotask(() => { void tick(); });
+    }
+}
+// Set of pending inflight promises — one per active runRequest. Used
+// by `drain()` to await graceful completion on SIGTERM.
+const inflightPromises = new Set();
+function start() {
+    if (timer || stopped)
+        return;
+    // Fire immediately so the first request after pairing isn't held up
+    // by a full polling interval.
+    void tick();
+    timer = setInterval(() => { void tick(); }, POLL_INTERVAL_MS);
+}
+function stop() {
+    stopped = true;
+    if (timer) {
+        clearInterval(timer);
+        timer = null;
+    }
+}
+/**
+ * Wait for all in-flight requests to finish, with a hard ceiling so a
+ * hung spawn can't block daemon shutdown forever. Resolves when either
+ * all promises settle OR the timeout elapses.
+ */
+async function drain(timeoutMs) {
+    if (inflightPromises.size === 0)
+        return { drained: 0, timedOut: 0 };
+    const started = inflightPromises.size;
+    const deadline = new Promise((resolve) => {
+        const t = setTimeout(resolve, timeoutMs);
+        t.unref?.();
+    });
+    await Promise.race([
+        Promise.allSettled([...inflightPromises]).then(() => undefined),
+        deadline,
+    ]);
+    const remaining = inflightPromises.size;
+    return { drained: started - remaining, timedOut: remaining };
+}

package/dist/run-retry.d.ts

@@ -0,0 +1,28 @@
+export interface FailIo {
+    /** retry_count from the request row at pick time — drives the backoff step. */
+    retryCount: number;
+    /** ladder_hops from the request row at pick time. */
+    ladderCount: number;
+    /** Ask the server to requeue (runtime_request_mark_retry). Resolves true if
+     *  requeued, false if the retry cap is already spent. */
+    markRetry: (code: string | undefined, message: string | undefined, delayMs: number) => Promise<boolean>;
+    /** Clear the stored resume session id (request row + parent chat) AND requeue
+     *  the request, in one server-side call. Resolves true if requeued fresh,
+     *  false if the retry cap is spent / row no longer retryable. Optional so
+     *  older call sites (and tests) can omit it — when absent, a session-invalid
+     *  failure falls through to the classify+ladder tail just as any other
+     *  failure would. */
+    clearSessionAndRetry?: (code: string | undefined, message: string | undefined, delayMs: number) => Promise<boolean>;
+    /** Server decides: hop to the next eligible provider ('laddered') or write
+     *  the terminal dead_letter status ('dead_letter'). This is BOTH the ladder
+     *  path AND the dead-letter writer — the server picks based on remaining
+     *  ladder budget + eligible online agents. */
+    ladderOrDeadLetter: (code: string | undefined, message: string | undefined) => Promise<'laddered' | 'dead_letter'>;
+    /** Write the terminal failed status. Only used as a last-resort local write
+     *  when ladderOrDeadLetter itself is unreachable — the server is otherwise
+     *  the single writer of terminal/dead_letter status. */
+    writeFailed: (code?: string, message?: string) => Promise<void>;
+    /** Fire-and-forget request event for observability. */
+    emit: (type: string, payload: unknown) => unknown;
+}
+export declare function failOrRetry(io: FailIo, errorCode?: string, errorMessage?: string): Promise<'retried' | 'laddered' | 'failed'>;

package/dist/run-retry.js

@@ -0,0 +1,81 @@
+"use strict";
+// Single choke point for "this run did not finish cleanly": classify the
+// error, and either requeue the request for a same-provider retry (transient,
+// server-side cap not reached), hop to the next provider in the ladder
+// (ladderable, or transient budget spent), or let the server write the
+// terminal dead_letter status (terminal, or ladder budget spent). The
+// requeue/ladder are server-side status flips back to 'pending' with a
+// next_attempt_at backoff, so retries and hops survive daemon restarts and
+// the relay trigger never fires a user-facing failure for an attempt we're
+// still going to make.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.failOrRetry = failOrRetry;
+const error_classify_1 = require("./error-classify");
+async function failOrRetry(io, errorCode, errorMessage) {
+    // 1. Stale/invalid resume id → clear the stored session and retry ONCE
+    //    fresh. Checked before the transient/ladderable/terminal split:
+    //    resuming the same missing id would fail identically, so the id MUST
+    //    be dropped before the requeue. The retry_count<3 budget (shared
+    //    server-side with mark_retry / stale-reclaim) bounds this — and a
+    //    fresh session can't reproduce this exact error, so it is a genuine
+    //    one-shot for this cause, not an infinite loop. If clearSessionAndRetry
+    //    is absent, returns false, or throws, fall through to the classify+
+    //    ladder tail below rather than writing failed directly — the server
+    //    remains the single dead-letter writer.
+    if (io.clearSessionAndRetry && (0, error_classify_1.isSessionInvalid)(errorCode, errorMessage)) {
+        const delayMs = (0, error_classify_1.retryDelayMs)(io.retryCount);
+        try {
+            if (await io.clearSessionAndRetry(errorCode, errorMessage, delayMs)) {
+                void io.emit('session_invalid_retry', {
+                    errorCode: errorCode ?? null,
+                    errorMessage: (errorMessage ?? '').slice(0, 500) || null,
+                    delayMs,
+                    attempt: io.retryCount + 1,
+                });
+                return 'retried';
+            }
+        }
+        catch {
+            // RPC unreachable / cap spent — fall through to classify+ladder so an
+            // honest terminal/ladder decision stands rather than wedging.
+        }
+    }
+    const verdict = (0, error_classify_1.classifyFailure)(errorCode, errorMessage);
+    // 2. Transient → same-provider retry until budget spent.
+    if (verdict === 'transient') {
+        const delayMs = (0, error_classify_1.retryDelayMs)(io.retryCount);
+        try {
+            if (await io.markRetry(errorCode, errorMessage, delayMs)) {
+                void io.emit('retry_scheduled', {
+                    errorCode: errorCode ?? null,
+                    errorMessage: (errorMessage ?? '').slice(0, 500) || null,
+                    delayMs,
+                    attempt: io.retryCount + 1,
+                });
+                return 'retried';
+            }
+        }
+        catch {
+            // RPC unreachable / budget spent — fall through to the ladder rather
+            // than dying on a transient signal.
+        }
+        // transient budget spent → try the ladder rather than dying.
+    }
+    // 3. ladderable, or transient-budget-spent → ask the server to hop to the
+    //    next eligible provider or dead-letter. terminal also routes here so
+    //    the server writes the dead_letter status (single writer) rather than
+    //    the daemon writing 'failed' directly.
+    try {
+        const outcome = await io.ladderOrDeadLetter(errorCode, errorMessage);
+        if (outcome === 'laddered') {
+            void io.emit('provider_laddered', { errorCode: errorCode ?? null, hop: io.ladderCount + 1 });
+            return 'laddered';
+        }
+        return 'failed'; // dead_letter written server-side
+    }
+    catch {
+        // RPC unreachable → last-resort local terminal write so the row settles.
+        await io.writeFailed(errorCode, errorMessage);
+        return 'failed';
+    }
+}