@addai/node 0.4.0

package/dist/diskguard.js

@@ -0,0 +1,435 @@
+"use strict";
+// DiskGuard — CCTV-style disk-threshold auto-purge for the per-run Codex
+// scratch dirs the runtime creates.
+//
+// Background: codex-spawn mints a throwaway CODEX_HOME per run under
+// os.tmpdir() (entities-codex-<rand>). Codex writes its state_5.sqlite,
+// logs, caches and shell snapshots there — 50-350 MB per run. On a
+// dedicated runtime host that does thousands of runs, these pile up and
+// fill the disk (observed: 3,006 dirs = ~158 GiB, hit 0 bytes free /
+// ENOSPC). codex-spawn now removes its own dir on exit; DiskGuard is the
+// safety net for the cases that skip that path — SIGKILL, crashes, and
+// orphans left by a previous runtime process.
+//
+// Model (like a CCTV DVR): watch free space on the filesystem holding the
+// temp root. When usage crosses a HIGH-water mark, delete the OLDEST
+// INACTIVE scratch dirs, oldest-first, until usage drops below a LOW-water
+// mark or no evictable dirs remain — leaving a safety buffer for the OS.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DiskGuard = void 0;
+exports.loadDiskGuardConfig = loadDiskGuardConfig;
+exports.runEvictionLoop = runEvictionLoop;
+exports.makeSingleFlight = makeSingleFlight;
+exports.removeSessionDir = removeSessionDir;
+exports.getDiskGuard = getDiskGuard;
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+function numEnv(env, key, def) {
+    const raw = env[key];
+    if (raw === undefined || raw === '')
+        return def;
+    const n = Number(raw);
+    return Number.isFinite(n) ? n : def;
+}
+function boolEnv(env, key, def) {
+    const raw = env[key];
+    if (raw === undefined || raw === '')
+        return def;
+    return /^(1|true|yes|on)$/i.test(raw.trim());
+}
+function loadDiskGuardConfig(env = process.env) {
+    let high = numEnv(env, 'DISKGUARD_HIGH_WATER_PCT', 90);
+    let low = numEnv(env, 'DISKGUARD_LOW_WATER_PCT', 80);
+    // Clamp to sane bounds and keep low strictly below high — otherwise the
+    // eviction loop can't converge (it stops when usage <= low).
+    high = Math.min(99, Math.max(1, high));
+    low = Math.min(99, Math.max(1, low));
+    if (low >= high)
+        low = Math.max(1, high - 1);
+    const minAgeMin = Math.max(0, numEnv(env, 'DISKGUARD_MIN_AGE_MINUTES', 30));
+    const checkSec = Math.max(5, numEnv(env, 'DISKGUARD_CHECK_INTERVAL_SECONDS', 60));
+    const prefixEnv = (env.DISKGUARD_PREFIXES ?? 'entities-codex-').trim();
+    const prefixes = prefixEnv.split(',').map(s => s.trim()).filter(Boolean);
+    return {
+        enabled: boolEnv(env, 'DISKGUARD_ENABLED', true),
+        highWaterPct: high,
+        lowWaterPct: low,
+        minAgeMs: minAgeMin * 60_000,
+        checkIntervalMs: checkSec * 1000,
+        dryRun: boolEnv(env, 'DISKGUARD_DRY_RUN', false),
+        tempRoot: env.DISKGUARD_TEMP_ROOT || os.tmpdir(),
+        prefixes: prefixes.length > 0 ? prefixes : ['entities-codex-'],
+    };
+}
+async function runEvictionLoop(deps, opts) {
+    const used = await deps.readUsedPct();
+    if (used === null) {
+        // Fail safe: if we can't read usage, do nothing.
+        return { ran: false, usedBefore: null, usedAfter: null, evicted: [], bytesFreed: 0, skippedActive: 0, skippedYoung: 0, reason: 'usage-unreadable' };
+    }
+    if (used < opts.highWaterPct) {
+        return { ran: false, usedBefore: used, usedAfter: used, evicted: [], bytesFreed: 0, skippedActive: 0, skippedYoung: 0, reason: 'below-high-water' };
+    }
+    const candidates = await deps.listCandidates();
+    let skippedActive = 0;
+    let skippedYoung = 0;
+    const eligible = [];
+    for (const c of candidates) {
+        if (c.active) {
+            skippedActive++;
+            continue;
+        }
+        if (c.ageMs < opts.minAgeMs) {
+            skippedYoung++;
+            continue;
+        }
+        eligible.push(c);
+    }
+    // Oldest activity first — CCTV overwrites the oldest footage.
+    eligible.sort((a, b) => a.activityMs - b.activityMs);
+    const evicted = [];
+    if (opts.dryRun) {
+        // Can't observe real usage drop without deleting — estimate from sizes
+        // when we know the volume size; otherwise just report all eligible.
+        let est = used;
+        for (const c of eligible) {
+            if (deps.totalBytes && est <= opts.lowWaterPct)
+                break;
+            evicted.push(c.path);
+            if (deps.totalBytes)
+                est -= (c.sizeBytes / deps.totalBytes) * 100;
+        }
+        return {
+            ran: true, usedBefore: used, usedAfter: deps.totalBytes ? est : used,
+            evicted, bytesFreed: eligible.filter(c => evicted.includes(c.path)).reduce((s, c) => s + c.sizeBytes, 0),
+            skippedActive, skippedYoung, reason: 'dry-run',
+        };
+    }
+    let current = used;
+    for (const c of eligible) {
+        if (current <= opts.lowWaterPct)
+            break;
+        // A failed delete (EBUSY/EPERM — common on Windows when a handle is
+        // still open in the tree) must not abort the sweep or bubble up as an
+        // unhandled rejection: skip the dir and keep evicting the rest.
+        try {
+            await deps.deleteDir(c.path);
+        }
+        catch (err) {
+            console.error(`[diskguard] failed to evict ${c.path}: ${err.message}`);
+            continue;
+        }
+        evicted.push(c.path);
+        const after = await deps.readUsedPct();
+        if (after !== null)
+            current = after;
+    }
+    const bytesFreed = deps.totalBytes ? Math.max(0, ((used - current) / 100) * deps.totalBytes) : 0;
+    return {
+        ran: true, usedBefore: used, usedAfter: current, evicted, bytesFreed,
+        skippedActive, skippedYoung,
+        reason: evicted.length > 0 ? 'evicted' : 'no-eligible',
+    };
+}
+/** Single-flight wrapper: while a call is in flight, concurrent callers get
+ *  the same promise instead of starting a second sweep (the mutex). */
+function makeSingleFlight() {
+    let inFlight = null;
+    return (fn) => {
+        if (inFlight)
+            return inFlight;
+        inFlight = fn().finally(() => { inFlight = null; });
+        return inFlight;
+    };
+}
+// ---------------------------------------------------------------------------
+// Filesystem helpers
+// ---------------------------------------------------------------------------
+function safeRealpath(p) {
+    try {
+        return fs.realpathSync(p);
+    }
+    catch {
+        return null;
+    }
+}
+function pidAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        // Signal 0 doesn't send a signal, just checks for existence/permission.
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        // EPERM means the process exists but is owned by someone else — alive.
+        return e.code === 'EPERM';
+    }
+}
+/** Best-effort recursive dir size. Never follows symlinks (the scratch dir's
+ *  `sessions`/`auth.json` are symlinks into the durable ~/.codex). */
+async function dirSize(dir) {
+    let total = 0;
+    let entries;
+    try {
+        entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return 0;
+    }
+    for (const e of entries) {
+        if (e.isSymbolicLink())
+            continue;
+        const full = path.join(dir, e.name);
+        try {
+            if (e.isDirectory())
+                total += await dirSize(full);
+            else {
+                total += (await fs.promises.stat(full)).size;
+            }
+        }
+        catch { /* skip unreadable entry */ }
+    }
+    return total;
+}
+/** Remove a scratch dir. `recursive+force` unlinks symlink ENTRIES without
+ *  following them, so the ~/.codex targets behind `sessions`/`auth.json`
+ *  are never touched. ENOENT is swallowed by `force`. */
+async function removeSessionDir(dir) {
+    await fs.promises.rm(dir, { recursive: true, force: true });
+}
+class DiskGuard {
+    config;
+    timer;
+    runExclusive = makeSingleFlight();
+    /** Realpath'd scratch dirs owned by a live in-process session. */
+    activeDirs = new Set();
+    constructor(overrides) {
+        this.config = { ...loadDiskGuardConfig(), ...overrides };
+    }
+    /** Start the periodic sweeper. No-op (beyond a log line) if disabled;
+     *  cleanup-on-exit via beginSession still works regardless. */
+    start() {
+        if (!this.config.enabled) {
+            console.log('[diskguard] disabled (DISKGUARD_ENABLED=false) — codex scratch dirs still cleaned on exit');
+            return;
+        }
+        void this.sweep('startup').catch(err => console.error('[diskguard] startup sweep failed:', err.message));
+        this.timer = setInterval(() => void this.sweep('periodic').catch(err => console.error('[diskguard] sweep failed:', err.message)), this.config.checkIntervalMs);
+        this.timer.unref?.();
+        console.log(`[diskguard] enabled — root=${this.config.tempRoot} high=${this.config.highWaterPct}% ` +
+            `low=${this.config.lowWaterPct}% minAge=${Math.round(this.config.minAgeMs / 60_000)}m ` +
+            `interval=${Math.round(this.config.checkIntervalMs / 1000)}s${this.config.dryRun ? ' DRY-RUN' : ''}`);
+    }
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = undefined;
+        }
+    }
+    /** Run one sweep. Concurrent calls share a single in-flight sweep. */
+    sweep(reason) {
+        return this.runExclusive(() => this.runSweep(reason));
+    }
+    async runSweep(reason) {
+        const totalBytes = this.readTotalBytes();
+        const res = await runEvictionLoop({
+            listCandidates: () => this.listCandidates(this.config.dryRun),
+            readUsedPct: async () => this.readUsedPct(),
+            deleteDir: (p) => removeSessionDir(p),
+            totalBytes,
+        }, this.config);
+        if (res.reason === 'usage-unreadable') {
+            console.warn(`[diskguard] ${reason}: could not read disk usage for ${this.config.tempRoot}; skipping`);
+        }
+        else if (res.ran && (res.evicted.length > 0 || this.config.dryRun)) {
+            const verb = this.config.dryRun ? 'would evict' : 'evicted';
+            console.log(`[diskguard] ${reason}: usage ${fmtPct(res.usedBefore)}→${fmtPct(res.usedAfter)}, ` +
+                `${verb} ${res.evicted.length} dir(s) (~${fmtMB(res.bytesFreed)} MB), ` +
+                `skipped ${res.skippedActive} active / ${res.skippedYoung} too-young`);
+        }
+        return res;
+    }
+    /** Register a live session: mark its dir active in-process and write a
+     *  PID lock (with a periodic heartbeat) so a DIFFERENT runtime process
+     *  can tell the dir is still owned and not evict it. */
+    beginSession(dir) {
+        const real = safeRealpath(dir) ?? dir;
+        this.activeDirs.add(real);
+        const lockPath = path.join(dir, 'session.lock');
+        const writeLock = () => {
+            try {
+                fs.writeFileSync(lockPath, JSON.stringify({ pid: process.pid, ts: Date.now() }), { mode: 0o600 });
+            }
+            catch { /* dir may already be gone */ }
+        };
+        writeLock();
+        const heartbeat = setInterval(writeLock, Math.max(30_000, Math.floor(this.config.checkIntervalMs / 2)));
+        heartbeat.unref?.();
+        let disposed = false;
+        return {
+            dispose: async (removeDir) => {
+                if (disposed)
+                    return;
+                disposed = true;
+                clearInterval(heartbeat);
+                this.activeDirs.delete(real);
+                if (removeDir) {
+                    try {
+                        await removeSessionDir(dir);
+                    }
+                    catch { /* best-effort */ }
+                }
+            },
+        };
+    }
+    readUsedPct() {
+        try {
+            const st = fs.statfsSync(this.config.tempRoot);
+            const blocks = Number(st.blocks);
+            const bavail = Number(st.bavail);
+            if (!Number.isFinite(blocks) || blocks <= 0)
+                return null;
+            return (1 - bavail / blocks) * 100;
+        }
+        catch {
+            return null;
+        }
+    }
+    readTotalBytes() {
+        try {
+            const st = fs.statfsSync(this.config.tempRoot);
+            return Number(st.blocks) * Number(st.bsize);
+        }
+        catch {
+            return 0;
+        }
+    }
+    async listCandidates(measureSize) {
+        const root = this.config.tempRoot;
+        const realRoot = safeRealpath(root);
+        let names;
+        try {
+            names = await fs.promises.readdir(root);
+        }
+        catch {
+            return [];
+        }
+        const now = Date.now();
+        const out = [];
+        for (const name of names) {
+            if (!this.config.prefixes.some(p => name.startsWith(p)))
+                continue;
+            const full = path.join(root, name);
+            let lst;
+            try {
+                lst = await fs.promises.lstat(full);
+            }
+            catch {
+                continue;
+            }
+            // Only real directories — never a symlink masquerading as one.
+            if (!lst.isDirectory())
+                continue;
+            // Safety: the resolved path must still sit directly under the temp
+            // root, so we can never rm somewhere else via a swapped symlink.
+            const real = safeRealpath(full);
+            if (!real || (realRoot !== null && path.dirname(real) !== realRoot))
+                continue;
+            const { active, activityMs } = this.classifyActivity(full, lst.mtimeMs);
+            const sizeBytes = measureSize ? await dirSize(full) : 0;
+            out.push({ path: full, activityMs, ageMs: now - lst.mtimeMs, sizeBytes, active });
+        }
+        return out;
+    }
+    /** Decide whether a scratch dir is owned by a live session, and compute
+     *  its last-activity time for oldest-first ordering. */
+    classifyActivity(dir, mtimeMs) {
+        const real = safeRealpath(dir);
+        if (real && this.activeDirs.has(real))
+            return { active: true, activityMs: Date.now() };
+        const lockPath = path.join(dir, 'session.lock');
+        let lockRaw;
+        try {
+            lockRaw = fs.readFileSync(lockPath, 'utf8');
+        }
+        catch {
+            return { active: false, activityMs: mtimeMs };
+        }
+        let lock;
+        try {
+            lock = JSON.parse(lockRaw);
+        }
+        catch {
+            return { active: false, activityMs: mtimeMs };
+        }
+        let lockMtime = mtimeMs;
+        try {
+            lockMtime = fs.statSync(lockPath).mtimeMs;
+        }
+        catch { /* keep dir mtime */ }
+        if (typeof lock.pid === 'number' && pidAlive(lock.pid)) {
+            return { active: true, activityMs: Math.max(lockMtime, lock.ts ?? 0) };
+        }
+        // Owning process is gone. Give a grace window after the last heartbeat
+        // (guards against PID reuse and races) before declaring it evictable.
+        const grace = Math.max(this.config.checkIntervalMs * 2, 120_000);
+        if (Date.now() - lockMtime < grace)
+            return { active: true, activityMs: lockMtime };
+        return { active: false, activityMs: Math.min(mtimeMs, lockMtime) };
+    }
+}
+exports.DiskGuard = DiskGuard;
+function fmtPct(v) {
+    return v === null ? 'n/a' : `${v.toFixed(1)}%`;
+}
+function fmtMB(bytes) {
+    return (bytes / (1024 * 1024)).toFixed(0);
+}
+// ---------------------------------------------------------------------------
+// Process-wide singleton so codex-spawn (session registration) and index.ts
+// (the sweeper) share one activeDirs set.
+// ---------------------------------------------------------------------------
+let singleton = null;
+function getDiskGuard() {
+    if (!singleton)
+        singleton = new DiskGuard();
+    return singleton;
+}

package/dist/error-classify.d.ts

@@ -0,0 +1,20 @@
+export type ErrorClass = 'transient' | 'terminal';
+/**
+ * True when the failure is a stale/invalid resume id — the run should clear
+ * the stored session id and retry ONCE fresh rather than fail terminally.
+ * Checked BEFORE classifyError's transient/terminal split (session-invalid
+ * needs the id cleared, which a plain transient requeue does not do).
+ */
+export declare function isSessionInvalid(code?: string | null, message?: string | null): boolean;
+export type Verdict = 'transient' | 'ladderable' | 'terminal';
+/**
+ * Three-way verdict driving the resilience policy:
+ *   transient  → same-provider retry (retry_count budget)
+ *   ladderable → hop to the next provider in the ladder
+ *   terminal   → give up (dead-letter); a hop would fail identically
+ * Order matters: transient capacity signals win over ladderable auth words,
+ * hard-terminal (content/context) wins over ladderable, and unknown → terminal.
+ */
+export declare function classifyFailure(code?: string | null, message?: string | null): Verdict;
+export declare function classifyError(code?: string | null, message?: string | null): ErrorClass;
+export declare function retryDelayMs(retryCount: number): number;

package/dist/error-classify.js

@@ -0,0 +1,159 @@
+"use strict";
+// Classifies a failed run's error into 'transient' (worth an automatic
+// same-provider retry: capacity, rate limits, network blips) vs 'terminal'
+// (retrying would just repeat the failure: auth, bad input, content policy,
+// context overflow). The default for anything unrecognized is 'terminal' —
+// a wrong retry re-runs a whole agent turn (and any side effects), which is
+// worse than a wrong give-up now that the failure reason is surfaced to the
+// user with a retry path.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isSessionInvalid = isSessionInvalid;
+exports.classifyFailure = classifyFailure;
+exports.classifyError = classifyError;
+exports.retryDelayMs = retryDelayMs;
+// Codes we assign ourselves that are never worth retrying, regardless of
+// message content.
+const TERMINAL_CODES = new Set([
+    'no_prompt',
+    'project_not_found',
+    'project_not_ready',
+    'watchdog_timeout',
+    'codex_not_found',
+]);
+const TERMINAL_CODE_SUFFIXES = ['_not_installed'];
+// Codes we assign ourselves that ARE worth a retry regardless of message —
+// e.g. a hang-watchdog kill: the child produced no output and we killed it,
+// so there's no provider error text, but a stuck tool call may well succeed
+// on a fresh attempt.
+const TRANSIENT_CODES = new Set([
+    'agent_hang',
+]);
+// Checked FIRST: a concrete capacity/network signal wins over generic
+// terminal-looking words that often ride along in the same error body.
+const TRANSIENT_PATTERNS = [
+    /\b429\b/, /\brate.?limit/i, /too many requests/i,
+    /\b529\b/, /overloaded/i, /capacity/i,
+    /\b50[0234]\b/, /internal server error/i, /service unavailable/i, /bad gateway/i, /gateway timeout/i,
+    /\bETIMEDOUT\b/, /\bECONNRESET\b/, /\bECONNREFUSED\b/, /\bENOTFOUND\b/, /\bEAI_AGAIN\b/, /\bEPIPE\b/,
+    /fetch failed/i, /socket hang up/i, /network error/i, /timed? ?out/i,
+    /spawn (EAGAIN|ENOMEM)/, /\b(EAGAIN|ENOMEM)\b/,
+];
+const TERMINAL_PATTERNS = [
+    /\b40[13]\b/, /unauthorized/i, /forbidden/i, /authentication/i, /api.?key/i, /log ?in/i, /credential/i,
+    /\bENOENT\b/, /not (installed|found|on PATH)/i,
+    /prompt is too long/i, /context (length|window|limit)/i, /too many tokens/i,
+    /content (filter|policy|blocked)/i, /\bpolicy\b/i, /invalid request/i, /bad request/i, /\b400\b/,
+];
+// A stored resume session/thread id that the local agent CLI no longer has —
+// recycled at 50 turns, cleared by a runtime switch, or created on another
+// machine. Resuming it fails hard, but the run is NOT dead: dropping the stale
+// id and retrying fresh self-heals. This is its OWN class, separate from
+// transient, because a plain same-session retry would resume the SAME missing
+// id and fail identically — the retry must clear the id first. Matched across
+// every agent CLI's invalid-resume phrasing (all surface via *_stderr):
+//   claude  → "No conversation found with session ID: <uuid>"
+//   codex   → "session/thread not found" on `exec resume <id>`
+//   kimi    → "session not found" on `--session <id>`
+//   grok    → "no such session" / "session not found" on `--resume <id>`
+//   gemini  → "Error resuming session: Invalid session identifier ..." (bad id)
+//             or "... No previous sessions found for this project." (empty/
+//             swept-then-recreated store) on `--resume <uuid>`
+const SESSION_INVALID_PATTERNS = [
+    // Hang-watchdog kill of a run that was RESUMING and produced zero output:
+    // the resume itself is wedged (observed: grok loading a heavy session).
+    // Retrying the same resume wedges identically, so route it through the
+    // clear-session-and-retry path like any other invalid-session failure.
+    /silent resume hang/i,
+    /no conversation found with session id/i,
+    /no conversation found/i,
+    /\b(session|thread|conversation)\b[^.]{0,40}\bnot found\b/i,
+    /\bnot found\b[^.]{0,40}\b(session|thread|conversation)\b/i,
+    /\bno such (session|thread|conversation)\b/i,
+    /\b(unknown|invalid|missing|stale|expired) session( id)?\b/i,
+    /could not (find|resume|load)[^.]{0,40}\b(session|thread|conversation)\b/i,
+    /(session|thread) id .{0,60}\b(does not exist|doesn'?t exist|is invalid)\b/i,
+    // gemini stale-resume phrasings. The plural "sessions" in the empty-store
+    // message escapes the \bsession\b patterns above, so it needs its own line.
+    /invalid session identifier/i,
+    /no previous sessions found/i,
+];
+/**
+ * True when the failure is a stale/invalid resume id — the run should clear
+ * the stored session id and retry ONCE fresh rather than fail terminally.
+ * Checked BEFORE classifyError's transient/terminal split (session-invalid
+ * needs the id cleared, which a plain transient requeue does not do).
+ */
+function isSessionInvalid(code, message) {
+    const msg = message ?? '';
+    if (!msg.trim())
+        return false;
+    return SESSION_INVALID_PATTERNS.some(re => re.test(msg));
+}
+// Provider/infra failures where the SAME input on a DIFFERENT provider is
+// likely to succeed: auth/token/credential, plan-or-usage limits, and
+// capacity once same-provider transient retries are spent. These read as
+// "terminal" to classifyError today; the ladder rescues them.
+const LADDERABLE_PATTERNS = [
+    /\b40[13]\b/, /unauthorized/i, /forbidden/i, /authentication/i, /api.?key/i, /credential/i, /log ?in/i,
+    /session limit/i, /usage.{0,20}limit/i, /quota/i, /\bplan\b.{0,20}\blimit/i, /resets? \d/i,
+];
+// Provider-independent failures — switching providers repeats them. These
+// stay terminal (never ladder).
+const HARD_TERMINAL_PATTERNS = [
+    /prompt is too long/i, /context (length|window|limit)/i, /too many tokens/i,
+    /content (filter|policy|blocked)/i,
+    /invalid request/i, /bad request/i, /\b400\b/,
+    /\bENOENT\b/, /not (installed|found|on PATH)/i,
+];
+/**
+ * Three-way verdict driving the resilience policy:
+ *   transient  → same-provider retry (retry_count budget)
+ *   ladderable → hop to the next provider in the ladder
+ *   terminal   → give up (dead-letter); a hop would fail identically
+ * Order matters: transient capacity signals win over ladderable auth words,
+ * hard-terminal (content/context) wins over ladderable, and unknown → terminal.
+ */
+function classifyFailure(code, message) {
+    const c = (code ?? '').toLowerCase();
+    if (TRANSIENT_CODES.has(c))
+        return 'transient';
+    if (TERMINAL_CODES.has(c))
+        return 'terminal';
+    if (TERMINAL_CODE_SUFFIXES.some(s => c.endsWith(s)))
+        return 'terminal';
+    const msg = message ?? '';
+    if (!msg.trim())
+        return 'terminal';
+    if (TRANSIENT_PATTERNS.some(re => re.test(msg)))
+        return 'transient';
+    if (HARD_TERMINAL_PATTERNS.some(re => re.test(msg)))
+        return 'terminal';
+    if (LADDERABLE_PATTERNS.some(re => re.test(msg)))
+        return 'ladderable';
+    return 'terminal';
+}
+function classifyError(code, message) {
+    const c = (code ?? '').toLowerCase();
+    if (TRANSIENT_CODES.has(c))
+        return 'transient';
+    if (TERMINAL_CODES.has(c))
+        return 'terminal';
+    if (TERMINAL_CODE_SUFFIXES.some(s => c.endsWith(s)))
+        return 'terminal';
+    const msg = message ?? '';
+    if (!msg.trim())
+        return 'terminal';
+    if (TRANSIENT_PATTERNS.some(re => re.test(msg)))
+        return 'transient';
+    return 'terminal';
+}
+// Exponential backoff with ±25% jitter: 10s, 30s, capped at 60s. Retries
+// must fit inside the server watchdog's 10-minute created_at ceiling for
+// chat runs, so the schedule stays short.
+const RETRY_BASE_MS = 10_000;
+const RETRY_CAP_MS = 60_000;
+function retryDelayMs(retryCount) {
+    const ideal = Math.min(RETRY_CAP_MS, RETRY_BASE_MS * Math.pow(3, Math.max(0, retryCount)));
+    const jitter = 1 + (Math.random() * 0.5 - 0.25); // ±25%
+    return Math.min(RETRY_CAP_MS, Math.round(ideal * jitter));
+}

package/dist/events.d.ts

@@ -0,0 +1,26 @@
+import type { RuntimeEvent } from './types';
+/**
+ * Event source for degenerate spawn handles (binary not installed / spawn
+ * threw synchronously). The old pattern — queueMicrotask over a listeners
+ * array — silently DROPPED the events: session-runner attaches onEvent only
+ * after `await setStatus(...)`, and the microtask fires at that first await,
+ * when the listeners array is still empty. The request then completed with
+ * no error ever surfaced to the chat.
+ *
+ * This buffers the events and delivers them (async, preserving the old
+ * post-return timing contract) to each listener when it attaches.
+ */
+export declare function bufferedEvents(events: RuntimeEvent[]): {
+    onEvent: (cb: (e: RuntimeEvent) => void) => void;
+};
+/**
+ * Pure transform from one line of Claude's JSONL transcript into zero or
+ * more RuntimeEvent objects.
+ *
+ *   - `user` lines: extract any tool_result content blocks (caller's own
+ *     prompt echoes are dropped — they already know what they sent).
+ *   - `assistant` lines: text blocks → assistant_text, tool_use blocks
+ *     → tool_use. String content → single assistant_text.
+ *   - Everything else → claude:<type> forward-compat passthrough.
+ */
+export declare function normalizeJsonlLine(line: Record<string, unknown>): RuntimeEvent[];