agent-lyceum 0.18.0 → 0.20.0

package/dist/fallback.js

@@ -0,0 +1,157 @@
+/** `<runtime>/<model>`, the model trimmed and lowercased (no model = ""), so entries naming the same model share one key. */
+const keyOf = (runtime, model) => `${runtime}/${(model ?? "").trim().toLowerCase()}`;
+/**
+ * The primary model followed by the agent's fallback chain, in order. An entry with the same key as the primary or as an
+ * earlier entry is dropped: a chain shared through `dispatcher.fallback` may list the agent's own primary (spec 20.12).
+ */
+export function candidatesFor(agent) {
+    // `validate` refuses an agent without a runtime before anything is woken.
+    const runtime = agent.runtime;
+    const primary = { runtime, model: agent.model, effort: agent.effort, primary: true, key: keyOf(runtime, agent.model) };
+    const out = [primary];
+    const seen = new Set([primary.key]);
+    for (const f of agent.fallback) {
+        const model = f.model || undefined;
+        const key = keyOf(f.runtime, model);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({ runtime: f.runtime, model, effort: f.effort, primary: false, key });
+    }
+    return out;
+}
+/**
+ * The agent as it runs on candidate `c`: itself for the primary; for a backup, the same agent on the backup's runtime, model
+ * and effort, at the primary's effective web level (spec 20.12). A set `allow_web` carries over. Unset, claude-code has no web
+ * tools while codex keeps its own default, which may search: a codex backup of such an agent gets `none`.
+ */
+export function agentForCandidate(agent, c) {
+    if (c.primary)
+        return agent;
+    const allowWeb = agent.allowWeb ?? (agent.runtime === "claude-code" && c.runtime === "codex" ? "none" : undefined);
+    return { ...agent, runtime: c.runtime, model: c.model, effort: c.effort, allowWeb };
+}
+/** Models that failed with `rate_limit`/`quota`, and until when (epoch ms) wakes skip them. Shared by a run, kept in `state.json`. */
+export class Cooldowns {
+    table = new Map();
+    /** Cool `key` down for `sec` seconds from `now`; never shortens a longer cooldown already in place. */
+    mark(key, kind, now, sec) {
+        if (sec <= 0)
+            return;
+        const until = now + sec * 1000;
+        if ((this.table.get(key)?.until ?? 0) >= until)
+            return;
+        this.table.set(key, { until, kind });
+    }
+    /** Milliseconds of cooldown left for `key` at `now`; 0 = none. */
+    active(key, now) {
+        const until = this.table.get(key)?.until ?? 0;
+        return Math.max(0, until - now);
+    }
+    toJSON() {
+        return Object.fromEntries([...this.table].map(([k, v]) => [k, { until: new Date(v.until).toISOString(), kind: v.kind }]));
+    }
+    /** Rebuild from `toJSON()` output; a missing table (older `state.json`) is an empty one. */
+    static from(json) {
+        const c = new Cooldowns();
+        for (const [k, v] of Object.entries(json ?? {})) {
+            const until = Date.parse(v.until);
+            if (Number.isFinite(until))
+                c.table.set(k, { until, kind: v.kind });
+        }
+        return c;
+    }
+}
+/** The candidates for one wake: the chain minus those cooling down, or the whole chain when every one is (something must be tried). */
+export function planAttempts(agent, cooldowns, now) {
+    const all = candidatesFor(agent);
+    const warm = all.filter((c) => cooldowns.active(c.key, now) === 0);
+    return warm.length ? warm : all;
+}
+/**
+ * What to do after a failed call (spec 20.2 decisions 2 and 5). `attemptsOnCurrent` counts the calls made on the current
+ * candidate, the failed one included; each candidate gets `retry` retries. `legacy` (the agent has no fallback chain)
+ * keeps the old loop: every kind is retried `retry` times, then the wake fails.
+ */
+export function nextStep(kind, attemptsOnCurrent, retry, hasNext, legacy) {
+    const retriesLeft = attemptsOnCurrent <= retry;
+    if (legacy)
+        return retriesLeft ? "retry-same" : "stop";
+    switch (kind) {
+        case "rate_limit":
+        case "quota":
+            return hasNext ? "next-candidate" : "stop";
+        case "timeout":
+        case "network":
+        case "crash":
+            return retriesLeft ? "retry-same" : hasNext ? "next-candidate" : "stop";
+        case "auth":
+            return "stop";
+        default:
+            // `unknown` and unclassified failures: same model only, as before fallback existed.
+            return retriesLeft ? "retry-same" : "stop";
+    }
+}
+/** Wait before the `n`th retry (1-based) on the same candidate: 5s, 15s, 45s, then 60s. */
+export function backoffMs(n) {
+    return Math.min(5 * 3 ** (n - 1), 60) * 1000;
+}
+/** `setTimeout` as a promise that resolves early (never rejects) when `signal` aborts. */
+export function defaultSleep(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal?.aborted)
+            return resolve();
+        const done = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener("abort", done);
+            resolve();
+        };
+        const timer = setTimeout(done, ms);
+        signal?.addEventListener("abort", done, { once: true });
+    });
+}
+/**
+ * The one retry/fallback loop shared by `RunSession.wake` and `call` (spec 20.2 decision 9): call each candidate,
+ * retry it or move on as `nextStep` says, back off between retries. Success and cancellation end the loop as they are.
+ */
+export async function runWithFallback(o) {
+    const first = o.candidates[0];
+    if (!first)
+        throw new Error("runWithFallback needs at least one candidate");
+    const sleep = o.sleep ?? defaultSleep;
+    const now = o.now ?? Date.now;
+    const tried = [];
+    let index = 0;
+    let used = first;
+    let ctx = { attempt: 0, candidateAttempt: 0 };
+    let result;
+    for (;;) {
+        if (o.beforeAttempt?.(used, ctx) === "stop")
+            return { result, tried, used, stopped: "beforeAttempt" };
+        result = await o.invoke(used, ctx);
+        await o.afterAttempt?.(used, result, ctx);
+        if (result.ok || result.cancelled)
+            return { result, tried, used };
+        tried.push({ runtime: used.runtime, model: used.model, kind: result.kind, error: result.error });
+        if (!o.legacy && (result.kind === "rate_limit" || result.kind === "quota"))
+            o.cooldowns.mark(used.key, result.kind, now(), o.cooldownSec);
+        const next = o.candidates[index + 1];
+        const step = nextStep(result.kind, ctx.candidateAttempt + 1, o.retry, !!next, o.legacy);
+        if (step === "stop")
+            return { result, tried, used };
+        if (step === "retry-same") {
+            if (!o.legacy) {
+                await sleep(backoffMs(ctx.candidateAttempt + 1), o.signal);
+                if (o.signal?.aborted)
+                    return { result, tried, used, stopped: "aborted" };
+            }
+            ctx = { attempt: ctx.attempt + 1, candidateAttempt: ctx.candidateAttempt + 1, previous: result };
+            continue;
+        }
+        const to = next;
+        o.onSwitch?.(used, to, result.kind, result);
+        ctx = { attempt: ctx.attempt + 1, candidateAttempt: 0, previous: result, switchedFrom: used };
+        index++;
+        used = to;
+    }
+}

package/dist/policy.d.ts

@@ -37,6 +37,8 @@ export declare function sharedIndex(project: ResolvedProject): string | undefine
 /** The repo directory this agent works in: its own worktree, or the main repo. */
 export declare function repoDirFor(project: ResolvedProject, agent: string): string;
 export declare function writePolicy(project: ResolvedProject, agent: ResolvedAgent): WritePolicy;
+/** Test seam: pretend the OS sandbox is (un)available; `undefined` re-probes the host. */
+export declare function setOsSandboxForTest(available: boolean | undefined): void;
 export declare function enforcementFor(project: ResolvedProject, agent: ResolvedAgent): AgentEnforcement;
 /** Can `agent` legitimately change this protected file? */
 export declare function mayEditProtected(project: ResolvedProject, agent: ResolvedAgent, file: string): boolean;

package/dist/policy.js

@@ -136,6 +136,10 @@ function osSandboxAvailable() {
         sandboxProbe = false;
     return sandboxProbe;
 }
+/** Test seam: pretend the OS sandbox is (un)available; `undefined` re-probes the host. */
+export function setOsSandboxForTest(available) {
+    sandboxProbe = available;
+}
 export function enforcementFor(project, agent) {
     const pol = writePolicy(project, agent);
     const os = osSandboxAvailable();

package/dist/process-runner.d.ts

@@ -13,6 +13,16 @@ export interface RunnerOptions {
     /** How long to wait for output pipes to close after the process is gone or killed, before giving up on them. Default 1000. */
     settleWaitMs?: number;
 }
+/** Pids below `root` (children, grandchildren, ...) by parent links; catches descendants that left the process group (setsid, daemons). */
+export declare function descendantsOf(root: number): number[];
+/** Sampling interval of the descendant timer (default 2000 ms); for tests. Takes effect for the next timer. */
+export declare function setSamplerIntervalMs(ms: number): void;
+/** Whether the descendant timer is running (for tests). */
+export declare function samplerActive(): boolean;
+/** How many runtimes are registered as running (for tests). */
+export declare function runningCount(): number;
+/** Synchronously SIGKILL every runtime still running, with the descendants seen when it was started or last checked. Used when agent-lyceum itself is going away. */
+export declare function killAllRunning(): void;
 /**
  * Run an invocation as a child process in its own process group, with a timeout, cancellation, bounded memory
  * and a bounded finish: whatever the runtime and its descendants do with the output pipes, this settles once,

package/dist/process-runner.js

@@ -1,4 +1,4 @@
-import { spawn } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 /** Most output kept in memory per stream; the rest only goes to the log files. */
@@ -31,6 +31,121 @@ function endStream(s) {
         });
     });
 }
+/**
+ * Runtimes that are running right now, with the descendants seen so far, so an exiting agent-lyceum can take them with it.
+ * A descendant that left the process group (setsid, daemon) is only found by parent links while its parent is alive, so a
+ * shared timer samples the process table while any runtime is registered. Remaining gap: a child that escapes and
+ * outlives its parent within one sampling interval is never seen.
+ */
+const live = new Map();
+let exitHook = false;
+let sampler;
+let samplerMs = 2000;
+/** Parent -> children from one `ps` run, or undefined when it fails or hangs (SIGKILL on timeout: a ps that ignores SIGTERM must not block, even in the exit hook). */
+function processTable() {
+    if (process.platform === "win32")
+        return undefined;
+    const ps = spawnSync("ps", ["-A", "-o", "pid=,ppid="], { encoding: "utf8", timeout: 2000, killSignal: "SIGKILL" });
+    if (ps.status !== 0 || !ps.stdout)
+        return undefined;
+    const kids = new Map();
+    for (const line of ps.stdout.split("\n")) {
+        const [pid, ppid] = line.trim().split(/\s+/).map(Number);
+        if (!pid || !ppid)
+            continue;
+        kids.set(ppid, [...(kids.get(ppid) ?? []), pid]);
+    }
+    return kids;
+}
+function below(kids, root) {
+    const out = [];
+    if (!kids)
+        return out;
+    for (let todo = [root]; todo.length;) {
+        const next = [];
+        for (const p of todo) {
+            for (const k of kids.get(p) ?? []) {
+                if (out.includes(k))
+                    continue;
+                out.push(k);
+                next.push(k);
+            }
+        }
+        todo = next;
+    }
+    return out;
+}
+/** Pids below `root` (children, grandchildren, ...) by parent links; catches descendants that left the process group (setsid, daemons). */
+export function descendantsOf(root) {
+    return below(processTable(), root);
+}
+/** Record the current descendants of every live runtime (one `ps` for all). */
+function sample() {
+    if (!live.size)
+        return;
+    const kids = processTable();
+    for (const [pid, seen] of live)
+        for (const d of below(kids, pid))
+            seen.add(d);
+}
+function register(pid) {
+    live.set(pid, new Set());
+    if (!exitHook) {
+        exitHook = true;
+        process.on("exit", killAllRunning); // process.exit() and crashes must not leave detached runtimes behind
+    }
+    if (!sampler) {
+        sampler = setInterval(sample, samplerMs);
+        sampler.unref();
+    }
+}
+function unregister(pid) {
+    live.delete(pid);
+    if (!live.size && sampler) {
+        clearInterval(sampler);
+        sampler = undefined;
+    }
+}
+/** Sampling interval of the descendant timer (default 2000 ms); for tests. Takes effect for the next timer. */
+export function setSamplerIntervalMs(ms) {
+    samplerMs = ms;
+}
+/** Whether the descendant timer is running (for tests). */
+export function samplerActive() {
+    return sampler !== undefined;
+}
+function signalPid(pid, sig, group) {
+    try {
+        process.kill(group && process.platform !== "win32" ? -pid : pid, sig);
+    }
+    catch {
+        /* already gone */
+    }
+}
+/** Signal the runtime's group, the descendants recorded earlier that are still in the process table, and its current descendants. */
+function killTree(pid, sig) {
+    const kids = processTable();
+    const seen = live.get(pid) ?? new Set();
+    const now = below(kids, pid);
+    const present = kids ? new Set([...kids.values()].flat()) : undefined; // without a table, trust the record
+    for (const d of now)
+        seen.add(d);
+    signalPid(pid, sig, true);
+    for (const d of seen)
+        if (!present || present.has(d))
+            signalPid(d, sig, false);
+}
+/** How many runtimes are registered as running (for tests). */
+export function runningCount() {
+    return live.size;
+}
+/** Synchronously SIGKILL every runtime still running, with the descendants seen when it was started or last checked. Used when agent-lyceum itself is going away. */
+export function killAllRunning() {
+    for (const pid of [...live.keys()]) {
+        killTree(pid, "SIGKILL");
+        unregister(pid);
+    }
+}
 /**
  * Run an invocation as a child process in its own process group, with a timeout, cancellation, bounded memory
  * and a bounded finish: whatever the runtime and its descendants do with the output pipes, this settles once,
@@ -49,6 +164,8 @@ export function runInvocation(inv, opts) {
         outLog.on("error", () => { });
         errLog.on("error", () => { });
         const child = spawn(inv.cmd, inv.args, { cwd: inv.cwd, env: inv.env, stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32" });
+        if (child.pid !== undefined)
+            register(child.pid);
         const stdoutTail = new Tail();
         const stderrTail = new Tail();
         const parser = inv.stream?.();
@@ -58,17 +175,8 @@ export function runInvocation(inv, opts) {
         let settled = false;
         const timers = [];
         const killGroup = (sig) => {
-            if (child.pid === undefined)
-                return;
-            try {
-                if (process.platform === "win32")
-                    child.kill(sig);
-                else
-                    process.kill(-child.pid, sig);
-            }
-            catch {
-                /* nothing left to signal */
-            }
+            if (child.pid !== undefined)
+                killTree(child.pid, sig);
         };
         const finish = (spawnError) => {
             if (settled)
@@ -78,17 +186,21 @@ export function runInvocation(inv, opts) {
                 clearTimeout(t);
             opts.signal?.removeEventListener("abort", onAbort);
             killGroup("SIGKILL"); // whatever of the tree is still around
+            if (child.pid !== undefined)
+                unregister(child.pid);
             child.stdout?.destroy();
             child.stderr?.destroy();
             void Promise.all([endStream(outLog), endStream(errLog)]).then(() => {
                 if (spawnError) {
                     const hint = spawnError.code === "ENOENT" ? ` ("${inv.cmd}" not found on PATH)` : "";
-                    return resolve({ ok: false, text: "", exitCode: null, timedOut: false, error: `${spawnError.message}${hint}` });
+                    // a missing binary is a crash the chain may route around (the next candidate can be another CLI); other spawn errors stay unclassified
+                    const kind = hint ? "crash" : "unknown";
+                    return resolve({ ok: false, text: "", exitCode: null, timedOut: false, error: `${spawnError.message}${hint}`, kind });
                 }
                 if (reason === "cancel")
                     return resolve({ ...cancelled, exitCode: exitCode ?? null });
                 if (reason === "timeout")
-                    return resolve({ ok: false, text: "", exitCode: exitCode ?? null, timedOut: true, error: `timed out after ${opts.timeoutSec}s` });
+                    return resolve({ ok: false, text: "", exitCode: exitCode ?? null, timedOut: true, error: `timed out after ${opts.timeoutSec}s`, kind: "timeout" });
                 const parsed = parser
                     ? parser.finish({ code: exitCode ?? null, stdoutTail: stdoutTail.text(), stderrTail: stderrTail.text() })
                     : inv.parse({ stdout: fallback.length ? fallback.toString("utf8") : stdoutTail.text(), stderr: stderrTail.text(), code: exitCode ?? null });

package/dist/run-outcome.d.ts

@@ -1,5 +1,5 @@
 import type { DoneContract } from "./format.js";
-import type { EndReason } from "./run-store.js";
+import type { EndReason, RunFailure } from "./run-store.js";
 import type { RunOutcome } from "./schema.js";
 export interface UnintegratedWork {
     agent: string;
@@ -16,17 +16,9 @@ export declare function resolveDoneOutcome(contract: DoneContract, rejections: n
 export declare function endOutcome(endReason: EndReason, done?: {
     outcome: RunOutcome;
     note?: string;
-}, failure?: {
-    agent: string;
-    error?: string;
-    attempts: number;
-}): {
+}, failure?: RunFailure): {
     outcome: RunOutcome;
     note?: string;
 };
 /** What to tell the user when an agent kept failing and the run stopped: where to look, what to fix, how to go on. */
-export declare function failureGuide(project: string, run: string, f: {
-    agent: string;
-    error?: string;
-    attempts: number;
-}, runDir: string): string;
+export declare function failureGuide(project: string, run: string, f: RunFailure, runDir: string): string;

package/dist/run-outcome.js

@@ -35,9 +35,14 @@ export function endOutcome(endReason, done, failure) {
 }
 /** What to tell the user when an agent kept failing and the run stopped: where to look, what to fix, how to go on. */
 export function failureGuide(project, run, f, runDir) {
+    // With a fallback chain: every model given up on, in order.
+    const tried = f.tried?.length
+        ? ["Models tried, in order:", ...f.tried.map((t, i) => `  ${i + 1}. ${t.model ? `${t.runtime}/${t.model}` : t.runtime}: ${t.kind ?? "unknown error"}${t.error ? ` (${t.error})` : ""}`)]
+        : [];
     return [
         "",
         `${f.agent} was called ${f.attempts} time(s) and failed every time (${f.error ?? "unknown error"}). The run stopped; nothing was lost: the message ${f.agent} was handling is still unread and the work done so far is kept.`,
+        ...tried,
         "To fix it:",
         `  1. Read why it failed: ${path.join(runDir, "log.jsonl")} (event "wake"), and the full output of each attempt in ${path.join(runDir, "mail", "attempts")}`,
         `  2. Fix the cause: the ${f.agent} CLI (installed, logged in, model name), a timeout (dispatcher.wake_timeout_sec), or its AGENT.md / the message it keeps failing on. Check with: agent-lyceum doctor -p ${project}`,

package/dist/run-session.d.ts

@@ -2,7 +2,7 @@ import { type Invoker } from "./adapters/index.js";
 import type { ResolvedProject } from "./config.js";
 import type { RunOutcome } from "./schema.js";
 import { type ResultBranch, type WorkspacePlan } from "./worktree.js";
-import { newRunId, type EndReason, type RunState } from "./run-store.js";
+import { newRunId, type EndReason, type RunFailure, type RunState } from "./run-store.js";
 import { type PreparedTask } from "./task.js";
 import { type CommandRunner, type PrOutcome } from "./open-pr.js";
 export { newRunId };
@@ -36,11 +36,7 @@ export interface RunSummary {
     /** The project's name, for the commands printed after the run. */
     project: string;
     /** The run stopped because this agent kept failing. */
-    failure?: {
-        agent: string;
-        error?: string;
-        attempts: number;
-    };
+    failure?: RunFailure;
 }
 export interface RunOptions {
     project: ResolvedProject;
@@ -65,6 +61,8 @@ export interface RunOptions {
     /** Stop the run when this fires: running agents are killed, unread mail is kept, the run ends `cancelled` and can be resumed. */
     signal?: AbortSignal;
     log?: (line: string) => void;
+    /** The wait between retries of a model with a fallback chain; tests pass one that does not wait. Default: a timer cut short by `signal`. */
+    sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
 }
 export declare class RunSession {
     private readonly opts;
@@ -89,6 +87,8 @@ export declare class RunSession {
     private doneOutcome;
     private doneNote;
     private failure;
+    /** Models cooling down after a rate limit or quota error; one table for the run, shared by parallel wake-ups. */
+    private readonly cooldowns;
     private doneVerification;
     /** The accepted `done` carried `open_pr: true` (only the lead can send `done`). */
     private openPrRequested;
@@ -113,6 +113,8 @@ export declare class RunSession {
     private pendingAgents;
     private pickBatch;
     private wake;
+    /** Copy the shared cooldown table into the state (only once something has cooled down); true when it was. */
+    private syncCooldowns;
     /**
      * A finished member's files go from its worktree into the repo before anyone reads its mail, so the lead
      * sees them. When that cannot be done, nothing is applied, the member's work stays on its branch, and the lead is told.