agent-lyceum 0.22.0 → 0.24.0

package/dist/redo.js

@@ -0,0 +1,281 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { cutUtf8, readRegularFileCappedNoFollow, trimPartialUtf8 } from "./fs-util.js";
+const METADATA_LIMIT = 64 * 1024;
+const DRAFT_BUDGET = 4096;
+const DRAFT_FILES = 5;
+const OUTBOX_SCAN = 64;
+const UNREADABLE = "unknown (attempt record unreadable)";
+const hasControl = (s) => [...s].some((ch) => ch.charCodeAt(0) < 0x20);
+/** Can `s` be one path segment (a claim or attempt id taken from a file)? */
+function isSegment(s) {
+    return typeof s === "string" && s.length >= 1 && s.length <= 200 && !/[/\\]/.test(s) && !s.includes("\0") && s !== "." && s !== ".." && !s.startsWith(".");
+}
+/** Read a claim or attempt JSON without following a symlink or blocking on a FIFO, at most 64 KiB. Never throws. */
+function readMetadataJson(file) {
+    const r = readRegularFileCappedNoFollow(file, METADATA_LIMIT);
+    if (!r.ok)
+        return { skip: r.reason };
+    if (r.truncated)
+        return { skip: "metadata_too_large" };
+    try {
+        return { value: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(r.buf)) };
+    }
+    catch {
+        return { skip: "metadata_unreadable" };
+    }
+}
+const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+const str = (v) => (typeof v === "string" ? v : undefined);
+/** The attempt record, or the reason it is not usable. The id that builds paths must equal the file name. */
+function validAttempt(v, fileBase) {
+    if (!isObject(v))
+        return "bad_record";
+    const { id, claim_id: claimId, agent, status, n } = v;
+    if (typeof id !== "string" || typeof agent !== "string" || typeof status !== "string")
+        return "bad_record";
+    if (!isSegment(claimId) || !Number.isSafeInteger(n) || n < 1)
+        return "bad_record";
+    if (id !== fileBase || id !== `${claimId}-a${n}`)
+        return "bad_record";
+    return { ...v, id, claim_id: claimId, agent, status, n, error: str(v.error), started_at: str(v.started_at) ?? "", finished_at: str(v.finished_at) };
+}
+function validClaim(v, claimId) {
+    if (!isObject(v) || v.id !== claimId || typeof v.agent !== "string" || typeof v.status !== "string")
+        return "bad_record";
+    if (!Array.isArray(v.message_ids) || !v.message_ids.every((x) => typeof x === "string"))
+        return "bad_record";
+    return v;
+}
+const rel = (...p) => path.join(...p);
+/** lstat each level: a real folder, or the reason it is not one ("missing" when it does not exist). */
+function realDir(file) {
+    try {
+        const st = fs.lstatSync(file);
+        if (st.isSymbolicLink())
+            return "ancestor_symlink";
+        return st.isDirectory() ? "ok" : "not_dir";
+    }
+    catch {
+        return "missing";
+    }
+}
+/** The attempt folder's mtime in ms, 0 when it cannot be read. */
+function dirMtime(dir) {
+    try {
+        return fs.lstatSync(dir).mtimeMs;
+    }
+    catch {
+        return 0;
+    }
+}
+/** Attempts of `agent` that failed before this invoke, on a claim for one of `messageIds`. Never throws. */
+export function priorFailedAttempts(runDir, agent, messageIds, exclude) {
+    const skipped = [];
+    const empty = () => ({ prior: [], skipped });
+    const levels = [".", "mail", rel("mail", "attempts"), rel("mail", "claims")];
+    for (const l of levels) {
+        const st = realDir(path.join(runDir, l));
+        if (st === "ancestor_symlink" || st === "not_dir") {
+            skipped.push({ path: l, reason: st });
+            return empty();
+        }
+        if (st === "missing") {
+            if (l === rel("mail", "claims") && realDir(path.join(runDir, "mail", "attempts")) === "ok")
+                skipped.push({ path: l, reason: "metadata_missing" });
+            return empty();
+        }
+    }
+    const dir = path.join(runDir, "mail", "attempts");
+    let names;
+    try {
+        names = fs.readdirSync(dir);
+    }
+    catch (e) {
+        skipped.push({ path: rel("mail", "attempts"), reason: `open_error:${e.code ?? "unknown"}` });
+        return empty();
+    }
+    // Candidates by claim id; an attempt without a usable record is kept as "unreadable".
+    const found = [];
+    const recorded = new Set();
+    for (const name of names.filter((x) => x.endsWith(".json")).sort()) {
+        const base = name.slice(0, -5);
+        const file = rel("mail", "attempts", name);
+        const r = readMetadataJson(path.join(dir, name));
+        if ("skip" in r) {
+            skipped.push({ path: file, reason: r.skip });
+            continue;
+        }
+        const rec = validAttempt(r.value, base);
+        if (typeof rec === "string") {
+            skipped.push({ path: file, reason: rec });
+            continue;
+        }
+        recorded.add(base);
+        if (rec.status !== "failed" || rec.agent !== agent || rec.id === exclude)
+            continue;
+        const t = [rec.finished_at, rec.started_at].map((s) => (s ? Date.parse(s) : NaN)).find((x) => Number.isFinite(x));
+        found.push({ id: rec.id, claimId: rec.claim_id, n: rec.n, error: rec.error ?? "unknown error", at: t ?? dirMtime(path.join(dir, rec.id)), parked: Array.isArray(rec.parked) ? rec.parked.filter((x) => typeof x === "string") : undefined });
+    }
+    for (const name of names.filter((x) => !x.endsWith(".json") && !recorded.has(x))) {
+        const m = /^(.+)-a([1-9]\d*)$/.exec(name);
+        if (!m || !isSegment(m[1]) || name === exclude || realDir(path.join(dir, name)) !== "ok")
+            continue;
+        found.push({ id: name, claimId: m[1], n: Number(m[2]), error: UNREADABLE, at: dirMtime(path.join(dir, name)) });
+    }
+    const accepted = new Map();
+    const usable = (claimId) => {
+        const known = accepted.get(claimId);
+        if (known !== undefined)
+            return known;
+        const file = rel("mail", "claims", `${claimId}.json`);
+        let ok = false;
+        const r = readMetadataJson(path.join(runDir, file));
+        if ("skip" in r)
+            skipped.push({ path: file, reason: r.skip });
+        else {
+            const claim = validClaim(r.value, claimId);
+            if (typeof claim === "string")
+                skipped.push({ path: file, reason: claim });
+            else
+                ok = claim.agent === agent && (claim.status === "claimed" || claim.status === "abandoned") && claim.message_ids.some((id) => messageIds.includes(id));
+        }
+        accepted.set(claimId, ok);
+        return ok;
+    };
+    const prior = found.filter((a) => usable(a.claimId)).sort((a, b) => b.at - a.at || b.n - a.n);
+    return { prior, skipped };
+}
+/** The drafts block for the attempts in `prior` (newest first), read from their folders, at most `budget` bytes and `maxFiles` files. */
+export function collectDrafts(runDir, prior, budget = DRAFT_BUDGET, maxFiles = DRAFT_FILES) {
+    const result = { text: "", injected: [], skipped: [], reconciled: [] };
+    const candidates = [];
+    for (const a of prior) {
+        const outboxRel = rel("mail", "attempts", a.id, "outbox");
+        const parts = [rel("mail", "attempts", a.id), outboxRel];
+        const bad = parts.map((p) => ({ p, st: realDir(path.join(runDir, p)) })).find((x) => x.st !== "ok");
+        if (bad) {
+            // A missing outbox just means nothing was set aside.
+            if (bad.st !== "missing")
+                result.skipped.push({ path: bad.p, reason: bad.st });
+            continue;
+        }
+        let entries;
+        try {
+            entries = fs.readdirSync(path.join(runDir, outboxRel), { withFileTypes: true }).sort((x, y) => (x.name < y.name ? -1 : 1));
+        }
+        catch (e) {
+            result.skipped.push({ path: outboxRel, reason: `open_error:${e.code ?? "unknown"}` });
+            continue;
+        }
+        if (entries.length > OUTBOX_SCAN)
+            result.skipped.push({ path: outboxRel, reason: "over_entry_cap" });
+        const onDisk = [];
+        for (const e of entries.slice(0, OUTBOX_SCAN)) {
+            const file = rel(outboxRel, e.name);
+            if (e.name.startsWith("."))
+                continue;
+            if (!e.isFile()) {
+                result.skipped.push({ path: file, reason: e.isSymbolicLink() ? "symlink" : "not_regular" });
+                continue;
+            }
+            if (hasControl(e.name) || e.name.length > 200) {
+                result.skipped.push({ path: file, reason: "bad_name" });
+                continue;
+            }
+            onDisk.push(e.name);
+            candidates.push({ attempt: a, rel: file, name: e.name });
+        }
+        if (a.parked) {
+            const onlyOnDisk = onDisk.filter((x) => !a.parked?.includes(x));
+            const onlyInRecord = a.parked.filter((x) => !onDisk.includes(x));
+            if (onlyOnDisk.length || onlyInRecord.length)
+                result.reconciled.push({ attempt: a.id, onlyOnDisk, onlyInRecord });
+        }
+    }
+    if (candidates.length === 0)
+        return result;
+    const header = "# Drafts from your failed attempt (unverified, not delivered)\nThese files were in your outbox when an earlier attempt at this message failed. They were set aside and never delivered; they may be incomplete or wrong. Check them against the repository before reusing any of them.\n";
+    const more = (n, c) => `(${n} more draft file(s) not shown: ${rel("mail", "attempts", c.attempt.id, "outbox")}/)`;
+    const render = (c, content, truncated, size) => {
+        const longest = Math.max(0, ...(content.match(/`+/g) ?? []).map((x) => x.length));
+        const fence = "`".repeat(Math.max(3, longest + 1));
+        return `\n## ${c.rel} (attempt ${c.attempt.n}, ${c.attempt.error})\n${fence}\n${content}\n${fence}\n${truncated ? `[truncated: ${c.rel}, ${size} B]\n` : ""}`;
+    };
+    const bytes = (s) => Buffer.byteLength(s);
+    const reserve = Math.max(...candidates.map((c) => bytes(`\n${more(999, c)}\n`)));
+    let used = bytes(header);
+    let text = header;
+    const seen = new Set();
+    let shown = 0;
+    let left = candidates.length;
+    for (const c of candidates) {
+        if (shown >= maxFiles)
+            break;
+        const room = budget - used - reserve;
+        const overhead = bytes(render(c, "", true, Number.MAX_SAFE_INTEGER));
+        if (room - overhead < 1)
+            break;
+        left--;
+        const r = readRegularFileCappedNoFollow(path.join(runDir, c.rel), room - overhead);
+        if (!r.ok) {
+            result.skipped.push({ path: c.rel, reason: r.reason });
+            continue;
+        }
+        // The capped read can end inside a character: drop the incomplete tail before decoding.
+        if (r.truncated)
+            r.buf = trimPartialUtf8(r.buf);
+        if (r.buf.includes(0)) {
+            result.skipped.push({ path: c.rel, reason: "binary" });
+            continue;
+        }
+        if (!r.truncated) {
+            const h = crypto.createHash("sha256").update(r.buf).digest("hex");
+            if (seen.has(h))
+                continue;
+            seen.add(h);
+        }
+        // Backtick-heavy content lengthens the fence; trim until the block fits.
+        let lim = r.buf.length;
+        let block = render(c, "", r.truncated, r.size);
+        for (;;) {
+            const cut = cutUtf8(r.buf, lim);
+            block = render(c, cut.toString("utf8"), r.truncated || cut.length < r.buf.length, r.size);
+            if (bytes(block) <= room || lim <= 0)
+                break;
+            lim = cut.length - (bytes(block) - room);
+        }
+        if (bytes(block) > room) {
+            left++;
+            break;
+        }
+        text += block;
+        used += bytes(block);
+        shown++;
+        result.injected.push({ path: c.rel, bytes: r.size, truncated: block.includes(`[truncated: ${c.rel},`) });
+    }
+    if (shown === 0)
+        return result;
+    const rest = candidates.filter((c) => !result.injected.some((i) => i.path === c.rel) && !result.skipped.some((s) => s.path === c.rel));
+    if (left > 0 && rest[0])
+        text += `${more(rest.length, rest[0])}\n`;
+    result.text = text.trimEnd();
+    return result;
+}
+const TASK_MEMORY = (dir) => (dir ? ` Read your task memory (\`${dir}\`) first: progress you recorded there survives a failed attempt.` : "");
+/** Prompt for a repeat attempt in the same wake-up: the earlier one failed and may have changed things already. */
+export function retryPrompt(prompt, error, o) {
+    const mail = o.drafts ? "Its mail was set aside, not delivered: see the drafts below." : "Its mail was discarded.";
+    return `${prompt}\n\n# Retry notice\nYour previous attempt at this wake-up failed${error ? ` (${error})` : ""} after possibly changing files already. ${mail} Check the current state of the repository before redoing the work, and send the mail again.${TASK_MEMORY(o.taskMemory)}`;
+}
+/** The user prompt for this invoke: `base`, a Retry or Redo notice when there is something to redo, then the drafts. No IO. */
+export function redoUserPrompt(base, o) {
+    const tail = o.drafts ? `\n\n${o.drafts}` : "";
+    if (o.attempt > 0)
+        return `${retryPrompt(base, o.error, { drafts: !!o.drafts, taskMemory: o.taskMemory })}${tail}`;
+    const last = o.prior[0];
+    if (!last)
+        return base;
+    return `${base}\n\n# Redo notice\nAn earlier attempt at this message did not finish (${last.error}; attempt ${last.id}). It may have changed files already, and its mail was set aside, not delivered${o.drafts ? ": see the drafts below" : ""}. Check the current state of the repository before redoing the work, and send the mail again.${TASK_MEMORY(o.taskMemory)}${tail}`;
+}

package/dist/run-session.d.ts

@@ -113,6 +113,11 @@ export declare class RunSession {
     private pendingAgents;
     private pickBatch;
     private wake;
+    /**
+     * The earlier failed attempts at these messages and the drafts they left, for the invoke that `attemptId` is about to make.
+     * Every skip and the outcome go to the log; a read problem never fails the wake-up.
+     */
+    private redoContext;
     /** Copy the shared cooldown table into the state (only once something has cooled down); true when it was. */
     private syncCooldowns;
     /**

package/dist/run-session.js

@@ -6,11 +6,12 @@ import { briefOf, digestOf, doneContract, mergeSteps, parseSteps } from "./forma
 import { ProtectedGuard } from "./guard.js";
 import { addBatch, askersWithAnswers, askReplyPath, checkAnswers, describeCheck, isComplete, markDelivered, parseAskBody, readAskReply, renderAnswers, resolveOptionLetters, writeAskReply } from "./ask-reply.js";
 import { deliver, deliverOnce, ensureProjectDirs, finishDone, listUnread, rejectDone, routeOutboxes, unreadFiles } from "./mailbox.js";
-import { abandonClaim, attemptLogDir, sourceIdOf, beginAttempt, claimMessages, commitClaim, finishAttempt, markOutputReady, parkOutbox, recoverRunMail, RouteJournal } from "./message-store.js";
+import { abandonClaim, attemptLogDir, sourceIdOf, beginAttempt, claimMessages, commitClaim, fault, finishAttempt, markOutputReady, parkOutbox, recoverRunMail, RouteJournal } from "./message-store.js";
 import { createRunLog } from "./run-log.js";
 import { endOutcome, resolveDoneOutcome } from "./run-outcome.js";
 import { outboxDir, sharedIndex } from "./policy.js";
 import { buildSystemPrompt, buildUserPrompt, pickMessages } from "./prompt.js";
+import { collectDrafts, priorFailedAttempts, redoUserPrompt } from "./redo.js";
 import { atomicWrite, readFileCapped } from "./fs-util.js";
 import { assertWorktreeRunnable, collectAgentChanges, dirtyPaths, existingAgentWorkspace, finishLeadBranch, integrateAgentChanges, prepareAgentWorkspace, recutLeadWorkspace, refusedIntegration, snapshotBase, snapshotHead, toplevel } from "./worktree.js";
 import { bindRunProject, newRunId, newRunState, roundLimit, roundsLabel, saveRunState } from "./run-store.js";
@@ -32,10 +33,8 @@ function readIndex(file) {
         return undefined;
     return readFileCapped(file, INDEX_COMPARE_LIMIT)?.text ?? "";
 }
-/** Prompt for a repeat attempt: the earlier one failed and may have changed things already. */
-function retryPrompt(prompt, error) {
-    return `${prompt}\n\n# Retry notice\nYour previous attempt at this wake-up failed${error ? ` (${error})` : ""} after possibly changing files already. Its mail was discarded. Check the current state of the repository before redoing the work, and send the mail again.`;
-}
+/** The `state.sessions` key of the session an agent runs in on candidate `c`: a backup has a session of its own. */
+const sessionKeyOf = (agent, c) => (c.primary ? agent.name : `${agent.name}@${c.key}`);
 /** `runtime/model`, or the runtime alone for its default model. */
 const modelLabel = (c) => (c.model ? `${c.runtime}/${c.model}` : c.runtime);
 /** Each model given up on once, in order, with its last failure (retries on the same model are one entry). */
@@ -343,7 +342,8 @@ export class RunSession {
                 if (opts.signal?.aborted)
                     return;
                 say(`  ${agent.name} falling back to ${modelLabel(to)} (${kind ?? "unknown error"})`);
-                this.note(`${agent.name} fell back from ${modelLabel(from)} to ${modelLabel(to)} (${kind ?? "unknown error"}); the backup starts a fresh session and redoes this wake-up. ` +
+                const resumes = agent.resume && !!state.sessions[sessionKeyOf(agent, to)];
+                this.note(`${agent.name} fell back from ${modelLabel(from)} to ${modelLabel(to)} (${kind ?? "unknown error"}); the backup ${resumes ? "resumes its own earlier session" : "starts a fresh session"} and redoes this wake-up. ` +
                     `Anything the failed attempts already did outside the mailbox (file edits, commands) was not undone: side effects may have happened.`);
             },
             invoke: async (c, ctx) => {
@@ -352,10 +352,13 @@ export class RunSession {
                 const attempt = ctx.attempt;
                 // A backup runs as the same agent on another runtime/model, in a session of its own.
                 const runAs = agentForCandidate(agent, c);
-                const sessionKey = c.primary ? agent.name : `${agent.name}@${c.key}`;
+                const sessionKey = sessionKeyOf(agent, c);
                 const round = ++state.rounds;
                 const started = Date.now();
                 const att = beginAttempt(runDir, claim.id);
+                // Looked up again for every call: a retry sees what the attempt before it just set aside.
+                const redo = this.redoContext(agent.name, unread.map((m) => m.meta.id), att.id, attempt);
+                const userPrompt = redoUserPrompt(base.userPrompt, { attempt, error: ctx.previous?.error, prior: redo.prior, drafts: redo.drafts, taskMemory: agent.memory.task });
                 const retryN = chain ? ctx.candidateAttempt : attempt;
                 say(`[round ${roundsLabel(state.rounds, state.max_rounds)}] waking ${agent.name} (${chain ? modelLabel(c) : agent.runtime})${retryN ? ` — retry ${retryN}` : ""}`);
                 const sessionId = agent.resume ? state.sessions[sessionKey] : undefined;
@@ -369,7 +372,7 @@ export class RunSession {
                     agent: runAs,
                     signal: opts.signal,
                     logDir: attemptLogDir(runDir, att.id),
-                    userPrompt: attempt ? retryPrompt(base.userPrompt, ctx.previous?.error) : base.userPrompt,
+                    userPrompt,
                     sessionId,
                 }).catch((e) => ({
                     ok: false,
@@ -411,6 +414,7 @@ export class RunSession {
                     duration_ms: Date.now() - started,
                     messages: unread.map((m) => m.meta.id),
                     error: result.error,
+                    ...(redo.prior.length ? { redo_of: redo.prior.map((a) => a.id) } : {}),
                     ...(chain ? { model: c.model, fallback_from: which.fallback_from, kind: result.kind } : {}),
                 });
                 this.saveState();
@@ -425,11 +429,13 @@ export class RunSession {
                 if (result.cancelled) {
                     // Stopped on purpose: not a failure to report. Its output is set aside (its input stays unread for the resume, below).
                     const parked = parkOutbox(runDir, outboxDir(project, agent.name), att.id);
+                    fault("park:before-finish");
                     finishAttempt(runDir, att.id, "failed", { error: "cancelled", parked });
                     return result;
                 }
                 // A failed attempt's output is never trusted: park it under the attempt, away from the next attempt's result.
                 const parked = parkOutbox(runDir, outboxDir(project, agent.name), att.id);
+                fault("park:before-finish");
                 finishAttempt(runDir, att.id, "failed", { error: result.error, parked });
                 say(`  ${agent.name} failed: ${result.error ?? "unknown error"}`);
                 this.note(`${agent.name} attempt ${att.n} failed (${result.error ?? "unknown error"})${parked.length ? `; ${parked.length} output file(s) set aside in ${path.join("mail", "attempts", att.id, "outbox")}` : ""}. ` +
@@ -456,6 +462,42 @@ export class RunSession {
         // The agent failed every time it was called: the run stops here (see settle) and its message stays unread for the resume.
         return { ok: false, error: result?.error, attempts: calls, ...(chain ? { tried: lastPerModel(outcome.tried) } : {}) };
     }
+    /**
+     * The earlier failed attempts at these messages and the drafts they left, for the invoke that `attemptId` is about to make.
+     * Every skip and the outcome go to the log; a read problem never fails the wake-up.
+     */
+    redoContext(agent, messageIds, attemptId, attempt) {
+        const { log, runDir } = this;
+        let prior = [];
+        let drafts;
+        const skipped = [];
+        try {
+            const found = priorFailedAttempts(runDir, agent, messageIds, attemptId);
+            prior = found.prior;
+            skipped.push(...found.skipped);
+            if (prior.length) {
+                drafts = collectDrafts(runDir, prior);
+                skipped.push(...drafts.skipped);
+            }
+        }
+        catch (e) {
+            prior = [];
+            drafts = undefined;
+            skipped.push({ path: "mail/attempts", reason: `error:${e.message}` });
+        }
+        for (const s of skipped)
+            log("draft_skip", { agent, attempt: attemptId, path: s.path, reason: s.reason });
+        if (!prior.length)
+            return { prior, drafts: "" };
+        log("redo", { agent, attempt: attemptId, prior: prior.map((a) => a.id), notice: attempt > 0 ? "retry" : "redo" });
+        for (const r of drafts?.reconciled ?? [])
+            log("draft_reconcile", { agent, attempt: r.attempt, only_on_disk: r.onlyOnDisk, only_in_record: r.onlyInRecord });
+        if (drafts?.injected.length)
+            log("draft_inject", { agent, attempt: attemptId, files: drafts.injected });
+        else
+            log("draft_none", { agent, attempt: attemptId, prior: prior.map((a) => a.id) });
+        return { prior, drafts: drafts?.text ?? "" };
+    }
     /** Copy the shared cooldown table into the state (only once something has cooled down); true when it was. */
     syncCooldowns() {
         const table = this.cooldowns.toJSON();

package/dist/scaffold.js

@@ -7,7 +7,7 @@ const PERSONAS = {
 You coordinate the team. You receive the user's task, break it into concrete pieces, and delegate each piece to the right teammate by mail.
 
 - Keep \`COMMON.md\` (shared team context: goal, conventions) accurate and short.
-- Before you send \`done\`, record what was decided and why in \`DECISIONS.md\` and what the project learned in \`LESSONS.md\` (next to \`COMMON.md\`); later runs start from them.
+- Before you send \`done\`, record what was decided and why as decision entries, and what the project learned as lesson entries, in the records folders named in your rules (one file per entry, with frontmatter); mark outdated entries \`superseded\`. Later runs start from the effective-active ones (about 6 KB per kind).
 - Write every task with the standard task headings, and cite the mail it derives from under Upstream so teammates can trace it.
 - When a teammate reports back, decide the next step: assign follow-up work, ask another teammate to verify, or accept the result.
 - Teammates cannot message each other by default; route everything through yourself.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-lyceum",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "Config + dispatcher for teams of Claude Code / Codex agents with scoped context and file-based mailboxes",
   "keywords": [
     "ai",