@jitsusama/agentic-harness.core 0.1.0

package/dist/review/ask/stack-round.js

@@ -0,0 +1,125 @@
+/**
+ * Asking a roster about a whole stack at once.
+ *
+ * The fan-out is the council's, reused rather than rewritten: the
+ * rules worth getting right, that participants are asked concurrently,
+ * that a thrown error and a reported one are the same event, and that
+ * findings are numbered in roster order, are the same rules here and
+ * would have to change together. What differs is what a reply means. A
+ * council reply is findings about one change; a stack reply is
+ * findings about a stack, each naming the changes it is about.
+ *
+ * Which is why `record` takes a ref. A stack round produces findings
+ * for several changes at once, and the store needs to know which
+ * change each belongs to. A spanning finding is filed once, at its
+ * earliest change, because filing it on every change it touches makes
+ * a reader answer one observation three times.
+ */
+import { askRoster, stopWarning, } from "./council.js";
+import { participantIdentity } from "./identity.js";
+import { settleReplies } from "./progress.js";
+import { newRunId, whatItGave, whatItRead, } from "./run.js";
+import { alsoRecordedInStack, harvestStackFindings, saidAt } from "./span.js";
+/** Ask a roster about a stack and file what it says. */
+export async function runStackCouncil(request, deps) {
+    const startedAt = deps.now();
+    const id = newRunId("stack", startedAt, request.seq);
+    const replies = await askRoster(request.roster, request.prompt, id, deps);
+    const warnings = [];
+    const outcomes = await settleReplies(replies, (reply) => fileReply(reply, id, request, deps, warnings), (outcome) => ({
+        participantId: outcome.participantId,
+        findings: outcome.findingIds.length,
+    }), deps.progress);
+    const participants = request.roster.reviewers.map((participant) => participantIdentity("reviewer", participant));
+    return {
+        run: {
+            id,
+            round: "stack",
+            startedAt: startedAt.toISOString(),
+            participants,
+            outcomes,
+            // The witness cannot be one commit here, which is why this
+            // round carries one per change. The caveat can: a stack is
+            // read in a single tree like everything else, so a fallback
+            // is a fact about the whole round. The exemption was written
+            // about the witness and does not reach this.
+            ...whatItRead({
+                ...(request.unpinned === undefined
+                    ? {}
+                    : { unpinned: request.unpinned }),
+            }),
+            ...whatItGave(request),
+        },
+        warnings,
+    };
+}
+/** Read one reply, file it change by change, and say how it went. */
+async function fileReply(reply, runId, request, deps, warnings) {
+    const { participant, answer } = reply;
+    if ("failure" in answer) {
+        return {
+            participantId: participant.id,
+            findingIds: [],
+            failure: answer.failure,
+        };
+    }
+    const origin = {
+        kind: "reviewer",
+        runId,
+        reviewerId: participant.id,
+    };
+    // The same rule as everywhere else, in this round's shape: an
+    // anchor may claim a commit only if the reviewer read it. The
+    // witnesses here are per change, and a fallback tree misses all of
+    // them at once, so the whole lookup goes rather than each answer.
+    const formedAt = request.unpinned === undefined ? request.witnessFor : undefined;
+    const harvest = alsoRecordedInStack(harvestStackFindings(answer.text, origin, request.stackRefs, formedAt), answer.recorded, origin, request.stackRefs, formedAt);
+    // The same rule the single-change round follows, and this round
+    // needs it most: it holds every change at once, so it is the one
+    // most likely to be interrupted, and it was passing the harvest's
+    // own complaints straight through. A reviewer we stopped was being
+    // reported as one that wrote a malformed answer.
+    const said = [
+        ...(answer.stopped === undefined
+            ? harvest.warnings
+            : [
+                stopWarning(answer.stopped, harvest.findings.length, answer.text.trim() !== "" && harvest.findings.length === 0),
+                ...(harvest.recordedWarnings ?? []),
+            ]),
+        ...(answer.notes ?? []),
+    ];
+    for (const warning of said) {
+        warnings.push(`${participant.id}: ${warning}`);
+    }
+    // Grouped by where each finding is said, and filed in stack order
+    // so ids run the way a reader walks the stack.
+    const byRef = new Map();
+    for (const { span, finding } of harvest.findings) {
+        const ref = saidAt(span, request.stackRefs);
+        const held = byRef.get(ref);
+        if (held === undefined)
+            byRef.set(ref, [finding]);
+        else
+            held.push(finding);
+    }
+    const findingIds = [];
+    for (const ref of request.stackRefs) {
+        const held = byRef.get(ref);
+        if (held === undefined)
+            continue;
+        const recorded = await deps.record(ref, held);
+        findingIds.push(...recorded.map((finding) => finding.id));
+    }
+    return {
+        participantId: participant.id,
+        findingIds,
+        // The round most likely to be stopped, since it is the one asked
+        // to hold every change at once, was the one throwing the stop
+        // away.
+        ...(answer.stopped === undefined ? {} : { stopped: answer.stopped }),
+        ...(answer.answerPath === undefined
+            ? {}
+            : { answerPath: answer.answerPath }),
+        ...(answer.usage === undefined ? {} : { usage: answer.usage }),
+    };
+}

package/dist/review/ask/start.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Starting a round and walking away from it.
+ *
+ * A council holds the session for as long as it runs, which for the
+ * models worth asking is tens of minutes. That is the cost the whole
+ * of this plan has been circling: not that a round can be lost, but
+ * that waiting for one is the only way to have one.
+ *
+ * Nothing about a reviewer actually needs the session. The supervisor
+ * is its own process, it writes its answer to disk, and since #455 a
+ * round can be finished from what it left there. So a round can be
+ * started and abandoned on purpose: the ledger entry says it exists,
+ * the reviewer directories fill up in their own time, and `collect`
+ * turns them back into findings whenever somebody asks.
+ *
+ * What is given up is real and worth saying plainly. Nobody is
+ * watching, so there is no progress to report, no wrap-up when a
+ * reviewer runs long, and no retry of a model that failed to start.
+ * The backstop timers inside each supervisor are the only thing
+ * keeping a detached round from running until the machine is
+ * rebooted, which is exactly why they were made to work first.
+ */
+import { type CouncilRequest, type CouncilResult } from "./council.js";
+import type { Participant } from "./identity.js";
+import type { AskRun } from "./run.js";
+/** What starting a round needs, which is much less than running one. */
+export interface StartDeps {
+    /** Dispatch one reviewer and return once it is running. */
+    start(participant: Participant, prompt: string, runId: string): Promise<void>;
+    now(): Date;
+    /** The round exists and nobody has answered. See `CouncilDeps`. */
+    opened?(run: AskRun): Promise<void>;
+}
+/** What starting a round came to. */
+export interface StartResult extends CouncilResult {
+    /**
+     * How many reviewers are actually running.
+     *
+     * Said rather than left to be worked out. The caller used to
+     * subtract the warnings from the roster, which assumes one warning
+     * per reviewer, and two of the warnings here are about the round
+     * rather than about anybody on it: a round that started nobody
+     * because its ledger write failed reported six running.
+     */
+    started: number;
+}
+/** Start every reviewer on the roster and hand back the open round. */
+export declare function startCouncil(request: CouncilRequest, deps: StartDeps): Promise<StartResult>;

package/dist/review/ask/start.js

@@ -0,0 +1,118 @@
+/**
+ * Starting a round and walking away from it.
+ *
+ * A council holds the session for as long as it runs, which for the
+ * models worth asking is tens of minutes. That is the cost the whole
+ * of this plan has been circling: not that a round can be lost, but
+ * that waiting for one is the only way to have one.
+ *
+ * Nothing about a reviewer actually needs the session. The supervisor
+ * is its own process, it writes its answer to disk, and since #455 a
+ * round can be finished from what it left there. So a round can be
+ * started and abandoned on purpose: the ledger entry says it exists,
+ * the reviewer directories fill up in their own time, and `collect`
+ * turns them back into findings whenever somebody asks.
+ *
+ * What is given up is real and worth saying plainly. Nobody is
+ * watching, so there is no progress to report, no wrap-up when a
+ * reviewer runs long, and no retry of a model that failed to start.
+ * The backstop timers inside each supervisor are the only thing
+ * keeping a detached round from running until the machine is
+ * rebooted, which is exactly why they were made to work first.
+ */
+import { openingRun, thrownAt, } from "./council.js";
+/** Start every reviewer on the roster and hand back the open round. */
+export async function startCouncil(request, deps) {
+    const run = openingRun(request, deps.now());
+    const id = run.id;
+    // Before anybody is started, and here it is load-bearing rather
+    // than prudent. A live council can still report what it found if
+    // the ledger write fails; this one cannot. Nothing is waiting for
+    // these reviewers, so the entry is the only thing that will ever
+    // say they were a round rather than seven directories.
+    try {
+        await deps.opened?.(run);
+    }
+    catch (error) {
+        // Every reviewer gets the outcome it actually had, which is that
+        // it was never asked. Handing back a settled round with an empty
+        // outcome list instead makes the summary read the silence as
+        // seven reviewers who dropped, and a round that asked nobody
+        // anything then accuses the whole roster of failing.
+        //
+        // The reason is the same for all of them and belongs to the
+        // round rather than to any one of them, which is exactly what an
+        // advisory is, so it is hoisted once and the roll call says so.
+        const notWritten = `${id} was not started: the round could not be written down first (${why(error)}), and a detached round nothing has recorded cannot be collected.`;
+        return {
+            run: settled({ ...run, outcomes: everyoneMissed(request, notWritten) }),
+            warnings: [notWritten],
+            started: 0,
+        };
+    }
+    // In roster order and one at a time. They overlap anyway, since
+    // starting one only waits for a process to exist, and doing it in
+    // order means a spawn storm of seven models hits the machine as a
+    // queue rather than all at once.
+    const warnings = [];
+    const missed = [];
+    let running = 0;
+    for (const participant of request.roster.reviewers) {
+        try {
+            await deps.start(participant, request.prompt, id);
+            running += 1;
+        }
+        catch (error) {
+            // One model that will not start must not cost the others
+            // their round. Recorded as an outcome as well as a warning:
+            // a reviewer that never started is a reviewer that failed,
+            // and leaving it silent means the collect that finishes this
+            // round has to rediscover from an empty directory something
+            // that was known here, with the reason no longer to hand.
+            const failure = `${participant.id} could not be started: ${why(error)}.`;
+            warnings.push(failure);
+            missed.push({
+                participantId: participant.id,
+                findingIds: [],
+                failure,
+            });
+        }
+    }
+    if (running === 0) {
+        // An open round with nobody behind it is an alarm that can never
+        // be answered: every collect would find nothing, forever, and
+        // leaving it open is how a listing fills with rounds nobody can
+        // do anything about.
+        return {
+            run: settled({ ...run, outcomes: missed }),
+            warnings: [
+                ...warnings,
+                `${id} was closed straight away: nobody could be started, so there will never be anything to collect.`,
+            ],
+            started: 0,
+        };
+    }
+    return { run: { ...run, outcomes: missed }, warnings, started: running };
+}
+/** The outcome every reviewer had when none of them was asked. */
+function everyoneMissed(request, reason) {
+    return request.roster.reviewers.map((participant) => ({
+        participantId: participant.id,
+        findingIds: [],
+        failure: reason,
+        advisory: reason,
+    }));
+}
+/** The same round, marked as having nothing left to wait for. */
+function settled(run) {
+    const { open: _open, ...rest } = run;
+    return rest;
+}
+/** What went wrong, in whatever words the thrower used. */
+function why(error) {
+    // The same reading a waiting round gives. A detached round is the
+    // one whose diagnosis is hardest to come by, since nobody watched
+    // it fail and the warning is all there is, so it is the last place
+    // that should say less.
+    return thrownAt(error);
+}

package/dist/review/ask/store.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Rounds kept per change, outliving the session that ran them.
+ *
+ * A judge consolidates a council, a critique pushes back on a judge,
+ * and a retry substitutes into a round that already happened. All
+ * three need to find a round somebody else ran, possibly in a
+ * different session, so rounds live on disk beside the findings
+ * rather than in session state.
+ *
+ * Every round is kept, not just the last. A judge that consolidated a
+ * council needs that council still to be there, or nothing can say
+ * what it consolidated.
+ */
+import type { ChangeRef } from "../change.js";
+import type { AskRound, AskRun } from "./run.js";
+/** Rounds held against a change. */
+export interface RunStore {
+    /** Add a round, after the ones already held. */
+    record(change: ChangeRef, run: AskRun): Promise<void>;
+    /** Every round on a change, in the order they ran. */
+    list(change: ChangeRef): Promise<AskRun[]>;
+    /** The most recent finished round of one kind, if there was one. */
+    latest(change: ChangeRef, round: AskRound): Promise<AskRun | undefined>;
+    /** One round by its id. */
+    byId(change: ChangeRef, runId: string): Promise<AskRun | undefined>;
+    /** Swap a round for a new version of itself, in place. */
+    replace(change: ChangeRef, run: AskRun): Promise<void>;
+    /**
+     * Hold this version of a round, whether or not one is held.
+     *
+     * A round is written down twice: once when it opens, before it has
+     * asked anybody, and once when it settles. Both are this call,
+     * rather than a record and then a replace, so neither site has to
+     * know whether the other one happened. It matters because the
+     * opening write is best-effort: bookkeeping must not cost a round,
+     * and if settling were a replace, an opening write that failed
+     * would turn the end of a long council into an exception and throw
+     * away everything it found.
+     *
+     * Distinct from `replace`, which refuses a round it has never seen
+     * on purpose: a retry patching a round that does not exist has
+     * invented one, and should say so rather than quietly adding it.
+     */
+    keep(change: ChangeRef, run: AskRun): Promise<void>;
+    /**
+     * Every round still open, across every change.
+     *
+     * The one question here that is not about a change, and it exists
+     * for the retention sweep. A detached round finishes on disk and
+     * stays open on the ledger until somebody collects it, so a sweep
+     * that judges a run by whether it is terminal will happily delete
+     * the answers a round is still waiting to be asked for. Before
+     * rounds outlived their sessions the two could not disagree.
+     *
+     * What could not be read is answered beside what could, rather
+     * than thrown or quietly skipped. Skipping was wrong in the one
+     * direction that costs something: a torn file drops protection
+     * from every round it held, silently and only for that change,
+     * which is a sweep deleting paid-for reviews and reporting a clean
+     * run. Throwing would be no better, since the answer for every
+     * other change is perfectly good and a sweep that ran would
+     * reclaim real disk. The caller decides, and it has what it needs
+     * to decide with.
+     */
+    openRunIds(): Promise<OpenRuns>;
+}
+/** Which rounds are open, and what could not be asked. */
+export interface OpenRuns {
+    readonly open: ReadonlySet<string>;
+    /**
+     * What exists and would not read, said the way a person can act on
+     * it: a full path, since a caller that declines over this has to
+     * tell somebody which file to go and deal with, and a bare name is
+     * not enough to find one under a state directory nobody memorized.
+     *
+     * Non-empty means the open set is incomplete and there is no way
+     * to tell which rounds are missing from it, so a caller deciding
+     * what to delete has to decline rather than read it as "none".
+     */
+    readonly unreadable: readonly string[];
+}
+/** Runs on disk, one file per change. */
+export declare function createRunStore(root: string): RunStore;

package/dist/review/ask/store.js

@@ -0,0 +1,226 @@
+/**
+ * Rounds kept per change, outliving the session that ran them.
+ *
+ * A judge consolidates a council, a critique pushes back on a judge,
+ * and a retry substitutes into a round that already happened. All
+ * three need to find a round somebody else ran, possibly in a
+ * different session, so rounds live on disk beside the findings
+ * rather than in session state.
+ *
+ * Every round is kept, not just the last. A judge that consolidated a
+ * council needs that council still to be there, or nothing can say
+ * what it consolidated.
+ */
+import { mkdir, readdir, readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { changeKey } from "../keys.js";
+/** Runs on disk, one file per change. */
+export function createRunStore(root) {
+    async function read(change) {
+        let raw;
+        const path = join(root, fileFor(change));
+        try {
+            raw = await readFile(path, "utf8");
+        }
+        catch (error) {
+            // No file. This change has no rounds behind it, which is an
+            // answer rather than a failure: most changes have never been
+            // reviewed.
+            //
+            // Only ENOENT, though. Every other errno says the history is
+            // there and could not be reached, and answering "no rounds" to
+            // those is worse here than in the sweep: the next write lays a
+            // one-round ledger over whatever was behind it. The reasoning
+            // twenty lines down, about a file that will not parse, applies
+            // exactly as well to a file that will not open.
+            if (!isMissing(error)) {
+                throw new Error(`The rounds held against ${change.label} could not be read from ${path}: ${error instanceof Error ? error.message : String(error)}. Fix or move that file: answering "no rounds" for it would let the next round overwrite the history behind it.`);
+            }
+            return { runs: [] };
+        }
+        // A file that will not parse is a different thing entirely, and
+        // answering "no rounds" for it is how a history disappears
+        // quietly: the caller is told the change is fresh, and the next
+        // write lays a one-round ledger over whatever was there. Said
+        // out loud instead, so a torn file is a problem somebody can
+        // see rather than a change that looks new.
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            // Falls through to the sentence below, which says where the
+            // file is and what to do. A parser's own message names a
+            // character offset in a file the reader has not been told
+            // about.
+        }
+        if (typeof parsed === "object" &&
+            parsed !== null &&
+            "runs" in parsed &&
+            Array.isArray(parsed.runs)) {
+            return { runs: parsed.runs };
+        }
+        throw new Error(`The rounds held against ${change.label} are at ${join(root, fileFor(change))} and could not be read as a ledger. Nothing was changed. Move that file aside to start a fresh one, keeping it if the rounds in it still matter.`);
+    }
+    async function write(change, ledger) {
+        await mkdir(root, { recursive: true });
+        const path = join(root, fileFor(change));
+        // Written beside and renamed over, because a plain write
+        // truncates in place and a process killed midway leaves half a
+        // document. This change writes at the start of an operation
+        // whose whole premise is that the session may not survive it,
+        // so the window stopped being theoretical. A rename within one
+        // directory is atomic: a reader sees the old ledger or the new
+        // one.
+        const pending = `${path}.${process.pid}.tmp`;
+        await writeFile(pending, JSON.stringify(ledger, null, 2), "utf8");
+        await rename(pending, path);
+    }
+    return {
+        async record(change, run) {
+            const ledger = await read(change);
+            await write(change, { runs: [...ledger.runs, run] });
+        },
+        async list(change) {
+            return (await read(change)).runs;
+        },
+        async openRunIds() {
+            const open = new Set();
+            const unreadable = [];
+            let files;
+            try {
+                files = await readdir(root);
+            }
+            catch (error) {
+                // No ledger directory at all, so no rounds are open. The
+                // caller is a sweep, and answering "none" is the honest
+                // reading of an empty history.
+                //
+                // Only for that reading, though. Every other errno says the
+                // history is there and cannot be seen, and answering "none"
+                // to those tells a sweep every round is collectable rubbish.
+                // The catch used to be untyped, so a permissions problem on
+                // this directory read as an empty history.
+                if (!isMissing(error)) {
+                    return { open, unreadable: [`${root} (the whole ledger)`] };
+                }
+                return { open, unreadable };
+            }
+            for (const file of files) {
+                if (!file.endsWith(".json"))
+                    continue;
+                const path = join(root, file);
+                let parsed;
+                try {
+                    parsed = JSON.parse(await readFile(path, "utf8"));
+                }
+                catch (error) {
+                    // A file that has gone since the listing is not a file
+                    // that will not read. Another session settling a round
+                    // deletes nothing, but a person tidying up does, and
+                    // treating that as a torn ledger stops the sweep on this
+                    // machine for good over a file nobody has any more.
+                    //
+                    // Nothing covers this branch, and the same is true of the
+                    // matching one in the artifact sweep's stat. Both windows
+                    // are between a listing and a read inside one call, with
+                    // no seam to hold open from outside, and a case that raced
+                    // for one would be a case that fails sometimes. Two
+                    // attempts at a test for this deleted the file before the
+                    // listing, which exercises nothing and passes.
+                    if (isMissing(error))
+                        continue;
+                    // Said, not skipped. Skipping used to be justified on the
+                    // grounds that the sweep errs towards deleting nothing it
+                    // was unsure about, and it does not: a run it is unsure
+                    // about is a run nothing protects, which is the ordinary
+                    // window. One torn file among twenty took protection off
+                    // that change's rounds with no error anywhere.
+                    unreadable.push(path);
+                    continue;
+                }
+                if (typeof parsed !== "object" ||
+                    parsed === null ||
+                    !("runs" in parsed) ||
+                    !Array.isArray(parsed.runs)) {
+                    // A file that parses into the wrong shape is no more
+                    // readable than one that does not parse at all.
+                    unreadable.push(path);
+                    continue;
+                }
+                for (const run of parsed.runs) {
+                    if (run?.open === true && typeof run.id === "string") {
+                        open.add(run.id);
+                    }
+                }
+            }
+            return { open, unreadable };
+        },
+        async latest(change, round) {
+            // Finished rounds only. A round is now on the ledger from the
+            // moment it opens, so the newest entry is routinely one that
+            // has answered nothing: a judge asked while a council was
+            // still running, or after one was interrupted, would have
+            // consolidated its empty outcomes and reported that the
+            // council found nothing.
+            const held = (await read(change)).runs.filter(
+            // Nor one a person gave up on, which is neither open nor
+            // finished. It has no outcomes, so handing it back here
+            // would have a judge consolidate nothing and report that
+            // the council found nothing, with the real council one
+            // entry behind and unreachable.
+            (r) => r.round === round && r.open !== true && r.closed !== true);
+            return held.at(-1);
+        },
+        async byId(change, runId) {
+            return (await read(change)).runs.find((r) => r.id === runId);
+        },
+        async keep(change, run) {
+            const ledger = await read(change);
+            const at = ledger.runs.findIndex((held) => held.id === run.id);
+            await write(change, {
+                runs: at === -1
+                    ? [...ledger.runs, run]
+                    : ledger.runs.map((held, index) => (index === at ? run : held)),
+            });
+        },
+        async replace(change, run) {
+            const ledger = await read(change);
+            const at = ledger.runs.findIndex((held) => held.id === run.id);
+            if (at === -1) {
+                // Adding it silently would make a retry look like it
+                // patched something when it invented a round instead.
+                throw new Error(`No run "${run.id}" is held against this change, so there is nothing to replace. Record it first, or check the id.`);
+            }
+            await write(change, {
+                runs: ledger.runs.map((held, index) => (index === at ? run : held)),
+            });
+        },
+    };
+}
+/**
+ * Whether a filesystem error means the thing is simply not there.
+ *
+ * The distinction matters wherever absence is an answer, because
+ * every other errno means the thing exists and could not be reached,
+ * and reporting that as absence is how a sweep concludes a full
+ * history is an empty one.
+ */
+function isMissing(error) {
+    return (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        error.code === "ENOENT");
+}
+/**
+ * The file a change's rounds live in.
+ *
+ * Slugged because the key names a file: a change id is somebody
+ * else's string and may hold a separator.
+ */
+function fileFor(change) {
+    const safe = changeKey(change)
+        .replace(/[^A-Za-z0-9._-]/g, "_")
+        .replace(/^\.+/, "_");
+    return `${safe}.json`;
+}