@jitsusama/agentic-harness.core 0.1.0

package/dist/review/ask/council.js

@@ -0,0 +1,386 @@
+/**
+ * Asking a whole roster at once.
+ *
+ * Participants are asked concurrently, because the wait is the whole
+ * cost and there is no reason for six models to queue. Their
+ * findings are recorded **in roster order** regardless of who
+ * answered first, because people say finding numbers out loud and a
+ * number that depends on which model happened to be quickest would
+ * make the same run describe itself differently every time.
+ *
+ * Nothing one participant does can take the round down. A failure
+ * reported by the runner and an exception thrown by it are the same
+ * event seen from two sides, and both are recorded against that
+ * participant so the rest of the round survives. Losing five good
+ * reviews to one unavailable model would be the worst possible
+ * trade, given what a round costs to run.
+ */
+import { count } from "../../ui/count.js";
+import { alsoRecorded, harvestFindings } from "./harvest.js";
+import { participantIdentity } from "./identity.js";
+import { noAskProgress, settleReplies } from "./progress.js";
+import { newRunId, whatItGave, whatItRead, } from "./run.js";
+/**
+ * Ask every reviewer at once and hand the answers back in roster
+ * order.
+ *
+ * Shared with the stack round rather than written twice, because this
+ * is the part with the rules in it: concurrent asking, a thrown error
+ * and a reported one becoming the same thing, and the ordering that
+ * makes finding numbers stable. A stack round differs in what a reply
+ * *means*, not in how a roster is asked, and if the ordering rule ever
+ * changes both rounds have to change with it.
+ */
+export async function askRoster(roster, prompt, runId, deps) {
+    const progress = deps.progress ?? noAskProgress;
+    progress.start(roster.reviewers);
+    return await Promise.all(roster.reviewers.map(async (participant) => {
+        progress.started(participant.id);
+        const answer = await asked(participant, prompt, deps, {
+            runId,
+            report: (activity) => progress.activity(participant.id, activity),
+        });
+        // A failure the runner hands back and one it throws are the
+        // same event to somebody watching a board.
+        if ("failure" in answer)
+            progress.failed(participant.id, answer.failure);
+        else
+            progress.answered(participant.id);
+        return { participant, answer };
+    }));
+}
+/**
+ * Ask one participant, reported the same way a roster is.
+ *
+ * A round with a single participant is still a round somebody is waiting
+ * on, and the wait is no shorter for being one subagent: a judge
+ * consolidating sixty findings takes as long as the council that raised
+ * them. Reporting was left to each round to remember, and the judge did
+ * not, so it ran to completion showing nothing at all.
+ *
+ * Beside `askRoster` rather than inside it, because a fan-out of one is
+ * not what this is: there is no ordering rule to share and no concurrency
+ * to arrange. What the two have in common is the beats they report, and
+ * this is the place that says what those are for one participant.
+ */
+export async function askOne(participant, prompt, runId, deps) {
+    const progress = deps.progress ?? noAskProgress;
+    progress.start([participant]);
+    progress.started(participant.id);
+    const answer = await asked(participant, prompt, deps, {
+        runId,
+        report: (activity) => progress.activity(participant.id, activity),
+    });
+    // A failure the runner hands back and one it throws are the same event
+    // to somebody watching a board.
+    if ("failure" in answer)
+        progress.failed(participant.id, answer.failure);
+    else
+        progress.answered(participant.id);
+    return answer;
+}
+/** Ask a roster about a change and record what it says. */
+export async function runCouncil(request, deps) {
+    const opening = openingRun(request, deps.now());
+    const { id, round, participants } = opening;
+    const startedAt = opening.startedAt;
+    // What this round says about itself beyond its own outcomes: the
+    // tree its reviewers read, and the conditions they read it under.
+    //
+    // Taken off the opening record rather than computed again from the
+    // request. The two are written three hundred lines apart and have to
+    // agree, and a second call agrees only for as long as nobody edits
+    // one of the two sites: reading it back from what was already
+    // written down makes disagreement impossible rather than unlikely.
+    const provenance = { ...whatItRead(opening), ...whatItGave(opening) };
+    // Written down before anything is asked, because everything after
+    // this line costs money and takes minutes, and until now the round
+    // was recorded only once it was over. A session that died halfway
+    // through left seven reviewer directories on disk and nothing
+    // saying which change they belonged to or that they were ever a
+    // round.
+    //
+    // Guarded here rather than trusted to each caller. The docstring
+    // on the callback promises a round is worth more than the
+    // bookkeeping around it, and awaiting it bare put that promise in
+    // the hands of whoever implements it: one that throws would take
+    // the council down before a single reviewer was dispatched. The
+    // seam is public, so the first caller in another package would
+    // have found that out the expensive way.
+    try {
+        await deps.opened?.(opening);
+    }
+    catch {
+        // Nothing to say to: the round has not started, so there is no
+        // warnings list yet. The cost is that an interrupted round may
+        // not be findable, which is what it was before this existed.
+    }
+    const replies = await askRoster(request.roster, request.prompt, id, deps);
+    const warnings = [];
+    const outcomes = await settleReplies(replies, (reply) => recordReply(reply, { runId: id, ...whatItRead(request) }, deps, warnings), (outcome) => ({
+        participantId: outcome.participantId,
+        findings: outcome.findingIds.length,
+    }), deps.progress);
+    return {
+        run: {
+            id,
+            round,
+            startedAt,
+            participants,
+            outcomes,
+            ...provenance,
+            // No `open`, which is what settles it: the field is present
+            // only while a round is unfinished.
+        },
+        warnings,
+    };
+}
+/**
+ * A round at the moment it exists and nobody has been asked.
+ *
+ * Shared by the runner that waits for a round and the one that walks
+ * away from it, because a round's identity is one concept and two
+ * copies of it would drift. A detached round is collected back into
+ * the same listing as a live one, so an id minted differently, or a
+ * participant identified differently, would show up as two kinds of
+ * round rather than one round run two ways.
+ */
+export function openingRun(request, startedAt) {
+    const round = request.round ?? "council";
+    return {
+        id: newRunId(round, startedAt, request.seq),
+        round,
+        startedAt: startedAt.toISOString(),
+        participants: request.roster.reviewers.map((participant) => participantIdentity("reviewer", participant)),
+        outcomes: [],
+        open: true,
+        ...whatItRead(request),
+        ...whatItGave(request),
+    };
+}
+/**
+ * Ask one participant, turning a thrown error into a reported one.
+ *
+ * A runner that rejects and one that answers with a failure are the
+ * same event, and a caller downstream should not have to care which
+ * shape it arrived in.
+ */
+async function asked(participant, prompt, deps, context) {
+    try {
+        return await deps.ask(participant, prompt, context);
+    }
+    catch (error) {
+        return { failure: thrownAt(error) };
+    }
+}
+/**
+ * What was thrown, said well enough to act on.
+ *
+ * A failure a runner reports is a sentence somebody wrote for a
+ * person, and it is left exactly as it is. A failure that was thrown
+ * is usually nobody's sentence: it is a message the runtime wrote,
+ * and on its own it does not say what kind of fault it was or where
+ * it happened.
+ *
+ * Measured, on a council that lost all seven reviewers to one
+ * TypeError. The ledger held "Cannot read properties of undefined
+ * (reading 'split')" and nothing else, which is true of every call to
+ * split in the dispatch path, so the diagnosis was an afternoon of
+ * reading them. The round is the expensive thing here and the ledger
+ * is what survives it, so the ledger has to carry enough to name the
+ * line.
+ *
+ * One frame, not the whole stack. The rest is this file's own
+ * machinery on every occasion, and a ledger entry is read in a table.
+ *
+ * A cause is carried when there is one. An error wrapped in another
+ * says the same little as the message this exists to improve on: the
+ * outer sentence names the layer that gave up and the inner one names
+ * what actually happened.
+ */
+export function thrownAt(error) {
+    // A thrown object that is not an Error still usually says
+    // something, and `String` of one is "[object Object]", the worst
+    // answer available: it names neither what was thrown nor where.
+    // Measured on the detached path, where a health check threw a
+    // plain record and seven reviewers were reported that way.
+    if (!(error instanceof Error))
+        return said(error) ?? String(error);
+    const kind = error.name === "Error" ? "" : `${error.name}: `;
+    const frame = firstFrame(error.stack, `${error.name}: ${error.message}`);
+    const where = frame === undefined ? "" : ` (${frame})`;
+    const cause = error.cause instanceof Error ? `, caused by ${thrownAt(error.cause)}` : "";
+    return `${kind}${error.message}${where}${cause}`;
+}
+/** Whatever a thrown non-Error has to say for itself. */
+function said(thrown) {
+    if (typeof thrown !== "object" || thrown === null)
+        return undefined;
+    const { message } = thrown;
+    return typeof message === "string" && message !== "" ? message : undefined;
+}
+/**
+ * The innermost frame of a stack, trimmed of its noise.
+ *
+ * The message is skipped rather than parsed past. A stack begins with
+ * the message, a message is free text, and one quoting a stack of its
+ * own is exactly what a failed subprocess hands back: its lines are
+ * shape-identical to real frames, so no pattern can tell them apart
+ * and the first quoted line would be reported as the place this error
+ * came from. What can be told apart is where the message ends, since
+ * the runtime wrote both.
+ *
+ * A frame still has to carry a location, for a runtime that leaves the
+ * message off the stack entirely and leaves nothing to skip.
+ *
+ * The path is said relative to where the process is running, when it
+ * is under it. An absolute path is somebody's home directory, and this
+ * string is read back by a model, shown on a panel and kept in a
+ * ledger that outlives the round.
+ */
+function firstFrame(stack, header) {
+    if (stack === undefined)
+        return undefined;
+    const frames = stack.startsWith(header) ? stack.slice(header.length) : stack;
+    for (const line of frames.split("\n")) {
+        const trimmed = line.trim();
+        // Node writes a frame as "at name (file:line:col)" or, with no
+        // name to give, "at file:line:col". Either is worth keeping
+        // whole: the name says what was running and the location says
+        // where to look, and neither is much use alone.
+        if (!/^at .*:\d+:\d+\)?$/.test(trimmed))
+            continue;
+        return shortenPaths(trimmed.slice("at ".length));
+    }
+    return undefined;
+}
+/** The same frame, without the part of the path that is somebody's home. */
+function shortenPaths(frame) {
+    const root = process.cwd();
+    return root === "/" ? frame : frame.split(`${root}/`).join("");
+}
+/**
+ * What to say about a reviewer we stopped.
+ *
+ * It names what was kept as well as what ended it, because those are
+ * two different decisions for the reader: whether to trust the pass,
+ * and whether to give it more room next time.
+ */
+/** Whichever reading of a participant's words found more. */
+function richer(said, earlier) {
+    if (earlier === undefined)
+        return said;
+    return earlier.findings.length > said.findings.length ? earlier : said;
+}
+/**
+ * What to say about a participant that did not get to finish.
+ *
+ * Shared with the stack round, which is the round most likely to be
+ * stopped and was saying nothing about it at all: it pushed the
+ * harvest's own warnings through, so a reviewer we interrupted was
+ * reported as having written a malformed answer.
+ */
+export function stopWarning(stop, kept, cutOff) {
+    const held = kept === 0
+        ? "Nothing had been read from it yet"
+        : `${count(kept, "finding")} had been read from it first`;
+    // A soft deadline is the round working as intended, so it does not
+    // get the sentence written for a reviewer that was cut off. Saying
+    // "stopped before it finished" about a reviewer we deliberately
+    // asked early would report our own design as an incident, and a
+    // reader who saw that on seven participants would go looking for
+    // the fault.
+    //
+    // It still gets the cut-off clause. Being asked early does not
+    // stop an answer being truncated on the way out, and that clause
+    // is the only thing that says how much is missing. Returning
+    // before it threw the sentence away in the case it was written
+    // for.
+    if (stop.limit === "soft-deadline") {
+        return `asked to wrap up early so it had time to answer: ${stop.detail} ${held}, so treat this pass as partial.${cutOff ? CUT_OFF : ""}`;
+    }
+    // Said here rather than left in the harvest's own warnings, which
+    // this branch replaces. Those describe the shape of the text and
+    // would blame the reviewer for our deadline; this one is about our
+    // deadline, and dropping it was hiding the fact in the exact case
+    // it was written for.
+    const cut = cutOff ? CUT_OFF : "";
+    return `stopped before it finished (${stop.limit}): ${stop.detail} ${held}, so treat this pass as partial.${cut}`;
+}
+/** Said whenever an answer stopped mid-entry, however it stopped. */
+const CUT_OFF = " Its answer was cut off mid-entry, so what it was writing when it stopped is gone and there is no telling how much more there would have been.";
+/**
+ * Harvest one reply, record what it held, and say how it went.
+ *
+ * Shared with collecting a round afterwards rather than written
+ * twice, because this is the part with the rules in it: reading a
+ * fragment against a wrap-up, folding in what was recorded, and
+ * deciding which warnings belong to the reviewer rather than to our
+ * deadline. A collected answer is the same answer arriving late, and
+ * a second copy of these rules would drift from this one.
+ */
+export async function recordReply(reply, run, deps, warnings) {
+    const { participant, answer } = reply;
+    if ("failure" in answer) {
+        return {
+            participantId: participant.id,
+            findingIds: [],
+            failure: answer.failure,
+            ...(answer.advisory === undefined ? {} : { advisory: answer.advisory }),
+            ...(answer.usage === undefined ? {} : { usage: answer.usage }),
+        };
+    }
+    const origin = {
+        kind: "reviewer",
+        runId: run.runId,
+        reviewerId: participant.id,
+    };
+    // A stopped reviewer can have said something twice: once as the
+    // fragment it was cut off in, and once when asked afterwards for
+    // what it had. Read both and keep whichever carried more, since the
+    // wrap-up is asked for only what the reviewer was sure of and may
+    // honestly be the shorter of the two. Ties go to the later answer,
+    // which is the considered one.
+    // The harvest is handed what the round read, not a commit, so it
+    // is the harvest that decides whether an anchor may claim one.
+    const harvest = alsoRecorded(richer(harvestFindings(answer.text, origin, run), answer.earlierText === undefined
+        ? undefined
+        : harvestFindings(answer.earlierText, origin, run)), answer.recorded, origin, run);
+    // A stopped reviewer answers for its own harvest warnings. Those
+    // warnings describe the shape of the text, and the text is a
+    // sentence we interrupted, so passing them on blames the reviewer
+    // for our deadline and tells the reader to fix the wrong thing.
+    //
+    // A warning about something it recorded is not that. That is a whole
+    // line the reviewer wrote deliberately, minutes before the stop, and
+    // it failed the contract on its own merits. Dropping it would be the
+    // silent drop this module opens by refusing to make: the reader
+    // would take the missing finding for the reviewer having nothing
+    // more to say.
+    const said = [
+        ...(answer.stopped === undefined
+            ? harvest.warnings
+            : [
+                stopWarning(answer.stopped, harvest.findings.length, harvest.truncated === true),
+                ...(harvest.recordedWarnings ?? []),
+            ]),
+        // Whatever went wrong that the answer cannot show for itself: a
+        // process that died after recording something, a journal that
+        // came back short. Outside the branch above, because a stop
+        // explains neither of them.
+        ...(answer.notes ?? []),
+    ];
+    for (const warning of said) {
+        warnings.push(`${participant.id}: ${warning}`);
+    }
+    const recorded = harvest.findings.length === 0 ? [] : await deps.record(harvest.findings);
+    return {
+        participantId: participant.id,
+        findingIds: recorded.map((finding) => finding.id),
+        ...(answer.stopped === undefined ? {} : { stopped: answer.stopped }),
+        ...(answer.answerPath === undefined
+            ? {}
+            : { answerPath: answer.answerPath }),
+        ...(answer.usage === undefined ? {} : { usage: answer.usage }),
+    };
+}

package/dist/review/ask/critique.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Pushing back on what a judge concluded.
+ *
+ * A critique records **positions, not findings**. A critic that could
+ * also raise findings would make the round both a discovery pass and a
+ * challenge to one, and neither could be read on its own afterwards.
+ * So the output is a position per finding with the argument for it,
+ * and the run's outcomes carry no finding ids at all.
+ *
+ * Silence about a finding is no position, never assent. Reading an
+ * absent critique as agreement would manufacture consensus out of a
+ * critic that ran out of budget, which is the worst way to be wrong
+ * here: it makes a weakly supported finding look corroborated.
+ */
+import { type Ask } from "./council.js";
+import type { AskProgress } from "./progress.js";
+import type { Roster } from "./roster.js";
+import { type AskRun, type RoundGiven } from "./run.js";
+/** Where a critic stands on a finding. */
+export type Position = "agree" | "disagree" | "qualify" | "unsure";
+/** One critic's position on one finding. */
+export interface Critique {
+    findingId: number;
+    participantId: string;
+    position: Position;
+    rationale: string;
+}
+/** What came out of one critic's answer. */
+export interface CritiqueHarvest {
+    critiques: Critique[];
+    warnings: string[];
+}
+/** The impure things a critique needs. */
+export interface CritiqueDeps {
+    ask: Ask;
+    now(): Date;
+    /** Told what is happening while it happens. Optional. */
+    progress?: AskProgress;
+}
+/** What to put to whom. */
+export interface CritiqueRequest {
+    roster: Roster;
+    prompt: string;
+    seq: number;
+    /** The findings being put up for challenge. */
+    findingIds: number[];
+    /**
+     * The commit this round was formed against.
+     *
+     * Recorded for the same reason a council records one. A position
+     * on a finding is about code at a moment, and a round that cannot
+     * say which moment cannot be weighed later.
+     */
+    witness?: string;
+    /** Said when the tree the reviewers read was not that commit. */
+    unpinned?: string;
+    /** What the critics were handed, besides the change. */
+    given?: RoundGiven;
+}
+/** The round, the positions taken, and anything worth saying. */
+export interface CritiqueResult {
+    run: AskRun;
+    critiques: Critique[];
+    warnings: string[];
+}
+/** Read one critic's answer, warning about what it dropped. */
+export declare function harvestCritiques(text: string, participantId: string, findingIds: readonly number[]): CritiqueHarvest;
+/** Put a judge's findings to the roster and gather what they say. */
+export declare function runCritique(request: CritiqueRequest, deps: CritiqueDeps): Promise<CritiqueResult>;

package/dist/review/ask/critique.js

@@ -0,0 +1,157 @@
+/**
+ * Pushing back on what a judge concluded.
+ *
+ * A critique records **positions, not findings**. A critic that could
+ * also raise findings would make the round both a discovery pass and a
+ * challenge to one, and neither could be read on its own afterwards.
+ * So the output is a position per finding with the argument for it,
+ * and the run's outcomes carry no finding ids at all.
+ *
+ * Silence about a finding is no position, never assent. Reading an
+ * absent critique as agreement would manufacture consensus out of a
+ * critic that ran out of budget, which is the worst way to be wrong
+ * here: it makes a weakly supported finding look corroborated.
+ */
+import { askRoster } from "./council.js";
+import { participantIdentity } from "./identity.js";
+import { newRunId, whatItGave, whatItRead, } from "./run.js";
+import { ANSWER_WAS_CUT_OFF, isRecord, readAnswer, wireText } from "./wire.js";
+/** The positions a critic may take. */
+const POSITIONS = ["agree", "disagree", "qualify", "unsure"];
+/** Read one critic's answer, warning about what it dropped. */
+export function harvestCritiques(text, participantId, findingIds) {
+    const { parsed, truncated } = readAnswer(text, "critiques");
+    const held = parsed?.critiques;
+    if (!Array.isArray(held)) {
+        return {
+            critiques: [],
+            warnings: [
+                "Nothing in this answer parsed as a critiques array, so no positions could be read from it.",
+            ],
+        };
+    }
+    const put = new Set(findingIds);
+    const critiques = [];
+    const warnings = [];
+    if (truncated)
+        warnings.push(ANSWER_WAS_CUT_OFF);
+    for (const [index, entry] of held.entries()) {
+        const one = readCritique(entry, index, participantId, put, warnings);
+        if (one !== undefined)
+            critiques.push(one);
+    }
+    return { critiques, warnings };
+}
+/** Put a judge's findings to the roster and gather what they say. */
+export async function runCritique(request, deps) {
+    const startedAt = deps.now();
+    const id = newRunId("critique", startedAt, request.seq);
+    // Asking six models to push back on an empty list is a bill for
+    // nothing, so a round with nothing to challenge asks nobody and
+    // says so by holding no participants.
+    if (request.findingIds.length === 0) {
+        return {
+            run: {
+                id,
+                round: "critique",
+                startedAt: startedAt.toISOString(),
+                participants: [],
+                outcomes: [],
+                ...whatItRead(request),
+                ...whatItGave(request),
+            },
+            critiques: [],
+            warnings: [],
+        };
+    }
+    // askRoster rather than a second fan-out of its own. Everything
+    // that makes asking correct lives there, and it would all have to
+    // change together: concurrent asking, a thrown failure and a
+    // reported one being one event, roster ordering, reporting.
+    const replies = await askRoster(request.roster, request.prompt, id, deps);
+    const critiques = [];
+    const warnings = [];
+    const outcomes = [];
+    // In roster order, so a report of the round reads the same way
+    // whoever answered first.
+    for (const { participant, answer } of replies) {
+        if ("failure" in answer) {
+            outcomes.push({
+                participantId: participant.id,
+                findingIds: [],
+                failure: answer.failure,
+            });
+            continue;
+        }
+        const harvest = harvestCritiques(answer.text, participant.id, request.findingIds);
+        critiques.push(...harvest.critiques);
+        for (const warning of harvest.warnings) {
+            warnings.push(`${participant.id}: ${warning}`);
+        }
+        outcomes.push({
+            participantId: participant.id,
+            // A critique raises no findings, on purpose.
+            findingIds: [],
+            ...(answer.stopped === undefined ? {} : { stopped: answer.stopped }),
+            ...(answer.answerPath === undefined
+                ? {}
+                : { answerPath: answer.answerPath }),
+            ...(answer.usage === undefined ? {} : { usage: answer.usage }),
+        });
+    }
+    // Raising no findings is why this has to say so itself: the rounds
+    // that record them finish through `settleReplies`, and a critique never
+    // goes near it, so the board it put up would have outlived the round
+    // and sat there describing work that had stopped.
+    deps.progress?.finish();
+    return {
+        run: {
+            id,
+            round: "critique",
+            startedAt: startedAt.toISOString(),
+            participants: request.roster.reviewers.map((participant) => participantIdentity("reviewer", participant)),
+            outcomes,
+            ...whatItRead(request),
+            ...whatItGave(request),
+        },
+        critiques,
+        warnings,
+    };
+}
+/** One critique, or nothing plus a warning saying why. */
+function readCritique(entry, index, participantId, put, warnings) {
+    const at = `critiques[${index}]`;
+    if (!isRecord(entry)) {
+        warnings.push(`${at} is not an object, so it was dropped.`);
+        return undefined;
+    }
+    const findingId = entry.findingId;
+    if (typeof findingId !== "number" || !Number.isInteger(findingId)) {
+        warnings.push(`${at} names no finding, so it was dropped.`);
+        return undefined;
+    }
+    if (!put.has(findingId)) {
+        // A critic inventing an id would attach an opinion to some
+        // other finding's number, which is worse than losing it.
+        warnings.push(`${at} takes a position on finding ${findingId}, which was not among the findings put up for challenge, so it was dropped.`);
+        return undefined;
+    }
+    const position = wireText(entry.position);
+    if (position === undefined || !POSITIONS.includes(position)) {
+        warnings.push(`${at} takes the position "${position ?? "nothing"}", which is not one of ${POSITIONS.join(", ")}, so it was dropped.`);
+        return undefined;
+    }
+    const rationale = wireText(entry.rationale);
+    if (rationale === undefined) {
+        // A position with no argument cannot be weighed against the
+        // finding it disputes, so it is worth less than silence.
+        warnings.push(`${at} gives no rationale, and a bare vote cannot be weighed against the finding it disputes, so it was dropped.`);
+        return undefined;
+    }
+    return {
+        findingId,
+        participantId,
+        position: position,
+        rationale,
+    };
+}