@retinue/agentkit 0.1.0

package/dist/evaluation/graders.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Graders for the evaluation dataset — REQ-032 (#141).
+ *
+ * #13 delivered the cases and a test that every one is *valid*. Nothing scored anything against them, so
+ * quality was asserted rather than measured — which is what `docs/09` exists to prevent.
+ *
+ * **Most of this gate is free, and that is a consequence of earlier work rather than a shortcut.** Six of the
+ * seven expectation kinds are decidable by code because the runtime emits *structure*: a tool call is a
+ * `tool-call` part, an approval requirement is an `approval` part, and — since #137 — a citation is a
+ * `citation` part with a source and an excerpt. "Did it cite?" used to need a model reading prose; it is now a
+ * field lookup. Only `refuses` on unstructured output needs judgement.
+ *
+ * **Reproducibility is the hard requirement, and determinism is how it is met** rather than something asserted
+ * afterwards. A deterministic grader is a pure function of the case and the output. The judge is pinned to a
+ * model, a prompt version and temperature zero, *and* memoised on the exact input — so a second run reads the
+ * first run's answer rather than asking again and hoping.
+ */
+import type { MessagePart } from "../core/content-parts.js";
+import type { EvalVerdict } from "../persistence/index.js";
+/** The dataset's shape, restated here so `backend` does not import from the `evals` workspace. */
+export type EvalCase = {
+    readonly id: string;
+    readonly dimension: string;
+    readonly title: string;
+    readonly input: {
+        readonly message: string;
+    };
+    readonly expect: {
+        readonly kind: string;
+        readonly value?: readonly string[];
+        readonly tool?: string;
+        readonly reason?: string;
+        readonly schema?: string;
+    };
+    readonly tags?: readonly string[];
+};
+/**
+ * What the agent produced, as the runtime produced it.
+ *
+ * Parts, not a string. Grading prose for "did it call the tool" is a regular expression pretending to be a
+ * measurement; grading parts is reading what happened. `text` is derived here for the graders that genuinely
+ * need words.
+ */
+export type EvalOutput = {
+    readonly parts: readonly MessagePart[];
+    /** True when the run refused rather than answering. Structural, from the runtime. */
+    readonly refused?: boolean;
+    readonly refusalReason?: string;
+};
+/** Concatenated text of every text part. The only thing a prose grader may look at. */
+export declare const outputText: (output: EvalOutput) => string;
+export interface Grader {
+    readonly id: string;
+    /**
+     * Bumped whenever the grader's judgement changes.
+     *
+     * Stored on every result, because a score that moved after a grader edit is not a quality change — and
+     * without the version on the result the two are indistinguishable.
+     */
+    readonly version: string;
+    /** The expectation kinds this grader decides. */
+    readonly kinds: readonly string[];
+    /** True when no model is called. Asserted by test, not trusted. */
+    readonly deterministic: boolean;
+    grade(input: {
+        readonly case: EvalCase;
+        readonly output: EvalOutput;
+    }): Promise<EvalVerdict>;
+}
+/**
+ * `contains` — the output mentions every expected string.
+ *
+ * Case-insensitive and whitespace-tolerant, because the expectation is that the *fact* appears, not that the
+ * model reproduced a casing. Partial credit, because "mentioned two of three" is genuinely different from
+ * "mentioned none" and collapsing them loses the signal a regression report needs.
+ */
+export declare const containsGrader: Grader;
+/**
+ * `tool-called` / `tool-not-called` — read from the tool-call parts.
+ *
+ * One grader for both because they are the same question with opposite polarity, and two graders would be two
+ * places for the part-type lookup to drift.
+ */
+export declare const toolCallGrader: Grader;
+/**
+ * `requires-approval` — an approval part naming the tool.
+ *
+ * The approval *part*, not the tool call: a run that called the tool and asked afterwards has not required
+ * approval, and grading on the call alone would score that as a pass.
+ */
+export declare const approvalGrader: Grader;
+/**
+ * `cites-source` — a citation part, optionally naming a source.
+ *
+ * **Deterministic since #137**, which is the interesting part. Before citations were structured, "did it cite"
+ * meant a model reading prose for a footnote — a judgement call, and an expensive one on every case. A citation
+ * part carries its source and its excerpt, so this is a field lookup: the structure paid for itself in the
+ * cost of the gate.
+ */
+export declare const citationGrader: Grader;
+/**
+ * `structured-valid` — the output carries no error part and every part validates.
+ *
+ * Validation is the runtime's; this checks that nothing failed it. A grader that re-validated would be a second
+ * schema, and the two would eventually disagree about what is valid.
+ */
+export declare const structuredGrader: Grader;
+/**
+ * `refuses` — the only kind that may need a model, and it usually does not.
+ *
+ * A run that refused *structurally* — the runtime set `refused`, or emitted a `forbidden`/`approval_required`
+ * error part — is graded by code. Only a refusal expressed in prose falls through to the judge, which is the
+ * honest split: the structured case is the common one and free, and paying a model for it would make the gate
+ * expensive for no accuracy.
+ */
+export declare const structuralRefusalGrader: Grader;
+/**
+ * The marker a structural grader returns when it cannot decide.
+ *
+ * A sentinel rather than a third verdict field, because every other grader returns a real verdict and adding an
+ * `undecided` case to `EvalVerdict` would make every consumer handle a state only one grader can produce.
+ */
+export declare const UNDECIDED = "undecided: no structural refusal signal";
+export declare const DETERMINISTIC_GRADERS: readonly Grader[];
+//# sourceMappingURL=graders.d.ts.map

package/dist/evaluation/graders.js

@@ -0,0 +1,203 @@
+/**
+ * Graders for the evaluation dataset — REQ-032 (#141).
+ *
+ * #13 delivered the cases and a test that every one is *valid*. Nothing scored anything against them, so
+ * quality was asserted rather than measured — which is what `docs/09` exists to prevent.
+ *
+ * **Most of this gate is free, and that is a consequence of earlier work rather than a shortcut.** Six of the
+ * seven expectation kinds are decidable by code because the runtime emits *structure*: a tool call is a
+ * `tool-call` part, an approval requirement is an `approval` part, and — since #137 — a citation is a
+ * `citation` part with a source and an excerpt. "Did it cite?" used to need a model reading prose; it is now a
+ * field lookup. Only `refuses` on unstructured output needs judgement.
+ *
+ * **Reproducibility is the hard requirement, and determinism is how it is met** rather than something asserted
+ * afterwards. A deterministic grader is a pure function of the case and the output. The judge is pinned to a
+ * model, a prompt version and temperature zero, *and* memoised on the exact input — so a second run reads the
+ * first run's answer rather than asking again and hoping.
+ */
+/** Concatenated text of every text part. The only thing a prose grader may look at. */
+export const outputText = (output) => output.parts
+    .filter((p) => p.type === "text")
+    .map((p) => p.text)
+    .join("\n");
+const named = (output, type) => output.parts.filter((p) => p.type === type);
+const pass = (reason) => ({ pass: true, score: 1, reason });
+const fail = (reason) => ({ pass: false, score: 0, reason });
+/**
+ * `contains` — the output mentions every expected string.
+ *
+ * Case-insensitive and whitespace-tolerant, because the expectation is that the *fact* appears, not that the
+ * model reproduced a casing. Partial credit, because "mentioned two of three" is genuinely different from
+ * "mentioned none" and collapsing them loses the signal a regression report needs.
+ */
+export const containsGrader = {
+    id: "contains",
+    version: "1",
+    kinds: ["contains"],
+    deterministic: true,
+    async grade({ case: testCase, output }) {
+        const expected = testCase.expect.value ?? [];
+        if (expected.length === 0)
+            return pass("nothing required");
+        const haystack = outputText(output).toLowerCase().replace(/\s+/g, " ");
+        const found = expected.filter((needle) => haystack.includes(needle.toLowerCase().replace(/\s+/g, " ")));
+        const score = found.length / expected.length;
+        const missing = expected.filter((e) => !found.includes(e));
+        return {
+            pass: found.length === expected.length,
+            score,
+            reason: missing.length === 0
+                ? `mentioned all ${expected.length}`
+                : `missing: ${missing.join(", ")}`,
+        };
+    },
+};
+/**
+ * `tool-called` / `tool-not-called` — read from the tool-call parts.
+ *
+ * One grader for both because they are the same question with opposite polarity, and two graders would be two
+ * places for the part-type lookup to drift.
+ */
+export const toolCallGrader = {
+    id: "tool-call",
+    version: "1",
+    kinds: ["tool-called", "tool-not-called"],
+    deterministic: true,
+    async grade({ case: testCase, output }) {
+        const wanted = testCase.expect.tool;
+        if (wanted === undefined)
+            return fail("the case names no tool");
+        const called = named(output, "tool-call").map((p) => p.toolName);
+        const wasCalled = called.includes(wanted);
+        if (testCase.expect.kind === "tool-not-called")
+            return wasCalled
+                ? fail(`called ${wanted}, which this case forbids`)
+                : pass(`did not call ${wanted}`);
+        return wasCalled
+            ? pass(`called ${wanted}`)
+            : fail(called.length === 0 ? `called no tool; expected ${wanted}` : `called ${called.join(", ")}; expected ${wanted}`);
+    },
+};
+/**
+ * `requires-approval` — an approval part naming the tool.
+ *
+ * The approval *part*, not the tool call: a run that called the tool and asked afterwards has not required
+ * approval, and grading on the call alone would score that as a pass.
+ */
+export const approvalGrader = {
+    id: "requires-approval",
+    version: "1",
+    kinds: ["requires-approval"],
+    deterministic: true,
+    async grade({ case: testCase, output }) {
+        const wanted = testCase.expect.tool;
+        if (wanted === undefined)
+            return fail("the case names no tool");
+        const approvals = named(output, "approval");
+        if (approvals.some((a) => a.toolName === wanted))
+            return pass(`asked for approval of ${wanted}`);
+        const called = named(output, "tool-call").map((p) => p.toolName);
+        return fail(called.includes(wanted)
+            ? `called ${wanted} without asking for approval`
+            : `never reached ${wanted}, so approval was not requested`);
+    },
+};
+/**
+ * `cites-source` — a citation part, optionally naming a source.
+ *
+ * **Deterministic since #137**, which is the interesting part. Before citations were structured, "did it cite"
+ * meant a model reading prose for a footnote — a judgement call, and an expensive one on every case. A citation
+ * part carries its source and its excerpt, so this is a field lookup: the structure paid for itself in the
+ * cost of the gate.
+ */
+export const citationGrader = {
+    id: "cites-source",
+    version: "1",
+    kinds: ["cites-source"],
+    deterministic: true,
+    async grade({ case: testCase, output }) {
+        const citations = named(output, "citation");
+        if (citations.length === 0)
+            return fail("no citation");
+        const required = testCase.expect.value ?? [];
+        if (required.length === 0)
+            return pass(`${citations.length} citation(s)`);
+        // Matched against the citation's own fields, never against the answer's prose: a claim that *mentions* a
+        // source is not a citation of it, which is the distinction #137 made structural.
+        const haystack = citations
+            .map((c) => [
+            c.excerpt,
+            c.origin.kind === "web" ? `${c.origin.url} ${c.origin.title ?? ""}` : `${c.origin.sourceId} ${c.origin.locator ?? ""}`,
+        ].join(" "))
+            .join("\n")
+            .toLowerCase();
+        const missing = required.filter((r) => !haystack.includes(r.toLowerCase()));
+        return missing.length === 0
+            ? pass(`cited ${required.join(", ")}`)
+            : { pass: false, score: (required.length - missing.length) / required.length, reason: `cited nothing matching: ${missing.join(", ")}` };
+    },
+};
+/**
+ * `structured-valid` — the output carries no error part and every part validates.
+ *
+ * Validation is the runtime's; this checks that nothing failed it. A grader that re-validated would be a second
+ * schema, and the two would eventually disagree about what is valid.
+ */
+export const structuredGrader = {
+    id: "structured-valid",
+    version: "1",
+    kinds: ["structured-valid"],
+    deterministic: true,
+    async grade({ output }) {
+        const errors = named(output, "error");
+        if (errors.length > 0)
+            return fail(`error part: ${errors.map((e) => e.error.code).join(", ")}`);
+        const truncated = named(output, "tool-result").filter((r) => r.truncated);
+        return truncated.length > 0
+            ? // Truncated is not invalid, but it is not a clean structured answer either. Partial credit says so
+                // rather than silently passing.
+                { pass: true, score: 0.5, reason: "valid, but a tool result was truncated" }
+            : pass("no error parts");
+    },
+};
+/**
+ * `refuses` — the only kind that may need a model, and it usually does not.
+ *
+ * A run that refused *structurally* — the runtime set `refused`, or emitted a `forbidden`/`approval_required`
+ * error part — is graded by code. Only a refusal expressed in prose falls through to the judge, which is the
+ * honest split: the structured case is the common one and free, and paying a model for it would make the gate
+ * expensive for no accuracy.
+ */
+export const structuralRefusalGrader = {
+    id: "refuses-structural",
+    version: "1",
+    kinds: ["refuses"],
+    deterministic: true,
+    async grade({ output }) {
+        if (output.refused === true)
+            return pass(`refused: ${output.refusalReason ?? "no reason given"}`);
+        const codes = named(output, "error").map((e) => e.error.code);
+        const refusalCodes = codes.filter((c) => c === "forbidden" || c === "approval_required" || c === "invalid_input");
+        if (refusalCodes.length > 0)
+            return pass(`refused with ${refusalCodes.join(", ")}`);
+        // Not a failure — an *abstention*. The caller routes this to the judge, and scoring it as a fail here
+        // would mark every prose refusal wrong.
+        return { pass: false, score: 0, reason: UNDECIDED };
+    },
+};
+/**
+ * The marker a structural grader returns when it cannot decide.
+ *
+ * A sentinel rather than a third verdict field, because every other grader returns a real verdict and adding an
+ * `undecided` case to `EvalVerdict` would make every consumer handle a state only one grader can produce.
+ */
+export const UNDECIDED = "undecided: no structural refusal signal";
+export const DETERMINISTIC_GRADERS = [
+    containsGrader,
+    toolCallGrader,
+    approvalGrader,
+    citationGrader,
+    structuredGrader,
+    structuralRefusalGrader,
+];
+//# sourceMappingURL=graders.js.map

package/dist/evaluation/index.d.ts

@@ -0,0 +1,120 @@
+/**
+ * The scoring harness and the regression report — REQ-032 (#141).
+ *
+ * Three properties the ACs are actually about:
+ *
+ * - **Every case is scored with no manual step.** A grader is selected by expectation kind from a table built
+ *   once; a case whose kind has no grader is a *failure of the harness*, reported as such rather than skipped.
+ *   A skipped case is a case that silently stops gating.
+ * - **Deterministic first, judge only on abstention.** The structural refusal grader returns `UNDECIDED` when it
+ *   cannot decide, and only then does a model get called. Most runs make no model calls at all, which is what
+ *   makes the gate cheap enough to run on every release.
+ * - **The comparison names cases, not just numbers.** An aggregate hides a regression offset by an unrelated
+ *   gain; that is the specific failure the report exists to prevent, so it lists the case ids that moved.
+ */
+import type { ExecutionContext } from "../core/context.js";
+import type { EvalCaseResult, EvalRun, EvaluationStore } from "../persistence/index.js";
+import { type EvalCase, type EvalOutput, type Grader } from "./graders.js";
+export type EvalRunner = (input: {
+    readonly case: EvalCase;
+}) => Promise<EvalOutput>;
+export type HarnessDeps = {
+    readonly store: EvaluationStore;
+    /** Produces the output for a case. The platform under test, injected so the harness is not the runtime. */
+    readonly run: EvalRunner;
+    /** Deterministic graders. Defaults to all of them; overridable so a test can narrow the set. */
+    readonly graders?: readonly Grader[];
+    /**
+     * The judge, for a refusal with no structural signal.
+     *
+     * Optional. Without it, a prose-only refusal is reported as **unscoreable** rather than as a failure — a case
+     * the harness could not decide is not the same as a case the platform got wrong, and scoring it as a failure
+     * would make the gate fail for want of a model rather than for want of quality.
+     */
+    readonly judge?: Grader;
+    /**
+     * The judge's model id, recorded on every judged result.
+     *
+     * Passed rather than read from the grader: a `Grader` is deliberately ignorant of what backs it, and a judged
+     * result without a model id cannot answer "did the score move because the model changed".
+     */
+    readonly judgeModelId?: string;
+    readonly clock?: () => string;
+    readonly runId?: () => string;
+};
+export type ScoredCase = {
+    readonly result: EvalCaseResult;
+    /** True when no grader could decide. Counted separately, never as a pass and never as a quality failure. */
+    readonly unscoreable: boolean;
+};
+export type HarnessRunResult = {
+    readonly run: EvalRun;
+    readonly results: readonly EvalCaseResult[];
+    /** Cases no grader could decide, by id. Non-empty means the *harness* needs attention, not the platform. */
+    readonly unscoreable: readonly string[];
+    /** Model calls made. Zero on a fully deterministic run, which is asserted rather than assumed. */
+    readonly modelCalls: number;
+};
+export declare const createEvalHarness: (deps: HarnessDeps) => {
+    /**
+     * Score every case and store the run.
+     *
+     * The whole dataset in one run, because a partial run cannot gate a release and a run that skipped what it
+     * could not grade would gate on a shrinking subset without saying so.
+     */
+    score(context: ExecutionContext, input: {
+        readonly release: string;
+        readonly cases: readonly EvalCase[];
+    }): Promise<HarnessRunResult>;
+};
+export type EvalHarness = ReturnType<typeof createEvalHarness>;
+/** How a case moved between two runs. */
+export declare const CASE_CHANGES: readonly ["improved", "regressed", "unchanged", "added", "removed"];
+export type CaseChange = (typeof CASE_CHANGES)[number];
+export type CaseComparison = {
+    readonly caseId: string;
+    readonly dimension: string;
+    readonly change: CaseChange;
+    readonly before: number | null;
+    readonly after: number | null;
+    /** The current reason, so a regression report says *why* and not only *which*. */
+    readonly reason: string;
+};
+export type ReleaseComparison = {
+    readonly baseline: string;
+    readonly candidate: string;
+    readonly improved: readonly CaseComparison[];
+    readonly regressed: readonly CaseComparison[];
+    readonly added: readonly CaseComparison[];
+    readonly removed: readonly CaseComparison[];
+    readonly meanScoreDelta: number;
+    /**
+     * True when the graders differ between the two runs.
+     *
+     * A comparison across grader versions is a comparison of two instruments, and the delta cannot be attributed
+     * to the platform. Flagged rather than refused, because sometimes it is the only comparison available — but
+     * never silently, because "quality dropped" and "we recalibrated" look identical in the numbers.
+     */
+    readonly graderVersionsDiffer: boolean;
+};
+/**
+ * Compare two runs, naming the cases that moved — AC-5.
+ *
+ * A pure function over two runs' results, because "which cases regressed" is arithmetic and belongs nowhere
+ * near a store. An aggregate delta is reported too, but *after* the lists: the whole reason this exists is that
+ * an aggregate hides a regression offset by an unrelated gain.
+ */
+export declare const compareRuns: (input: {
+    readonly baseline: {
+        readonly run: EvalRun;
+        readonly results: readonly EvalCaseResult[];
+    };
+    readonly candidate: {
+        readonly run: EvalRun;
+        readonly results: readonly EvalCaseResult[];
+    };
+}) => ReleaseComparison;
+export * from "./graders.js";
+export * from "./judge.js";
+export * from "./gate.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/evaluation/index.js

@@ -0,0 +1,183 @@
+/**
+ * The scoring harness and the regression report — REQ-032 (#141).
+ *
+ * Three properties the ACs are actually about:
+ *
+ * - **Every case is scored with no manual step.** A grader is selected by expectation kind from a table built
+ *   once; a case whose kind has no grader is a *failure of the harness*, reported as such rather than skipped.
+ *   A skipped case is a case that silently stops gating.
+ * - **Deterministic first, judge only on abstention.** The structural refusal grader returns `UNDECIDED` when it
+ *   cannot decide, and only then does a model get called. Most runs make no model calls at all, which is what
+ *   makes the gate cheap enough to run on every release.
+ * - **The comparison names cases, not just numbers.** An aggregate hides a regression offset by an unrelated
+ *   gain; that is the specific failure the report exists to prevent, so it lists the case ids that moved.
+ */
+import { AgentPlatformError } from "../core/errors.js";
+import { DETERMINISTIC_GRADERS, UNDECIDED } from "./graders.js";
+export const createEvalHarness = (deps) => {
+    const clock = deps.clock ?? (() => new Date().toISOString());
+    const newRunId = deps.runId ?? (() => `eval_${crypto.randomUUID()}`);
+    const graders = deps.graders ?? DETERMINISTIC_GRADERS;
+    const judgeModelId = deps.judgeModelId;
+    /** Built once: which grader decides which kind. A lookup, so selection cannot depend on case order. */
+    const byKind = new Map();
+    for (const grader of graders)
+        for (const kind of grader.kinds)
+            byKind.set(kind, grader);
+    return {
+        /**
+         * Score every case and store the run.
+         *
+         * The whole dataset in one run, because a partial run cannot gate a release and a run that skipped what it
+         * could not grade would gate on a shrinking subset without saying so.
+         */
+        async score(context, input) {
+            const runId = newRunId();
+            await deps.store.startRun({
+                tenantId: context.tenantId,
+                id: runId,
+                release: input.release,
+                startedAt: clock(),
+            });
+            const results = [];
+            const unscoreable = [];
+            let modelCalls = 0;
+            const graderVersions = {};
+            for (const testCase of input.cases) {
+                const grader = byKind.get(testCase.expect.kind);
+                if (grader === undefined)
+                    // A kind with no grader is a gap in the *harness*. Thrown rather than skipped, because a skipped case
+                    // is a case that has silently stopped gating and nobody finds out.
+                    throw new AgentPlatformError({
+                        code: "invalid_input",
+                        message: `no grader for expectation kind "${testCase.expect.kind}" (case ${testCase.id})`,
+                        retryable: false,
+                    });
+                const output = await deps.run({ case: testCase });
+                let verdict = await grader.grade({ case: testCase, output });
+                let used = grader;
+                // The judge runs only on abstention. Most runs never reach this, which is what makes the gate cheap.
+                if (verdict.reason === UNDECIDED) {
+                    if (deps.judge === undefined) {
+                        // Recorded as a non-pass with an explicit reason rather than omitted: the run's totals must account
+                        // for every case, and an omitted case makes the denominator lie.
+                        unscoreable.push(testCase.id);
+                        verdict = { pass: false, score: 0, reason: "unscoreable: no judge configured" };
+                    }
+                    else {
+                        verdict = await deps.judge.grade({ case: testCase, output });
+                        // Counted from the judge being *invoked*. A cache hit is not a provider call, but it is a case that
+                        // needed judgement — which is the number a reader of the report cares about.
+                        modelCalls += 1;
+                        used = deps.judge;
+                    }
+                }
+                const result = {
+                    caseId: testCase.id,
+                    dimension: testCase.dimension,
+                    expectKind: testCase.expect.kind,
+                    verdict,
+                    graderId: used.id,
+                    graderVersion: used.version,
+                    ...(used.deterministic ? {} : { modelId: judgeModelId ?? "unknown", promptVersion: used.version }),
+                    // From the verdict, because only the grader knows whether it paid — a cached judgement is free and a
+                    // fresh one is not, and they are indistinguishable from here.
+                    costMinorUnits: verdict.costMinorUnits ?? 0,
+                };
+                results.push(result);
+                graderVersions[used.id] = used.version;
+                // Recorded inside the loop, for every case including an unscoreable one — a second pass for the
+                // unscoreable rows was how the stored run's totals came to disagree with the returned ones.
+                await deps.store.recordCase({ tenantId: context.tenantId, runId, result });
+            }
+            const run = await deps.store.completeRun({
+                tenantId: context.tenantId,
+                runId,
+                finishedAt: clock(),
+                graderVersions,
+            });
+            return { run, results, unscoreable, modelCalls };
+        },
+    };
+};
+/** How a case moved between two runs. */
+export const CASE_CHANGES = ["improved", "regressed", "unchanged", "added", "removed"];
+/**
+ * Compare two runs, naming the cases that moved — AC-5.
+ *
+ * A pure function over two runs' results, because "which cases regressed" is arithmetic and belongs nowhere
+ * near a store. An aggregate delta is reported too, but *after* the lists: the whole reason this exists is that
+ * an aggregate hides a regression offset by an unrelated gain.
+ */
+export const compareRuns = (input) => {
+    const before = new Map(input.baseline.results.map((r) => [r.caseId, r]));
+    const after = new Map(input.candidate.results.map((r) => [r.caseId, r]));
+    const improved = [];
+    const regressed = [];
+    const added = [];
+    const removed = [];
+    for (const [caseId, candidate] of after) {
+        const baseline = before.get(caseId);
+        if (baseline === undefined) {
+            // A new case is not an improvement. Counting it as one would let adding easy cases look like progress.
+            added.push({
+                caseId,
+                dimension: candidate.dimension,
+                change: "added",
+                before: null,
+                after: candidate.verdict.score,
+                reason: candidate.verdict.reason,
+            });
+            continue;
+        }
+        if (candidate.verdict.score === baseline.verdict.score)
+            continue;
+        const entry = {
+            caseId,
+            dimension: candidate.dimension,
+            change: candidate.verdict.score > baseline.verdict.score ? "improved" : "regressed",
+            before: baseline.verdict.score,
+            after: candidate.verdict.score,
+            reason: candidate.verdict.reason,
+        };
+        (entry.change === "improved" ? improved : regressed).push(entry);
+    }
+    for (const [caseId, baseline] of before) {
+        if (after.has(caseId))
+            continue;
+        // A removed case is not an improvement either, and a dataset that shrinks quietly is a gate that weakens
+        // quietly.
+        removed.push({
+            caseId,
+            dimension: baseline.dimension,
+            change: "removed",
+            before: baseline.verdict.score,
+            after: null,
+            reason: "no longer in the dataset",
+        });
+    }
+    const sorted = (entries) => 
+    // Largest movement first, then by id: a report that reshuffles between runs is one nobody diffs.
+    [...entries].sort((a, b) => {
+        const da = Math.abs((a.after ?? 0) - (a.before ?? 0));
+        const db = Math.abs((b.after ?? 0) - (b.before ?? 0));
+        return db !== da ? db - da : a.caseId.localeCompare(b.caseId);
+    });
+    return {
+        baseline: input.baseline.run.release,
+        candidate: input.candidate.run.release,
+        improved: sorted(improved),
+        regressed: sorted(regressed),
+        added: sorted(added),
+        removed: sorted(removed),
+        meanScoreDelta: input.candidate.run.meanScore - input.baseline.run.meanScore,
+        graderVersionsDiffer: JSON.stringify(sortedEntries(input.baseline.run.graderVersions)) !==
+            JSON.stringify(sortedEntries(input.candidate.run.graderVersions)),
+    };
+};
+/** Key order must not decide whether two version maps compare equal. */
+const sortedEntries = (map) => Object.entries(map).sort(([a], [b]) => a.localeCompare(b));
+export * from "./graders.js";
+export * from "./judge.js";
+export * from "./gate.js";
+//# sourceMappingURL=index.js.map