@namzu/sdk 2.0.0 → 3.0.0

package/dist/eval/__tests__/scorer-gates.test.js

@@ -0,0 +1,136 @@
+import { describe, expect, it } from 'vitest';
+import { formatReport, runExperiment } from '../experiment.js';
+/**
+ * A case's verdict was one unweighted mean over every scorer against one
+ * suite-wide threshold, and the two halves of that fought each other.
+ *
+ * At the default threshold of 1 the harness never reports a false pass —
+ * but a trajectory F1 and a graded judge can essentially never reach 1, so
+ * every real suite lowers it. And every step down buys tolerance for the
+ * fuzzy scorers by buying exactly the same tolerance for the deterministic
+ * ones: at 0.75, a trajectory score of 0 alongside three perfect scores
+ * averages to 0.75 and reports PASSED. The regression the harness exists
+ * to catch comes back green.
+ *
+ * The workarounds were lossy too — folding every hard check into one
+ * custom scorer destroys per-dimension reporting, and two experiments at
+ * two thresholds re-runs the dataset and splits one case's verdict across
+ * two reports.
+ */
+const run = () => ({
+    output: 'answer',
+    steps: [],
+    toolCalls: [],
+    totalTokens: 0,
+    totalCostUsd: 0,
+    durationMs: 0,
+});
+const scorer = (name, score, extra = {}) => ({
+    name,
+    score: () => ({ score, reason: `${name} scored ${score}` }),
+    ...extra,
+});
+const experiment = (scorers, passThreshold = 0.75) => runExperiment({
+    name: 'gates',
+    cases: [{ name: 'one', input: null }],
+    scorers,
+    passThreshold,
+    run: async () => run(),
+});
+describe('a gate scorer', () => {
+    it('fails the case even when the mean clears the threshold', async () => {
+        const report = await experiment([
+            scorer('trajectory', 0, { severity: 'gate' }),
+            scorer('completion', 1),
+            scorer('contains', 1),
+            scorer('judge', 1),
+        ]);
+        // Mean is exactly 0.75 against a 0.75 threshold — reported passed
+        // before, with the one deterministic check at zero.
+        expect(report.cases[0]?.mean).toBeCloseTo(0.75);
+        expect(report.cases[0]?.status).toBe('failed');
+    });
+    it('names itself, so the reader does not have to guess', async () => {
+        const report = await experiment([
+            scorer('trajectory', 0, { severity: 'gate' }),
+            scorer('judge', 1),
+        ]);
+        expect(report.cases[0]?.failedGates).toEqual(['trajectory']);
+    });
+    it('honours its own threshold when it sets one', async () => {
+        // "Good enough" is not one number across dimensions: a trajectory
+        // match at 0.8 may be fine while a completion check at 0.8 is
+        // meaningless.
+        const lenient = await experiment([
+            scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.7 }),
+        ]);
+        expect(lenient.cases[0]?.status).toBe('passed');
+        const strict = await experiment([
+            scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.9 }),
+        ]);
+        expect(strict.cases[0]?.status).toBe('failed');
+    });
+    it('falls back to the suite threshold when it sets none', async () => {
+        const report = await experiment([scorer('trajectory', 0.5, { severity: 'gate' })], 0.75);
+        expect(report.cases[0]?.failedGates).toEqual(['trajectory']);
+    });
+    it('does not fail the case when it could not judge at all', async () => {
+        // Unavailable is not zero. A gate that threw did not judge the run
+        // badly, it failed to judge it — the inconclusive path, not a
+        // failure, and the two demand opposite responses.
+        const report = await runExperiment({
+            name: 'gates',
+            cases: [{ name: 'one', input: null }],
+            scorers: [
+                {
+                    name: 'judge-gate',
+                    severity: 'gate',
+                    score: () => {
+                        throw new Error('provider unreachable');
+                    },
+                },
+            ],
+            run: async () => run(),
+        });
+        expect(report.cases[0]?.status).toBe('inconclusive');
+        expect(report.cases[0]?.failedGates).toBeUndefined();
+    });
+    it('lets a passing gate through', async () => {
+        const report = await experiment([scorer('trajectory', 1, { severity: 'gate' })]);
+        expect(report.cases[0]?.status).toBe('passed');
+    });
+});
+describe('a soft scorer', () => {
+    it('only moves the mean', async () => {
+        // The fuzzy dimensions, where a number below 1 is normal and a
+        // threshold is a judgement call.
+        const report = await experiment([scorer('judge', 0.5), scorer('other', 1)]);
+        expect(report.cases[0]?.status).toBe('passed');
+        expect(report.cases[0]?.failedGates).toBeUndefined();
+    });
+    it('is the default, so a suite that sets nothing is unchanged', async () => {
+        const report = await experiment([
+            scorer('a', 0),
+            scorer('b', 1),
+            scorer('c', 1),
+            scorer('d', 1),
+        ]);
+        expect(report.cases[0]?.status).toBe('passed');
+    });
+    it('can still fail the case through the mean', async () => {
+        const report = await experiment([scorer('a', 0), scorer('b', 0.5)]);
+        expect(report.cases[0]?.status).toBe('failed');
+        expect(report.cases[0]?.failedGates).toBeUndefined();
+    });
+});
+describe('the report', () => {
+    it('says which gate missed, above the individual scores', async () => {
+        const report = await experiment([
+            scorer('trajectory', 0, { severity: 'gate' }),
+            scorer('judge', 1),
+        ]);
+        const text = formatReport(report);
+        expect(text).toContain('gate missed: trajectory');
+    });
+});
+//# sourceMappingURL=scorer-gates.test.js.map

package/dist/eval/__tests__/scorer-gates.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer-gates.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/scorer-gates.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAG9D;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,GAAG,GAAG,GAAY,EAAE,CAAC,CAAC;IAC3B,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,EAAE;IACT,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,CAAC;CACb,CAAC,CAAA;AAEF,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,QAAyB,EAAE,EAAU,EAAE,CAAC,CAAC;IACrF,IAAI;IACJ,KAAK,EAAE,GAAU,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,WAAW,KAAK,EAAE,EAAE,CAAC;IAClE,GAAG,KAAK;CACR,CAAC,CAAA;AAEF,MAAM,UAAU,GAAG,CAAC,OAAiB,EAAE,aAAa,GAAG,IAAI,EAAE,EAAE,CAC9D,aAAa,CAAC;IACb,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACrC,OAAO;IACP,aAAa;IACb,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,EAAE;CACtB,CAAC,CAAA;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC/B,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;YACvB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YACrB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;SAClB,CAAC,CAAA;QAEF,kEAAkE;QAClE,oDAAoD;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QAC/C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC/B,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;SAClB,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC3D,kEAAkE;QAClE,8DAA8D;QAC9D,eAAe;QACf,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;YAChC,MAAM,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;SAC/D,CAAC,CAAA;QACF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAE/C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC/B,MAAM,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;SAC/D,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACxF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACtE,mEAAmE;QACnE,8DAA8D;QAC9D,kDAAkD;QAClD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YACrC,OAAO,EAAE;gBACR;oBACC,IAAI,EAAE,YAAY;oBAClB,QAAQ,EAAE,MAAM;oBAChB,KAAK,EAAE,GAAG,EAAE;wBACX,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;oBACxC,CAAC;iBACD;aACD;YACD,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,EAAE;SACtB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,aAAa,EAAE,CAAA;IACrD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;QAChF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;QACpC,+DAA+D;QAC/D,iCAAiC;QACjC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC3E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,aAAa,EAAE,CAAA;IACrD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC/B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;SACd,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;QACnE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,aAAa,EAAE,CAAA;IACrD,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC/B,MAAM,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YAC7C,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;SAClB,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/experiment.d.ts

@@ -0,0 +1,54 @@
+import type { CaseResult, EvalCase, EvalRun, ExperimentReport, Scorer } from './types.js';
+export interface ExperimentConfig<TInput = unknown> {
+    name: string;
+    cases: readonly EvalCase<TInput>[];
+    /** Applied to every case unless the case overrides them. */
+    scorers: readonly Scorer[];
+    /**
+     * Execute one case. Returning an `EvalRun` rather than driving `query()`
+     * here keeps the harness independent of how you construct a run —
+     * scripted mock, real provider, or a whole agent behind a facade.
+     */
+    run: (input: TInput, evalCase: EvalCase<TInput>, signal: AbortSignal) => Promise<EvalRun>;
+    /**
+     * Deadline for a single case. Unset means no deadline.
+     *
+     * `executeCase` used to be a bare await, so a `run` closure that never
+     * settled blocked its worker and `runExperiment` never returned — no
+     * report, no partial results, nothing to read. The documented path
+     * inherits deadlines from the runtime it drives, so the residual is a
+     * closure that does not go through `query()` and a mid-iteration
+     * provider stall the between-iterations guard cannot see. Both are
+     * reachable, and neither is the suite's fault to absorb silently.
+     *
+     * A timed-out case is REPORTED and the suite continues, exactly like a
+     * case that threw: forty cases should not be lost to one that hung.
+     */
+    timeoutMs?: number;
+    /** Mean score a case must reach to count as passed. Default 1. */
+    passThreshold?: number;
+    /** Cases to run at once. Default 1 — deterministic ordering by default. */
+    concurrency?: number;
+    onCaseFinish?: (result: CaseResult) => void;
+}
+/**
+ * Run a dataset and score it.
+ *
+ * There was no evaluation harness of any kind: no dataset, no scorer, no
+ * judge, no trajectory assertion. So every behavior change in this SDK —
+ * a tool description, the `search_tools` top-k, the compaction threshold —
+ * shipped with no regression signal behind it, and the only way to notice
+ * a degradation was for a user to hit it.
+ *
+ * The scorer contract requires a `reason` on every score, so a report says
+ * what got worse rather than only that something did.
+ */
+export declare function runExperiment<TInput>(config: ExperimentConfig<TInput>): Promise<ExperimentReport>;
+/**
+ * Render a report as text.
+ *
+ * Failures print their scorer reasons, because a CI log that says
+ * "0.62" is a log that sends someone back to reproduce it by hand.
+ */
+export declare function formatReport(report: ExperimentReport): string;
+//# sourceMappingURL=experiment.d.ts.map

package/dist/eval/experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,UAAU,EAEV,QAAQ,EACR,OAAO,EACP,gBAAgB,EAEhB,MAAM,EACN,MAAM,YAAY,CAAA;AAEnB,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAA;IAClC,4DAA4D;IAC5D,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B;;;;OAIG;IACH,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IACzF;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,CAAA;CAC3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,aAAa,CAAC,MAAM,EACzC,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAsG3B;AAmHD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAsD7D"}

package/dist/eval/experiment.js

@@ -0,0 +1,263 @@
+/**
+ * Run a dataset and score it.
+ *
+ * There was no evaluation harness of any kind: no dataset, no scorer, no
+ * judge, no trajectory assertion. So every behavior change in this SDK —
+ * a tool description, the `search_tools` top-k, the compaction threshold —
+ * shipped with no regression signal behind it, and the only way to notice
+ * a degradation was for a user to hit it.
+ *
+ * The scorer contract requires a `reason` on every score, so a report says
+ * what got worse rather than only that something did.
+ */
+export async function runExperiment(config) {
+    const startedAt = Date.now();
+    const threshold = config.passThreshold ?? 1;
+    const concurrency = Math.max(1, config.concurrency ?? 1);
+    const results = new Array(config.cases.length);
+    let cursor = 0;
+    const worker = async () => {
+        for (;;) {
+            const index = cursor++;
+            const evalCase = config.cases[index];
+            if (!evalCase)
+                return;
+            const run = await executeCase(config, evalCase);
+            const scorers = evalCase.scorers ?? config.scorers;
+            const scores = {};
+            // Scores are keyed by name, so two scorers sharing one collapse:
+            // the mean's denominator becomes the count of distinct NAMES and
+            // the surviving score is whichever ran last. Two
+            // `containsScorer(...)` instances are both called 'contains', so
+            // this is easy to hit by accident and silently halves the
+            // evidence. Ambiguous results are worse than a loud failure.
+            const seen = new Set();
+            for (const scorer of scorers) {
+                if (seen.has(scorer.name)) {
+                    throw new Error(`Duplicate scorer name "${scorer.name}" for case "${evalCase.name}". Scores are keyed by name, so the second would overwrite the first and the case mean would be computed over the wrong denominator. Give each scorer a distinct name.`);
+                }
+                seen.add(scorer.name);
+                scores[scorer.name] = await safeScore(scorer, run, evalCase);
+            }
+            // Only scores that were actually produced count — an unavailable
+            // scorer leaves the denominator alone rather than dragging the
+            // mean toward zero with a measurement that never happened.
+            const values = Object.values(scores)
+                .filter((s) => s.unavailable !== true)
+                .map((s) => s.score);
+            const mean = values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+            // A gate miss fails the case whatever the mean says. Averaging a
+            // hard check together with a fuzzy one lets three good scores
+            // carry a zero: trajectory 0 + completion 1 + contains 1 +
+            // judge 1 averages to 0.75 and reports passed at a threshold of
+            // 0.75 — the exact regression the harness exists to catch,
+            // reported green. An UNAVAILABLE gate does not fail the case;
+            // it did not judge the run at all, which is the inconclusive
+            // path, not a failure.
+            const failedGates = scorers
+                .filter((s) => s.severity === 'gate')
+                .filter((s) => {
+                const score = scores[s.name];
+                if (!score || score.unavailable === true)
+                    return false;
+                return score.score < (s.threshold ?? threshold);
+            })
+                .map((s) => s.name);
+            const status = Object.keys(scores).length > 0 && values.length === 0
+                ? 'inconclusive'
+                : failedGates.length > 0
+                    ? 'failed'
+                    : mean >= threshold
+                        ? 'passed'
+                        : 'failed';
+            const result = {
+                case: evalCase.name,
+                run,
+                scores,
+                mean,
+                status,
+                passed: status === 'passed',
+                // Named, not just counted: "failed" with a mean of 0.75 sends
+                // somebody to read four scores and guess which one mattered.
+                ...(failedGates.length > 0 ? { failedGates } : {}),
+            };
+            results[index] = result;
+            config.onCaseFinish?.(result);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, config.cases.length) }, worker));
+    const settled = results.filter(Boolean);
+    // An inconclusive case has no mean to contribute; averaging its zero in
+    // would report a suite-wide regression caused by the harness.
+    const scored = settled.filter((r) => r.status !== 'inconclusive');
+    const mean = scored.length === 0 ? 0 : scored.reduce((sum, r) => sum + r.mean, 0) / scored.length;
+    return {
+        name: config.name,
+        cases: settled,
+        mean,
+        passed: settled.filter((r) => r.status === 'passed').length,
+        failed: settled.filter((r) => r.status === 'failed').length,
+        inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
+        byScorer: meanByScorer(settled),
+        durationMs: Date.now() - startedAt,
+    };
+}
+/**
+ * A case that throws is a RESULT, not a crash. An eval suite whose first
+ * broken case aborts the run tells you nothing about the other forty.
+ */
+async function executeCase(config, evalCase) {
+    const startedAt = Date.now();
+    const controller = new AbortController();
+    const timeoutMs = config.timeoutMs;
+    // The signal is handed to `run` so a closure that drives `query()` can
+    // pass it through and actually stop working. A closure that ignores it
+    // is merely detached rather than stopped — the same bargain every other
+    // deadline in the SDK makes — but the SUITE is unblocked either way,
+    // which is the part that was missing.
+    const timer = timeoutMs !== undefined && timeoutMs > 0
+        ? setTimeout(() => controller.abort(new Error(`case timed out after ${timeoutMs}ms`)), timeoutMs)
+        : undefined;
+    const deadline = timer === undefined
+        ? undefined
+        : new Promise((_resolve, reject) => {
+            controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {
+                once: true,
+            });
+        });
+    try {
+        const work = config.run(evalCase.input, evalCase, controller.signal);
+        // A rejection of the loser must never surface as an unhandled
+        // rejection after the race has already been decided.
+        if (deadline)
+            work.catch(() => { });
+        return await (deadline ? Promise.race([work, deadline]) : work);
+    }
+    catch (err) {
+        return {
+            output: null,
+            steps: [],
+            toolCalls: [],
+            totalTokens: 0,
+            totalCostUsd: 0,
+            // Real elapsed time, not zero: a case that burned its whole
+            // deadline is the most interesting number in the report, and
+            // zero would hide it.
+            durationMs: Date.now() - startedAt,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    finally {
+        // Only the timer needs clearing. Aborting here would fire a spurious
+        // signal at a `run` that has already settled — the deadline branch
+        // has aborted already, and every other path is done.
+        clearTimeout(timer);
+    }
+}
+/** A throwing scorer scores zero with the throw as its reason. */
+async function safeScore(scorer, run, evalCase) {
+    // A run that THREW scores zero, whatever the scorer would have said.
+    // `executeCase` catches the failure and returns an empty run, and an
+    // empty run walks straight into every scorer's happy path:
+    // `stepBudgetScorer` sees 0 steps against its allowance and returns 1,
+    // `trajectoryScorer` sees no tools expected and none called and returns
+    // 1. So a suite whose runs were all dying reported green. The failure is
+    // recorded on `run.error` and nothing consulted it.
+    if (run.error !== undefined) {
+        return { score: 0, reason: `run failed: ${run.error}`, details: { error: run.error } };
+    }
+    try {
+        return await scorer.score(run, evalCase);
+    }
+    catch (err) {
+        // UNAVAILABLE, not zero. A scorer that threw did not judge the run
+        // badly — it failed to judge it at all, and the two call for
+        // opposite responses. Scoring the throw zero was survivable while
+        // every scorer was a pure function; a scorer that reaches a provider
+        // makes it actively misleading, because one rate limit would look
+        // exactly like a behavioural regression.
+        return {
+            score: 0,
+            unavailable: true,
+            reason: `scorer "${scorer.name}" could not judge this run: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+}
+function meanByScorer(results) {
+    const sums = new Map();
+    for (const result of results) {
+        for (const [name, score] of Object.entries(result.scores)) {
+            if (score.unavailable === true)
+                continue;
+            const entry = sums.get(name) ?? { total: 0, count: 0 };
+            entry.total += score.score;
+            entry.count++;
+            sums.set(name, entry);
+        }
+    }
+    const out = {};
+    // A scorer that was never available anywhere is omitted rather than
+    // reported as 0 — a dimension with no measurements is not a dimension
+    // that scored badly.
+    for (const [name, { total, count }] of sums) {
+        if (count > 0)
+            out[name] = total / count;
+    }
+    return out;
+}
+/**
+ * Render a report as text.
+ *
+ * Failures print their scorer reasons, because a CI log that says
+ * "0.62" is a log that sends someone back to reproduce it by hand.
+ */
+export function formatReport(report) {
+    const lines = [
+        `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
+        '',
+    ];
+    // Surfaced above the failures, because an inconclusive case means the
+    // harness is broken and every number below it is measured over less
+    // evidence than it looks like.
+    if (report.inconclusive > 0) {
+        lines.push(`  ${report.inconclusive} case${report.inconclusive === 1 ? '' : 's'} could not be judged — the numbers below cover the rest`, '');
+    }
+    for (const [name, mean] of Object.entries(report.byScorer)) {
+        lines.push(`  ${name}: ${mean.toFixed(2)}`);
+    }
+    const failures = report.cases.filter((c) => c.status === 'failed');
+    if (failures.length > 0) {
+        lines.push('', 'Failures:');
+        for (const failure of failures) {
+            lines.push(`  ✗ ${failure.case} (${failure.mean.toFixed(2)})`);
+            if (failure.failedGates && failure.failedGates.length > 0) {
+                // First line under the case, because it says WHY this failed.
+                // A mean of 0.75 next to four scores leaves the reader
+                // guessing which one mattered.
+                lines.push(`      gate missed: ${failure.failedGates.join(', ')}`);
+            }
+            for (const [name, score] of Object.entries(failure.scores)) {
+                if (score.unavailable === true) {
+                    lines.push(`      ${name}: not judged — ${score.reason}`);
+                }
+                else if (score.score < 1) {
+                    lines.push(`      ${name}: ${score.reason}`);
+                }
+            }
+            if (failure.run.error)
+                lines.push(`      error: ${failure.run.error}`);
+        }
+    }
+    const unjudged = report.cases.filter((c) => c.status === 'inconclusive');
+    if (unjudged.length > 0) {
+        lines.push('', 'Not judged:');
+        for (const item of unjudged) {
+            lines.push(`  ? ${item.case}`);
+            for (const [name, score] of Object.entries(item.scores)) {
+                lines.push(`      ${name}: ${score.reason}`);
+            }
+        }
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=experiment.js.map

package/dist/eval/experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AA2CA;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IAExD,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC5D,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpC,IAAI,CAAC,QAAQ;gBAAE,OAAM;YAErB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;YAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;YAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;YAExC,iEAAiE;YACjE,iEAAiE;YACjE,iDAAiD;YACjD,iEAAiE;YACjE,0DAA0D;YAC1D,6DAA6D;YAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;YAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;gBACF,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;YAC7D,CAAC;YAED,iEAAiE;YACjE,+DAA+D;YAC/D,2DAA2D;YAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;iBAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;iBACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;YACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;YAExF,iEAAiE;YACjE,8DAA8D;YAC9D,2DAA2D;YAC3D,gEAAgE;YAChE,2DAA2D;YAC3D,8DAA8D;YAC9D,6DAA6D;YAC7D,uBAAuB;YACvB,MAAM,WAAW,GAAG,OAAO;iBACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;iBACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;gBAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;oBAAE,OAAO,KAAK,CAAA;gBACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;YAChD,CAAC,CAAC;iBACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBACpD,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;oBACvB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,IAAI,IAAI,SAAS;wBAClB,CAAC,CAAC,QAAQ;wBACV,CAAC,CAAC,QAAQ,CAAA;YACd,MAAM,MAAM,GAAe;gBAC1B,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,GAAG;gBACH,MAAM;gBACN,IAAI;gBACJ,MAAM;gBACN,MAAM,EAAE,MAAM,KAAK,QAAQ;gBAC3B,8DAA8D;gBAC9D,6DAA6D;gBAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAClD,CAAA;YACD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE7F,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAClC,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B;IAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAA;IAClC,uEAAuE;IACvE,uEAAuE;IACvE,wEAAwE;IACxE,qEAAqE;IACrE,sCAAsC;IACtC,MAAM,KAAK,GACV,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC;QACvC,CAAC,CAAC,UAAU,CACV,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT;QACF,CAAC,CAAC,SAAS,CAAA;IAEb,MAAM,QAAQ,GACb,KAAK,KAAK,SAAS;QAClB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YACxC,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnF,IAAI,EAAE,IAAI;aACV,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IAEL,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;QACpE,8DAA8D;QAC9D,qDAAqD;QACrD,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAClC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAChE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,qEAAqE;QACrE,mEAAmE;QACnE,qDAAqD;QACrD,YAAY,CAAC,KAAK,CAAC,CAAA;IACpB,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,GAAY,EAAE,QAAkB;IACxE,qEAAqE;IACrE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,yEAAyE;IACzE,oDAAoD;IACpD,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,eAAe,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,CAAA;IACvF,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,mEAAmE;QACnE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,+BAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,KAAK,GAAa;QACvB,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;QAC3H,EAAE;KACF,CAAA;IAED,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YAC9D,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/from-run.d.ts

@@ -0,0 +1,30 @@
+import type { Run } from '../types/run/entity.js';
+import type { EvalRun } from './types.js';
+/**
+ * Project a finished `Run` into the shape scorers consume.
+ *
+ * This is the whole bridge between the harness and the runtime, and it is
+ * three lines of mapping because `Run.steps` now exists. Before that, a
+ * trajectory scorer would have had to correlate raw `RunEvent`s by
+ * iteration number and diff cumulative counters — which is why the step
+ * record had to land before the harness could.
+ */
+export declare function evalRunFromRun(run: Run, opts?: {
+    durationMs?: number;
+}): EvalRun;
+/**
+ * Drain a `query()` generator to its returned `Run`, then project it.
+ *
+ * The common case, so it does not need writing per suite:
+ *
+ * ```ts
+ * runExperiment({
+ *   name: 'file-editing',
+ *   cases,
+ *   scorers: [trajectoryScorer(), completionScorer()],
+ *   run: (input) => evalRunFromQuery(query({ provider, tools, messages: input, … })),
+ * })
+ * ```
+ */
+export declare function evalRunFromQuery(generator: AsyncGenerator<unknown, Run>): Promise<EvalRun>;
+//# sourceMappingURL=from-run.d.ts.map

package/dist/eval/from-run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"from-run.d.ts","sourceRoot":"","sources":["../../src/eval/from-run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEzC;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAcpF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAKhG"}

package/dist/eval/from-run.js

@@ -0,0 +1,46 @@
+/**
+ * Project a finished `Run` into the shape scorers consume.
+ *
+ * This is the whole bridge between the harness and the runtime, and it is
+ * three lines of mapping because `Run.steps` now exists. Before that, a
+ * trajectory scorer would have had to correlate raw `RunEvent`s by
+ * iteration number and diff cumulative counters — which is why the step
+ * record had to land before the harness could.
+ */
+export function evalRunFromRun(run, opts = {}) {
+    const steps = run.steps ?? [];
+    return {
+        output: run.result ?? null,
+        ...(run.structuredOutput !== undefined ? { structuredOutput: run.structuredOutput } : {}),
+        steps,
+        // Flattened in call order across steps — the trajectory.
+        toolCalls: steps.flatMap((s) => s.toolCalls.map((c) => c.function.name)),
+        ...(run.stopReason ? { stopReason: run.stopReason } : {}),
+        totalTokens: run.tokenUsage.totalTokens,
+        totalCostUsd: run.costInfo.totalCost,
+        durationMs: opts.durationMs ?? (run.endedAt ?? Date.now()) - run.startedAt,
+        ...(run.lastError ? { error: run.lastError } : {}),
+    };
+}
+/**
+ * Drain a `query()` generator to its returned `Run`, then project it.
+ *
+ * The common case, so it does not need writing per suite:
+ *
+ * ```ts
+ * runExperiment({
+ *   name: 'file-editing',
+ *   cases,
+ *   scorers: [trajectoryScorer(), completionScorer()],
+ *   run: (input) => evalRunFromQuery(query({ provider, tools, messages: input, … })),
+ * })
+ * ```
+ */
+export async function evalRunFromQuery(generator) {
+    const startedAt = Date.now();
+    let next = await generator.next();
+    while (!next.done)
+        next = await generator.next();
+    return evalRunFromRun(next.value, { durationMs: Date.now() - startedAt });
+}
+//# sourceMappingURL=from-run.js.map

package/dist/eval/from-run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"from-run.js","sourceRoot":"","sources":["../../src/eval/from-run.ts"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,GAAQ,EAAE,OAAgC,EAAE;IAC1E,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAA;IAC7B,OAAO;QACN,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,GAAG,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzF,KAAK;QACL,yDAAyD;QACzD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,WAAW,EAAE,GAAG,CAAC,UAAU,CAAC,WAAW;QACvC,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,SAAS;QACpC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,SAAS;QAC1E,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAuC;IAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IACjC,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IAChD,OAAO,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;AAC1E,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,8 @@
+export { evalRunFromQuery, evalRunFromRun } from './from-run.js';
+export { formatReport, runExperiment } from './experiment.js';
+export type { ExperimentConfig } from './experiment.js';
+export { judgeScorer } from './judge.js';
+export type { JudgeScorerConfig } from './judge.js';
+export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export type { CaseResult, CaseStatus, EvalCase, EvalRun, ExperimentReport, Score, Scorer, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,YAAY,EACX,UAAU,EACV,UAAU,EACV,QAAQ,EACR,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}

package/dist/eval/index.js

@@ -0,0 +1,5 @@
+export { evalRunFromQuery, evalRunFromRun } from './from-run.js';
+export { formatReport, runExperiment } from './experiment.js';
+export { judgeScorer } from './judge.js';
+export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA"}