@namzu/sdk 2.0.0 → 3.0.0

package/dist/eval/__tests__/harness-honesty.test.js

@@ -0,0 +1,85 @@
+import { describe, expect, it } from 'vitest';
+import { runExperiment } from '../experiment.js';
+import { containsScorer, stepBudgetScorer, trajectoryScorer } from '../scorers.js';
+/**
+ * An eval harness that reports green on a broken suite is worse than no
+ * harness: it converts "we do not know" into "we checked". Both defects
+ * here did exactly that.
+ */
+const emptyRun = () => ({
+    output: 'ok',
+    steps: [],
+    toolCalls: [],
+    totalTokens: 0,
+    totalCostUsd: 0,
+    durationMs: 0,
+});
+describe('a case whose run THREW cannot score above zero', () => {
+    it('scores 0 instead of a perfect step budget', async () => {
+        // `executeCase` catches the throw and returns an empty run, and an
+        // empty run walks into every scorer's happy path: 0 steps is within
+        // any allowance, and "no tools expected, none called" is a match. The
+        // suite reported 1.0 for a case that never ran.
+        const report = await runExperiment({
+            name: 'crash',
+            cases: [{ name: 'boom', input: 'x' }],
+            scorers: [stepBudgetScorer(5), trajectoryScorer()],
+            run: () => {
+                throw new Error('provider exploded');
+            },
+        });
+        expect(report.cases[0]?.scores['step-budget']?.score).toBe(0);
+        expect(report.cases[0]?.scores.trajectory?.score).toBe(0);
+        expect(report.mean).toBe(0);
+    });
+    it('says why, so the green-to-red change is diagnosable', async () => {
+        const report = await runExperiment({
+            name: 'crash',
+            cases: [{ name: 'boom', input: 'x' }],
+            scorers: [stepBudgetScorer(5)],
+            run: () => {
+                throw new Error('provider exploded');
+            },
+        });
+        expect(report.cases[0]?.scores['step-budget']?.reason).toContain('provider exploded');
+    });
+    it('still scores a healthy run normally', async () => {
+        const report = await runExperiment({
+            name: 'fine',
+            cases: [{ name: 'ok', input: 'x' }],
+            scorers: [stepBudgetScorer(5)],
+            run: () => Promise.resolve(emptyRun()),
+        });
+        expect(report.cases[0]?.scores['step-budget']?.score).toBe(1);
+    });
+});
+describe('two scorers cannot silently collapse into one', () => {
+    it('refuses a duplicate scorer name rather than overwriting', async () => {
+        // Scores are keyed by name, so two `containsScorer(...)` instances —
+        // both called 'contains' — meant the second overwrote the first and
+        // the case mean was computed over the wrong denominator. With 'a'
+        // scoring 0 and 'b' scoring 1 the suite reported 1.0 where the honest
+        // answer is 0.5.
+        await expect(runExperiment({
+            name: 'dupe',
+            cases: [{ name: 'c', input: 'x' }],
+            scorers: [containsScorer('a'), containsScorer('b')],
+            run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
+        })).rejects.toThrow(/Duplicate scorer name "contains"/);
+    });
+    it('accepts distinct names', async () => {
+        const report = await runExperiment({
+            name: 'ok',
+            cases: [{ name: 'c', input: 'x' }],
+            scorers: [
+                { ...containsScorer('a'), name: 'contains-a' },
+                { ...containsScorer('b'), name: 'contains-b' },
+            ],
+            run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
+        });
+        expect(report.cases[0]?.scores['contains-a']?.score).toBe(0);
+        expect(report.cases[0]?.scores['contains-b']?.score).toBe(1);
+        expect(report.cases[0]?.mean).toBe(0.5);
+    });
+});
+//# sourceMappingURL=harness-honesty.test.js.map

package/dist/eval/__tests__/harness-honesty.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness-honesty.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/harness-honesty.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAGlF;;;;GAIG;AAEH,MAAM,QAAQ,GAAG,GAAY,EAAE,CAAC,CAAC;IAChC,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,EAAE;IACT,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,CAAC;CACb,CAAC,CAAA;AAEF,QAAQ,CAAC,gDAAgD,EAAE,GAAG,EAAE;IAC/D,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,mEAAmE;QACnE,oEAAoE;QACpE,sEAAsE;QACtE,gDAAgD;QAChD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC;YAClD,GAAG,EAAE,GAAG,EAAE;gBACT,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;YACrC,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC9B,GAAG,EAAE,GAAG,EAAE;gBACT,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;YACrC,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAA;IACtF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC9B,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SACtC,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9D,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,+CAA+C,EAAE,GAAG,EAAE;IAC9D,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACxE,qEAAqE;QACrE,oEAAoE;QACpE,kEAAkE;QAClE,sEAAsE;QACtE,iBAAiB;QACjB,MAAM,MAAM,CACX,aAAa,CAAC;YACb,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,OAAO,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;YACnD,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,OAAO,EAAE;gBACR,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE;gBAC9C,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE;aAC9C;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/harness.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=harness.test.d.ts.map

package/dist/eval/__tests__/harness.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/harness.test.ts"],"names":[],"mappings":""}

package/dist/eval/__tests__/harness.test.js

@@ -0,0 +1,207 @@
+import { describe, expect, it } from 'vitest';
+import { formatReport, runExperiment } from '../experiment.js';
+import { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from '../scorers.js';
+/**
+ * There was no evaluation harness of any kind — no dataset, no scorer, no
+ * judge, no trajectory assertion. So the constants that govern namzu's
+ * most load-bearing behavior (`search_tools` top-k, the compaction
+ * threshold, six state-list caps, every builtin tool description) were
+ * tunable but not measurable: change one and the only way to learn the
+ * agent now takes four tool calls where it took one was for a user to
+ * hit it.
+ */
+function run(overrides = {}) {
+    return {
+        output: 'done',
+        steps: [],
+        toolCalls: [],
+        stopReason: 'end_turn',
+        totalTokens: 0,
+        totalCostUsd: 0,
+        durationMs: 0,
+        ...overrides,
+    };
+}
+const CASE = { name: 'c', input: null };
+describe('trajectoryScorer', () => {
+    // `Scorer.score` may be async (a model-graded judge is just an async
+    // predicate), so every call site awaits even when the built-in is sync.
+    const score = async (expectedTools, toolCalls) => await trajectoryScorer().score(run({ toolCalls }), { ...CASE, expectedTools });
+    it('scores an exact trajectory 1', async () => {
+        expect((await score(['read', 'edit'], ['read', 'edit'])).score).toBe(1);
+    });
+    it('distinguishes ORDER — a set-based score could not', async () => {
+        // Reading before editing is not the same run as editing then reading.
+        const reversed = await score(['read', 'edit'], ['edit', 'read']);
+        expect(reversed.score).toBeLessThan(1);
+        expect(reversed.score).toBeGreaterThan(0);
+    });
+    it('penalises an extra call through precision', async () => {
+        const wasteful = await score(['read'], ['read', 'read', 'read']);
+        expect(wasteful.score).toBeLessThan(1);
+        expect(wasteful.details?.recall).toBe(1);
+        expect(wasteful.details?.precision).toBeLessThan(1);
+    });
+    it('penalises a missing call through recall', async () => {
+        const lazy = await score(['read', 'edit', 'bash'], ['read']);
+        expect(lazy.details?.precision).toBe(1);
+        expect(lazy.details?.recall).toBeLessThan(1);
+    });
+    it('separates "wasteful" from "skipped" — a final-answer score collapses them', async () => {
+        const wasteful = await score(['read', 'edit'], ['read', 'ls', 'edit']);
+        const skipped = await score(['read', 'edit'], ['read']);
+        expect(wasteful.score).not.toBe(skipped.score);
+    });
+    it('scores 1 when no tools were expected and none ran', async () => {
+        expect((await score([], [])).score).toBe(1);
+    });
+    it('scores 0 when tools were expected and none ran', async () => {
+        const s = await score(['read'], []);
+        expect(s.score).toBe(0);
+        expect(s.reason).toContain('no tools were called');
+    });
+    it('scores 0 when none were expected and some ran', async () => {
+        expect((await score([], ['bash'])).score).toBe(0);
+    });
+    it('always explains itself', async () => {
+        expect((await score(['read'], ['edit'])).reason.length).toBeGreaterThan(0);
+    });
+});
+describe('completionScorer', () => {
+    it('accepts a clean settle', async () => {
+        expect((await completionScorer().score(run({ stopReason: 'end_turn' }), CASE)).score).toBe(1);
+    });
+    it('rejects a budget-forced stop', async () => {
+        const s = await completionScorer().score(run({ stopReason: 'max_iterations' }), CASE);
+        expect(s.score).toBe(0);
+        expect(s.reason).toContain('max_iterations');
+    });
+    it('reports the error when the run threw', async () => {
+        const s = await completionScorer().score(run({ error: 'boom' }), CASE);
+        expect(s.score).toBe(0);
+        expect(s.reason).toContain('boom');
+    });
+});
+describe('stepBudgetScorer', () => {
+    const withSteps = (n) => run({ steps: Array.from({ length: n }, () => ({})) });
+    it('passes within budget', async () => {
+        expect((await stepBudgetScorer(5).score(withSteps(3), CASE)).score).toBe(1);
+    });
+    it('degrades proportionally past it, rather than snapping to zero', async () => {
+        // A run that took 6 steps against a 5 allowance is worse than one
+        // that took 20; a binary score cannot say so.
+        const slight = (await stepBudgetScorer(5).score(withSteps(6), CASE)).score;
+        const severe = (await stepBudgetScorer(5).score(withSteps(20), CASE)).score;
+        expect(slight).toBeGreaterThan(severe);
+        expect(slight).toBeLessThan(1);
+    });
+});
+describe('containsScorer', () => {
+    it('scores the fraction found and names what is missing', async () => {
+        const s = await containsScorer('alpha', 'beta').score(run({ output: 'alpha only' }), CASE);
+        expect(s.score).toBe(0.5);
+        expect(s.reason).toContain('beta');
+    });
+});
+describe('runExperiment', () => {
+    const cases = [
+        { name: 'good', input: 'a', expectedTools: ['read'] },
+        { name: 'bad', input: 'b', expectedTools: ['read', 'edit'] },
+    ];
+    it('scores every case and aggregates', async () => {
+        const report = await runExperiment({
+            name: 'demo',
+            cases,
+            scorers: [trajectoryScorer()],
+            run: async (input) => run({ toolCalls: input === 'a' ? ['read'] : ['bash'] }),
+        });
+        expect(report.cases).toHaveLength(2);
+        expect(report.passed).toBe(1);
+        expect(report.failed).toBe(1);
+        expect(report.byScorer.trajectory).toBeCloseTo(0.5, 5);
+    });
+    it('a throwing case is a RESULT, not a crash', async () => {
+        // A suite whose first broken case aborts tells you nothing about the
+        // other forty.
+        const report = await runExperiment({
+            name: 'demo',
+            cases,
+            scorers: [completionScorer()],
+            run: async (input) => {
+                if (input === 'a')
+                    throw new Error('exploded');
+                return run();
+            },
+        });
+        expect(report.cases).toHaveLength(2);
+        expect(report.cases[0]?.run.error).toBe('exploded');
+        expect(report.cases[1]?.passed).toBe(true);
+    });
+    it('a throwing scorer scores zero with the throw as its reason', async () => {
+        const report = await runExperiment({
+            name: 'demo',
+            cases: [cases[0]],
+            scorers: [
+                customScorer('boom', () => {
+                    throw new Error('scorer broke');
+                }),
+            ],
+            run: async () => run(),
+        });
+        expect(report.cases[0]?.scores.boom?.score).toBe(0);
+        expect(report.cases[0]?.scores.boom?.reason).toContain('scorer broke');
+    });
+    it('preserves case order under concurrency', async () => {
+        const many = Array.from({ length: 8 }, (_, i) => ({
+            name: `case-${i}`,
+            input: i,
+        }));
+        const report = await runExperiment({
+            name: 'demo',
+            cases: many,
+            concurrency: 4,
+            scorers: [completionScorer()],
+            run: async (input) => {
+                await new Promise((r) => setTimeout(r, (8 - input) * 2));
+                return run();
+            },
+        });
+        expect(report.cases.map((c) => c.case)).toEqual(many.map((c) => c.name));
+    });
+    it('honours a per-case scorer override', async () => {
+        const report = await runExperiment({
+            name: 'demo',
+            cases: [{ name: 'special', input: null, scorers: [containsScorer('xyz')] }],
+            scorers: [completionScorer()],
+            run: async () => run({ output: 'no match here' }),
+        });
+        expect(Object.keys(report.cases[0]?.scores ?? {})).toEqual(['contains']);
+    });
+    it('respects a partial pass threshold', async () => {
+        const report = await runExperiment({
+            name: 'demo',
+            cases: [{ name: 'partial', input: null, expectedTools: ['read', 'edit'] }],
+            scorers: [trajectoryScorer()],
+            passThreshold: 0.5,
+            run: async () => run({ toolCalls: ['read'] }),
+        });
+        expect(report.passed).toBe(1);
+    });
+});
+describe('formatReport', () => {
+    it('prints scorer reasons for failures, not just numbers', async () => {
+        // A CI log that says "0.62" sends someone back to reproduce it by hand.
+        const report = await runExperiment({
+            name: 'demo',
+            cases: [{ name: 'bad', input: null, expectedTools: ['read', 'edit'] }],
+            scorers: [trajectoryScorer()],
+            run: async () => run({ toolCalls: ['bash'] }),
+        });
+        const text = formatReport(report);
+        expect(text).toContain('0/1 passed');
+        expect(text).toContain('✗ bad');
+        expect(text).toContain('trajectory:');
+        expect(text).toContain('expected read → edit');
+    });
+});
+//# sourceMappingURL=harness.test.js.map

package/dist/eval/__tests__/harness.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/harness.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,eAAe,CAAA;AAGtB;;;;;;;;GAQG;AAEH,SAAS,GAAG,CAAC,YAA8B,EAAE;IAC5C,OAAO;QACN,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,UAAU;QACtB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,GAAG,SAAS;KACZ,CAAA;AACF,CAAC;AAED,MAAM,IAAI,GAAa,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;AAEjD,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,qEAAqE;IACrE,wEAAwE;IACxE,MAAM,KAAK,GAAG,KAAK,EAAE,aAAuB,EAAE,SAAmB,EAAE,EAAE,CACpE,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,CAAC,CAAA;IAE/E,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACxE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QAChE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QAChE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;QAC1F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QACtE,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;QACvD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAA;QACnC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC3E,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,CAAC,CAAC,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9F,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,CAAC,GAAG,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QACrF,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,CAAC,GAAG,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QACtE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAU,CAAC,EAAE,CAAC,CAAA;IAE/F,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC9E,kEAAkE;QAClE,8CAA8C;QAC9C,MAAM,MAAM,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;QAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QAC1F,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,MAAM,KAAK,GAAe;QACzB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,MAAM,CAAC,EAAE;QACrD,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;KAC5D,CAAA;IAED,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACzD,qEAAqE;QACrE,eAAe;QACf,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACpB,IAAI,KAAK,KAAK,GAAG;oBAAE,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAA;gBAC9C,OAAO,GAAG,EAAE,CAAA;YACb,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAa,CAAC;YAC7B,OAAO,EAAE;gBACR,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE;oBACzB,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAA;gBAChC,CAAC,CAAC;aACF;YACD,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,EAAE;SACtB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;IACvE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,IAAI,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,QAAQ,CAAC,EAAE;YACjB,KAAK,EAAE,CAAC;SACR,CAAC,CAAC,CAAA;QACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,GAAI,KAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBACpE,OAAO,GAAG,EAAE,CAAA;YACb,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACzE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;SACjD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAA;IACzE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1E,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,aAAa,EAAE,GAAG;YAClB,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7C,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,wEAAwE;QACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YACtE,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7C,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAA;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;QAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACrC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/judge.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=judge.test.d.ts.map

package/dist/eval/__tests__/judge.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":""}

package/dist/eval/__tests__/judge.test.js

@@ -0,0 +1,255 @@
+import { describe, expect, it } from 'vitest';
+import { runExperiment } from '../experiment.js';
+import { judgeScorer } from '../judge.js';
+/**
+ * Every other scorer is a pure function over the run, which is what makes
+ * them reproducible and what makes them unable to say whether an answer is
+ * GOOD. A judge closes that, and brings a failure mode none of the others
+ * have: it can fail to answer at all. A failed measurement scored zero
+ * reads exactly like a regression, so the distinction is what most of this
+ * file is about.
+ */
+function fakeProvider(reply, tokens = 42) {
+    const seen = [];
+    const provider = {
+        id: 'fake',
+        name: 'fake',
+        capabilities: {
+            supportsTools: false,
+            supportsStreaming: true,
+            supportsFunctionCalling: false,
+        },
+        seen,
+        async *chatStream(params) {
+            seen.push(params);
+            if (typeof reply !== 'string')
+                reply();
+            yield { id: 'j', delta: { content: reply } };
+            yield {
+                id: 'j',
+                delta: {},
+                finishReason: 'stop',
+                usage: {
+                    promptTokens: tokens,
+                    completionTokens: 0,
+                    totalTokens: tokens,
+                    cachedTokens: 0,
+                    cacheWriteTokens: 0,
+                },
+            };
+        },
+        async listModels() {
+            return [];
+        },
+        async healthCheck() {
+            return true;
+        },
+    };
+    return provider;
+}
+const RUN = {
+    output: 'The capital of France is Paris.',
+    steps: [],
+    toolCalls: ['search'],
+    stopReason: 'end_turn',
+    totalTokens: 100,
+    totalCostUsd: 0,
+    durationMs: 10,
+};
+const CASE = { name: 'capital', input: 'What is the capital of France?' };
+const RUBRIC = 'The answer names the correct city.';
+describe('grading', () => {
+    it('normalises the grade onto 0..1 against the scale it asked for', async () => {
+        const scorer = judgeScorer({
+            provider: fakeProvider('{"grade": 3, "reason": "names the right city"}'),
+            model: 'm',
+            rubric: RUBRIC,
+            scale: 4,
+        });
+        const score = await scorer.score(RUN, CASE);
+        expect(score.score).toBe(0.75);
+        expect(score.reason).toBe('names the right city');
+        expect(score.details?.grade).toBe(3);
+    });
+    it('reads a verdict wrapped in prose or fences', async () => {
+        const scorer = judgeScorer({
+            provider: fakeProvider('Here is my grade:\n```json\n{"grade": 4, "reason": "correct"}\n```'),
+            model: 'm',
+            rubric: RUBRIC,
+        });
+        expect((await scorer.score(RUN, CASE)).score).toBe(1);
+    });
+    it('handles a brace inside the reason string', async () => {
+        const scorer = judgeScorer({
+            provider: fakeProvider('{"grade": 2, "reason": "mentions {city} placeholder"}'),
+            model: 'm',
+            rubric: RUBRIC,
+            scale: 4,
+        });
+        // A naive scan to the first `}` would cut the object in half here.
+        expect((await scorer.score(RUN, CASE)).score).toBe(0.5);
+    });
+    it('carries what the judging itself cost', async () => {
+        const scorer = judgeScorer({
+            provider: fakeProvider('{"grade": 4, "reason": "ok"}', 1234),
+            model: 'm',
+            rubric: RUBRIC,
+        });
+        // A judge is the most expensive scorer there is; a bill nobody can
+        // attribute is a bill nobody controls.
+        expect((await scorer.score(RUN, CASE)).details?.judgeTokens).toBe(1234);
+    });
+    it('grades the same run the same way twice', async () => {
+        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
+        // Sampling noise is indistinguishable from a regression, so the
+        // judge does not sample.
+        expect(provider.seen[0]?.temperature).toBe(0);
+    });
+});
+describe('what the judge is shown', () => {
+    const promptOf = (provider) => String(provider.seen[0]?.messages[0]?.content);
+    it('shows the rubric, the task and the answer', async () => {
+        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
+        const prompt = promptOf(provider);
+        expect(prompt).toContain(RUBRIC);
+        expect(prompt).toContain('What is the capital of France?');
+        expect(prompt).toContain('The capital of France is Paris.');
+    });
+    it('withholds the trajectory unless asked', async () => {
+        const off = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({ provider: off, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
+        expect(promptOf(off)).not.toContain('TOOLS CALLED');
+        const on = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({
+            provider: on,
+            model: 'm',
+            rubric: RUBRIC,
+            includeTrajectory: true,
+        }).score(RUN, CASE);
+        expect(promptOf(on)).toContain('search');
+    });
+    it('says when it truncated the answer', async () => {
+        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({
+            provider,
+            model: 'm',
+            rubric: RUBRIC,
+            maxOutputChars: 10,
+        }).score({ ...RUN, output: 'x'.repeat(500) }, CASE);
+        // A judge shown a silently cut answer marks it down for stopping
+        // mid-sentence, which scores our truncation rather than the run.
+        expect(promptOf(provider)).toContain('cut at 10 characters by the harness');
+    });
+    it('offers the reference answer when the case has one', async () => {
+        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
+        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, {
+            ...CASE,
+            expected: 'Paris',
+        });
+        expect(promptOf(provider)).toContain('Paris');
+    });
+});
+describe('a judge that cannot judge', () => {
+    const failing = (reply) => judgeScorer({ provider: fakeProvider(reply), model: 'm', rubric: RUBRIC, scale: 4 });
+    it('throws on a reply with no JSON at all', async () => {
+        await expect(failing('I think it was pretty good!').score(RUN, CASE)).rejects.toThrow(/no JSON/);
+    });
+    it('throws on a grade outside the scale it was given', async () => {
+        // Clamping would turn a judge that misread the scale into a
+        // confident score, and a judge that misread the scale did not apply
+        // the rubric either.
+        await expect(failing('{"grade": 9, "reason": "great"}').score(RUN, CASE)).rejects.toThrow(/outside the 0\.\.4 scale/);
+    });
+    it('throws on a non-numeric grade', async () => {
+        await expect(failing('{"grade": "good", "reason": "x"}').score(RUN, CASE)).rejects.toThrow(/no numeric grade/);
+    });
+    it('throws on an unterminated object', async () => {
+        await expect(failing('{"grade": 3, "reason": "cut off').score(RUN, CASE)).rejects.toThrow(/unterminated/);
+    });
+    it('refuses to exist without a rubric', () => {
+        expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: '   ' })).toThrow(/rubric is required/);
+    });
+    it('refuses a nonsense scale', () => {
+        expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: RUBRIC, scale: 0 })).toThrow(/positive integer/);
+    });
+});
+describe('a broken judge is not a bad run', () => {
+    const dataset = {
+        name: 'suite',
+        cases: [CASE],
+        run: async () => RUN,
+    };
+    it('reports the case as inconclusive rather than failed', async () => {
+        const report = await runExperiment({
+            ...dataset,
+            scorers: [
+                judgeScorer({
+                    provider: fakeProvider(() => {
+                        throw new Error('429 rate limited');
+                    }),
+                    model: 'm',
+                    rubric: RUBRIC,
+                }),
+            ],
+        });
+        expect(report.inconclusive).toBe(1);
+        expect(report.failed).toBe(0);
+        expect(report.passed).toBe(0);
+        expect(report.cases[0]?.scores.judge?.unavailable).toBe(true);
+        expect(report.cases[0]?.scores.judge?.reason).toContain('429');
+    });
+    it('keeps an unavailable score out of the mean entirely', async () => {
+        const report = await runExperiment({
+            ...dataset,
+            scorers: [
+                { name: 'always-one', score: () => ({ score: 1, reason: 'fine' }) },
+                judgeScorer({
+                    provider: fakeProvider(() => {
+                        throw new Error('down');
+                    }),
+                    model: 'm',
+                    rubric: RUBRIC,
+                }),
+            ],
+        });
+        // Not 0.5. A measurement that never happened is not a zero, and
+        // averaging it in would report a regression the harness caused.
+        expect(report.cases[0]?.mean).toBe(1);
+        expect(report.cases[0]?.status).toBe('passed');
+        expect(report.mean).toBe(1);
+    });
+    it('omits a scorer that was never available from the per-dimension means', async () => {
+        const report = await runExperiment({
+            ...dataset,
+            scorers: [
+                judgeScorer({
+                    provider: fakeProvider(() => {
+                        throw new Error('down');
+                    }),
+                    model: 'm',
+                    rubric: RUBRIC,
+                }),
+            ],
+        });
+        // A dimension with no measurements is not a dimension that scored 0.
+        expect(report.byScorer.judge).toBeUndefined();
+    });
+    it('still fails a case the judge actually judged badly', async () => {
+        const report = await runExperiment({
+            ...dataset,
+            scorers: [
+                judgeScorer({
+                    provider: fakeProvider('{"grade": 0, "reason": "names the wrong city"}'),
+                    model: 'm',
+                    rubric: RUBRIC,
+                }),
+            ],
+        });
+        expect(report.failed).toBe(1);
+        expect(report.inconclusive).toBe(0);
+        expect(report.cases[0]?.scores.judge?.unavailable).toBeUndefined();
+    });
+});
+//# sourceMappingURL=judge.test.js.map

package/dist/eval/__tests__/judge.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAI7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAGzC;;;;;;;GAOG;AAEH,SAAS,YAAY,CAAC,KAA6B,EAAE,MAAM,GAAG,EAAE;IAC/D,MAAM,IAAI,GAA2B,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAmD;QAChE,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,MAAM;QACZ,YAAY,EAAE;YACb,aAAa,EAAE,KAAK;YACpB,iBAAiB,EAAE,IAAI;YACvB,uBAAuB,EAAE,KAAK;SAC9B;QACD,IAAI;QACJ,KAAK,CAAC,CAAC,UAAU,CAAC,MAA4B;YAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACjB,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,KAAK,EAAE,CAAA;YACtC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAe,EAAE,EAAE,CAAA;YACtD,MAAM;gBACL,EAAE,EAAE,GAAG;gBACP,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,MAAM;gBACpB,KAAK,EAAE;oBACN,YAAY,EAAE,MAAM;oBACpB,gBAAgB,EAAE,CAAC;oBACnB,WAAW,EAAE,MAAM;oBACnB,YAAY,EAAE,CAAC;oBACf,gBAAgB,EAAE,CAAC;iBACnB;aACD,CAAA;QACF,CAAC;QACD,KAAK,CAAC,UAAU;YACf,OAAO,EAAE,CAAA;QACV,CAAC;QACD,KAAK,CAAC,WAAW;YAChB,OAAO,IAAI,CAAA;QACZ,CAAC;KACD,CAAA;IACD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,MAAM,GAAG,GAAY;IACpB,MAAM,EAAE,iCAAiC;IACzC,KAAK,EAAE,EAAE;IACT,SAAS,EAAE,CAAC,QAAQ,CAAC;IACrB,UAAU,EAAE,UAAU;IACtB,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,EAAE;CACd,CAAA;AAED,MAAM,IAAI,GAAa,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAA;AAEnF,MAAM,MAAM,GAAG,oCAAoC,CAAA;AAEnD,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,gDAAgD,CAAC;YACxE,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,CAAC;SACR,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAE3C,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;QACjD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,oEAAoE,CAAC;YAC5F,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;SACd,CAAC,CAAA;QACF,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,uDAAuD,CAAC;YAC/E,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,CAAC;SACR,CAAC,CAAA;QACF,mEAAmE;QACnE,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,8BAA8B,EAAE,IAAI,CAAC;YAC5D,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;SACd,CAAC,CAAA;QACF,mEAAmE;QACnE,uCAAuC;QACvC,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAE3D,CAAA;QACD,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAC5E,gEAAgE;QAChE,yBAAyB;QACzB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACxC,MAAM,QAAQ,GAAG,CAAC,QAAqB,EAAE,EAAE,CAC1C,MAAM,CAAE,QAAwD,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IAEhG,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAE5E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACjC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;QAChC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAA;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iCAAiC,CAAC,CAAA;IAC5D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,GAAG,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QACxD,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACjF,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;QAEnD,MAAM,EAAE,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QACvD,MAAM,WAAW,CAAC;YACjB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,iBAAiB,EAAE,IAAI;SACvB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACnB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC;YACjB,QAAQ;YACR,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,cAAc,EAAE,EAAE;SAClB,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAEnD,iEAAiE;QACjE,iEAAiE;QACjE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,qCAAqC,CAAC,CAAA;IAC5E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACtE,GAAG,IAAI;YACP,QAAQ,EAAE,OAAO;SACjB,CAAC,CAAA;QACF,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IAC1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CACjC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAErF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACjG,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QACjE,4DAA4D;QAC5D,oEAAoE;QACpE,qBAAqB;QACrB,MAAM,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACxF,0BAA0B,CAC1B,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzF,kBAAkB,CAClB,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACxF,cAAc,CACd,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAC7F,oBAAoB,CACpB,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACnC,MAAM,CAAC,GAAG,EAAE,CACX,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACnF,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAChD,MAAM,OAAO,GAAG;QACf,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,GAAG,EAAE,KAAK,IAAsB,EAAE,CAAC,GAAG;KACtC,CAAA;IAED,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;oBACpC,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE;gBACnE,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;oBACxB,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,gEAAgE;QAChE,gEAAgE;QAChE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;oBACxB,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,qEAAqE;QACrE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,gDAAgD,CAAC;oBACxE,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,aAAa,EAAE,CAAA;IACnE,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/scorer-gates.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=scorer-gates.test.d.ts.map

package/dist/eval/__tests__/scorer-gates.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer-gates.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/scorer-gates.test.ts"],"names":[],"mappings":""}