@namzu/sdk 14.0.7 → 15.0.0

package/dist/eval/__tests__/end-to-end.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"end-to-end.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/end-to-end.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAEjD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAA;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,wCAAwC,CAAA;AAC9E,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAA;AAK9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAA;AAGrD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAGlE;;;;;;;;;GASG;AAEH,MAAM,MAAM,GAAG,UAAmB,CAAA;AAElC,SAAS,UAAU;IAClB,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAA;IAC7E,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAuB,CAAA;AAC3F,CAAC;AAED,0EAA0E;AAC1E,KAAK,UAAU,UAAU,CAAC,KAAgB;IACzC,MAAM,QAAQ,GAAc,EAAE,CAAA;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,UAA8B,CAAA;IAClC,MAAM,GAAG,GAAG,UAAU,EAAE,CAAA;IAExB,MAAM,KAAK,GAAG;QACb,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI;YACJ,iBAAiB,EAAE,GAAG,EAAE,CAAC,IAAI;YAC7B,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI;YACtB,aAAa,EAAE,GAAG,EAAE,CAAC,KAAK;SAC1B,CAAC,CAAC;QACH,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;QACjF,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QACtB,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;QAC1B,eAAe,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC;QACtC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;QAC3B,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE;QACjB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE;KACgB,CAAA;IAEpC,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,MAAM,EAAE;QAC/C,OAAO,EAAE,KAAK;QACd,cAAc,EAAE,KAAK;QACrB,aAAa,EAAE,KAAK;KACpB,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG;QACd,EAAE,EAAE,MAAM;QACV,QAAQ;QACR,UAAU,EAAE;YACX,YAAY,EAAE,CAAC;YACf,gBAAgB,EAAE,CAAC;YACnB,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,gBAAgB,EAAE,CAAC;SACnB;QACD,QAAQ,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;QACnF,IAAI,gBAAgB;YACnB,OAAO,SAAS,CAAA;QACjB,CAAC;QACD,kBAAkB,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS;QACrC,WAAW,EAAE,CAAC,CAAU,EAAE,EAAE;YAC3B,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QACD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;QACxB,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE;QACxB,qBAAqB,EAAE,EAAE,CAAC,EAAE,EAAE;QAC9B,gBAAgB,EAAE,SAAS;QAC3B,aAAa,EAAE,CAAC,CAAS,EAAE,EAAE;YAC5B,UAAU,GAAG,CAAC,CAAA;QACf,CAAC;QACD,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE;QAC5B,aAAa,EAAE,EAAE,CAAC,EAAE,EAAE;KACtB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,qBAAqB,CAAC;QAC9C,QAAQ,EAAE,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,KAAc,EAAE,CAAC;QACxD,SAAS,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE;QAC/C,KAAK;QACL,MAAM;QACN,YAAY,EAAE,IAAI,YAAY,CAC7B;YACC,KAAK;YACL,KAAK,EAAE,MAAM;YACb,gBAAgB,EAAE,MAAM;YACxB,cAAc,EAAE,MAAM;YACtB,GAAG,EAAE,EAAE;YACP,WAAW,EAAE,IAAI,eAAe,EAAE,CAAC,MAAM;SACzC,EACD,aAAa,EACb,KAAK,IAAI,EAAE,GAAE,CAAC,EACd,GAAG,CACH;QACD,aAAa;QACb,eAAe,EAAE,IAAI,eAAe,EAAE;QACtC,GAAG;QACH,SAAS,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC;QACzB,YAAY,EAAE,QAAQ,CAAC,MAA0B,CAAC;QAClD,aAAa,EAAE,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;QACvD,WAAW,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE;QAClC,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;QACxD,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE;YACN,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;gBACvB,UAAU,EAAE,SAAS,IAAI,EAAE;gBAC3B,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,KAAK;gBAClB,UAAU,EAAE,gBAAgB;aAC5B,CAAC;SACF;KACQ,CAAC,CAAA;IAEX,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,CAAA;IAClC,IAAI,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAC3B,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAE1C,OAAO;QACN,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,WAAW;QACnB,QAAQ;QACR,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,gBAAgB,EAAE,SAAS;QAC3B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE;QAC1B,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;QACnB,KAAK,EAAE,YAAY,CAAC,QAAQ,EAAE;QAC9B,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,MAAM,EAAE,MAAM;KACI,CAAA;AACpB,CAAC;AAED,0DAA0D;AAC1D,MAAM,MAAM,GAAe;IAC1B;QACC,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,eAAe;QACtB,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC;KACzC;CACD,CAAA;AAED,MAAM,OAAO,GAAG;IACf,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE;CACnC,CAAA;AAED,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QACrC,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,CAAA;QAErC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAA;QAC/D,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,CAAC,gBAAgB,EAAE,EAAE,gBAAgB,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACnE,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;SAC1D,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,2CAA2C,EAAE,GAAG,EAAE;IAC1D,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC5D,uEAAuE;QACvE,oEAAoE;QACpE,mCAAmC;QACnC,MAAM,SAAS,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAA;QAE1F,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;SAC5D,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACrE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAChE,uEAAuE;QACvE,sEAAsE;QACtE,OAAO;QACP,MAAM,QAAQ,GAAG;YAChB,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAC/B,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE;SACnC,CAAA;QAED,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;SAC3D,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACnC,sEAAsE;QACtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC/E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,KAAK,GAAG,KAAK,EAAE,KAAgB,EAAE,EAAE;YACxC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;gBAClC,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;gBAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;aACxD,CAAC,CAAA;YACF,OAAO,MAAM,CAAC,IAAI,CAAA;QACnB,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC;YAC3B,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE;SACnC,CAAC,CAAA;QACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC;YAC5B,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;YAC/B,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YACjC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE;SACnC,CAAC,CAAA;QAEF,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClC,+DAA+D;QAC/D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/harness-honesty.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=harness-honesty.test.d.ts.map

package/dist/eval/__tests__/harness-honesty.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"harness-honesty.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/harness-honesty.test.ts"],"names":[],"mappings":""}

package/dist/eval/__tests__/harness-honesty.test.js

@@ -1,85 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { runExperiment } from '../experiment.js';
-import { containsScorer, stepBudgetScorer, trajectoryScorer } from '../scorers.js';
-/**
- * An eval harness that reports green on a broken suite is worse than no
- * harness: it converts "we do not know" into "we checked". Both defects
- * here did exactly that.
- */
-const emptyRun = () => ({
-    output: 'ok',
-    steps: [],
-    toolCalls: [],
-    totalTokens: 0,
-    totalCostUsd: 0,
-    durationMs: 0,
-});
-describe('a case whose run THREW cannot score above zero', () => {
-    it('scores 0 instead of a perfect step budget', async () => {
-        // `executeCase` catches the throw and returns an empty run, and an
-        // empty run walks into every scorer's happy path: 0 steps is within
-        // any allowance, and "no tools expected, none called" is a match. The
-        // suite reported 1.0 for a case that never ran.
-        const report = await runExperiment({
-            name: 'crash',
-            cases: [{ name: 'boom', input: 'x' }],
-            scorers: [stepBudgetScorer(5), trajectoryScorer()],
-            run: () => {
-                throw new Error('provider exploded');
-            },
-        });
-        expect(report.cases[0]?.scores['step-budget']?.score).toBe(0);
-        expect(report.cases[0]?.scores.trajectory?.score).toBe(0);
-        expect(report.mean).toBe(0);
-    });
-    it('says why, so the green-to-red change is diagnosable', async () => {
-        const report = await runExperiment({
-            name: 'crash',
-            cases: [{ name: 'boom', input: 'x' }],
-            scorers: [stepBudgetScorer(5)],
-            run: () => {
-                throw new Error('provider exploded');
-            },
-        });
-        expect(report.cases[0]?.scores['step-budget']?.reason).toContain('provider exploded');
-    });
-    it('still scores a healthy run normally', async () => {
-        const report = await runExperiment({
-            name: 'fine',
-            cases: [{ name: 'ok', input: 'x' }],
-            scorers: [stepBudgetScorer(5)],
-            run: () => Promise.resolve(emptyRun()),
-        });
-        expect(report.cases[0]?.scores['step-budget']?.score).toBe(1);
-    });
-});
-describe('two scorers cannot silently collapse into one', () => {
-    it('refuses a duplicate scorer name rather than overwriting', async () => {
-        // Scores are keyed by name, so two `containsScorer(...)` instances —
-        // both called 'contains' — meant the second overwrote the first and
-        // the case mean was computed over the wrong denominator. With 'a'
-        // scoring 0 and 'b' scoring 1 the suite reported 1.0 where the honest
-        // answer is 0.5.
-        await expect(runExperiment({
-            name: 'dupe',
-            cases: [{ name: 'c', input: 'x' }],
-            scorers: [containsScorer('a'), containsScorer('b')],
-            run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
-        })).rejects.toThrow(/Duplicate scorer name "contains"/);
-    });
-    it('accepts distinct names', async () => {
-        const report = await runExperiment({
-            name: 'ok',
-            cases: [{ name: 'c', input: 'x' }],
-            scorers: [
-                { ...containsScorer('a'), name: 'contains-a' },
-                { ...containsScorer('b'), name: 'contains-b' },
-            ],
-            run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
-        });
-        expect(report.cases[0]?.scores['contains-a']?.score).toBe(0);
-        expect(report.cases[0]?.scores['contains-b']?.score).toBe(1);
-        expect(report.cases[0]?.mean).toBe(0.5);
-    });
-});
-//# sourceMappingURL=harness-honesty.test.js.map

package/dist/eval/__tests__/harness-honesty.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"harness-honesty.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/harness-honesty.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAGlF;;;;GAIG;AAEH,MAAM,QAAQ,GAAG,GAAY,EAAE,CAAC,CAAC;IAChC,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,EAAE;IACT,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,CAAC;CACb,CAAC,CAAA;AAEF,QAAQ,CAAC,gDAAgD,EAAE,GAAG,EAAE;IAC/D,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,mEAAmE;QACnE,oEAAoE;QACpE,sEAAsE;QACtE,gDAAgD;QAChD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC;YAClD,GAAG,EAAE,GAAG,EAAE;gBACT,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;YACrC,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC9B,GAAG,EAAE,GAAG,EAAE;gBACT,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;YACrC,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAA;IACtF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC9B,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SACtC,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9D,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,+CAA+C,EAAE,GAAG,EAAE;IAC9D,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACxE,qEAAqE;QACrE,oEAAoE;QACpE,kEAAkE;QAClE,sEAAsE;QACtE,iBAAiB;QACjB,MAAM,MAAM,CACX,aAAa,CAAC;YACb,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,OAAO,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC;YACnD,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,OAAO,EAAE;gBACR,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE;gBAC9C,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE;aAC9C;YACD,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/harness.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=harness.test.d.ts.map

package/dist/eval/__tests__/harness.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"harness.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/harness.test.ts"],"names":[],"mappings":""}

package/dist/eval/__tests__/harness.test.js

@@ -1,207 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { formatReport, runExperiment } from '../experiment.js';
-import { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from '../scorers.js';
-/**
- * There was no evaluation harness of any kind — no dataset, no scorer, no
- * judge, no trajectory assertion. So the constants that govern namzu's
- * most load-bearing behavior (`search_tools` top-k, the compaction
- * threshold, six state-list caps, every builtin tool description) were
- * tunable but not measurable: change one and the only way to learn the
- * agent now takes four tool calls where it took one was for a user to
- * hit it.
- */
-function run(overrides = {}) {
-    return {
-        output: 'done',
-        steps: [],
-        toolCalls: [],
-        stopReason: 'end_turn',
-        totalTokens: 0,
-        totalCostUsd: 0,
-        durationMs: 0,
-        ...overrides,
-    };
-}
-const CASE = { name: 'c', input: null };
-describe('trajectoryScorer', () => {
-    // `Scorer.score` may be async (a model-graded judge is just an async
-    // predicate), so every call site awaits even when the built-in is sync.
-    const score = async (expectedTools, toolCalls) => await trajectoryScorer().score(run({ toolCalls }), { ...CASE, expectedTools });
-    it('scores an exact trajectory 1', async () => {
-        expect((await score(['read', 'edit'], ['read', 'edit'])).score).toBe(1);
-    });
-    it('distinguishes ORDER — a set-based score could not', async () => {
-        // Reading before editing is not the same run as editing then reading.
-        const reversed = await score(['read', 'edit'], ['edit', 'read']);
-        expect(reversed.score).toBeLessThan(1);
-        expect(reversed.score).toBeGreaterThan(0);
-    });
-    it('penalises an extra call through precision', async () => {
-        const wasteful = await score(['read'], ['read', 'read', 'read']);
-        expect(wasteful.score).toBeLessThan(1);
-        expect(wasteful.details?.recall).toBe(1);
-        expect(wasteful.details?.precision).toBeLessThan(1);
-    });
-    it('penalises a missing call through recall', async () => {
-        const lazy = await score(['read', 'edit', 'bash'], ['read']);
-        expect(lazy.details?.precision).toBe(1);
-        expect(lazy.details?.recall).toBeLessThan(1);
-    });
-    it('separates "wasteful" from "skipped" — a final-answer score collapses them', async () => {
-        const wasteful = await score(['read', 'edit'], ['read', 'ls', 'edit']);
-        const skipped = await score(['read', 'edit'], ['read']);
-        expect(wasteful.score).not.toBe(skipped.score);
-    });
-    it('scores 1 when no tools were expected and none ran', async () => {
-        expect((await score([], [])).score).toBe(1);
-    });
-    it('scores 0 when tools were expected and none ran', async () => {
-        const s = await score(['read'], []);
-        expect(s.score).toBe(0);
-        expect(s.reason).toContain('no tools were called');
-    });
-    it('scores 0 when none were expected and some ran', async () => {
-        expect((await score([], ['bash'])).score).toBe(0);
-    });
-    it('always explains itself', async () => {
-        expect((await score(['read'], ['edit'])).reason.length).toBeGreaterThan(0);
-    });
-});
-describe('completionScorer', () => {
-    it('accepts a clean settle', async () => {
-        expect((await completionScorer().score(run({ stopReason: 'end_turn' }), CASE)).score).toBe(1);
-    });
-    it('rejects a budget-forced stop', async () => {
-        const s = await completionScorer().score(run({ stopReason: 'max_iterations' }), CASE);
-        expect(s.score).toBe(0);
-        expect(s.reason).toContain('max_iterations');
-    });
-    it('reports the error when the run threw', async () => {
-        const s = await completionScorer().score(run({ error: 'boom' }), CASE);
-        expect(s.score).toBe(0);
-        expect(s.reason).toContain('boom');
-    });
-});
-describe('stepBudgetScorer', () => {
-    const withSteps = (n) => run({ steps: Array.from({ length: n }, () => ({})) });
-    it('passes within budget', async () => {
-        expect((await stepBudgetScorer(5).score(withSteps(3), CASE)).score).toBe(1);
-    });
-    it('degrades proportionally past it, rather than snapping to zero', async () => {
-        // A run that took 6 steps against a 5 allowance is worse than one
-        // that took 20; a binary score cannot say so.
-        const slight = (await stepBudgetScorer(5).score(withSteps(6), CASE)).score;
-        const severe = (await stepBudgetScorer(5).score(withSteps(20), CASE)).score;
-        expect(slight).toBeGreaterThan(severe);
-        expect(slight).toBeLessThan(1);
-    });
-});
-describe('containsScorer', () => {
-    it('scores the fraction found and names what is missing', async () => {
-        const s = await containsScorer('alpha', 'beta').score(run({ output: 'alpha only' }), CASE);
-        expect(s.score).toBe(0.5);
-        expect(s.reason).toContain('beta');
-    });
-});
-describe('runExperiment', () => {
-    const cases = [
-        { name: 'good', input: 'a', expectedTools: ['read'] },
-        { name: 'bad', input: 'b', expectedTools: ['read', 'edit'] },
-    ];
-    it('scores every case and aggregates', async () => {
-        const report = await runExperiment({
-            name: 'demo',
-            cases,
-            scorers: [trajectoryScorer()],
-            run: async (input) => run({ toolCalls: input === 'a' ? ['read'] : ['bash'] }),
-        });
-        expect(report.cases).toHaveLength(2);
-        expect(report.passed).toBe(1);
-        expect(report.failed).toBe(1);
-        expect(report.byScorer.trajectory).toBeCloseTo(0.5, 5);
-    });
-    it('a throwing case is a RESULT, not a crash', async () => {
-        // A suite whose first broken case aborts tells you nothing about the
-        // other forty.
-        const report = await runExperiment({
-            name: 'demo',
-            cases,
-            scorers: [completionScorer()],
-            run: async (input) => {
-                if (input === 'a')
-                    throw new Error('exploded');
-                return run();
-            },
-        });
-        expect(report.cases).toHaveLength(2);
-        expect(report.cases[0]?.run.error).toBe('exploded');
-        expect(report.cases[1]?.passed).toBe(true);
-    });
-    it('a throwing scorer scores zero with the throw as its reason', async () => {
-        const report = await runExperiment({
-            name: 'demo',
-            cases: [cases[0]],
-            scorers: [
-                customScorer('boom', () => {
-                    throw new Error('scorer broke');
-                }),
-            ],
-            run: async () => run(),
-        });
-        expect(report.cases[0]?.scores.boom?.score).toBe(0);
-        expect(report.cases[0]?.scores.boom?.reason).toContain('scorer broke');
-    });
-    it('preserves case order under concurrency', async () => {
-        const many = Array.from({ length: 8 }, (_, i) => ({
-            name: `case-${i}`,
-            input: i,
-        }));
-        const report = await runExperiment({
-            name: 'demo',
-            cases: many,
-            concurrency: 4,
-            scorers: [completionScorer()],
-            run: async (input) => {
-                await new Promise((r) => setTimeout(r, (8 - input) * 2));
-                return run();
-            },
-        });
-        expect(report.cases.map((c) => c.case)).toEqual(many.map((c) => c.name));
-    });
-    it('honours a per-case scorer override', async () => {
-        const report = await runExperiment({
-            name: 'demo',
-            cases: [{ name: 'special', input: null, scorers: [containsScorer('xyz')] }],
-            scorers: [completionScorer()],
-            run: async () => run({ output: 'no match here' }),
-        });
-        expect(Object.keys(report.cases[0]?.scores ?? {})).toEqual(['contains']);
-    });
-    it('respects a partial pass threshold', async () => {
-        const report = await runExperiment({
-            name: 'demo',
-            cases: [{ name: 'partial', input: null, expectedTools: ['read', 'edit'] }],
-            scorers: [trajectoryScorer()],
-            passThreshold: 0.5,
-            run: async () => run({ toolCalls: ['read'] }),
-        });
-        expect(report.passed).toBe(1);
-    });
-});
-describe('formatReport', () => {
-    it('prints scorer reasons for failures, not just numbers', async () => {
-        // A CI log that says "0.62" sends someone back to reproduce it by hand.
-        const report = await runExperiment({
-            name: 'demo',
-            cases: [{ name: 'bad', input: null, expectedTools: ['read', 'edit'] }],
-            scorers: [trajectoryScorer()],
-            run: async () => run({ toolCalls: ['bash'] }),
-        });
-        const text = formatReport(report);
-        expect(text).toContain('0/1 passed');
-        expect(text).toContain('✗ bad');
-        expect(text).toContain('trajectory:');
-        expect(text).toContain('expected read → edit');
-    });
-});
-//# sourceMappingURL=harness.test.js.map

package/dist/eval/__tests__/harness.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"harness.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/harness.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAE7C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,eAAe,CAAA;AAGtB;;;;;;;;GAQG;AAEH,SAAS,GAAG,CAAC,YAA8B,EAAE;IAC5C,OAAO;QACN,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,UAAU;QACtB,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,UAAU,EAAE,CAAC;QACb,GAAG,SAAS;KACZ,CAAA;AACF,CAAC;AAED,MAAM,IAAI,GAAa,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;AAEjD,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,qEAAqE;IACrE,wEAAwE;IACxE,MAAM,KAAK,GAAG,KAAK,EAAE,aAAuB,EAAE,SAAmB,EAAE,EAAE,CACpE,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,CAAC,CAAA;IAE/E,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACxE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QAChE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QAChE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACtC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;QAC5D,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;QAC1F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QACtE,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;QACvD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAA;QACnC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC3E,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACvC,MAAM,CAAC,CAAC,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9F,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,CAAC,GAAG,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QACrF,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAA;IAC7C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,CAAC,GAAG,MAAM,gBAAgB,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QACtE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAU,CAAC,EAAE,CAAC,CAAA;IAE/F,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC9E,kEAAkE;QAClE,8CAA8C;QAC9C,MAAM,MAAM,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,MAAM,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAA;QAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QAC1F,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACzB,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,MAAM,KAAK,GAAe;QACzB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,MAAM,CAAC,EAAE;QACrD,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;KAC5D,CAAA;IAED,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACzD,qEAAqE;QACrE,eAAe;QACf,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACpB,IAAI,KAAK,KAAK,GAAG;oBAAE,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAA;gBAC9C,OAAO,GAAG,EAAE,CAAA;YACb,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,CAAa,CAAC;YAC7B,OAAO,EAAE;gBACR,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE;oBACzB,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAA;gBAChC,CAAC,CAAC;aACF;YACD,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,EAAE;SACtB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;IACvE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,IAAI,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,QAAQ,CAAC,EAAE;YACjB,KAAK,EAAE,CAAC;SACR,CAAC,CAAC,CAAA;QACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,GAAI,KAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBACpE,OAAO,GAAG,EAAE,CAAA;YACb,CAAC;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACzE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;SACjD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAA;IACzE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1E,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,aAAa,EAAE,GAAG;YAClB,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7C,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,wEAAwE;QACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YACtE,OAAO,EAAE,CAAC,gBAAgB,EAAE,CAAC;YAC7B,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7C,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAA;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;QAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACrC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/judge.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=judge.test.d.ts.map

package/dist/eval/__tests__/judge.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"judge.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":""}

package/dist/eval/__tests__/judge.test.js

@@ -1,255 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { runExperiment } from '../experiment.js';
-import { judgeScorer } from '../judge.js';
-/**
- * Every other scorer is a pure function over the run, which is what makes
- * them reproducible and what makes them unable to say whether an answer is
- * GOOD. A judge closes that, and brings a failure mode none of the others
- * have: it can fail to answer at all. A failed measurement scored zero
- * reads exactly like a regression, so the distinction is what most of this
- * file is about.
- */
-function fakeProvider(reply, tokens = 42) {
-    const seen = [];
-    const provider = {
-        id: 'fake',
-        name: 'fake',
-        capabilities: {
-            supportsTools: false,
-            supportsStreaming: true,
-            supportsFunctionCalling: false,
-        },
-        seen,
-        async *chatStream(params) {
-            seen.push(params);
-            if (typeof reply !== 'string')
-                reply();
-            yield { id: 'j', delta: { content: reply } };
-            yield {
-                id: 'j',
-                delta: {},
-                finishReason: 'stop',
-                usage: {
-                    promptTokens: tokens,
-                    completionTokens: 0,
-                    totalTokens: tokens,
-                    cachedTokens: 0,
-                    cacheWriteTokens: 0,
-                },
-            };
-        },
-        async listModels() {
-            return [];
-        },
-        async healthCheck() {
-            return true;
-        },
-    };
-    return provider;
-}
-const RUN = {
-    output: 'The capital of France is Paris.',
-    steps: [],
-    toolCalls: ['search'],
-    stopReason: 'end_turn',
-    totalTokens: 100,
-    totalCostUsd: 0,
-    durationMs: 10,
-};
-const CASE = { name: 'capital', input: 'What is the capital of France?' };
-const RUBRIC = 'The answer names the correct city.';
-describe('grading', () => {
-    it('normalises the grade onto 0..1 against the scale it asked for', async () => {
-        const scorer = judgeScorer({
-            provider: fakeProvider('{"grade": 3, "reason": "names the right city"}'),
-            model: 'm',
-            rubric: RUBRIC,
-            scale: 4,
-        });
-        const score = await scorer.score(RUN, CASE);
-        expect(score.score).toBe(0.75);
-        expect(score.reason).toBe('names the right city');
-        expect(score.details?.grade).toBe(3);
-    });
-    it('reads a verdict wrapped in prose or fences', async () => {
-        const scorer = judgeScorer({
-            provider: fakeProvider('Here is my grade:\n```json\n{"grade": 4, "reason": "correct"}\n```'),
-            model: 'm',
-            rubric: RUBRIC,
-        });
-        expect((await scorer.score(RUN, CASE)).score).toBe(1);
-    });
-    it('handles a brace inside the reason string', async () => {
-        const scorer = judgeScorer({
-            provider: fakeProvider('{"grade": 2, "reason": "mentions {city} placeholder"}'),
-            model: 'm',
-            rubric: RUBRIC,
-            scale: 4,
-        });
-        // A naive scan to the first `}` would cut the object in half here.
-        expect((await scorer.score(RUN, CASE)).score).toBe(0.5);
-    });
-    it('carries what the judging itself cost', async () => {
-        const scorer = judgeScorer({
-            provider: fakeProvider('{"grade": 4, "reason": "ok"}', 1234),
-            model: 'm',
-            rubric: RUBRIC,
-        });
-        // A judge is the most expensive scorer there is; a bill nobody can
-        // attribute is a bill nobody controls.
-        expect((await scorer.score(RUN, CASE)).details?.judgeTokens).toBe(1234);
-    });
-    it('grades the same run the same way twice', async () => {
-        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
-        // Sampling noise is indistinguishable from a regression, so the
-        // judge does not sample.
-        expect(provider.seen[0]?.temperature).toBe(0);
-    });
-});
-describe('what the judge is shown', () => {
-    const promptOf = (provider) => String(provider.seen[0]?.messages[0]?.content);
-    it('shows the rubric, the task and the answer', async () => {
-        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
-        const prompt = promptOf(provider);
-        expect(prompt).toContain(RUBRIC);
-        expect(prompt).toContain('What is the capital of France?');
-        expect(prompt).toContain('The capital of France is Paris.');
-    });
-    it('withholds the trajectory unless asked', async () => {
-        const off = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({ provider: off, model: 'm', rubric: RUBRIC }).score(RUN, CASE);
-        expect(promptOf(off)).not.toContain('TOOLS CALLED');
-        const on = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({
-            provider: on,
-            model: 'm',
-            rubric: RUBRIC,
-            includeTrajectory: true,
-        }).score(RUN, CASE);
-        expect(promptOf(on)).toContain('search');
-    });
-    it('says when it truncated the answer', async () => {
-        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({
-            provider,
-            model: 'm',
-            rubric: RUBRIC,
-            maxOutputChars: 10,
-        }).score({ ...RUN, output: 'x'.repeat(500) }, CASE);
-        // A judge shown a silently cut answer marks it down for stopping
-        // mid-sentence, which scores our truncation rather than the run.
-        expect(promptOf(provider)).toContain('cut at 10 characters by the harness');
-    });
-    it('offers the reference answer when the case has one', async () => {
-        const provider = fakeProvider('{"grade": 4, "reason": "ok"}');
-        await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, {
-            ...CASE,
-            expected: 'Paris',
-        });
-        expect(promptOf(provider)).toContain('Paris');
-    });
-});
-describe('a judge that cannot judge', () => {
-    const failing = (reply) => judgeScorer({ provider: fakeProvider(reply), model: 'm', rubric: RUBRIC, scale: 4 });
-    it('throws on a reply with no JSON at all', async () => {
-        await expect(failing('I think it was pretty good!').score(RUN, CASE)).rejects.toThrow(/no JSON/);
-    });
-    it('throws on a grade outside the scale it was given', async () => {
-        // Clamping would turn a judge that misread the scale into a
-        // confident score, and a judge that misread the scale did not apply
-        // the rubric either.
-        await expect(failing('{"grade": 9, "reason": "great"}').score(RUN, CASE)).rejects.toThrow(/outside the 0\.\.4 scale/);
-    });
-    it('throws on a non-numeric grade', async () => {
-        await expect(failing('{"grade": "good", "reason": "x"}').score(RUN, CASE)).rejects.toThrow(/no numeric grade/);
-    });
-    it('throws on an unterminated object', async () => {
-        await expect(failing('{"grade": 3, "reason": "cut off').score(RUN, CASE)).rejects.toThrow(/unterminated/);
-    });
-    it('refuses to exist without a rubric', () => {
-        expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: '   ' })).toThrow(/rubric is required/);
-    });
-    it('refuses a nonsense scale', () => {
-        expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: RUBRIC, scale: 0 })).toThrow(/positive integer/);
-    });
-});
-describe('a broken judge is not a bad run', () => {
-    const dataset = {
-        name: 'suite',
-        cases: [CASE],
-        run: async () => RUN,
-    };
-    it('reports the case as inconclusive rather than failed', async () => {
-        const report = await runExperiment({
-            ...dataset,
-            scorers: [
-                judgeScorer({
-                    provider: fakeProvider(() => {
-                        throw new Error('429 rate limited');
-                    }),
-                    model: 'm',
-                    rubric: RUBRIC,
-                }),
-            ],
-        });
-        expect(report.inconclusive).toBe(1);
-        expect(report.failed).toBe(0);
-        expect(report.passed).toBe(0);
-        expect(report.cases[0]?.scores.judge?.unavailable).toBe(true);
-        expect(report.cases[0]?.scores.judge?.reason).toContain('429');
-    });
-    it('keeps an unavailable score out of the mean entirely', async () => {
-        const report = await runExperiment({
-            ...dataset,
-            scorers: [
-                { name: 'always-one', score: () => ({ score: 1, reason: 'fine' }) },
-                judgeScorer({
-                    provider: fakeProvider(() => {
-                        throw new Error('down');
-                    }),
-                    model: 'm',
-                    rubric: RUBRIC,
-                }),
-            ],
-        });
-        // Not 0.5. A measurement that never happened is not a zero, and
-        // averaging it in would report a regression the harness caused.
-        expect(report.cases[0]?.mean).toBe(1);
-        expect(report.cases[0]?.status).toBe('passed');
-        expect(report.mean).toBe(1);
-    });
-    it('omits a scorer that was never available from the per-dimension means', async () => {
-        const report = await runExperiment({
-            ...dataset,
-            scorers: [
-                judgeScorer({
-                    provider: fakeProvider(() => {
-                        throw new Error('down');
-                    }),
-                    model: 'm',
-                    rubric: RUBRIC,
-                }),
-            ],
-        });
-        // A dimension with no measurements is not a dimension that scored 0.
-        expect(report.byScorer.judge).toBeUndefined();
-    });
-    it('still fails a case the judge actually judged badly', async () => {
-        const report = await runExperiment({
-            ...dataset,
-            scorers: [
-                judgeScorer({
-                    provider: fakeProvider('{"grade": 0, "reason": "names the wrong city"}'),
-                    model: 'm',
-                    rubric: RUBRIC,
-                }),
-            ],
-        });
-        expect(report.failed).toBe(1);
-        expect(report.inconclusive).toBe(0);
-        expect(report.cases[0]?.scores.judge?.unavailable).toBeUndefined();
-    });
-});
-//# sourceMappingURL=judge.test.js.map

package/dist/eval/__tests__/judge.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"judge.test.js","sourceRoot":"","sources":["../../../src/eval/__tests__/judge.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAI7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAGzC;;;;;;;GAOG;AAEH,SAAS,YAAY,CAAC,KAA6B,EAAE,MAAM,GAAG,EAAE;IAC/D,MAAM,IAAI,GAA2B,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAmD;QAChE,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,MAAM;QACZ,YAAY,EAAE;YACb,aAAa,EAAE,KAAK;YACpB,iBAAiB,EAAE,IAAI;YACvB,uBAAuB,EAAE,KAAK;SAC9B;QACD,IAAI;QACJ,KAAK,CAAC,CAAC,UAAU,CAAC,MAA4B;YAC7C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACjB,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,KAAK,EAAE,CAAA;YACtC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAe,EAAE,EAAE,CAAA;YACtD,MAAM;gBACL,EAAE,EAAE,GAAG;gBACP,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE,MAAM;gBACpB,KAAK,EAAE;oBACN,YAAY,EAAE,MAAM;oBACpB,gBAAgB,EAAE,CAAC;oBACnB,WAAW,EAAE,MAAM;oBACnB,YAAY,EAAE,CAAC;oBACf,gBAAgB,EAAE,CAAC;iBACnB;aACD,CAAA;QACF,CAAC;QACD,KAAK,CAAC,UAAU;YACf,OAAO,EAAE,CAAA;QACV,CAAC;QACD,KAAK,CAAC,WAAW;YAChB,OAAO,IAAI,CAAA;QACZ,CAAC;KACD,CAAA;IACD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,MAAM,GAAG,GAAY;IACpB,MAAM,EAAE,iCAAiC;IACzC,KAAK,EAAE,EAAE;IACT,SAAS,EAAE,CAAC,QAAQ,CAAC;IACrB,UAAU,EAAE,UAAU;IACtB,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,EAAE;CACd,CAAA;AAED,MAAM,IAAI,GAAa,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAA;AAEnF,MAAM,MAAM,GAAG,oCAAoC,CAAA;AAEnD,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,gDAAgD,CAAC;YACxE,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,CAAC;SACR,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAE3C,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;QACjD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,oEAAoE,CAAC;YAC5F,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;SACd,CAAC,CAAA;QACF,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,uDAAuD,CAAC;YAC/E,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,CAAC;SACR,CAAC,CAAA;QACF,mEAAmE;QACnE,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG,WAAW,CAAC;YAC1B,QAAQ,EAAE,YAAY,CAAC,8BAA8B,EAAE,IAAI,CAAC;YAC5D,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;SACd,CAAC,CAAA;QACF,mEAAmE;QACnE,uCAAuC;QACvC,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxE,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAE3D,CAAA;QACD,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAC5E,gEAAgE;QAChE,yBAAyB;QACzB,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACxC,MAAM,QAAQ,GAAG,CAAC,QAAqB,EAAE,EAAE,CAC1C,MAAM,CAAE,QAAwD,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IAEhG,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAE5E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACjC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;QAChC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAA;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,iCAAiC,CAAC,CAAA;IAC5D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,GAAG,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QACxD,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACjF,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;QAEnD,MAAM,EAAE,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QACvD,MAAM,WAAW,CAAC;YACjB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,iBAAiB,EAAE,IAAI;SACvB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACnB,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC;YACjB,QAAQ;YACR,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,MAAM;YACd,cAAc,EAAE,EAAE;SAClB,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAEnD,iEAAiE;QACjE,iEAAiE;QACjE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,qCAAqC,CAAC,CAAA;IAC5E,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,QAAQ,GAAG,YAAY,CAAC,8BAA8B,CAAC,CAAA;QAC7D,MAAM,WAAW,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACtE,GAAG,IAAI;YACP,QAAQ,EAAE,OAAO;SACjB,CAAC,CAAA;QACF,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IAC1C,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CACjC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAErF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACjG,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QACjE,4DAA4D;QAC5D,oEAAoE;QACpE,qBAAqB;QACrB,MAAM,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACxF,0BAA0B,CAC1B,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACzF,kBAAkB,CAClB,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACxF,cAAc,CACd,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAC7F,oBAAoB,CACpB,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACnC,MAAM,CAAC,GAAG,EAAE,CACX,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CACnF,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAChD,MAAM,OAAO,GAAG;QACf,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,GAAG,EAAE,KAAK,IAAsB,EAAE,CAAC,GAAG;KACtC,CAAA;IAED,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;oBACpC,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAC/D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE;gBACnE,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;oBACxB,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,gEAAgE;QAChE,gEAAgE;QAChE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACrF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,GAAG,EAAE;wBAC3B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAA;oBACxB,CAAC,CAAC;oBACF,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,qEAAqE;QACrE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC;YAClC,GAAG,OAAO;YACV,OAAO,EAAE;gBACR,WAAW,CAAC;oBACX,QAAQ,EAAE,YAAY,CAAC,gDAAgD,CAAC;oBACxE,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,MAAM;iBACd,CAAC;aACF;SACD,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,aAAa,EAAE,CAAA;IACnE,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/eval/__tests__/scorer-gates.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=scorer-gates.test.d.ts.map

package/dist/eval/__tests__/scorer-gates.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scorer-gates.test.d.ts","sourceRoot":"","sources":["../../../src/eval/__tests__/scorer-gates.test.ts"],"names":[],"mappings":""}