jevkit-bench 0.1.0

package/README.md

@@ -0,0 +1,64 @@
+# jevkit-bench
+
+Score a labeled Jev suite for accuracy **and** cost, and compare two runs.
+
+Any one number alone is easy to win. Accuracy without cost hides that you spent
+ten times the tokens; cost without accuracy hides that you broke the task. This
+reports them together.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-bench
+```
+
+## Use
+
+```python
+from jevkit_core import read_records
+from jevkit_bench import compare_suites, score_records
+
+suite = score_records(read_records("suite.jevl"))
+print(suite.summary())
+
+for failure in suite.failures()[:5]:
+    print(failure.question_id, failure.predicted, "should be", failure.label)
+```
+
+`failures()` sorts by probability descending, so the most confident wrong answers
+come first. Those are the interesting bugs: a wrong answer at 0.35 is the model
+telling you it was unsure, while a wrong answer at 0.98 is a question that needs
+rewriting.
+
+## Comparing runs
+
+```python
+comparison = compare_suites(baseline, candidate)
+print(comparison.summary())
+print(comparison.regressions)   # right before, wrong now
+```
+
+Aggregate accuracy can hold perfectly steady while the set of things you get
+right churns underneath. That matters when a specific case is the one you
+promised someone would work, so regressions and fixes are tracked individually
+rather than netted off.
+
+## CLI
+
+```bash
+jevkit-bench suite.jevl
+jevkit-bench suite.jevl --baseline last-week.jevl
+jevkit-bench suite.jevl --min-accuracy 0.90              # CI gate
+jevkit-bench suite.jevl --baseline last-week.jevl --max-regressions 0
+jevkit-bench suite.jevl --show-failures 10
+```
+
+## Cost
+
+Computed from input tokens at $0.042 per million, Jev's published price. Output
+tokens are free on Jev, so they are reported but never billed. Override with
+`--price-per-mtok` if your plan differs.
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+/** `jevkit-bench` command line interface. */
+export declare const EXIT_OK = 0;
+export declare const EXIT_FAILED_GATE = 1;
+export declare const EXIT_USAGE = 2;
+export declare function main(argv?: string[]): number;
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,6CAA6C;AAM7C,eAAO,MAAM,OAAO,IAAI,CAAC;AACzB,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,UAAU,IAAI,CAAC;AAe5B,wBAAgB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,MAAM,CA+GnE"}

package/dist/cli.js

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/** `jevkit-bench` command line interface. */
+import { RecordFormatError, readRecords } from "jevkit-core";
+import { PRICE_PER_MTOK, compareSuites, scoreRecords } from "./score.js";
+export const EXIT_OK = 0;
+export const EXIT_FAILED_GATE = 1;
+export const EXIT_USAGE = 2;
+const USAGE = `usage: jevkit-bench <suite.jevl> [options]
+
+Score a labeled .jevl suite for accuracy and cost, and compare two runs of it.
+Never calls the API.
+
+  --baseline FILE        a previous run, to compare against
+  --question ID          restrict to this question id (repeatable)
+  --price-per-mtok F     input price per million tokens (default ${PRICE_PER_MTOK})
+  --min-accuracy F       exit non-zero below this
+  --max-regressions N    exit non-zero above this many (needs --baseline)
+  --show-failures N      print the N most confident wrong answers
+  --format text|json`;
+export function main(argv = process.argv.slice(2)) {
+    const files = [];
+    const questions = [];
+    let baselinePath = null;
+    let pricePerMtok = PRICE_PER_MTOK;
+    let minAccuracy = null;
+    let maxRegressions = null;
+    let showFailures = 0;
+    let format = "text";
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        switch (arg) {
+            case "--baseline":
+                baselinePath = String(argv[++i]);
+                break;
+            case "--question":
+                questions.push(String(argv[++i]));
+                break;
+            case "--price-per-mtok":
+                pricePerMtok = Number(argv[++i]);
+                break;
+            case "--min-accuracy":
+                minAccuracy = Number(argv[++i]);
+                break;
+            case "--max-regressions":
+                maxRegressions = Number(argv[++i]);
+                break;
+            case "--show-failures":
+                showFailures = Number(argv[++i]);
+                break;
+            case "--format": {
+                const v = argv[++i];
+                if (v !== "text" && v !== "json") {
+                    console.error("jevkit-bench: --format expects text or json");
+                    return EXIT_USAGE;
+                }
+                format = v;
+                break;
+            }
+            case "-h":
+            case "--help":
+                console.log(USAGE);
+                return EXIT_OK;
+            default:
+                if (arg.startsWith("--")) {
+                    console.error(`jevkit-bench: unknown option ${arg}`);
+                    return EXIT_USAGE;
+                }
+                files.push(arg);
+        }
+    }
+    if (files.length !== 1) {
+        console.error(USAGE);
+        return EXIT_USAGE;
+    }
+    if (maxRegressions !== null && !baselinePath) {
+        console.error("jevkit-bench: --max-regressions needs --baseline");
+        return EXIT_USAGE;
+    }
+    const options = {
+        questionIds: questions.length ? questions : undefined,
+        pricePerMtok,
+    };
+    let suite, baseline;
+    try {
+        suite = scoreRecords(readRecords(files[0]), options);
+        baseline = baselinePath ? scoreRecords(readRecords(baselinePath), options) : null;
+    }
+    catch (err) {
+        const message = err instanceof RecordFormatError ? err.message : err.message;
+        console.error(`jevkit-bench: ${message}`);
+        return EXIT_USAGE;
+    }
+    if (!suite.count) {
+        console.error("jevkit-bench: nothing scored. Records need a 'label' object keyed by question id.");
+        return EXIT_USAGE;
+    }
+    const comparison = baseline ? compareSuites(baseline, suite) : null;
+    if (format === "json") {
+        const payload = { ...suite.toJSON() };
+        if (comparison) {
+            payload["comparison"] = {
+                baselineAccuracy: comparison.baseline.accuracy,
+                accuracyDelta: comparison.accuracyDelta,
+                costDelta: comparison.costDelta,
+                regressions: comparison.regressions,
+                fixes: comparison.fixes,
+            };
+        }
+        console.log(JSON.stringify(payload, null, 2));
+    }
+    else {
+        console.log(suite.summary());
+        if (comparison) {
+            console.log();
+            console.log(comparison.summary());
+            if (comparison.regressions.length) {
+                console.log("\nregressions:");
+                for (const [rid, qid] of comparison.regressions.slice(0, 20)) {
+                    console.log(`  ${rid.slice(7, 19)}  ${qid}`);
+                }
+            }
+        }
+        if (showFailures) {
+            const failures = suite.failures().slice(0, showFailures);
+            if (failures.length) {
+                console.log(`\nmost confident wrong answers (${failures.length}):`);
+                for (const f of failures) {
+                    console.log(`  ${f.requestId.slice(7, 19)}  ${f.questionId}: said ` +
+                        `${JSON.stringify(f.predicted)}, label ${JSON.stringify(f.label)}, ` +
+                        `p=${f.probability.toFixed(3)}`);
+                }
+            }
+        }
+    }
+    if (minAccuracy !== null && suite.accuracy < minAccuracy) {
+        console.error(`jevkit-bench: accuracy ${suite.accuracy.toFixed(4)} below ${minAccuracy}`);
+        return EXIT_FAILED_GATE;
+    }
+    if (maxRegressions !== null && comparison && comparison.regressions.length > maxRegressions) {
+        console.error(`jevkit-bench: ${comparison.regressions.length} regressions exceed ${maxRegressions}`);
+        return EXIT_FAILED_GATE;
+    }
+    return EXIT_OK;
+}
+const invokedDirectly = process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`;
+if (invokedDirectly)
+    process.exit(main());
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,6CAA6C;AAE7C,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEzE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC;AACzB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,MAAM,KAAK,GAAG;;;;;;;mEAOqD,cAAc;;;;qBAI5D,CAAC;AAEtB,MAAM,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,YAAY,GAAG,cAAc,CAAC;IAClC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAoB,MAAM,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,YAAY;gBAAE,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC3D,KAAK,YAAY;gBAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC5D,KAAK,kBAAkB;gBAAE,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACjE,KAAK,gBAAgB;gBAAE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC9D,KAAK,mBAAmB;gBAAE,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACpE,KAAK,iBAAiB;gBAAE,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAChE,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACpB,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBACtH,MAAM,GAAG,CAAC,CAAC;gBAAC,MAAM;YACpB,CAAC;YACD,KAAK,IAAI,CAAC;YAAC,KAAK,QAAQ;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAAC,OAAO,OAAO,CAAC;YAC7D;gBACE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBACtG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAAC,OAAO,UAAU,CAAC;IAAC,CAAC;IACpE,IAAI,cAAc,KAAK,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QAClE,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,OAAO,GAAG;QACd,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QACrD,YAAY;KACb,CAAC;IAEF,IAAI,KAAK,EAAE,QAAQ,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,OAAO,CAAC,CAAC;QACtD,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAE,GAAa,CAAC,OAAO,CAAC;QACxF,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,KAAK,CACX,mFAAmF,CACpF,CAAC;QACF,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpE,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,OAAO,GAA4B,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QAC/D,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,YAAY,CAAC,GAAG;gBACtB,gBAAgB,EAAE,UAAU,CAAC,QAAQ,CAAC,QAAQ;gBAC9C,aAAa,EAAE,UAAU,CAAC,aAAa;gBACvC,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,KAAK,EAAE,UAAU,CAAC,KAAK;aACxB,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7B,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;YAClC,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;oBAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;YACzD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO,CAAC,GAAG,CAAC,mCAAmC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;gBACpE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,UAAU,SAAS;wBACvD,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI;wBACpE,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAChC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,QAAQ,GAAG,WAAW,EAAE,CAAC;QACzD,OAAO,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,WAAW,EAAE,CAAC,CAAC;QAC1F,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,IAAI,cAAc,KAAK,IAAI,IAAI,UAAU,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC5F,OAAO,CAAC,KAAK,CACX,iBAAiB,UAAU,CAAC,WAAW,CAAC,MAAM,uBAAuB,cAAc,EAAE,CACtF,CAAC;QACF,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,eAAe,GACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AACnF,IAAI,eAAe;IAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Score a labeled Jev suite for accuracy and cost, and compare runs.
+ *
+ * Answers the question you have to defend: for this task, on my data, is Jev
+ * good enough and what does it cost? Accuracy, tokens and dollars together,
+ * because any one of them alone is easy to win.
+ */
+export { PRICE_PER_MTOK, SuiteComparison, SuiteResult, compareSuites, scoreRecords, type QuestionResult, type ScoreOptions, } from "./score.js";
+export declare const VERSION = "0.1.0";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EACzE,KAAK,cAAc,EAAE,KAAK,YAAY,GACvC,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,OAAO,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,10 @@
+/**
+ * Score a labeled Jev suite for accuracy and cost, and compare runs.
+ *
+ * Answers the question you have to defend: for this task, on my data, is Jev
+ * good enough and what does it cost? Accuracy, tokens and dollars together,
+ * because any one of them alone is easy to win.
+ */
+export { PRICE_PER_MTOK, SuiteComparison, SuiteResult, compareSuites, scoreRecords, } from "./score.js";
+export const VERSION = "0.1.0";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,GAE1E,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/dist/score.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Scoring a labeled `.jevl` suite, and comparing two runs of it.
+ *
+ * The question this answers is the one you actually have to defend: for this
+ * task, on my data, is Jev good enough, and what does it cost? That needs
+ * accuracy and cost together, because any one of them alone is easy to win.
+ *
+ * Costs come from a price per million input tokens, defaulting to Jev's
+ * published $0.042. Output tokens are free on Jev and are reported but never
+ * billed.
+ */
+import { type Record as JevlRecord } from "jevkit-core";
+export declare const PRICE_PER_MTOK = 0.042;
+/** One scored question from one record. */
+export interface QuestionResult {
+    requestId: string;
+    questionId: string;
+    type: string;
+    predicted: unknown;
+    label: unknown;
+    correct: boolean;
+    probability: number;
+    confidence: number | null;
+    tags: readonly string[];
+}
+/** Everything scored from one run of a suite. */
+export declare class SuiteResult {
+    readonly pricePerMtok: number;
+    results: QuestionResult[];
+    inputTokens: number;
+    outputTokens: number;
+    records: number;
+    unlabeled: number;
+    models: Set<string>;
+    constructor(pricePerMtok?: number);
+    get count(): number;
+    get correct(): number;
+    get accuracy(): number;
+    /** Input-token cost in dollars. Jev bills input only. */
+    get cost(): number;
+    get costPerQuestion(): number;
+    /** tag -> [n, accuracy]. A result with several tags counts under each. */
+    byTag(): Record<string, [number, number]>;
+    /** question id -> [n, accuracy]. Finds the one question dragging the suite. */
+    byQuestion(): Record<string, [number, number]>;
+    /** Wrong answers, most confident first: the most interesting bugs. */
+    failures(): QuestionResult[];
+    summary(): string;
+    toJSON(): {
+        records: number;
+        unlabeled: number;
+        scored: number;
+        models: string[];
+        accuracy: number;
+        correct: number;
+        inputTokens: number;
+        outputTokens: number;
+        cost: number;
+        byTag: {
+            [k: string]: {
+                n: number;
+                accuracy: number;
+            };
+        };
+        byQuestion: {
+            [k: string]: {
+                n: number;
+                accuracy: number;
+            };
+        };
+    };
+}
+export interface ScoreOptions {
+    questionIds?: Iterable<string>;
+    pricePerMtok?: number;
+}
+/**
+ * Score every labeled answer in a suite.
+ *
+ * Records with no `label` are counted and skipped rather than silently dropped,
+ * so a suite that quietly lost its labels shows up in the summary instead of
+ * producing a suspiciously perfect score over three records.
+ */
+export declare function scoreRecords(records: Iterable<JevlRecord>, options?: ScoreOptions): SuiteResult;
+/** Two runs of the same suite, side by side. */
+export declare class SuiteComparison {
+    readonly baseline: SuiteResult;
+    readonly candidate: SuiteResult;
+    readonly regressions: Array<[string, string]>;
+    readonly fixes: Array<[string, string]>;
+    constructor(baseline: SuiteResult, candidate: SuiteResult, regressions?: Array<[string, string]>, fixes?: Array<[string, string]>);
+    get accuracyDelta(): number;
+    get costDelta(): number;
+    summary(): string;
+}
+/**
+ * Compare two scored runs by (request, question).
+ *
+ * Aggregate accuracy can hold steady while the *set* of things you get right
+ * churns underneath, which matters when a specific case is the one you promised
+ * someone would work. Regressions and fixes are tracked individually for that
+ * reason.
+ */
+export declare function compareSuites(baseline: SuiteResult, candidate: SuiteResult): SuiteComparison;
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,MAAM,IAAI,UAAU,EAAgB,MAAM,aAAa,CAAC;AAEtE,eAAO,MAAM,cAAc,QAAQ,CAAC;AAEpC,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,iDAAiD;AACjD,qBAAa,WAAW;IAQV,QAAQ,CAAC,YAAY;IAPjC,OAAO,EAAE,cAAc,EAAE,CAAM;IAC/B,WAAW,SAAK;IAChB,YAAY,SAAK;IACjB,OAAO,SAAK;IACZ,SAAS,SAAK;IACd,MAAM,cAAqB;gBAEN,YAAY,SAAiB;IAElD,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,yDAAyD;IACzD,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,eAAe,IAAI,MAAM,CAE5B;IAED,0EAA0E;IAC1E,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAgBzC,+EAA+E;IAC/E,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAc9C,sEAAsE;IACtE,QAAQ,IAAI,cAAc,EAAE;IAI5B,OAAO,IAAI,MAAM;IA6BjB,MAAM;;;;;;;;;;;;;;;;;;;;;;;CAiBP;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,EAC7B,OAAO,GAAE,YAAiB,GACzB,WAAW,CAmCb;AAED,gDAAgD;AAChD,qBAAa,eAAe;IAExB,QAAQ,CAAC,QAAQ,EAAE,WAAW;IAC9B,QAAQ,CAAC,SAAS,EAAE,WAAW;IAC/B,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAH9B,QAAQ,EAAE,WAAW,EACrB,SAAS,EAAE,WAAW,EACtB,WAAW,GAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM,EACzC,KAAK,GAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAM;IAG9C,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,OAAO,IAAI,MAAM;CAWlB;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,GAAG,eAAe,CAc5F"}

package/dist/score.js

@@ -0,0 +1,211 @@
+/**
+ * Scoring a labeled `.jevl` suite, and comparing two runs of it.
+ *
+ * The question this answers is the one you actually have to defend: for this
+ * task, on my data, is Jev good enough, and what does it cost? That needs
+ * accuracy and cost together, because any one of them alone is easy to win.
+ *
+ * Costs come from a price per million input tokens, defaulting to Jev's
+ * published $0.042. Output tokens are free on Jev and are reported but never
+ * billed.
+ */
+import { parseAnswers } from "jevkit-core";
+export const PRICE_PER_MTOK = 0.042;
+/** Everything scored from one run of a suite. */
+export class SuiteResult {
+    pricePerMtok;
+    results = [];
+    inputTokens = 0;
+    outputTokens = 0;
+    records = 0;
+    unlabeled = 0;
+    models = new Set();
+    constructor(pricePerMtok = PRICE_PER_MTOK) {
+        this.pricePerMtok = pricePerMtok;
+    }
+    get count() {
+        return this.results.length;
+    }
+    get correct() {
+        return this.results.filter((r) => r.correct).length;
+    }
+    get accuracy() {
+        return this.count ? this.correct / this.count : 0;
+    }
+    /** Input-token cost in dollars. Jev bills input only. */
+    get cost() {
+        return (this.inputTokens / 1_000_000) * this.pricePerMtok;
+    }
+    get costPerQuestion() {
+        return this.count ? this.cost / this.count : 0;
+    }
+    /** tag -> [n, accuracy]. A result with several tags counts under each. */
+    byTag() {
+        const buckets = new Map();
+        for (const r of this.results) {
+            for (const tag of r.tags) {
+                if (!buckets.has(tag))
+                    buckets.set(tag, []);
+                buckets.get(tag).push(r);
+            }
+        }
+        const out = {};
+        for (const tag of [...buckets.keys()].sort()) {
+            const rs = buckets.get(tag);
+            out[tag] = [rs.length, rs.filter((r) => r.correct).length / rs.length];
+        }
+        return out;
+    }
+    /** question id -> [n, accuracy]. Finds the one question dragging the suite. */
+    byQuestion() {
+        const buckets = new Map();
+        for (const r of this.results) {
+            if (!buckets.has(r.questionId))
+                buckets.set(r.questionId, []);
+            buckets.get(r.questionId).push(r);
+        }
+        const out = {};
+        for (const qid of [...buckets.keys()].sort()) {
+            const rs = buckets.get(qid);
+            out[qid] = [rs.length, rs.filter((r) => r.correct).length / rs.length];
+        }
+        return out;
+    }
+    /** Wrong answers, most confident first: the most interesting bugs. */
+    failures() {
+        return this.results.filter((r) => !r.correct).sort((a, b) => b.probability - a.probability);
+    }
+    summary() {
+        const lines = [
+            `records:   ${this.records}` +
+                (this.unlabeled ? ` (${this.unlabeled} unlabeled, skipped)` : ""),
+            `scored:    ${this.count} question(s)`,
+            `model(s):  ${[...this.models].sort().join(", ") || "?"}`,
+            `accuracy:  ${this.accuracy.toFixed(4)}  (${this.correct}/${this.count})`,
+            `tokens:    ${this.inputTokens.toLocaleString("en-US")} in, ` +
+                `${this.outputTokens.toLocaleString("en-US")} out (out is free)`,
+            `cost:      $${this.cost.toFixed(6)}  ($${this.costPerQuestion.toFixed(8)}/question ` +
+                `at $${this.pricePerMtok}/Mtok)`,
+        ];
+        const tags = this.byTag();
+        if (Object.keys(tags).length) {
+            lines.push("by tag:");
+            for (const [tag, [n, acc]] of Object.entries(tags)) {
+                lines.push(`  ${tag.padEnd(24)} ${acc.toFixed(4)}  (n=${n})`);
+            }
+        }
+        const questions = this.byQuestion();
+        if (Object.keys(questions).length > 1) {
+            lines.push("by question:");
+            for (const [qid, [n, acc]] of Object.entries(questions)) {
+                lines.push(`  ${qid.padEnd(24)} ${acc.toFixed(4)}  (n=${n})`);
+            }
+        }
+        return lines.join("\n");
+    }
+    toJSON() {
+        const mapped = (src) => Object.fromEntries(Object.entries(src).map(([k, [n, a]]) => [k, { n, accuracy: a }]));
+        return {
+            records: this.records,
+            unlabeled: this.unlabeled,
+            scored: this.count,
+            models: [...this.models].sort(),
+            accuracy: this.accuracy,
+            correct: this.correct,
+            inputTokens: this.inputTokens,
+            outputTokens: this.outputTokens,
+            cost: this.cost,
+            byTag: mapped(this.byTag()),
+            byQuestion: mapped(this.byQuestion()),
+        };
+    }
+}
+/**
+ * Score every labeled answer in a suite.
+ *
+ * Records with no `label` are counted and skipped rather than silently dropped,
+ * so a suite that quietly lost its labels shows up in the summary instead of
+ * producing a suspiciously perfect score over three records.
+ */
+export function scoreRecords(records, options = {}) {
+    const wanted = options.questionIds ? new Set(options.questionIds) : null;
+    const suite = new SuiteResult(options.pricePerMtok ?? PRICE_PER_MTOK);
+    for (const record of records) {
+        suite.records += 1;
+        suite.models.add(record.model);
+        const usage = (record.usage ?? {});
+        suite.inputTokens += Number(usage["input_tokens"] ?? 0);
+        suite.outputTokens += Number(usage["output_tokens"] ?? 0);
+        if (!record.label || !Object.keys(record.label).length) {
+            suite.unlabeled += 1;
+            continue;
+        }
+        const answers = parseAnswers(record.answers);
+        for (const [qid, label] of Object.entries(record.label)) {
+            if (wanted && !wanted.has(qid))
+                continue;
+            const answer = answers[qid];
+            if (!answer)
+                continue;
+            suite.results.push({
+                requestId: record.requestId,
+                questionId: qid,
+                type: answer.type,
+                predicted: answer.predicted(),
+                label,
+                correct: answer.isCorrect(label),
+                probability: answer.topProbability,
+                confidence: answer.confidence,
+                tags: [...record.tags],
+            });
+        }
+    }
+    return suite;
+}
+/** Two runs of the same suite, side by side. */
+export class SuiteComparison {
+    baseline;
+    candidate;
+    regressions;
+    fixes;
+    constructor(baseline, candidate, regressions = [], fixes = []) {
+        this.baseline = baseline;
+        this.candidate = candidate;
+        this.regressions = regressions;
+        this.fixes = fixes;
+    }
+    get accuracyDelta() {
+        return this.candidate.accuracy - this.baseline.accuracy;
+    }
+    get costDelta() {
+        return this.candidate.cost - this.baseline.cost;
+    }
+    summary() {
+        const sign = (n, digits) => `${n >= 0 ? "+" : ""}${n.toFixed(digits)}`;
+        return [
+            `accuracy:  ${this.baseline.accuracy.toFixed(4)} -> ` +
+                `${this.candidate.accuracy.toFixed(4)} (${sign(this.accuracyDelta, 4)})`,
+            `cost:      $${this.baseline.cost.toFixed(6)} -> ` +
+                `$${this.candidate.cost.toFixed(6)} (${sign(this.costDelta, 6)})`,
+            `regressed: ${this.regressions.length} (right before, wrong now)`,
+            `fixed:     ${this.fixes.length} (wrong before, right now)`,
+        ].join("\n");
+    }
+}
+/**
+ * Compare two scored runs by (request, question).
+ *
+ * Aggregate accuracy can hold steady while the *set* of things you get right
+ * churns underneath, which matters when a specific case is the one you promised
+ * someone would work. Regressions and fixes are tracked individually for that
+ * reason.
+ */
+export function compareSuites(baseline, candidate) {
+    const index = (suite) => new Map(suite.results.map((r) => [`${r.requestId}\u0000${r.questionId}`, r]));
+    const before = index(baseline);
+    const after = index(candidate);
+    const shared = [...before.keys()].filter((k) => after.has(k));
+    const split = (k) => k.split("\u0000");
+    return new SuiteComparison(baseline, candidate, shared.filter((k) => before.get(k).correct && !after.get(k).correct).sort().map(split), shared.filter((k) => !before.get(k).correct && after.get(k).correct).sort().map(split));
+}
+//# sourceMappingURL=score.js.map

package/dist/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAA6B,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC;AAepC,iDAAiD;AACjD,MAAM,OAAO,WAAW;IAQD;IAPrB,OAAO,GAAqB,EAAE,CAAC;IAC/B,WAAW,GAAG,CAAC,CAAC;IAChB,YAAY,GAAG,CAAC,CAAC;IACjB,OAAO,GAAG,CAAC,CAAC;IACZ,SAAS,GAAG,CAAC,CAAC;IACd,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3B,YAAqB,eAAe,cAAc;QAA7B,iBAAY,GAAZ,YAAY,CAAiB;IAAG,CAAC;IAEtD,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACtD,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,yDAAyD;IACzD,IAAI,IAAI;QACN,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;IAC5D,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,0EAA0E;IAC1E,KAAK;QACH,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBACzB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAqC,EAAE,CAAC;QACjD,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,+EAA+E;IAC/E,UAAU;QACR,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4B,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,MAAM,GAAG,GAAqC,EAAE,CAAC;QACjD,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;YAC7B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,sEAAsE;IACtE,QAAQ;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAC9F,CAAC;IAED,OAAO;QACL,MAAM,KAAK,GAAG;YACZ,cAAc,IAAI,CAAC,OAAO,EAAE;gBAC1B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,cAAc,IAAI,CAAC,KAAK,cAAc;YACtC,cAAc,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE;YACzD,cAAc,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,GAAG;YACzE,cAAc,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO;gBAC3D,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,oBAAoB;YAClE,eAAe,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY;gBACnF,OAAO,IAAI,CAAC,YAAY,QAAQ;SACnC,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnD,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxD,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM;QACJ,MAAM,MAAM,GAAG,CAAC,GAAqC,EAAE,EAAE,CACvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACxF,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,IAAI,CAAC,KAAK;YAClB,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;YAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3B,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;SACtC,CAAC;IACJ,CAAC;CACF;AAOD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,OAA6B,EAC7B,UAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,YAAY,IAAI,cAAc,CAAC,CAAC;IAEtE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QACnB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAC;QAC9D,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACxD,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QAE1D,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;YACvD,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,UAAU,EAAE,GAAG;gBACf,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE;gBAC7B,KAAK;gBACL,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;gBAChC,WAAW,EAAE,MAAM,CAAC,cAAc;gBAClC,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC;aACvB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,MAAM,OAAO,eAAe;IAEf;IACA;IACA;IACA;IAJX,YACW,QAAqB,EACrB,SAAsB,EACtB,cAAuC,EAAE,EACzC,QAAiC,EAAE;QAHnC,aAAQ,GAAR,QAAQ,CAAa;QACrB,cAAS,GAAT,SAAS,CAAa;QACtB,gBAAW,GAAX,WAAW,CAA8B;QACzC,UAAK,GAAL,KAAK,CAA8B;IAC3C,CAAC;IAEJ,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC1D,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAClD,CAAC;IAED,OAAO;QACL,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,MAAc,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACvF,OAAO;YACL,cAAc,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;gBACnD,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG;YAC1E,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;gBAChD,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG;YACnE,cAAc,IAAI,CAAC,WAAW,CAAC,MAAM,4BAA4B;YACjE,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,4BAA4B;SAC5D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,SAAsB;IACzE,MAAM,KAAK,GAAG,CAAC,KAAkB,EAAE,EAAE,CACnC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,SAAS,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAqB,CAAC;IAEnE,OAAO,IAAI,eAAe,CACxB,QAAQ,EACR,SAAS,EACT,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EACxF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CACzF,CAAC;AACJ,CAAC"}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "jevkit-bench",
+  "version": "0.1.0",
+  "description": "Score a labeled TypeSafe Jev suite for accuracy and cost, and compare two runs. Catches regressions aggregate accuracy hides.",
+  "keywords": [
+    "jev",
+    "typesafe",
+    "system-one",
+    "benchmark",
+    "evaluation",
+    "accuracy",
+    "cost"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "jevkit-bench": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pjdurden/jevkit-js.git"
+  },
+  "dependencies": {
+    "jevkit-core": "^0.2.0"
+  },
+  "scripts": {
+    "build": "tsc -b"
+  }
+}