@tuned-tensor/local 0.1.4 → 0.1.6

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to Tuned Tensor Local will be documented in this file.
 
+## 0.1.6 - 2026-07-02
+
+### Added
+
+- Added `tt-local runs compare <run-id-a> <run-id-b>`: aligns two runs on their shared eval prompts and reports per-subset baseline/candidate averages, shared-subset deltas, new-example effects, and judge score noise measured on identical baseline outputs. Headline `avg_score_delta` values are not comparable across different eval subsets; the shared subset is.
+- Added a baseline evaluation cache (default on, `evaluation.baselineCache: false` to disable): re-running a spec with an unchanged base model, eval examples, generation settings, and scoring config reuses the previous baseline report from `<storeRoot>/cache/baseline-evals/` instead of re-running inference and judge calls. Reports served from cache carry `cached: true` and a `cache_key`. Reports containing fallback-scored examples are never cached.
+- Eval reports now record how each example was scored (`results[].scored_by`) and aggregate it as `judge_scored_count`, `fallback_scored_count`, and `judge_only_avg_score`, so transient judge failures that fall back to exact match can no longer silently skew `avg_score` without a trace. The comparison report includes `judge_only_avg_score_delta`.
+- Comparison reports now classify each regressed example's judge reasoning into a coarse category (`factual`, `omission`, `style`, `fallback`, `other`) and aggregate counts as `regression_taxonomy`, answering "what kind of worse?" at a glance.
+
+## 0.1.5 - 2026-07-02
+
+### Added
+
+- Added `avg_token_f1` to eval reports and `token_f1_delta` to the comparison report: a cheap deterministic token-overlap reference-similarity metric for free-text tasks where exact match is always 0 and an LLM judge can be noisy.
+- The OpenRouter judge now receives the spec's compiled system message (system prompt, guidelines, constraints) as `task_instructions` and scores conformance to the task instead of treating the expected output as a fact checklist. Previously a model fine-tuned toward a concise style was penalized for omitting reference details even when the spec asked for brevity.
+- `tt-local` now loads `.env` from the working directory (without overriding already-set variables), so `OPENROUTER_API_KEY` works without manual sourcing.
+- `validate` and `run` now warn about unknown hyperparameter keys (for example `per_device_train_batch_size` instead of `batch_size`), which were previously stripped silently and trained with defaults.
+
+### Fixed
+
+- Judge scoring with `scoring.fallback: "exact_match"` no longer fails the whole run when a judge request fails transiently; the affected example falls back to normalized exact match and records the judge failure in its reasoning.
+- Fixed a crash-level race in atomic store writes: concurrent run-state updates in the same millisecond (typical when tqdm flushes several progress lines at training completion) collided on the same temp filename and killed the run with `ENOENT: rename` after training had succeeded. Temp filenames now include a UUID.
+- The local evaluator no longer passes `temperature`/`top_p` to `model.generate` when decoding greedily, silencing the spurious transformers "generation flags are not valid" warning at `temperature: 0`.
+- `--verbose` no longer prints consecutive duplicate subprocess log lines (tqdm redraws the same progress line several times per step).
+
 ## 0.1.4 - 2026-07-01
 
 ### Added

package/README.md

@@ -186,7 +186,63 @@ exact matching:
 The evaluation report still includes `exact_match_rate`, but `avg_score`,
 `pass_rate`, and `json_field_metrics` are based on the selected JSON fields.
 If `fields` is omitted, the evaluator scores every key in the expected JSON
-object for each example.
+object for each example. A configured field that is missing from the expected
+JSON is always scored as incorrect (with a reasoning note), so misconfigured
+field lists cannot inflate scores.
+
+The judge receives the spec's compiled system message (system prompt,
+guidelines, and constraints) as task instructions and scores how well each
+output fulfills the task; the expected output is treated as a reference
+answer, not a fact checklist. Every eval report also includes `avg_token_f1`
+(and the comparison includes `token_f1_delta`), a deterministic token-overlap
+similarity against the reference outputs that is useful for free-text tasks
+such as summarization, where exact match is always 0 and judge scores can be
+noisy.
+
+Each eval example records how it was scored (`results[].scored_by`), and the
+report aggregates `judge_scored_count`, `fallback_scored_count`, and
+`judge_only_avg_score`. When a judge request fails transiently and the
+example falls back to exact match, `avg_score` mixes two metrics; use
+`judge_only_avg_score` (and the comparison's `judge_only_avg_score_delta`)
+when comparing judge quality. The comparison also classifies each regressed
+example's judge reasoning as `factual`, `omission`, `style`, `fallback`, or
+`other` and reports the counts in `regression_taxonomy`.
+
+Baseline evaluations are cached under `<storeRoot>/cache/baseline-evals/`:
+re-running a spec with an unchanged base model, eval examples, generation
+settings, and scoring config reuses the previous baseline report instead of
+re-running inference and judge calls. Cached reports carry `cached: true`.
+Set `evaluation.baselineCache: false` to disable. Reports containing
+fallback-scored examples are never cached, so a transient judge failure is
+not replayed into future runs.
+
+To compare two runs whose eval sets may differ (for example after raising
+`evaluation.maxExamples`), use:
+
+```bash
+tt-local runs compare <run-id-a> <run-id-b> --config spark-runner.json
+```
+
+It aligns the runs on their shared eval prompts, reports both runs' averages
+on that shared subset plus the new-example subset separately, and measures
+judge score noise on baseline outputs that are byte-identical across the two
+runs.
+
+With `"mode": "llm_judge"`, `scoring.fallback` controls what happens when the
+judge is unavailable, a judge request fails, or the judge returns malformed
+JSON. With `"fallback": "exact_match"` (the default), the affected example is
+scored by normalized exact match and its reasoning records the judge failure;
+the run does not fail. With `"fallback": "fail"`, judge errors fail the run.
+
+`evaluation.maxExamples` caps how many evaluation examples are scored. If it
+is unset, the request hyperparameter `max_eval_examples` is used instead;
+an explicit `maxExamples` in the config always wins. When the cap truncates
+the eval set, the runner draws a deterministic seeded random sample (not a
+prefix), so sorted or grouped eval files do not bias the subset. The seed
+defaults to a hash of the run id, can be pinned with `evaluation.sampleSeed`,
+and is recorded in each eval report as `eval_sample_seed`. Baseline and
+candidate evaluations always score the same sampled examples in the same
+order.
 
 If you want OpenRouter judging, set your key:
 
@@ -194,6 +250,14 @@ If you want OpenRouter judging, set your key:
 export OPENROUTER_API_KEY="your_openrouter_key"
 ```
 
+`tt-local` also loads a `.env` file from the working directory (existing
+environment variables win), so the key can live in the project's `.env`
+instead.
+
+`validate` and `run` warn about hyperparameter keys the schema does not
+recognize (for example `per_device_train_batch_size` instead of
+`batch_size`); unknown keys are ignored rather than passed to training.
+
 Run the job:
 
 ```bash
@@ -209,10 +273,30 @@ recorded as run events, so `tt-local runs watch` and per-run `progress.jsonl`
 show epoch, loss, step, percentage, and ETA updates when the trainer emits
 them.
 
+For spec-based runs (no `dataset_prebuilt`), the runner automatically holds
+out about 20% of the spec examples (at least 1 holdout and at least 1 training
+example) for evaluation, so baseline and candidate scores are not measured on
+the training data. The split is deterministic: it uses the per-run sample seed
+(a hash of the run id, or `evaluation.sampleSeed` when set), so re-running the
+same run id reproduces the same split and baseline/candidate always score the
+identical holdout. The training JSONL artifact contains only the training
+split, `run_metadata.training_example_count` records the training split size,
+and `eval_examples_total` records the holdout size. These runs report
+`eval_split: "spec_holdout"`. With only 1 spec example there is nothing to
+hold out, so evaluation runs on the training set and is reported as
+`eval_split: "spec_examples"` (training-set evaluation due to insufficient
+examples).
+
 For prebuilt datasets, `dataset_prebuilt.training` is always copied into the
 run artifact as the training set. Evaluation uses `dataset_prebuilt.test` when
-present, then `dataset_prebuilt.validation`, and otherwise falls back to the
-training file.
+present, then `dataset_prebuilt.validation`. If neither is provided, a real
+(non-dry) run fails with an error, because evaluating on the training split
+overstates improvement; set `evaluation.allowPrebuiltTrainingEval: true` to
+evaluate on the training file anyway. Dry runs are allowed to fall back to the
+training file without the override. Each eval report and the run's
+`run_metadata` record the split that was evaluated in `eval_split`
+(`spec_holdout`, `spec_examples`, `prebuilt_test`, `prebuilt_validation`, or
+`prebuilt_training`), so training-set evaluation is always visible.
 
 Multimodal runs use the same chat JSONL shape with structured user content.
 For `image_text_to_text` models such as `Qwen/Qwen3-VL-2B-Instruct`, image
@@ -281,6 +365,7 @@ tt-local runs list --config spark-runner.json
 tt-local runs get <run-id> --config spark-runner.json
 tt-local runs events <run-id> --config spark-runner.json
 tt-local runs report <run-id> --config spark-runner.json
+tt-local runs compare <run-id-a> <run-id-b> --config spark-runner.json
 tt-local models list --config spark-runner.json
 tt-local serve --config spark-runner.json
 ```

package/dist/compare.d.ts

@@ -0,0 +1,47 @@
+import type { RunReport } from "./contracts.js";
+interface SubsetSide {
+    baseline_avg_score: number | null;
+    candidate_avg_score: number | null;
+    candidate_avg_token_f1: number | null;
+}
+interface RunSummary {
+    run_id: string;
+    started_at: string;
+    base_model: string;
+    training_example_count: number | null;
+    eval_examples_used: number;
+    baseline_avg_score: number;
+    candidate_avg_score: number;
+    candidate_avg_token_f1: number | null;
+    avg_score_delta: number;
+    fallback_scored_count: number;
+}
+export interface RunComparison {
+    run_a: RunSummary;
+    run_b: RunSummary;
+    shared: {
+        examples: number;
+        run_a: SubsetSide;
+        run_b: SubsetSide;
+        candidate_avg_score_delta: number | null;
+        candidate_avg_token_f1_delta: number | null;
+    };
+    b_only: SubsetSide & {
+        examples: number;
+    };
+    judge_noise: {
+        identical_baseline_outputs: number;
+        mean_score_spread: number | null;
+        max_score_spread: number | null;
+    };
+    notes: string[];
+}
+/**
+ * Aligns two run reports on their shared eval prompts so recipe changes can
+ * be compared apples-to-apples even when the eval set grew or shifted between
+ * runs. Headline avg_score_delta values are not comparable across different
+ * eval subsets; the shared subset is. Identical baseline outputs that appear
+ * in both runs also give a free measurement of judge score noise.
+ */
+export declare function compareRuns(a: RunReport, b: RunReport): RunComparison;
+export {};

package/dist/compare.js

@@ -0,0 +1,95 @@
+import { tokenF1 } from "./evaluation.js";
+function byPrompt(report) {
+    return new Map(report.results.map((result) => [result.prompt, result]));
+}
+function avg(values) {
+    return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : null;
+}
+function summarizeRun(report) {
+    return {
+        run_id: report.run_id,
+        started_at: report.run_metadata.started_at,
+        base_model: report.base_model,
+        training_example_count: report.run_metadata.training_example_count,
+        eval_examples_used: report.run_metadata.eval_examples_used,
+        baseline_avg_score: report.baseline.avg_score,
+        candidate_avg_score: report.candidate.avg_score,
+        candidate_avg_token_f1: report.candidate.avg_token_f1 ?? null,
+        avg_score_delta: report.comparison.avg_score_delta,
+        fallback_scored_count: (report.baseline.fallback_scored_count ?? 0)
+            + (report.candidate.fallback_scored_count ?? 0),
+    };
+}
+function subsetSide(prompts, baseline, candidate) {
+    const candidateResults = prompts
+        .map((prompt) => candidate.get(prompt))
+        .filter((result) => Boolean(result));
+    return {
+        baseline_avg_score: avg(prompts
+            .map((prompt) => baseline.get(prompt)?.score)
+            .filter((score) => typeof score === "number")),
+        candidate_avg_score: avg(candidateResults.map((result) => result.score)),
+        candidate_avg_token_f1: avg(candidateResults.map((result) => tokenF1(result.expected, result.actual))),
+    };
+}
+/**
+ * Aligns two run reports on their shared eval prompts so recipe changes can
+ * be compared apples-to-apples even when the eval set grew or shifted between
+ * runs. Headline avg_score_delta values are not comparable across different
+ * eval subsets; the shared subset is. Identical baseline outputs that appear
+ * in both runs also give a free measurement of judge score noise.
+ */
+export function compareRuns(a, b) {
+    const aCandidate = byPrompt(a.candidate);
+    const bCandidate = byPrompt(b.candidate);
+    const aBaseline = byPrompt(a.baseline);
+    const bBaseline = byPrompt(b.baseline);
+    const sharedPrompts = [...bCandidate.keys()].filter((prompt) => aCandidate.has(prompt));
+    const bOnlyPrompts = [...bCandidate.keys()].filter((prompt) => !aCandidate.has(prompt));
+    const spreads = [];
+    let identical = 0;
+    for (const prompt of sharedPrompts) {
+        const first = aBaseline.get(prompt);
+        const second = bBaseline.get(prompt);
+        if (!first || !second || first.actual !== second.actual)
+            continue;
+        identical += 1;
+        spreads.push(Math.abs(first.score - second.score));
+    }
+    const sharedA = subsetSide(sharedPrompts, aBaseline, aCandidate);
+    const sharedB = subsetSide(sharedPrompts, bBaseline, bCandidate);
+    const runA = summarizeRun(a);
+    const runB = summarizeRun(b);
+    const notes = [];
+    if (bOnlyPrompts.length > 0) {
+        notes.push(`run_b evaluated ${bOnlyPrompts.length} example(s) run_a never saw; headline avg_score_delta values `
+            + "are not directly comparable across the two runs. Use the shared subset for recipe comparisons.");
+    }
+    if (runA.fallback_scored_count + runB.fallback_scored_count > 0) {
+        notes.push(`${runA.fallback_scored_count + runB.fallback_scored_count} example(s) across both runs were `
+            + "fallback-scored after judge failures; prefer judge_only_avg_score when comparing judge quality.");
+    }
+    return {
+        run_a: runA,
+        run_b: runB,
+        shared: {
+            examples: sharedPrompts.length,
+            run_a: sharedA,
+            run_b: sharedB,
+            candidate_avg_score_delta: sharedA.candidate_avg_score !== null && sharedB.candidate_avg_score !== null
+                ? sharedB.candidate_avg_score - sharedA.candidate_avg_score
+                : null,
+            candidate_avg_token_f1_delta: sharedA.candidate_avg_token_f1 !== null && sharedB.candidate_avg_token_f1 !== null
+                ? sharedB.candidate_avg_token_f1 - sharedA.candidate_avg_token_f1
+                : null,
+        },
+        b_only: { examples: bOnlyPrompts.length, ...subsetSide(bOnlyPrompts, bBaseline, bCandidate) },
+        judge_noise: {
+            identical_baseline_outputs: identical,
+            mean_score_spread: avg(spreads),
+            max_score_spread: spreads.length > 0 ? Math.max(...spreads) : null,
+        },
+        notes,
+    };
+}
+//# sourceMappingURL=compare.js.map

package/dist/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../src/compare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAwC1C,SAAS,QAAQ,CAAC,MAAkB;IAClC,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAClG,CAAC;AAED,SAAS,YAAY,CAAC,MAAiB;IACrC,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,YAAY,CAAC,UAAU;QAC1C,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,sBAAsB,EAAE,MAAM,CAAC,YAAY,CAAC,sBAAsB;QAClE,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,kBAAkB;QAC1D,kBAAkB,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;QAC7C,mBAAmB,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS;QAC/C,sBAAsB,EAAE,MAAM,CAAC,SAAS,CAAC,YAAY,IAAI,IAAI;QAC7D,eAAe,EAAE,MAAM,CAAC,UAAU,CAAC,eAAe;QAClD,qBAAqB,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,IAAI,CAAC,CAAC;cAC/D,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,IAAI,CAAC,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,OAAiB,EACjB,QAAwC,EACxC,SAAyC;IAEzC,MAAM,gBAAgB,GAAG,OAAO;SAC7B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACtC,MAAM,CAAC,CAAC,MAAM,EAA+B,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,kBAAkB,EAAE,GAAG,CAAC,OAAO;aAC5B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC;aAC5C,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;QACjE,mBAAmB,EAAE,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,sBAAsB,EAAE,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;KACvG,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,CAAY,EAAE,CAAY;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACxF,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAExF,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,SAAS;QAClE,SAAS,IAAI,CAAC,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAE7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CACR,mBAAmB,YAAY,CAAC,MAAM,+DAA+D;cACnG,gGAAgG,CACnG,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;QAChE,KAAK,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,oCAAoC;cAC5F,iGAAiG,CACpG,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,IAAI;QACX,MAAM,EAAE;YACN,QAAQ,EAAE,aAAa,CAAC,MAAM;YAC9B,KAAK,EAAE,OAAO;YACd,KAAK,EAAE,OAAO;YACd,yBAAyB,EACvB,OAAO,CAAC,mBAAmB,KAAK,IAAI,IAAI,OAAO,CAAC,mBAAmB,KAAK,IAAI;gBAC1E,CAAC,CAAC,OAAO,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB;gBAC3D,CAAC,CAAC,IAAI;YACV,4BAA4B,EAC1B,OAAO,CAAC,sBAAsB,KAAK,IAAI,IAAI,OAAO,CAAC,sBAAsB,KAAK,IAAI;gBAChF,CAAC,CAAC,OAAO,CAAC,sBAAsB,GAAG,OAAO,CAAC,sBAAsB;gBACjE,CAAC,CAAC,IAAI;SACX;QACD,MAAM,EAAE,EAAE,QAAQ,EAAE,YAAY,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE;QAC7F,WAAW,EAAE;YACX,0BAA0B,EAAE,SAAS;YACrC,iBAAiB,EAAE,GAAG,CAAC,OAAO,CAAC;YAC/B,gBAAgB,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;SACnE;QACD,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/contracts.d.ts

@@ -183,6 +183,13 @@ export declare const evalExampleResultSchema: z.ZodObject<{
     score: z.ZodNumber;
     reasoning: z.ZodNullable<z.ZodString>;
     latency_ms: z.ZodNumber;
+    scored_by: z.ZodOptional<z.ZodEnum<{
+        llm_judge: "llm_judge";
+        exact_match: "exact_match";
+        json_fields: "json_fields";
+        heuristic: "heuristic";
+        exact_match_fallback: "exact_match_fallback";
+    }>>;
 }, z.core.$strip>;
 export declare const jsonFieldMetricsSchema: z.ZodObject<{
     fields: z.ZodArray<z.ZodString>;
@@ -198,6 +205,13 @@ export declare const jsonFieldMetricsSchema: z.ZodObject<{
         accuracy: z.ZodNumber;
     }, z.core.$strip>>;
 }, z.core.$strip>;
+export declare const evalSplitSchema: z.ZodEnum<{
+    spec_examples: "spec_examples";
+    spec_holdout: "spec_holdout";
+    prebuilt_test: "prebuilt_test";
+    prebuilt_validation: "prebuilt_validation";
+    prebuilt_training: "prebuilt_training";
+}>;
 export declare const evalReportSchema: z.ZodObject<{
     kind: z.ZodEnum<{
         candidate: "candidate";
@@ -208,10 +222,24 @@ export declare const evalReportSchema: z.ZodObject<{
     eval_examples_total: z.ZodNumber;
     eval_examples_used: z.ZodNumber;
     eval_truncated: z.ZodBoolean;
+    eval_split: z.ZodOptional<z.ZodEnum<{
+        spec_examples: "spec_examples";
+        spec_holdout: "spec_holdout";
+        prebuilt_test: "prebuilt_test";
+        prebuilt_validation: "prebuilt_validation";
+        prebuilt_training: "prebuilt_training";
+    }>>;
+    eval_sample_seed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
     avg_score: z.ZodNumber;
     pass_rate: z.ZodNumber;
     exact_match_rate: z.ZodNumber;
+    avg_token_f1: z.ZodOptional<z.ZodNumber>;
     avg_latency_ms: z.ZodNumber;
+    judge_scored_count: z.ZodOptional<z.ZodNumber>;
+    fallback_scored_count: z.ZodOptional<z.ZodNumber>;
+    judge_only_avg_score: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    cached: z.ZodOptional<z.ZodBoolean>;
+    cache_key: z.ZodOptional<z.ZodString>;
     results: z.ZodArray<z.ZodObject<{
         prompt: z.ZodString;
         expected: z.ZodString;
@@ -220,14 +248,21 @@ export declare const evalReportSchema: z.ZodObject<{
         score: z.ZodNumber;
         reasoning: z.ZodNullable<z.ZodString>;
         latency_ms: z.ZodNumber;
+        scored_by: z.ZodOptional<z.ZodEnum<{
+            llm_judge: "llm_judge";
+            exact_match: "exact_match";
+            json_fields: "json_fields";
+            heuristic: "heuristic";
+            exact_match_fallback: "exact_match_fallback";
+        }>>;
     }, z.core.$strip>>;
     artifact_uri: z.ZodString;
     scoring_method: z.ZodEnum<{
         command: "command";
-        heuristic: "heuristic";
         llm_judge: "llm_judge";
         exact_match: "exact_match";
         json_fields: "json_fields";
+        heuristic: "heuristic";
     }>;
     judge_model_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
     inference_provider: z.ZodOptional<z.ZodEnum<{
@@ -257,16 +292,39 @@ export declare const evalReportSchema: z.ZodObject<{
     generation_config: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     log_uri: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
+export declare const regressionCategorySchema: z.ZodEnum<{
+    fallback: "fallback";
+    other: "other";
+    style: "style";
+    factual: "factual";
+    omission: "omission";
+}>;
 export declare const comparisonReportSchema: z.ZodObject<{
     avg_score_delta: z.ZodNumber;
     pass_rate_delta: z.ZodNumber;
     exact_match_rate_delta: z.ZodNumber;
+    token_f1_delta: z.ZodOptional<z.ZodNumber>;
+    judge_only_avg_score_delta: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
     regressions: z.ZodNumber;
     improvements: z.ZodNumber;
+    regression_taxonomy: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
+        fallback: "fallback";
+        other: "other";
+        style: "style";
+        factual: "factual";
+        omission: "omission";
+    }>, z.ZodNumber>>;
     regressed_examples: z.ZodArray<z.ZodObject<{
         prompt: z.ZodString;
         old_score: z.ZodNumber;
         new_score: z.ZodNumber;
+        category: z.ZodOptional<z.ZodEnum<{
+            fallback: "fallback";
+            other: "other";
+            style: "style";
+            factual: "factual";
+            omission: "omission";
+        }>>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 export declare const trainingReportSchema: z.ZodObject<{
@@ -300,10 +358,24 @@ export declare const runReportSchema: z.ZodObject<{
         eval_examples_total: z.ZodNumber;
         eval_examples_used: z.ZodNumber;
         eval_truncated: z.ZodBoolean;
+        eval_split: z.ZodOptional<z.ZodEnum<{
+            spec_examples: "spec_examples";
+            spec_holdout: "spec_holdout";
+            prebuilt_test: "prebuilt_test";
+            prebuilt_validation: "prebuilt_validation";
+            prebuilt_training: "prebuilt_training";
+        }>>;
+        eval_sample_seed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
         avg_score: z.ZodNumber;
         pass_rate: z.ZodNumber;
         exact_match_rate: z.ZodNumber;
+        avg_token_f1: z.ZodOptional<z.ZodNumber>;
         avg_latency_ms: z.ZodNumber;
+        judge_scored_count: z.ZodOptional<z.ZodNumber>;
+        fallback_scored_count: z.ZodOptional<z.ZodNumber>;
+        judge_only_avg_score: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+        cached: z.ZodOptional<z.ZodBoolean>;
+        cache_key: z.ZodOptional<z.ZodString>;
         results: z.ZodArray<z.ZodObject<{
             prompt: z.ZodString;
             expected: z.ZodString;
@@ -312,14 +384,21 @@ export declare const runReportSchema: z.ZodObject<{
             score: z.ZodNumber;
             reasoning: z.ZodNullable<z.ZodString>;
             latency_ms: z.ZodNumber;
+            scored_by: z.ZodOptional<z.ZodEnum<{
+                llm_judge: "llm_judge";
+                exact_match: "exact_match";
+                json_fields: "json_fields";
+                heuristic: "heuristic";
+                exact_match_fallback: "exact_match_fallback";
+            }>>;
         }, z.core.$strip>>;
         artifact_uri: z.ZodString;
         scoring_method: z.ZodEnum<{
             command: "command";
-            heuristic: "heuristic";
             llm_judge: "llm_judge";
             exact_match: "exact_match";
             json_fields: "json_fields";
+            heuristic: "heuristic";
         }>;
         judge_model_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
         inference_provider: z.ZodOptional<z.ZodEnum<{
@@ -359,10 +438,24 @@ export declare const runReportSchema: z.ZodObject<{
         eval_examples_total: z.ZodNumber;
         eval_examples_used: z.ZodNumber;
         eval_truncated: z.ZodBoolean;
+        eval_split: z.ZodOptional<z.ZodEnum<{
+            spec_examples: "spec_examples";
+            spec_holdout: "spec_holdout";
+            prebuilt_test: "prebuilt_test";
+            prebuilt_validation: "prebuilt_validation";
+            prebuilt_training: "prebuilt_training";
+        }>>;
+        eval_sample_seed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
         avg_score: z.ZodNumber;
         pass_rate: z.ZodNumber;
         exact_match_rate: z.ZodNumber;
+        avg_token_f1: z.ZodOptional<z.ZodNumber>;
         avg_latency_ms: z.ZodNumber;
+        judge_scored_count: z.ZodOptional<z.ZodNumber>;
+        fallback_scored_count: z.ZodOptional<z.ZodNumber>;
+        judge_only_avg_score: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+        cached: z.ZodOptional<z.ZodBoolean>;
+        cache_key: z.ZodOptional<z.ZodString>;
         results: z.ZodArray<z.ZodObject<{
             prompt: z.ZodString;
             expected: z.ZodString;
@@ -371,14 +464,21 @@ export declare const runReportSchema: z.ZodObject<{
             score: z.ZodNumber;
             reasoning: z.ZodNullable<z.ZodString>;
             latency_ms: z.ZodNumber;
+            scored_by: z.ZodOptional<z.ZodEnum<{
+                llm_judge: "llm_judge";
+                exact_match: "exact_match";
+                json_fields: "json_fields";
+                heuristic: "heuristic";
+                exact_match_fallback: "exact_match_fallback";
+            }>>;
         }, z.core.$strip>>;
         artifact_uri: z.ZodString;
         scoring_method: z.ZodEnum<{
             command: "command";
-            heuristic: "heuristic";
             llm_judge: "llm_judge";
             exact_match: "exact_match";
             json_fields: "json_fields";
+            heuristic: "heuristic";
         }>;
         judge_model_id: z.ZodOptional<z.ZodNullable<z.ZodString>>;
         inference_provider: z.ZodOptional<z.ZodEnum<{
@@ -412,12 +512,28 @@ export declare const runReportSchema: z.ZodObject<{
         avg_score_delta: z.ZodNumber;
         pass_rate_delta: z.ZodNumber;
         exact_match_rate_delta: z.ZodNumber;
+        token_f1_delta: z.ZodOptional<z.ZodNumber>;
+        judge_only_avg_score_delta: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
         regressions: z.ZodNumber;
         improvements: z.ZodNumber;
+        regression_taxonomy: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
+            fallback: "fallback";
+            other: "other";
+            style: "style";
+            factual: "factual";
+            omission: "omission";
+        }>, z.ZodNumber>>;
         regressed_examples: z.ZodArray<z.ZodObject<{
             prompt: z.ZodString;
             old_score: z.ZodNumber;
             new_score: z.ZodNumber;
+            category: z.ZodOptional<z.ZodEnum<{
+                fallback: "fallback";
+                other: "other";
+                style: "style";
+                factual: "factual";
+                omission: "omission";
+            }>>;
         }, z.core.$strip>>;
     }, z.core.$strip>;
     training: z.ZodObject<{
@@ -445,6 +561,14 @@ export declare const runReportSchema: z.ZodObject<{
         training_example_count: z.ZodNullable<z.ZodNumber>;
         eval_examples_total: z.ZodNumber;
         eval_examples_used: z.ZodNumber;
+        eval_split: z.ZodOptional<z.ZodEnum<{
+            spec_examples: "spec_examples";
+            spec_holdout: "spec_holdout";
+            prebuilt_test: "prebuilt_test";
+            prebuilt_validation: "prebuilt_validation";
+            prebuilt_training: "prebuilt_training";
+        }>>;
+        eval_sample_seed: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
         started_at: z.ZodString;
         completed_at: z.ZodString;
         elapsed_ms: z.ZodNumber;
@@ -474,8 +598,8 @@ export declare const localRunnerConfigSchema: z.ZodObject<{
     evaluation: z.ZodDefault<z.ZodObject<{
         mode: z.ZodDefault<z.ZodEnum<{
             command: "command";
-            heuristic: "heuristic";
             llm_judge: "llm_judge";
+            heuristic: "heuristic";
         }>>;
         baselineCommand: z.ZodOptional<z.ZodArray<z.ZodString>>;
         candidateCommand: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -515,6 +639,9 @@ export declare const localRunnerConfigSchema: z.ZodObject<{
         }, z.core.$strip>>;
         timeoutMs: z.ZodDefault<z.ZodNumber>;
         maxExamples: z.ZodOptional<z.ZodNumber>;
+        sampleSeed: z.ZodOptional<z.ZodNumber>;
+        allowPrebuiltTrainingEval: z.ZodDefault<z.ZodBoolean>;
+        baselineCache: z.ZodDefault<z.ZodBoolean>;
     }, z.core.$strip>>;
     llm: z.ZodOptional<z.ZodObject<{
         provider: z.ZodDefault<z.ZodLiteral<"openrouter">>;
@@ -531,6 +658,7 @@ export type FineTuneHyperparameters = z.infer<typeof fineTuneHyperparametersSche
 export type FineTuneRunRequest = z.infer<typeof fineTuneRunRequestSchema>;
 export type LocalBehaviorSpecFile = z.infer<typeof localBehaviorSpecFileSchema>;
 export type EvalExampleResult = z.infer<typeof evalExampleResultSchema>;
+export type EvalSplit = z.infer<typeof evalSplitSchema>;
 export type EvalReport = z.infer<typeof evalReportSchema>;
 export type ComparisonReport = z.infer<typeof comparisonReportSchema>;
 export type TrainingReport = z.infer<typeof trainingReportSchema>;