crewhaus 0.1.8 → 0.2.0

package/dist/eval-history.js

@@ -0,0 +1,157 @@
+/**
+ * Post-eval run-history orchestration for `crewhaus eval` — append the run
+ * to the index, diff against the pinned (spec, dataset) baseline, gate, and
+ * promote. Kept in a side-effect-free module so it is unit-testable (this
+ * package's entry file runs an argv switch on import).
+ *
+ * Policy (AUTOMATION-OPPORTUNITIES.md item 3):
+ * - Every run is appended to `.crewhaus/evals/index.jsonl`.
+ * - First run for a (spec, dataset) key pins the baseline.
+ * - A keyset mismatch from `diffReports` (sample ids changed) means the
+ *   dataset itself changed — start a new baseline lineage instead of
+ *   failing the run.
+ * - The regression gate is strict by default: ANY pass-rate drop fails, and
+ *   any sample-level pass→fail flip fails even when a recovery elsewhere
+ *   cancels it out in the aggregates.
+ * - Gate pass → auto-promote the new run to baseline (opt out with
+ *   `--no-promote`); gate fail → never promote. `--gate` only controls
+ *   whether a failing verdict maps to a non-zero exit.
+ */
+import { resolve } from "node:path";
+import { ReportError, appendRunIndex, diffReports, getBaseline, loadRun, setBaseline, } from "@crewhaus/eval-report";
+import { gate } from "@crewhaus/regression-runner";
+/**
+ * `eval-report history|baseline show --dataset <filter>` matching. Unioned
+ * runs are recorded under `<primary>+regressions@vX`, so the filter matches
+ * a stored datasetName that equals it exactly OR continues it with a `+`
+ * suffix segment — `--dataset smoke` finds `smoke` and
+ * `smoke+regressions@v1`, but not `smoke2`.
+ */
+export function datasetFilterMatches(filter, datasetName) {
+    return datasetName === filter || datasetName.startsWith(`${filter}+`);
+}
+/**
+ * Strict gate between a baseline run and a new run. Wraps
+ * `regression-runner`'s `gate()` with `regressionThreshold: 0` (any
+ * pass-rate drop fails) and additionally fails on sample-level regressions
+ * that net out to a flat pass rate. The latency criterion is DISABLED by
+ * default (threshold Infinity): the documented gate is pass-rate/flip-only,
+ * and regression-runner's +5000ms p95 default would fail runs the docs say
+ * pass. Latency gating arrives with explicit CLI flags later — callers can
+ * already opt in via `thresholds.latencyThreshold`.
+ */
+export function gateRuns(prev, next, thresholds = {}) {
+    const verdict = gate(prev, next, {
+        regressionThreshold: 0,
+        latencyThreshold: Number.POSITIVE_INFINITY,
+        ...thresholds,
+    });
+    if (verdict.verdict === "pass" && verdict.report.regressions.length > 0) {
+        const ids = verdict.report.regressions.map((r) => r.sampleId).join(", ");
+        return {
+            verdict: "fail",
+            reason: `sample regressions despite a flat pass rate: [${ids}]`,
+            report: verdict.report,
+        };
+    }
+    return verdict;
+}
+/**
+ * Record a completed eval run in the history index, then resolve/diff/gate
+ * against the pinned baseline for its (spec, dataset) key. Returns whether
+ * the caller should exit non-zero (gate requested + failed).
+ */
+export async function finishEvalRun(opts) {
+    const write = opts.write ?? ((line) => process.stdout.write(`${line}\n`));
+    const { summary, specName } = opts;
+    const datasetName = summary.config.datasetName;
+    const absOut = resolve(opts.outDir);
+    // Belt (the runner already refuses to run zero samples): a 0-sample run
+    // carries no signal and a passRate-0 "clean" entry would poison the index
+    // and could pin an empty baseline — refuse loudly instead of recording it.
+    if (summary.samples.length === 0) {
+        throw new Error(`refusing to record 0-sample eval run ${summary.runId} — dataset "${datasetName}" produced no samples`);
+    }
+    const entry = {
+        runId: summary.runId,
+        specName,
+        specHash: summary.config.specHash,
+        datasetName,
+        datasetHash: opts.datasetHash,
+        passRate: summary.aggregates.passRate,
+        meanScore: summary.aggregates.meanScore,
+        sampleCount: summary.samples.length,
+        retriedCount: summary.samples.filter((s) => s.retried === true).length,
+        ts: summary.endedAt,
+        outDir: absOut,
+    };
+    appendRunIndex(entry, opts.evalsDir);
+    const pinCurrentRun = (label) => {
+        if (!opts.promote) {
+            write(`[eval] baseline not set (${label}; --no-promote)`);
+            return;
+        }
+        const pin = {
+            specName,
+            datasetName,
+            runId: summary.runId,
+            outDir: absOut,
+            datasetHash: opts.datasetHash,
+            ts: entry.ts,
+        };
+        setBaseline(pin, opts.evalsDir);
+        write(`[eval] baseline set: ${summary.runId} (${label})`);
+    };
+    const baseline = getBaseline(specName, datasetName, opts.evalsDir);
+    if (baseline === undefined) {
+        pinCurrentRun(`first run for ${specName}/${datasetName}`);
+        return { gateFailed: false };
+    }
+    let prevLoaded;
+    try {
+        prevLoaded = await loadRun(baseline.outDir);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        write(`[eval] baseline run ${baseline.runId} unreadable (${msg}) — starting new baseline lineage`);
+        pinCurrentRun("new lineage");
+        return { gateFailed: false };
+    }
+    const nextLoaded = await loadRun(absOut);
+    let diff;
+    try {
+        diff = diffReports(prevLoaded, nextLoaded).diff;
+    }
+    catch (err) {
+        if (err instanceof ReportError) {
+            write("[eval] dataset changed — starting new baseline lineage");
+            pinCurrentRun("new lineage");
+            return { gateFailed: false };
+        }
+        throw err;
+    }
+    write(`[eval] vs baseline ${baseline.runId}: regressions=${diff.regressions.length} ` +
+        `recoveries=${diff.recoveries.length} score_shifts=${diff.scoreShifts.length} ` +
+        `unchanged=${diff.unchanged}`);
+    for (const r of diff.regressions) {
+        write(`[eval]   regression: ${r.sampleId} (score ${r.prev.score.toFixed(2)} → ${r.next.score.toFixed(2)})`);
+    }
+    for (const r of diff.recoveries) {
+        write(`[eval]   recovery: ${r.sampleId} (score ${r.prev.score.toFixed(2)} → ${r.next.score.toFixed(2)})`);
+    }
+    const verdict = gateRuns(prevLoaded.summary, nextLoaded.summary);
+    if (verdict.verdict === "fail") {
+        const reason = verdict.reason ?? "regression gate failed";
+        write(`[eval] gate: FAIL — ${reason}`);
+        write(`[eval] baseline kept: ${baseline.runId}`);
+        return opts.gateRequested ? { gateFailed: true, gateReason: reason } : { gateFailed: false };
+    }
+    write("[eval] gate: PASS");
+    if (opts.promote) {
+        pinCurrentRun("promoted");
+    }
+    else {
+        write(`[eval] baseline kept: ${baseline.runId} (--no-promote)`);
+    }
+    return { gateFailed: false };
+}

package/dist/eval-matrix.d.ts

@@ -0,0 +1,80 @@
+import type { MatrixCell, MatrixPricingFn } from "@crewhaus/eval-report";
+import type { EvalRunSummary } from "@crewhaus/eval-runner";
+/** Thrown on a malformed `--models` value or an incompatible flag combo.
+ *  The CLI entry file catches it and routes the message through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export declare class MatrixArgError extends Error {
+    readonly name = "MatrixArgError";
+}
+/**
+ * Parse `--models m1,m2,...`: split on commas, trim whitespace, drop empty
+ * segments (trailing-comma tolerance), reject duplicates, and validate
+ * every entry against the model-router grammar UP FRONT — a typo must fail
+ * before cell 1 burns a single token.
+ */
+export declare function parseModelsFlag(value: string): string[];
+/**
+ * `--gate` / `--no-promote` steer the item-3 baseline lineage, which matrix
+ * cells skip entirely — reject the combination instead of silently ignoring
+ * the flags.
+ */
+export declare function assertMatrixFlagsCompatible(flags: {
+    readonly gate: boolean;
+    readonly noPromote: boolean;
+}): void;
+/**
+ * Filesystem-safe cell directory name for a model string: every character
+ * outside `[A-Za-z0-9._-]` collapses to a single `_` (so the slug stays
+ * readable/reversible-enough: `openai/gpt-4o` → `openai_gpt-4o`,
+ * `local/llama3.2@http://localhost:11434/v1` →
+ * `local_llama3.2_http_localhost_11434_v1`). A slug with no alphanumeric
+ * left (nothing a path could safely be named after — e.g. all dots) falls
+ * back to a stable content-hash name.
+ */
+export declare function modelSlug(model: string): string;
+/**
+ * model → cell-directory slug for the whole matrix. Distinct models whose
+ * slugs collide (e.g. `openai/gpt.4o` vs `openai/gpt_4o`) get a `_2`, `_3`,
+ * … suffix in list order, so every cell keeps its own directory.
+ */
+export declare function assignCellSlugs(models: ReadonlyArray<string>): Map<string, string>;
+/**
+ * The real `MatrixPricingFn` for eval-report's matrix renderer: model
+ * string → provider via the model-router grammar, then cost-tracker's
+ * versioned pricing table over the cell's token totals (cached-read tokens
+ * are 0 — eval aggregates don't track them). Any miss — unparseable model,
+ * unknown provider row (e.g. a groq/ or local/ model billed elsewhere) —
+ * returns `undefined`, which renders as "n/a" rather than crashing.
+ */
+export declare function defaultMatrixPricing(): MatrixPricingFn;
+export type RunMatrixCellsOptions = {
+    readonly models: ReadonlyArray<string>;
+    /** model → cell directory name (see {@link assignCellSlugs}). */
+    readonly slugs: ReadonlyMap<string, string>;
+    /** Matrix root; each cell runs in `<rootDir>/<slug>`. */
+    readonly rootDir: string;
+    /** Execute one cell's eval and return its summary. Injected so tests can
+     *  stub the runner; the CLI passes a `runEval` wrapper that patches the
+     *  lowered ir's `agent.model` in-memory (mirroring `run --model`). */
+    readonly runCell: (model: string, cellOutDir: string) => Promise<EvalRunSummary>;
+    /** Line sink; defaults to stdout. */
+    readonly write?: (line: string) => void;
+};
+/**
+ * "The cell never really ran": eval-runner isolates per-sample invoker
+ * errors (a summary always comes back), so a bad credential / 404 model
+ * surfaces as EVERY sample erroring rather than as a throw. Map that to a
+ * cell failure — its 0% pass rate and 0ms latencies are artifacts of never
+ * producing output, not a comparison result. Partial sample errors (a 529
+ * blip on 1 of 20) leave the cell a normal "ran with failing samples" row.
+ */
+export declare function cellCrashReason(summary: EvalRunSummary): string | undefined;
+/**
+ * Run every cell sequentially, isolating failures: one model erroring (bad
+ * credentials, 404 model, …) — whether thrown or absorbed by the runner as
+ * all-samples-errored (see {@link cellCrashReason}) — records an error cell
+ * and the loop continues. The caller maps "any error cell" to a non-zero
+ * exit AFTER rendering the matrix — a crashed cell is distinct from a cell
+ * that ran with failing samples, which is a normal result.
+ */
+export declare function runMatrixCells(opts: RunMatrixCellsOptions): Promise<MatrixCell[]>;

package/dist/eval-matrix.js

@@ -0,0 +1,182 @@
+/**
+ * Item 11 — `crewhaus eval --models <m1,m2,...>`: one-command model
+ * benchmark matrix. The CLI resolves the dataset (registry ref or file,
+ * item-9 regression union included) ONCE, then runs the same samples +
+ * graders once per model, each cell writing a full run directory to
+ * `<out>/<model-slug>/` so `eval-report diff` works on any pair of cells.
+ *
+ * Deliberate non-interactions with the item-3 run-history features:
+ * matrix cells are model comparisons, not lineage runs — they never touch
+ * `finishEvalRun` (index append / baseline pin / gate / promote), because a
+ * shared (spec, dataset) baseline key across N models would corrupt the
+ * lineage. `--gate` / `--no-promote` are rejected up front for the same
+ * reason (see {@link assertMatrixFlagsCompatible}).
+ *
+ * Kept in a side-effect-free module (the CLI entry file runs an argv
+ * switch on import) mirroring `eval-history.ts` / `datasets.ts`: the
+ * per-cell eval execution is injected as `runCell`, so failure isolation
+ * is unit-testable without an LLM.
+ */
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { DEFAULT_PRICING, computeCostMicros, resolvePricing } from "@crewhaus/cost-tracker";
+import { parseModelString } from "@crewhaus/model-router";
+/** Thrown on a malformed `--models` value or an incompatible flag combo.
+ *  The CLI entry file catches it and routes the message through `die()`;
+ *  tests assert on `.message` without the process exiting. */
+export class MatrixArgError extends Error {
+    name = "MatrixArgError";
+}
+// -------- flag parsing / validation --------
+/**
+ * Parse `--models m1,m2,...`: split on commas, trim whitespace, drop empty
+ * segments (trailing-comma tolerance), reject duplicates, and validate
+ * every entry against the model-router grammar UP FRONT — a typo must fail
+ * before cell 1 burns a single token.
+ */
+export function parseModelsFlag(value) {
+    const models = value
+        .split(",")
+        .map((m) => m.trim())
+        .filter((m) => m.length > 0);
+    if (models.length === 0) {
+        throw new MatrixArgError("--models: expected a comma-separated list of model strings (e.g. claude-sonnet-4-5,openai/gpt-4o)");
+    }
+    const seen = new Set();
+    for (const model of models) {
+        if (seen.has(model)) {
+            throw new MatrixArgError(`--models: duplicate model "${model}"`);
+        }
+        seen.add(model);
+        try {
+            parseModelString(model);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new MatrixArgError(`--models: ${msg}`);
+        }
+    }
+    return models;
+}
+/**
+ * `--gate` / `--no-promote` steer the item-3 baseline lineage, which matrix
+ * cells skip entirely — reject the combination instead of silently ignoring
+ * the flags.
+ */
+export function assertMatrixFlagsCompatible(flags) {
+    if (flags.gate) {
+        throw new MatrixArgError("--models is incompatible with --gate — matrix cells are model comparisons and skip the (spec, dataset) baseline lineage; gate a single-model eval instead");
+    }
+    if (flags.noPromote) {
+        throw new MatrixArgError("--models is incompatible with --no-promote — matrix cells never touch the run index or baselines, so there is nothing to promote");
+    }
+}
+// -------- slugs --------
+/**
+ * Filesystem-safe cell directory name for a model string: every character
+ * outside `[A-Za-z0-9._-]` collapses to a single `_` (so the slug stays
+ * readable/reversible-enough: `openai/gpt-4o` → `openai_gpt-4o`,
+ * `local/llama3.2@http://localhost:11434/v1` →
+ * `local_llama3.2_http_localhost_11434_v1`). A slug with no alphanumeric
+ * left (nothing a path could safely be named after — e.g. all dots) falls
+ * back to a stable content-hash name.
+ */
+export function modelSlug(model) {
+    const slug = model.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "");
+    if (!/[A-Za-z0-9]/.test(slug)) {
+        return `model_${createHash("sha256").update(model).digest("hex").slice(0, 8)}`;
+    }
+    return slug;
+}
+/**
+ * model → cell-directory slug for the whole matrix. Distinct models whose
+ * slugs collide (e.g. `openai/gpt.4o` vs `openai/gpt_4o`) get a `_2`, `_3`,
+ * … suffix in list order, so every cell keeps its own directory.
+ */
+export function assignCellSlugs(models) {
+    const taken = new Set();
+    const out = new Map();
+    for (const model of models) {
+        const base = modelSlug(model);
+        let candidate = base;
+        for (let i = 2; taken.has(candidate); i += 1)
+            candidate = `${base}_${i}`;
+        taken.add(candidate);
+        out.set(model, candidate);
+    }
+    return out;
+}
+// -------- pricing seam --------
+/**
+ * The real `MatrixPricingFn` for eval-report's matrix renderer: model
+ * string → provider via the model-router grammar, then cost-tracker's
+ * versioned pricing table over the cell's token totals (cached-read tokens
+ * are 0 — eval aggregates don't track them). Any miss — unparseable model,
+ * unknown provider row (e.g. a groq/ or local/ model billed elsewhere) —
+ * returns `undefined`, which renders as "n/a" rather than crashing.
+ */
+export function defaultMatrixPricing() {
+    return (model, tokens) => {
+        try {
+            const parsed = parseModelString(model);
+            const row = resolvePricing(DEFAULT_PRICING, parsed.providerId, parsed.modelId);
+            if (row === undefined)
+                return undefined;
+            return computeCostMicros(row, tokens.input, tokens.output, 0);
+        }
+        catch {
+            return undefined;
+        }
+    };
+}
+/**
+ * "The cell never really ran": eval-runner isolates per-sample invoker
+ * errors (a summary always comes back), so a bad credential / 404 model
+ * surfaces as EVERY sample erroring rather than as a throw. Map that to a
+ * cell failure — its 0% pass rate and 0ms latencies are artifacts of never
+ * producing output, not a comparison result. Partial sample errors (a 529
+ * blip on 1 of 20) leave the cell a normal "ran with failing samples" row.
+ */
+export function cellCrashReason(summary) {
+    const n = summary.samples.length;
+    if (n === 0 || summary.aggregates.errorCount < n)
+        return undefined;
+    const first = summary.samples.find((s) => s.error !== undefined)?.error;
+    return `all ${n} sample(s) errored${first !== undefined ? ` (first: ${first})` : ""}`;
+}
+/**
+ * Run every cell sequentially, isolating failures: one model erroring (bad
+ * credentials, 404 model, …) — whether thrown or absorbed by the runner as
+ * all-samples-errored (see {@link cellCrashReason}) — records an error cell
+ * and the loop continues. The caller maps "any error cell" to a non-zero
+ * exit AFTER rendering the matrix — a crashed cell is distinct from a cell
+ * that ran with failing samples, which is a normal result.
+ */
+export async function runMatrixCells(opts) {
+    const write = opts.write ?? ((line) => process.stdout.write(`${line}\n`));
+    const cells = [];
+    for (const model of opts.models) {
+        const slug = opts.slugs.get(model) ?? modelSlug(model);
+        const outDir = join(opts.rootDir, slug);
+        write(`[eval] cell ${model} → ${outDir}`);
+        try {
+            const summary = await opts.runCell(model, outDir);
+            const crashed = cellCrashReason(summary);
+            if (crashed !== undefined) {
+                cells.push({ model, slug, outDir, error: crashed });
+                write(`[eval]   cell FAILED (${crashed}) — continuing with remaining models`);
+                continue;
+            }
+            cells.push({ model, slug, outDir, summary });
+            write(`[eval]   pass_rate=${(summary.aggregates.passRate * 100).toFixed(1)}% ` +
+                `mean_score=${summary.aggregates.meanScore.toFixed(3)} ` +
+                `errors=${summary.aggregates.errorCount}`);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            cells.push({ model, slug, outDir, error: msg });
+            write(`[eval]   cell FAILED (${msg}) — continuing with remaining models`);
+        }
+    }
+    return cells;
+}

package/dist/eval-sentinel.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Item 30 — nightly model-drift sentinel evals against a FROZEN baseline.
+ *
+ * `crewhaus eval --sentinel --baseline <run-dir>` re-runs a seed-pinned
+ * sentinel dataset against the UNCHANGED spec, then compares the fresh run to
+ * a stored baseline run. The premise: when the spec (specHash), the dataset
+ * (content hash), the judge model, AND the graders config (content hash) are
+ * ALL byte-identical to the baseline, ANY pass/fail flip or score shift can
+ * only be attributed to the PROVIDER silently changing model behaviour — the
+ * one variable the harness did not hold fixed. So the sentinel:
+ *
+ *   1. asserts `specHash` equality (baseline vs fresh run) — a mismatch means
+ *      the spec changed, so this is NOT a clean drift probe: report and exit
+ *      non-zero so a mis-pointed sentinel is loud rather than silently green.
+ *   2. asserts DATASET-HASH equality. run.json only records `datasetName`, so
+ *      the caller sha256's the dataset content itself and passes both hashes;
+ *      a mismatch (different sentinel data) is likewise NOT a clean probe.
+ *   3. asserts `config.judgeModel` equality AND `config.gradersHash` equality
+ *      (sha256 of the parsed GradersConfig) — a different `--judge-model` or
+ *      an edited graders.yaml changes what "pass" MEANS independent of the
+ *      provider under test, so either mismatch is likewise NOT a clean probe
+ *      (F2: without this, a judge/grader change silently reads as "provider
+ *      drift" because neither ever touched specHash or the dataset).
+ *   4. with every hash equal, diffs the two runs (`diffReports`) and gates on
+ *      the strict regression-runner defaults. Any regression or score shift is
+ *      flagged as provider drift; the command exits non-zero so a CI cron /
+ *      scheduler alerts.
+ *
+ * Side-effect-free (this package's entry file runs an argv switch on import):
+ * the fresh run + baseline load are performed by the caller and passed in as
+ * `LoadedRun`s, so the drift logic is unit-testable with seeded run dirs.
+ */
+import { type LoadedRun, type ReportDiff } from "@crewhaus/eval-report";
+export type SentinelVerdict = "clean" | "drift" | "not-comparable";
+export type SentinelResult = {
+    readonly verdict: SentinelVerdict;
+    /** True when the caller should exit non-zero (drift OR not-comparable). */
+    readonly alert: boolean;
+    readonly reason: string;
+    /** The score/flip diff, present only when the runs were comparable. */
+    readonly diff?: ReportDiff;
+};
+/**
+ * Compare a fresh run against a frozen baseline and decide whether the
+ * provider drifted. `baselineDatasetHash` / `currentDatasetHash` are the
+ * sha256 of the dataset CONTENT (run.json records only `datasetName`), so the
+ * caller hashes the dataset file/registry record and passes both in.
+ *
+ * Verdicts:
+ *   - `not-comparable` (alert): specHash, dataset-hash, judgeModel, or
+ *     gradersHash differs — the spec, the sentinel data, or what "pass" means
+ *     changed, so a diff cannot be attributed to the provider. Loud, not
+ *     silent: a mis-pointed sentinel must not read green.
+ *   - `drift` (alert): every hash/model equal AND the diff shows a regression
+ *     or a score shift ⇒ the provider silently changed model behaviour.
+ *   - `clean` (no alert): every hash/model equal AND no regression / no score
+ *     shift (recoveries alone are not drift-alerting — a provider getting
+ *     BETTER on frozen inputs is still worth noting but is not a failure).
+ */
+export declare function evaluateSentinel(opts: {
+    readonly baseline: LoadedRun;
+    readonly current: LoadedRun;
+    readonly baselineDatasetHash: string;
+    readonly currentDatasetHash: string;
+}): SentinelResult;

package/dist/eval-sentinel.js

@@ -0,0 +1,132 @@
+/**
+ * Item 30 — nightly model-drift sentinel evals against a FROZEN baseline.
+ *
+ * `crewhaus eval --sentinel --baseline <run-dir>` re-runs a seed-pinned
+ * sentinel dataset against the UNCHANGED spec, then compares the fresh run to
+ * a stored baseline run. The premise: when the spec (specHash), the dataset
+ * (content hash), the judge model, AND the graders config (content hash) are
+ * ALL byte-identical to the baseline, ANY pass/fail flip or score shift can
+ * only be attributed to the PROVIDER silently changing model behaviour — the
+ * one variable the harness did not hold fixed. So the sentinel:
+ *
+ *   1. asserts `specHash` equality (baseline vs fresh run) — a mismatch means
+ *      the spec changed, so this is NOT a clean drift probe: report and exit
+ *      non-zero so a mis-pointed sentinel is loud rather than silently green.
+ *   2. asserts DATASET-HASH equality. run.json only records `datasetName`, so
+ *      the caller sha256's the dataset content itself and passes both hashes;
+ *      a mismatch (different sentinel data) is likewise NOT a clean probe.
+ *   3. asserts `config.judgeModel` equality AND `config.gradersHash` equality
+ *      (sha256 of the parsed GradersConfig) — a different `--judge-model` or
+ *      an edited graders.yaml changes what "pass" MEANS independent of the
+ *      provider under test, so either mismatch is likewise NOT a clean probe
+ *      (F2: without this, a judge/grader change silently reads as "provider
+ *      drift" because neither ever touched specHash or the dataset).
+ *   4. with every hash equal, diffs the two runs (`diffReports`) and gates on
+ *      the strict regression-runner defaults. Any regression or score shift is
+ *      flagged as provider drift; the command exits non-zero so a CI cron /
+ *      scheduler alerts.
+ *
+ * Side-effect-free (this package's entry file runs an argv switch on import):
+ * the fresh run + baseline load are performed by the caller and passed in as
+ * `LoadedRun`s, so the drift logic is unit-testable with seeded run dirs.
+ */
+import { ReportError, diffReports } from "@crewhaus/eval-report";
+/**
+ * Compare a fresh run against a frozen baseline and decide whether the
+ * provider drifted. `baselineDatasetHash` / `currentDatasetHash` are the
+ * sha256 of the dataset CONTENT (run.json records only `datasetName`), so the
+ * caller hashes the dataset file/registry record and passes both in.
+ *
+ * Verdicts:
+ *   - `not-comparable` (alert): specHash, dataset-hash, judgeModel, or
+ *     gradersHash differs — the spec, the sentinel data, or what "pass" means
+ *     changed, so a diff cannot be attributed to the provider. Loud, not
+ *     silent: a mis-pointed sentinel must not read green.
+ *   - `drift` (alert): every hash/model equal AND the diff shows a regression
+ *     or a score shift ⇒ the provider silently changed model behaviour.
+ *   - `clean` (no alert): every hash/model equal AND no regression / no score
+ *     shift (recoveries alone are not drift-alerting — a provider getting
+ *     BETTER on frozen inputs is still worth noting but is not a failure).
+ */
+export function evaluateSentinel(opts) {
+    const baseSpecHash = opts.baseline.summary.config.specHash;
+    const curSpecHash = opts.current.summary.config.specHash;
+    if (baseSpecHash !== curSpecHash) {
+        return {
+            verdict: "not-comparable",
+            alert: true,
+            reason: `spec changed since the baseline (specHash ${short(baseSpecHash)} → ${short(curSpecHash)}) — a sentinel probe requires an UNCHANGED spec; re-pin the baseline against the current spec`,
+        };
+    }
+    if (opts.baselineDatasetHash !== opts.currentDatasetHash) {
+        return {
+            verdict: "not-comparable",
+            alert: true,
+            reason: `sentinel dataset changed since the baseline (datasetHash ${short(opts.baselineDatasetHash)} → ${short(opts.currentDatasetHash)}) — a sentinel probe requires byte-identical data; re-pin the baseline against the current dataset`,
+        };
+    }
+    // F2 — a different judge model or an edited graders.yaml changes what
+    // "pass" MEANS independent of the provider under test. Neither touches
+    // specHash or the dataset hash, so without this check the sentinel would
+    // misattribute the resulting score shift to "provider drift".
+    const baseJudgeModel = opts.baseline.summary.config.judgeModel;
+    const curJudgeModel = opts.current.summary.config.judgeModel;
+    if (baseJudgeModel !== curJudgeModel) {
+        return {
+            verdict: "not-comparable",
+            alert: true,
+            reason: `judge model changed since the baseline (judgeModel ${judgeModelLabel(baseJudgeModel)} → ${judgeModelLabel(curJudgeModel)}) — a sentinel probe requires an UNCHANGED judge model; re-pin the baseline against the current --judge-model`,
+        };
+    }
+    const baseGradersHash = opts.baseline.summary.config.gradersHash;
+    const curGradersHash = opts.current.summary.config.gradersHash;
+    if (baseGradersHash !== curGradersHash) {
+        return {
+            verdict: "not-comparable",
+            alert: true,
+            reason: `graders config changed since the baseline (gradersHash ${short(baseGradersHash)} → ${short(curGradersHash)}) — a sentinel probe requires byte-identical graders; re-pin the baseline against the current graders.yaml`,
+        };
+    }
+    let diff;
+    try {
+        diff = diffReports(opts.baseline, opts.current).diff;
+    }
+    catch (err) {
+        if (err instanceof ReportError) {
+            // Same specHash + dataset-hash but a keyset mismatch is a contradiction
+            // (identical data can't change sample ids) — surface it as not-comparable
+            // rather than crash the nightly job.
+            return {
+                verdict: "not-comparable",
+                alert: true,
+                reason: `sentinel diff failed despite equal hashes: ${err.message}`,
+            };
+        }
+        throw err;
+    }
+    const drifted = diff.regressions.length > 0 || diff.scoreShifts.length > 0;
+    if (drifted) {
+        return {
+            verdict: "drift",
+            alert: true,
+            reason: `provider drift on a frozen spec+dataset+judge+graders: ${diff.regressions.length} regression(s), ` +
+                `${diff.scoreShifts.length} score-shift(s), ${diff.recoveries.length} recovery(-ies) ` +
+                `(specHash ${short(baseSpecHash)}, datasetHash ${short(opts.baselineDatasetHash)} unchanged)`,
+            diff,
+        };
+    }
+    return {
+        verdict: "clean",
+        alert: false,
+        reason: `no drift: spec+dataset+judge+graders frozen and every sample held its verdict (${diff.recoveries.length} recovery(-ies), ${diff.unchanged} unchanged)`,
+        diff,
+    };
+}
+function judgeModelLabel(model) {
+    return model === undefined ? "(default)" : model;
+}
+function short(hash) {
+    if (hash === undefined)
+        return "(none)";
+    return hash.length > 12 ? `${hash.slice(0, 12)}…` : hash;
+}