@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/esm/EvalError.js

@@ -0,0 +1,65 @@
+/**
+ * The typed failures evaluation raises, with the stable codes a gate or a
+ * report branches on.
+ *
+ * @since 0.1.0
+ */
+import * as Schema from "effect/Schema";
+/**
+ * Stable evaluation failure codes.
+ *
+ * Each code names one thing a caller can do about it, so a gate that branches
+ * on the code routes the failure to the right owner:
+ *
+ * - `invalid_suite`: the suite declaration is wrong. Fix the suite.
+ * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.
+ * - `invalid_baseline`: the committed baseline is unreadable, belongs to
+ *   another suite, or holds a record the schema rejects. Regenerate it.
+ * - `invalid_tolerance`: the caller passed a tolerance that is not a finite
+ *   non-negative number. Fix the call, not the baseline.
+ * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`
+ *   outside their bounds. Fix the call.
+ * - `executor`: the target flow failed for a case.
+ * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so
+ *   an order-only batch runner's results cannot be attributed to a case. Give
+ *   each case its own step key, or use a runner that implements
+ *   `runBatchCorrelated`.
+ * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning
+ *   the wrong number of observations, or observations that identify jobs other
+ *   than the ones it was given. Nothing it returned can be trusted.
+ * - `scorer_unavailable`: no batch runner was available to score with.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const EvalErrorCode = Schema.Literals([
+    "invalid_suite",
+    "invalid_run_options",
+    "invalid_baseline",
+    "invalid_tolerance",
+    "invalid_trials",
+    "executor",
+    "ambiguous_score_job",
+    "scorer_protocol",
+    "scorer_unavailable"
+]);
+/**
+ * A typed failure raised while loading or executing an evaluation.
+ *
+ * `code` is the stable branch point, `message` is the sentence a CI log shows,
+ * and `path` locates the offending value inside the input the caller supplied
+ * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the
+ * original failure; it is never rendered on its own, so a message that matters
+ * to an operator has to say so itself.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export class EvalError extends Schema.TaggedError()("flows/evals/EvalError", {
+    code: EvalErrorCode,
+    message: Schema.String,
+    path: Schema.optional(Schema.String),
+    cause: Schema.optional(Schema.Defect())
+}) {
+}
+//# sourceMappingURL=EvalError.js.map

package/dist/esm/EvalError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalError.js","sourceRoot":"","sources":["../../src/EvalError.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,eAAe;IACf,qBAAqB;IACrB,kBAAkB;IAClB,mBAAmB;IACnB,gBAAgB;IAChB,UAAU;IACV,qBAAqB;IACrB,iBAAiB;IACjB,oBAAoB;CACrB,CAAC,CAAA;AAUF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,SAAU,SAAQ,MAAM,CAAC,WAAW,EAAa,CAAC,uBAAuB,EAAE;IACtF,IAAI,EAAE,aAAa;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM;IACtB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;IACpC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;CACxC,CAAC;CAAG","sourcesContent":["/**\n * The typed failures evaluation raises, with the stable codes a gate or a\n * report branches on.\n *\n * @since 0.1.0\n */\n\nimport * as Schema from \"effect/Schema\"\n\n/**\n * Stable evaluation failure codes.\n *\n * Each code names one thing a caller can do about it, so a gate that branches\n * on the code routes the failure to the right owner:\n *\n * - `invalid_suite`: the suite declaration is wrong. Fix the suite.\n * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.\n * - `invalid_baseline`: the committed baseline is unreadable, belongs to\n *   another suite, or holds a record the schema rejects. Regenerate it.\n * - `invalid_tolerance`: the caller passed a tolerance that is not a finite\n *   non-negative number. Fix the call, not the baseline.\n * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`\n *   outside their bounds. Fix the call.\n * - `executor`: the target flow failed for a case.\n * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so\n *   an order-only batch runner's results cannot be attributed to a case. Give\n *   each case its own step key, or use a runner that implements\n *   `runBatchCorrelated`.\n * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning\n *   the wrong number of observations, or observations that identify jobs other\n *   than the ones it was given. Nothing it returned can be trusted.\n * - `scorer_unavailable`: no batch runner was available to score with.\n *\n * @category models\n * @since 0.1.0\n */\nexport const EvalErrorCode = Schema.Literals([\n  \"invalid_suite\",\n  \"invalid_run_options\",\n  \"invalid_baseline\",\n  \"invalid_tolerance\",\n  \"invalid_trials\",\n  \"executor\",\n  \"ambiguous_score_job\",\n  \"scorer_protocol\",\n  \"scorer_unavailable\"\n])\n\n/**\n * Stable evaluation failure code.\n *\n * @category models\n * @since 0.1.0\n */\nexport type EvalErrorCode = typeof EvalErrorCode.Type\n\n/**\n * A typed failure raised while loading or executing an evaluation.\n *\n * `code` is the stable branch point, `message` is the sentence a CI log shows,\n * and `path` locates the offending value inside the input the caller supplied\n * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the\n * original failure; it is never rendered on its own, so a message that matters\n * to an operator has to say so itself.\n *\n * @category errors\n * @since 0.1.0\n */\nexport class EvalError extends Schema.TaggedError<EvalError>()(\"flows/evals/EvalError\", {\n  code: EvalErrorCode,\n  message: Schema.String,\n  path: Schema.optional(Schema.String),\n  cause: Schema.optional(Schema.Defect())\n}) {}\n"]}

package/dist/esm/Gate.d.ts

@@ -0,0 +1,55 @@
+/**
+ * The pass/fail decision over a finished evaluation run: how a threshold
+ * is applied and what a failing gate reports.
+ *
+ * @since 0.1.0
+ */
+import { type ScoreGateError, type Verdict } from "@smthrs/scorers/ScoreGate";
+import * as Effect from "effect/Effect";
+import type { Report } from "./Regression.ts";
+/**
+ * Thresholds accepted by a CI score gate.
+ *
+ * `mean` gates the arithmetic mean of every score observation, `min` gates the
+ * lowest one, and `perCase` gates each named case's lowest score. With none of
+ * them set the gate still runs, as `mean(0)`, so a run with no score at all is
+ * undecidable rather than a pass.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Options {
+    readonly mean?: number | undefined;
+    readonly min?: number | undefined;
+    readonly perCase?: Readonly<Record<string, number>> | undefined;
+}
+/**
+ * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.
+ *
+ * The threshold gates always run: an unobserved case cannot excuse the cases
+ * that were observed. Faults and findings are kept apart, and the verdict
+ * carries both, so a regression is a red while an unusable harness stays
+ * undecided.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const check: (report: Report, options?: Options) => Effect.Effect<Verdict, ScoreGateError>;
+/**
+ * Maps a gate verdict to the shared CI convention: a finding is exit code 1,
+ * an undecidable run is exit code 5.
+ *
+ * The summary is one log line: every C0 control and DEL in it becomes a
+ * space, the way a report cell is written, so a value that reached a verdict
+ * reason from a baseline, a step key or a target's failure cannot emit its own
+ * line, which on GitHub Actions would be a workflow command, or a terminal
+ * escape.
+ *
+ * @category grading
+ * @since 0.1.0
+ */
+export declare const ciGrade: (verdict: Verdict) => {
+    readonly exitCode: 0 | 1 | 5;
+    readonly summary: string;
+};
+//# sourceMappingURL=Gate.d.ts.map

package/dist/esm/Gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Gate.d.ts","sourceRoot":"","sources":["../../src/Gate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAIL,KAAK,cAAc,EAEnB,KAAK,OAAO,EACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;AAE7C;;;;;;;;;;GAUG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAClC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACjC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAA;CAChE;AAkDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAAI,QAAQ,MAAM,EAAE,UAAS,OAAY,KAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAa/F,CAAA;AAEJ;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,OAAO,GAAI,SAAS,OAAO,KAAG;IAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAGlG,CAAA"}

package/dist/esm/Gate.js

@@ -0,0 +1,90 @@
+/**
+ * The pass/fail decision over a finished evaluation run: how a threshold
+ * is applied and what a failing gate reports.
+ *
+ * @since 0.1.0
+ */
+import { combine, expectScores, grade } from "@smthrs/scorers/ScoreGate";
+import * as Effect from "effect/Effect";
+import { flattenControlCharacters } from "./internal/controlCharacters.js";
+import { scorerLabel } from "./internal/scorerLabel.js";
+const samples = (report) => report.run.observations.map((observation) => observation.kind === "score"
+    ? {
+        case: observation.case,
+        scorer: observation.scorer,
+        stepKey: observation.stepKey,
+        kind: "score",
+        value: observation.score,
+        reason: observation.reason
+    }
+    : {
+        case: observation.case,
+        scorer: observation.scorer,
+        stepKey: observation.stepKey,
+        kind: "inconclusive",
+        reason: observation.reason
+    });
+/**
+ * An environment fault: something the comparison needed was never observed, so
+ * the harness owes an answer it cannot give. A fault withholds a decision;
+ * it does not make one.
+ */
+const environmentFaults = (report) => [
+    ...report.run.cases.flatMap((result) => result.error === undefined ? [] : [`case '${result.case}' failed: ${result.error.code}: ${result.error.message}`]),
+    ...report.missing.map((item) => `missing ${item.side} observation for ${item.case}/${scorerLabel(item)}/${item.stepKey}`)
+];
+/**
+ * A finding: the run measured something the baseline says it should not have.
+ * A regression scored lower at a changed step key, and nondeterminism moved a
+ * score at an unchanged one. Both are results, so both are red.
+ */
+const findings = (report) => [
+    ...report.regressions.map((item) => `regression for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`),
+    ...report.nondeterminism.map((item) => `nondeterminism for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`)
+];
+/**
+ * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.
+ *
+ * The threshold gates always run: an unobserved case cannot excuse the cases
+ * that were observed. Faults and findings are kept apart, and the verdict
+ * carries both, so a regression is a red while an unusable harness stays
+ * undecided.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const check = (report, options = {}) => Effect.gen(function* () {
+    const expectation = expectScores(samples(report));
+    const verdicts = [];
+    if (options.mean !== undefined)
+        verdicts.push(yield* expectation.mean(options.mean));
+    if (options.min !== undefined)
+        verdicts.push(yield* expectation.min(options.min));
+    if (options.perCase !== undefined)
+        verdicts.push(yield* expectation.perCase(options.perCase));
+    if (options.mean === undefined && options.min === undefined && options.perCase === undefined) {
+        verdicts.push(yield* expectation.mean(0));
+    }
+    const reasons = findings(report);
+    if (reasons.length > 0)
+        verdicts.push({ _tag: "Failed", reasons, inconclusive: [] });
+    return combine(verdicts, environmentFaults(report));
+});
+/**
+ * Maps a gate verdict to the shared CI convention: a finding is exit code 1,
+ * an undecidable run is exit code 5.
+ *
+ * The summary is one log line: every C0 control and DEL in it becomes a
+ * space, the way a report cell is written, so a value that reached a verdict
+ * reason from a baseline, a step key or a target's failure cannot emit its own
+ * line, which on GitHub Actions would be a workflow command, or a terminal
+ * escape.
+ *
+ * @category grading
+ * @since 0.1.0
+ */
+export const ciGrade = (verdict) => {
+    const graded = grade(verdict);
+    return { exitCode: graded.exitCode, summary: flattenControlCharacters(graded.summary) };
+};
+//# sourceMappingURL=Gate.js.map

package/dist/esm/Gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Gate.js","sourceRoot":"","sources":["../../src/Gate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,OAAO,EACP,YAAY,EACZ,KAAK,EAIN,MAAM,2BAA2B,CAAA;AAClC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAoBvD,MAAM,OAAO,GAAG,CAAC,MAAc,EAA8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAC1C,WAAW,CAAC,IAAI,KAAK,OAAO;IAC1B,CAAC,CAAC;QACA,IAAI,EAAE,WAAW,CAAC,IAAI;QACtB,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,MAAM,EAAE,WAAW,CAAC,MAAM;KAC3B;IACD,CAAC,CAAC;QACA,IAAI,EAAE,WAAW,CAAC,IAAI;QACtB,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,OAAO,EAAE,WAAW,CAAC,OAAO;QAC5B,IAAI,EAAE,cAAc;QACpB,MAAM,EAAE,WAAW,CAAC,MAAM;KAC3B,CACJ,CAAA;AAEH;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,CAAC,MAAc,EAAyB,EAAE,CAAC;IACnE,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CACrC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAClH;IACD,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7B,WAAW,IAAI,CAAC,IAAI,oBAAoB,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CACzF;CACF,CAAA;AAED;;;;GAIG;AACH,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAyB,EAAE,CAAC;IAC1D,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACjC,kBAAkB,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,CAC1G;IACD,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACpC,sBAAsB,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,CAC9G;CACF,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,UAAmB,EAAE,EAA0C,EAAE,CACrG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;IACjD,MAAM,QAAQ,GAAmB,EAAE,CAAA;IACnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IACpF,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;IACjF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;IAC7F,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC7F,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3C,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAA;IAChC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAA;IACpF,OAAO,OAAO,CAAC,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAA;AACrD,CAAC,CAAC,CAAA;AAEJ;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAgB,EAA8D,EAAE;IACtG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAA;IAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,wBAAwB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;AACzF,CAAC,CAAA","sourcesContent":["/**\n * The pass/fail decision over a finished evaluation run: how a threshold\n * is applied and what a failing gate reports.\n *\n * @since 0.1.0\n */\n\nimport {\n  combine,\n  expectScores,\n  grade,\n  type ScoreGateError,\n  type ScoreSample,\n  type Verdict\n} from \"@smthrs/scorers/ScoreGate\"\nimport * as Effect from \"effect/Effect\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport type { Report } from \"./Regression.ts\"\n\n/**\n * Thresholds accepted by a CI score gate.\n *\n * `mean` gates the arithmetic mean of every score observation, `min` gates the\n * lowest one, and `perCase` gates each named case's lowest score. With none of\n * them set the gate still runs, as `mean(0)`, so a run with no score at all is\n * undecidable rather than a pass.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Options {\n  readonly mean?: number | undefined\n  readonly min?: number | undefined\n  readonly perCase?: Readonly<Record<string, number>> | undefined\n}\n\nconst samples = (report: Report): ReadonlyArray<ScoreSample> =>\n  report.run.observations.map((observation) =>\n    observation.kind === \"score\"\n      ? {\n        case: observation.case,\n        scorer: observation.scorer,\n        stepKey: observation.stepKey,\n        kind: \"score\",\n        value: observation.score,\n        reason: observation.reason\n      }\n      : {\n        case: observation.case,\n        scorer: observation.scorer,\n        stepKey: observation.stepKey,\n        kind: \"inconclusive\",\n        reason: observation.reason\n      }\n  )\n\n/**\n * An environment fault: something the comparison needed was never observed, so\n * the harness owes an answer it cannot give. A fault withholds a decision;\n * it does not make one.\n */\nconst environmentFaults = (report: Report): ReadonlyArray<string> => [\n  ...report.run.cases.flatMap((result) =>\n    result.error === undefined ? [] : [`case '${result.case}' failed: ${result.error.code}: ${result.error.message}`]\n  ),\n  ...report.missing.map((item) =>\n    `missing ${item.side} observation for ${item.case}/${scorerLabel(item)}/${item.stepKey}`\n  )\n]\n\n/**\n * A finding: the run measured something the baseline says it should not have.\n * A regression scored lower at a changed step key, and nondeterminism moved a\n * score at an unchanged one. Both are results, so both are red.\n */\nconst findings = (report: Report): ReadonlyArray<string> => [\n  ...report.regressions.map((item) =>\n    `regression for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`\n  ),\n  ...report.nondeterminism.map((item) =>\n    `nondeterminism for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`\n  )\n]\n\n/**\n * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.\n *\n * The threshold gates always run: an unobserved case cannot excuse the cases\n * that were observed. Faults and findings are kept apart, and the verdict\n * carries both, so a regression is a red while an unusable harness stays\n * undecided.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const check = (report: Report, options: Options = {}): Effect.Effect<Verdict, ScoreGateError> =>\n  Effect.gen(function*() {\n    const expectation = expectScores(samples(report))\n    const verdicts: Array<Verdict> = []\n    if (options.mean !== undefined) verdicts.push(yield* expectation.mean(options.mean))\n    if (options.min !== undefined) verdicts.push(yield* expectation.min(options.min))\n    if (options.perCase !== undefined) verdicts.push(yield* expectation.perCase(options.perCase))\n    if (options.mean === undefined && options.min === undefined && options.perCase === undefined) {\n      verdicts.push(yield* expectation.mean(0))\n    }\n    const reasons = findings(report)\n    if (reasons.length > 0) verdicts.push({ _tag: \"Failed\", reasons, inconclusive: [] })\n    return combine(verdicts, environmentFaults(report))\n  })\n\n/**\n * Maps a gate verdict to the shared CI convention: a finding is exit code 1,\n * an undecidable run is exit code 5.\n *\n * The summary is one log line: every C0 control and DEL in it becomes a\n * space, the way a report cell is written, so a value that reached a verdict\n * reason from a baseline, a step key or a target's failure cannot emit its own\n * line, which on GitHub Actions would be a workflow command, or a terminal\n * escape.\n *\n * @category grading\n * @since 0.1.0\n */\nexport const ciGrade = (verdict: Verdict): { readonly exitCode: 0 | 1 | 5; readonly summary: string } => {\n  const graded = grade(verdict)\n  return { exitCode: graded.exitCode, summary: flattenControlCharacters(graded.summary) }\n}\n"]}

package/dist/esm/Regression.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Step-key-aware evaluation regression comparison.
+ *
+ * The step key is what separates the two findings this module reports. A score
+ * that moved at a *changed* step key is a regression: the target produced
+ * different work and it graded worse. A score that moved at an *unchanged* step
+ * key is nondeterminism: the same work graded differently twice. Both are
+ * results, and a gate reads both as red.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import type { Baseline, BaselineRecord } from "./Baseline.ts";
+import { EvalError } from "./EvalError.ts";
+import type { Observation, RunResult } from "./Runner.ts";
+/**
+ * Tolerances used for score comparisons.
+ *
+ * A move is reported only when it exceeds both tolerances, so either one alone
+ * is enough to silence it. Both default to `0`, which reports every move.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Tolerances {
+    readonly absolute?: number | undefined;
+    readonly relative?: number | undefined;
+}
+/**
+ * A score drop at a changed step key.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Regression {
+    readonly case: string;
+    readonly scorer: string;
+    readonly baseline: BaselineRecord;
+    readonly actual: Extract<Observation, {
+        readonly kind: "score";
+    }>;
+    readonly drop: number;
+}
+/**
+ * A changed score at the same step key, indicating nondeterminism.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Nondeterminism {
+    readonly case: string;
+    readonly scorer: string;
+    readonly baseline: BaselineRecord;
+    readonly actual: Extract<Observation, {
+        readonly kind: "score";
+    }>;
+    readonly delta: number;
+}
+/**
+ * An observation present on only one side of a comparison.
+ *
+ * `side` names the side it is missing from: `"run"` for a baseline record the
+ * run never reproduced, `"baseline"` for a score the run produced that no
+ * baseline record accounts for. `scorerName` comes from the observation that
+ * is present, when that artifact carried one.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MissingObservation {
+    readonly side: "baseline" | "run";
+    readonly case: string;
+    readonly scorer: string;
+    readonly scorerName?: string | undefined;
+    readonly stepKey: string;
+}
+/**
+ * Complete regression comparison.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Report {
+    readonly suite: string;
+    readonly baseline: Baseline;
+    readonly run: RunResult;
+    readonly regressions: ReadonlyArray<Regression>;
+    readonly nondeterminism: ReadonlyArray<Nondeterminism>;
+    readonly missing: ReadonlyArray<MissingObservation>;
+    readonly samples: ReadonlyArray<Extract<Observation, {
+        readonly kind: "score";
+    }>>;
+    readonly inconclusive: ReadonlyArray<Extract<Observation, {
+        readonly kind: "inconclusive";
+    }>>;
+}
+/**
+ * Compares a run to a baseline, preserving missing and inconclusive observations.
+ *
+ * Records and observations are grouped by `(case, scorer)` and then paired by
+ * step key, so a scorer that ran several times against one case is compared
+ * pairwise instead of by array position. A pair whose step key is unchanged and
+ * whose score moved is nondeterminism; a pair whose step key changed and whose
+ * score dropped is a regression; an unpaired record on either side is a missing
+ * observation. Inconclusive observations are carried through untouched, because
+ * they decide nothing and a gate has to see them.
+ *
+ * Fails with `invalid_tolerance` when a tolerance is not finite and
+ * non-negative, and with `invalid_baseline` when the artifact or any of its
+ * records belongs to a suite other than the one the run reports.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const compare: (baseline: Baseline, run: RunResult, tolerances?: Tolerances) => Effect.Effect<Report, EvalError>;
+//# sourceMappingURL=Regression.d.ts.map

package/dist/esm/Regression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AA2ED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAwHjC,CAAA"}

package/dist/esm/Regression.js

@@ -0,0 +1,195 @@
+/**
+ * Step-key-aware evaluation regression comparison.
+ *
+ * The step key is what separates the two findings this module reports. A score
+ * that moved at a *changed* step key is a regression: the target produced
+ * different work and it graded worse. A score that moved at an *unchanged* step
+ * key is nondeterminism: the same work graded differently twice. Both are
+ * results, and a gate reads both as red.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import { EvalError } from "./EvalError.js";
+import { CanonicalJson } from "./internal/CanonicalJson.js";
+import { flattenControlCharacters } from "./internal/controlCharacters.js";
+import { tupleKey } from "./internal/tupleKey.js";
+// Injective: two distinct (case, scorer) pairs can never encode to one key,
+// which a delimiter join could not promise once a name may contain the
+// delimiter.
+const key = (caseName, scorer) => tupleKey(caseName, scorer);
+const validTolerance = (value) => Number.isFinite(value) && value >= 0;
+const groupBy = (items, of) => {
+    const groups = new Map();
+    for (const item of items) {
+        const bucket = groups.get(of(item));
+        if (bucket === undefined)
+            groups.set(of(item), [item]);
+        else
+            bucket.push(item);
+    }
+    return groups;
+};
+const byScore = (items) => [...items].sort((left, right) => left.score - right.score);
+const byStepThenScore = (items) => [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score);
+/**
+ * Pairs one case's baseline records with one case's observations.
+ *
+ * Same-step-key pairs are matched first, lowest score to lowest score, so a
+ * repeated scorer does not report a move that never happened. Whatever is left
+ * over is paired in stable order, and an unpaired record on either side is a
+ * missing observation rather than a silent drop.
+ */
+const pairsFor = (expected, observed) => {
+    const pairs = [];
+    const expectedByStep = groupBy(expected, (record) => record.stepKey);
+    const observedByStep = groupBy(observed, (record) => record.stepKey);
+    const leftoverExpected = [];
+    const leftoverObserved = [];
+    for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {
+        const atStep = byScore(expectedByStep.get(stepKey));
+        const observedAtStep = byScore(observedByStep.get(stepKey) ?? []);
+        const count = Math.min(atStep.length, observedAtStep.length);
+        for (let index = 0; index < count; index++) {
+            pairs.push({ kind: "matched", expected: atStep[index], observed: observedAtStep[index] });
+        }
+        leftoverExpected.push(...atStep.slice(count));
+        leftoverObserved.push(...observedAtStep.slice(count));
+    }
+    for (const [stepKey, atStep] of observedByStep) {
+        if (!expectedByStep.has(stepKey))
+            leftoverObserved.push(...atStep);
+    }
+    const remainingExpected = byStepThenScore(leftoverExpected);
+    const remainingObserved = byStepThenScore(leftoverObserved);
+    const matched = Math.min(remainingExpected.length, remainingObserved.length);
+    for (let index = 0; index < matched; index++) {
+        pairs.push({ kind: "matched", expected: remainingExpected[index], observed: remainingObserved[index] });
+    }
+    for (const record of remainingExpected.slice(matched))
+        pairs.push({ kind: "missing-run", expected: record });
+    for (const record of remainingObserved.slice(matched))
+        pairs.push({ kind: "missing-baseline", observed: record });
+    return pairs;
+};
+/**
+ * Compares a run to a baseline, preserving missing and inconclusive observations.
+ *
+ * Records and observations are grouped by `(case, scorer)` and then paired by
+ * step key, so a scorer that ran several times against one case is compared
+ * pairwise instead of by array position. A pair whose step key is unchanged and
+ * whose score moved is nondeterminism; a pair whose step key changed and whose
+ * score dropped is a regression; an unpaired record on either side is a missing
+ * observation. Inconclusive observations are carried through untouched, because
+ * they decide nothing and a gate has to see them.
+ *
+ * Fails with `invalid_tolerance` when a tolerance is not finite and
+ * non-negative, and with `invalid_baseline` when the artifact or any of its
+ * records belongs to a suite other than the one the run reports.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const compare = (baseline, run, tolerances = {}) => {
+    const absolute = tolerances.absolute ?? 0;
+    const relative = tolerances.relative ?? 0;
+    if (!validTolerance(absolute)) {
+        return Effect.fail(new EvalError({
+            code: "invalid_tolerance",
+            message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,
+            path: "tolerances.absolute"
+        }));
+    }
+    if (!validTolerance(relative)) {
+        return Effect.fail(new EvalError({
+            code: "invalid_tolerance",
+            message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,
+            path: "tolerances.relative"
+        }));
+    }
+    const baselineSuite = baseline.suite;
+    if (baselineSuite !== run.suite) {
+        return Effect.fail(new EvalError({
+            code: "invalid_baseline",
+            message: flattenControlCharacters(`Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`),
+            path: "baseline.suite"
+        }));
+    }
+    const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))];
+    if (foreign.length > 0) {
+        return Effect.fail(new EvalError({
+            code: "invalid_baseline",
+            message: flattenControlCharacters(`Baseline holds records for suite ${foreign.map((suite) => `'${suite}'`).join(", ")}, but the run is suite '${run.suite}'`),
+            path: "baseline.records"
+        }));
+    }
+    const actual = run.observations.filter((observation) => observation.kind === "score");
+    const inconclusive = run.observations.filter((observation) => observation.kind === "inconclusive");
+    const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer));
+    const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer));
+    const regressions = [];
+    const nondeterminism = [];
+    const missing = [];
+    for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {
+        const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? []);
+        for (const pair of pairs) {
+            if (pair.kind === "missing-baseline") {
+                missing.push({
+                    side: "baseline",
+                    case: pair.observed.case,
+                    scorer: pair.observed.scorer,
+                    ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),
+                    stepKey: pair.observed.stepKey
+                });
+                continue;
+            }
+            if (pair.kind === "missing-run") {
+                missing.push({
+                    side: "run",
+                    case: pair.expected.case,
+                    scorer: pair.expected.scorer,
+                    ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),
+                    stepKey: pair.expected.stepKey
+                });
+                continue;
+            }
+            const baselineRecord = pair.expected;
+            const observation = pair.observed;
+            const delta = observation.score - baselineRecord.score;
+            // Both branches divide by the same guarded magnitude. A baseline score of
+            // exactly 0 used to be special-cased in the regression branch, which was
+            // dead: a score is validated into [0, 1], so nothing can drop below 0.
+            if (observation.stepKey === baselineRecord.stepKey) {
+                if (Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative) {
+                    nondeterminism.push({
+                        case: observation.case,
+                        scorer: observation.scorer,
+                        baseline: baselineRecord,
+                        actual: observation,
+                        delta
+                    });
+                }
+            }
+            else if (delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative) {
+                regressions.push({
+                    case: observation.case,
+                    scorer: observation.scorer,
+                    baseline: baselineRecord,
+                    actual: observation,
+                    drop: -delta
+                });
+            }
+        }
+    }
+    return Effect.succeed({
+        suite: run.suite,
+        baseline,
+        run,
+        regressions,
+        nondeterminism,
+        missing,
+        samples: actual,
+        inconclusive
+    });
+};
+//# sourceMappingURL=Regression.js.map

package/dist/esm/Regression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Regression.js","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAmFjD,4EAA4E;AAC5E,uEAAuE;AACvE,aAAa;AACb,MAAM,GAAG,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAU,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;AACpF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAEvF,MAAM,OAAO,GAAG,CAAI,KAAuB,EAAE,EAAuB,EAAyB,EAAE;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAA;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;;YACjD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAuC,KAAuB,EAAY,EAAE,CAC1F,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAE5D,MAAM,eAAe,GAAG,CACtB,KAAuB,EACb,EAAE,CACZ,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAQtH;;;;;;;GAOG;AACH,MAAM,QAAQ,GAAG,CACf,QAAuC,EACvC,QAA8B,EACT,EAAE;IACvB,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,gBAAgB,GAA0B,EAAE,CAAA;IAClD,MAAM,gBAAgB,GAAiB,EAAE,CAAA;IACzC,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QACjF,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;QACjE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC5D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;QAC7F,CAAC;QACD,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7C,gBAAgB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;QAC/C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAC5E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;IAC3G,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC5G,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjH,OAAO,KAAK,CAAA;AACd,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,QAAkB,EAClB,GAAc,EACd,aAAyB,EAAE,EACO,EAAE;IACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAA;IACpC,IAAI,aAAa,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,8BAA8B,aAAa,4BAA4B,GAAG,CAAC,KAAK,GAAG,CACpF;YACD,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,oCACE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAChD,2BAA2B,GAAG,CAAC,KAAK,GAAG,CACxC;YACD,IAAI,EAAE,kBAAkB;SACzB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAwB,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;IAC3G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAC3C,WAAW,EAC6D,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,CAAC,CAAA;IACjH,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/F,MAAM,WAAW,GAAsB,EAAE,CAAA;IACzC,MAAM,cAAc,GAA0B,EAAE,CAAA;IAChD,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,KAAK,MAAM,UAAU,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;QAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAA;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAA;YACjC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAA;YACtD,0EAA0E;YAC1E,yEAAyE;YACzE,uEAAuE;YACvE,IAAI,WAAW,CAAC,OAAO,KAAK,cAAc,CAAC,OAAO,EAAE,CAAC;gBACnD,IACE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,QAAQ,EAC1G,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC;wBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;wBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;wBAC1B,QAAQ,EAAE,cAAc;wBACxB,MAAM,EAAE,WAAW;wBACnB,KAAK;qBACN,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;iBAAM,IACL,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,QAAQ,EAC3F,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,QAAQ,EAAE,cAAc;oBACxB,MAAM,EAAE,WAAW;oBACnB,IAAI,EAAE,CAAC,KAAK;iBACb,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;QACR,GAAG;QACH,WAAW;QACX,cAAc;QACd,OAAO;QACP,OAAO,EAAE,MAAM;QACf,YAAY;KACb,CAAC,CAAA;AACJ,CAAC,CAAA","sourcesContent":["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"]}