@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/Regression.js

@@ -0,0 +1,191 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key2 of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key2) && key2 !== except)
+        __defProp(to, key2, { get: () => from[key2], enumerable: !(desc = __getOwnPropDesc(from, key2)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var Regression_exports = {};
+__export(Regression_exports, {
+  compare: () => compare
+});
+module.exports = __toCommonJS(Regression_exports);
+var Effect = __toESM(require("effect/Effect"), 1);
+var import_EvalError = require("./EvalError.js");
+var import_CanonicalJson = require("./internal/CanonicalJson.js");
+var import_controlCharacters = require("./internal/controlCharacters.js");
+var import_tupleKey = require("./internal/tupleKey.js");
+const key = (caseName, scorer) => (0, import_tupleKey.tupleKey)(caseName, scorer);
+const validTolerance = (value) => Number.isFinite(value) && value >= 0;
+const groupBy = (items, of) => {
+  const groups = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    const bucket = groups.get(of(item));
+    if (bucket === void 0) groups.set(of(item), [item]);
+    else bucket.push(item);
+  }
+  return groups;
+};
+const byScore = (items) => [...items].sort((left, right) => left.score - right.score);
+const byStepThenScore = (items) => [...items].sort((left, right) => import_CanonicalJson.CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score);
+const pairsFor = (expected, observed) => {
+  const pairs = [];
+  const expectedByStep = groupBy(expected, (record) => record.stepKey);
+  const observedByStep = groupBy(observed, (record) => record.stepKey);
+  const leftoverExpected = [];
+  const leftoverObserved = [];
+  for (const stepKey of [...expectedByStep.keys()].sort(import_CanonicalJson.CanonicalJson.compareText)) {
+    const atStep = byScore(expectedByStep.get(stepKey));
+    const observedAtStep = byScore(observedByStep.get(stepKey) ?? []);
+    const count = Math.min(atStep.length, observedAtStep.length);
+    for (let index = 0; index < count; index++) {
+      pairs.push({ kind: "matched", expected: atStep[index], observed: observedAtStep[index] });
+    }
+    leftoverExpected.push(...atStep.slice(count));
+    leftoverObserved.push(...observedAtStep.slice(count));
+  }
+  for (const [stepKey, atStep] of observedByStep) {
+    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep);
+  }
+  const remainingExpected = byStepThenScore(leftoverExpected);
+  const remainingObserved = byStepThenScore(leftoverObserved);
+  const matched = Math.min(remainingExpected.length, remainingObserved.length);
+  for (let index = 0; index < matched; index++) {
+    pairs.push({ kind: "matched", expected: remainingExpected[index], observed: remainingObserved[index] });
+  }
+  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: "missing-run", expected: record });
+  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: "missing-baseline", observed: record });
+  return pairs;
+};
+const compare = (baseline, run, tolerances = {}) => {
+  const absolute = tolerances.absolute ?? 0;
+  const relative = tolerances.relative ?? 0;
+  if (!validTolerance(absolute)) {
+    return Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_tolerance",
+        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,
+        path: "tolerances.absolute"
+      })
+    );
+  }
+  if (!validTolerance(relative)) {
+    return Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_tolerance",
+        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,
+        path: "tolerances.relative"
+      })
+    );
+  }
+  const baselineSuite = baseline.suite;
+  if (baselineSuite !== run.suite) {
+    return Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_baseline",
+        message: (0, import_controlCharacters.flattenControlCharacters)(
+          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`
+        ),
+        path: "baseline.suite"
+      })
+    );
+  }
+  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))];
+  if (foreign.length > 0) {
+    return Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_baseline",
+        message: (0, import_controlCharacters.flattenControlCharacters)(
+          `Baseline holds records for suite ${foreign.map((suite) => `'${suite}'`).join(", ")}, but the run is suite '${run.suite}'`
+        ),
+        path: "baseline.records"
+      })
+    );
+  }
+  const actual = run.observations.filter((observation) => observation.kind === "score");
+  const inconclusive = run.observations.filter((observation) => observation.kind === "inconclusive");
+  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer));
+  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer));
+  const regressions = [];
+  const nondeterminism = [];
+  const missing = [];
+  for (const groupedKey of /* @__PURE__ */ new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {
+    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? []);
+    for (const pair of pairs) {
+      if (pair.kind === "missing-baseline") {
+        missing.push({
+          side: "baseline",
+          case: pair.observed.case,
+          scorer: pair.observed.scorer,
+          ...pair.observed.scorerName === void 0 ? {} : { scorerName: pair.observed.scorerName },
+          stepKey: pair.observed.stepKey
+        });
+        continue;
+      }
+      if (pair.kind === "missing-run") {
+        missing.push({
+          side: "run",
+          case: pair.expected.case,
+          scorer: pair.expected.scorer,
+          ...pair.expected.scorerName === void 0 ? {} : { scorerName: pair.expected.scorerName },
+          stepKey: pair.expected.stepKey
+        });
+        continue;
+      }
+      const baselineRecord = pair.expected;
+      const observation = pair.observed;
+      const delta = observation.score - baselineRecord.score;
+      if (observation.stepKey === baselineRecord.stepKey) {
+        if (Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative) {
+          nondeterminism.push({
+            case: observation.case,
+            scorer: observation.scorer,
+            baseline: baselineRecord,
+            actual: observation,
+            delta
+          });
+        }
+      } else if (delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative) {
+        regressions.push({
+          case: observation.case,
+          scorer: observation.scorer,
+          baseline: baselineRecord,
+          actual: observation,
+          drop: -delta
+        });
+      }
+    }
+  }
+  return Effect.succeed({
+    suite: run.suite,
+    baseline,
+    run,
+    regressions,
+    nondeterminism,
+    missing,
+    samples: actual,
+    inconclusive
+  });
+};
+//# sourceMappingURL=Regression.js.map

package/dist/cjs/Regression.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/Regression.ts"],
+  "sourcesContent": ["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAYA,aAAwB;AAExB,uBAA0B;AAC1B,2BAA8B;AAC9B,+BAAyC;AACzC,sBAAyB;AAsFzB,MAAM,MAAM,CAAC,UAAkB,eAA2B,0BAAS,UAAU,MAAM;AACnF,MAAM,iBAAiB,CAAC,UAA2B,OAAO,SAAS,KAAK,KAAK,SAAS;AAEtF,MAAM,UAAU,CAAI,OAAyB,OAAmD;AAC9F,QAAM,SAAS,oBAAI,IAAsB;AACzC,aAAW,QAAQ,OAAO;AACxB,UAAM,SAAS,OAAO,IAAI,GAAG,IAAI,CAAC;AAClC,QAAI,WAAW,OAAW,QAAO,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC;AAAA,QAChD,QAAO,KAAK,IAAI;AAAA,EACvB;AACA,SAAO;AACT;AAEA,MAAM,UAAU,CAAuC,UACrD,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,KAAK,QAAQ,MAAM,KAAK;AAE3D,MAAM,kBAAkB,CACtB,UAEA,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,mCAAc,YAAY,KAAK,SAAS,MAAM,OAAO,KAAK,KAAK,QAAQ,MAAM,KAAK;AAgBrH,MAAM,WAAW,CACf,UACA,aACwB;AACxB,QAAM,QAAqB,CAAC;AAC5B,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,mBAA0C,CAAC;AACjD,QAAM,mBAAiC,CAAC;AACxC,aAAW,WAAW,CAAC,GAAG,eAAe,KAAK,CAAC,EAAE,KAAK,mCAAc,WAAW,GAAG;AAChF,UAAM,SAAS,QAAQ,eAAe,IAAI,OAAO,CAAE;AACnD,UAAM,iBAAiB,QAAQ,eAAe,IAAI,OAAO,KAAK,CAAC,CAAC;AAChE,UAAM,QAAQ,KAAK,IAAI,OAAO,QAAQ,eAAe,MAAM;AAC3D,aAAS,QAAQ,GAAG,QAAQ,OAAO,SAAS;AAC1C,YAAM,KAAK,EAAE,MAAM,WAAW,UAAU,OAAO,KAAK,GAAI,UAAU,eAAe,KAAK,EAAG,CAAC;AAAA,IAC5F;AACA,qBAAiB,KAAK,GAAG,OAAO,MAAM,KAAK,CAAC;AAC5C,qBAAiB,KAAK,GAAG,eAAe,MAAM,KAAK,CAAC;AAAA,EACtD;AACA,aAAW,CAAC,SAAS,MAAM,KAAK,gBAAgB;AAC9C,QAAI,CAAC,eAAe,IAAI,OAAO,EAAG,kBAAiB,KAAK,GAAG,MAAM;AAAA,EACnE;AACA,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,UAAU,KAAK,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AAC3E,WAAS,QAAQ,GAAG,QAAQ,SAAS,SAAS;AAC5C,UAAM,KAAK,EAAE,MAAM,WAAW,UAAU,kBAAkB,KAAK,GAAI,UAAU,kBAAkB,KAAK,EAAG,CAAC;AAAA,EAC1G;AACA,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,eAAe,UAAU,OAAO,CAAC;AAC3G,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,oBAAoB,UAAU,OAAO,CAAC;AAChH,SAAO;AACT;AAoBO,MAAM,UAAU,CACrB,UACA,KACA,aAAyB,CAAC,MACW;AACrC,QAAM,WAAW,WAAW,YAAY;AACxC,QAAM,WAAW,WAAW,YAAY;AACxC,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,gBAAgB,SAAS;AAC/B,MAAI,kBAAkB,IAAI,OAAO;AAC/B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,8BAA8B,aAAa,4BAA4B,IAAI,KAAK;AAAA,QAClF;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,SAAS,QAAQ,OAAO,CAAC,WAAW,OAAO,UAAU,IAAI,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAChH,MAAI,QAAQ,SAAS,GAAG;AACtB,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,oCACE,QAAQ,IAAI,CAAC,UAAU,IAAI,KAAK,GAAG,EAAE,KAAK,IAAI,CAChD,2BAA2B,IAAI,KAAK;AAAA,QACtC;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,SAAS,IAAI,aAAa,OAAO,CAAC,gBAAsC,YAAY,SAAS,OAAO;AAC1G,QAAM,eAAe,IAAI,aAAa,OAAO,CAC3C,gBAC2E,YAAY,SAAS,cAAc;AAChH,QAAM,gBAAgB,QAAQ,SAAS,SAAS,CAAC,WAAW,IAAI,OAAO,MAAM,OAAO,MAAM,CAAC;AAC3F,QAAM,cAAc,QAAQ,QAAQ,CAAC,gBAAgB,IAAI,YAAY,MAAM,YAAY,MAAM,CAAC;AAC9F,QAAM,cAAiC,CAAC;AACxC,QAAM,iBAAwC,CAAC;AAC/C,QAAM,UAAqC,CAAC;AAC5C,aAAW,cAAc,oBAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,GAAG;AAClF,UAAM,QAAQ,SAAS,cAAc,IAAI,UAAU,KAAK,CAAC,GAAG,YAAY,IAAI,UAAU,KAAK,CAAC,CAAC;AAC7F,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,SAAS,oBAAoB;AACpC,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,UAAI,KAAK,SAAS,eAAe;AAC/B,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,YAAM,iBAAiB,KAAK;AAC5B,YAAM,cAAc,KAAK;AACzB,YAAM,QAAQ,YAAY,QAAQ,eAAe;AAIjD,UAAI,YAAY,YAAY,eAAe,SAAS;AAClD,YACE,KAAK,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,eAAe,KAAK,GAAG,KAAK,IAAI,UAClG;AACA,yBAAe,KAAK;AAAA,YAClB,MAAM,YAAY;AAAA,YAClB,QAAQ,YAAY;AAAA,YACpB,UAAU;AAAA,YACV,QAAQ;AAAA,YACR;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,WACE,QAAQ,KAAK,CAAC,QAAQ,YAAY,CAAC,QAAQ,KAAK,IAAI,eAAe,OAAO,KAAK,IAAI,UACnF;AACA,oBAAY,KAAK;AAAA,UACf,MAAM,YAAY;AAAA,UAClB,QAAQ,YAAY;AAAA,UACpB,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,MAAM,CAAC;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,OAAO,QAAQ;AAAA,IACpB,OAAO,IAAI;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;",
+  "names": []
+}

package/dist/cjs/Report.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Canonical JSON and Markdown regression reports.
+ *
+ * The JSON report is the machine-readable artifact; the Markdown report is what
+ * an operator reads in a CI log, so it names every case the run could not
+ * decide rather than only counting them.
+ *
+ * @since 0.1.0
+ */
+import type { Baseline } from "./Baseline.ts";
+import type { MissingObservation, Nondeterminism, Regression, Report as RegressionReport } from "./Regression.ts";
+import type { CaseResult, RunResult } from "./Runner.ts";
+/**
+ * Current serialized report format version.
+ *
+ * It versions {@link Data}, the wire shape, independently of the nested
+ * `Baseline.version`: the comparison can change shape without the committed
+ * baseline artifact changing at all.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const version: 1;
+/** An index into {@link Data.run}'s `observations` table. */
+type ObservationRef = number;
+/**
+ * The serialized shape of a regression report.
+ *
+ * `run.observations` is the observation table: every observation the report
+ * mentions appears in it exactly once, and everything else refers to one by
+ * index. `run.cases[i].observations` is a list of indexes, and a regression or
+ * nondeterminism entry's `actual` is a single index. An in-memory
+ * `Regression.Report` carries the same observation object in up to four
+ * places, so encoding it directly repeated every reason and every `meta` that
+ * many times.
+ *
+ * `samples` and `inconclusive` are absent by construction: both are filters of
+ * `run.observations` by `kind`, so a reader recomputes them rather than
+ * downloading a third copy.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export interface Data {
+    readonly reportVersion: typeof version;
+    readonly suite: string;
+    readonly baseline: Baseline;
+    readonly run: Omit<RunResult, "cases"> & {
+        readonly cases: ReadonlyArray<Omit<CaseResult, "observations"> & {
+            readonly observations: ReadonlyArray<ObservationRef>;
+        }>;
+    };
+    readonly regressions: ReadonlyArray<Omit<Regression, "actual"> & {
+        readonly actual: ObservationRef;
+    }>;
+    readonly nondeterminism: ReadonlyArray<Omit<Nondeterminism, "actual"> & {
+        readonly actual: ObservationRef;
+    }>;
+    readonly missing: ReadonlyArray<MissingObservation>;
+}
+/**
+ * Projects a comparison into its serialized form.
+ *
+ * The observation table starts as `run.observations` and grows to hold any
+ * observation reached only through a case or a comparison entry, so a
+ * hand-built report whose case lists are not flattened still serializes every
+ * reference. Observations are pooled by identity, not by value: two distinct
+ * objects that happen to be equal stay two rows, because collapsing them would
+ * claim one grading where there were two.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const data: (report: RegressionReport) => Data;
+/**
+ * Serializes a regression report as stable, sorted-key JSON.
+ *
+ * The wire shape is {@link Data}, stamped with {@link version}: each
+ * observation is encoded once and referred to by index everywhere else. The
+ * report embeds each case's raw `execution.output`, which comes from an
+ * arbitrary target flow, so the encoding is total rather than trusting: keys
+ * are sorted by code unit, embedded strings are capped, total traversal is
+ * bounded, and anything JSON cannot express becomes a named marker
+ * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a
+ * `RangeError` or a silent `null`. Two identical runs therefore produce
+ * byte-identical JSON.
+ *
+ * Nothing redacts the embedded output. A suite whose cases carry secrets must
+ * not print this report where the log is readable.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const json: (report: RegressionReport) => string;
+/**
+ * Renders a concise stable Markdown regression report.
+ *
+ * Every count in the summary that is not zero has a section naming its rows:
+ * the regressions and the nondeterminism a gate reads as red, and the case
+ * failures, missing observations, and inconclusive observations that leave a
+ * gate undecided. A report that only counted them named nothing an operator
+ * could act on, which is exactly the run that needs debugging.
+ *
+ * Every cell and the suite heading value replace C0 controls and DEL with
+ * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,
+ * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,
+ * plus an ellipsis when truncated, without splitting escapes or code points.
+ *
+ * @category rendering
+ * @since 0.1.0
+ */
+export declare const markdown: (report: RegressionReport) => string;
+export {};
+//# sourceMappingURL=Report.d.ts.map

package/dist/cjs/Report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Report.d.ts","sourceRoot":"","sources":["../../src/Report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAI7C,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACjH,OAAO,KAAK,EAAE,UAAU,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAErE;;;;;;;;;GASG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC,6DAA6D;AAC7D,KAAK,cAAc,GAAG,MAAM,CAAA;AAE5B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,aAAa,EAAE,OAAO,OAAO,CAAA;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EACR,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,GACxB;QACA,QAAQ,CAAC,KAAK,EAAE,aAAa,CACzB,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,GAChC;YAAE,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;SAAE,CAC3D,CAAA;KACF,CAAA;IACH,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAA;IACrG,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAA;IAC5G,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;CACpD;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,IAAI,GAAI,QAAQ,gBAAgB,KAAG,IAsB/C,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,IAAI,GAAI,QAAQ,gBAAgB,KAAG,MAC2C,CAAA;AAkD3F;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,QAAQ,GAAI,QAAQ,gBAAgB,KAAG,MA4EnD,CAAA"}

package/dist/cjs/Report.js

@@ -0,0 +1,161 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var Report_exports = {};
+__export(Report_exports, {
+  data: () => data,
+  json: () => json,
+  markdown: () => markdown,
+  version: () => version
+});
+module.exports = __toCommonJS(Report_exports);
+var import_CanonicalJson = require("./internal/CanonicalJson.js");
+var import_controlCharacters = require("./internal/controlCharacters.js");
+var import_scorerLabel = require("./internal/scorerLabel.js");
+const version = 1;
+const data = (report) => {
+  const positions = /* @__PURE__ */ new Map();
+  const observations = [];
+  const refer = (observation) => {
+    const known = positions.get(observation);
+    if (known !== void 0) return known;
+    const position = observations.length;
+    observations.push(observation);
+    positions.set(observation, position);
+    return position;
+  };
+  for (const observation of report.run.observations) refer(observation);
+  const cases = report.run.cases.map((result) => ({ ...result, observations: result.observations.map(refer) }));
+  return {
+    reportVersion: version,
+    suite: report.suite,
+    baseline: report.baseline,
+    run: { ...report.run, cases, observations },
+    regressions: report.regressions.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+    nondeterminism: report.nondeterminism.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+    missing: report.missing
+  };
+};
+const json = (report) => import_CanonicalJson.CanonicalJson.stringify(data(report), { maxStringLength: import_CanonicalJson.CanonicalJson.maxStringLength });
+const maxCellLength = 240;
+const cell = (value) => {
+  let text = "";
+  if (value !== void 0 && value !== null) {
+    if (typeof value === "string") text = value;
+    else {
+      try {
+        text = String(value);
+      } catch {
+        text = "[unreadable]";
+      }
+    }
+  }
+  let escaped = "";
+  for (const character of text) {
+    const code = character.codePointAt(0);
+    const token = (0, import_controlCharacters.isControlCodePoint)(code) ? " " : /[\\|`*_[\]<>!#~&.:@]/u.test(character) ? `\\${character}` : character;
+    if (escaped.length + token.length > maxCellLength) return `${escaped}\u2026`;
+    escaped += token;
+  }
+  return escaped;
+};
+const scorerCell = (scorer, scorerName) => cell((0, import_scorerLabel.scorerLabel)({ scorer, scorerName }));
+const section = (heading, header, alignment, rows) => rows.length === 0 ? [] : [
+  `## ${heading}`,
+  "",
+  `| ${header.join(" | ")} |`,
+  `| ${alignment.join(" | ")} |`,
+  ...rows.map((row) => `| ${row.join(" | ")} |`),
+  ""
+];
+const markdown = (report) => {
+  const lines = [
+    `# Evaluation report: ${cell(report.suite)}`,
+    "",
+    `- Regressions: ${report.regressions.length}`,
+    `- Nondeterminism: ${report.nondeterminism.length}`,
+    `- Missing observations: ${report.missing.length}`,
+    `- Inconclusive observations: ${report.inconclusive.length}`,
+    `- Failed cases: ${report.run.cases.filter((result) => result.error !== void 0).length}`,
+    ...report.run.trials === void 0 ? [] : [
+      `- pass@1: ${report.run.trials.passAt1.toFixed(6)}`,
+      `- pass@${report.run.k ?? 1}: ${report.run.trials.passAtK.toFixed(6)}`,
+      `- pass^${report.run.k ?? 1}: ${report.run.trials.passHatK.toFixed(6)}`,
+      `- stderr: ${(Object.values(report.run.trials.perCase).reduce((sum, item) => sum + item.stderr, 0) / report.run.trials.cases).toFixed(6)}`
+    ],
+    "",
+    ...section(
+      "Regressions",
+      ["Case", "Scorer", "Baseline", "Actual", "Drop"],
+      ["---", "---", "---:", "---:", "---:"],
+      report.regressions.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.actual.scorerName),
+        item.baseline.score.toFixed(6),
+        item.actual.score.toFixed(6),
+        item.drop.toFixed(6)
+      ])
+    ),
+    ...section(
+      "Nondeterminism",
+      ["Case", "Scorer", "Step key", "Baseline", "Actual", "Delta"],
+      ["---", "---", "---", "---:", "---:", "---:"],
+      report.nondeterminism.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.actual.scorerName),
+        cell(item.actual.stepKey),
+        item.baseline.score.toFixed(6),
+        item.actual.score.toFixed(6),
+        item.delta.toFixed(6)
+      ])
+    ),
+    ...section(
+      "Case failures",
+      ["Case", "Code", "Message"],
+      ["---", "---", "---"],
+      report.run.cases.flatMap(
+        (result) => result.error === void 0 ? [] : [[cell(result.case), cell(result.error.code), cell(result.error.message)]]
+      )
+    ),
+    ...section(
+      "Missing observations",
+      ["Side", "Case", "Scorer", "Step key"],
+      ["---", "---", "---", "---"],
+      report.missing.map((item) => [
+        cell(item.side),
+        cell(item.case),
+        scorerCell(item.scorer, item.scorerName),
+        cell(item.stepKey)
+      ])
+    ),
+    ...section(
+      "Inconclusive",
+      ["Case", "Scorer", "Step key", "Reason"],
+      ["---", "---", "---", "---"],
+      report.inconclusive.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.scorerName),
+        cell(item.stepKey),
+        cell(item.reason)
+      ])
+    )
+  ];
+  return `${lines.join("\n").trim()}
+`;
+};
+//# sourceMappingURL=Report.js.map

package/dist/cjs/Report.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/Report.ts"],
+  "sourcesContent": ["/**\n * Canonical JSON and Markdown regression reports.\n *\n * The JSON report is the machine-readable artifact; the Markdown report is what\n * an operator reads in a CI log, so it names every case the run could not\n * decide rather than only counting them.\n *\n * @since 0.1.0\n */\n\nimport type { Baseline } from \"./Baseline.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { isControlCodePoint } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport type { MissingObservation, Nondeterminism, Regression, Report as RegressionReport } from \"./Regression.ts\"\nimport type { CaseResult, Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current serialized report format version.\n *\n * It versions {@link Data}, the wire shape, independently of the nested\n * `Baseline.version`: the comparison can change shape without the committed\n * baseline artifact changing at all.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/** An index into {@link Data.run}'s `observations` table. */\ntype ObservationRef = number\n\n/**\n * The serialized shape of a regression report.\n *\n * `run.observations` is the observation table: every observation the report\n * mentions appears in it exactly once, and everything else refers to one by\n * index. `run.cases[i].observations` is a list of indexes, and a regression or\n * nondeterminism entry's `actual` is a single index. An in-memory\n * `Regression.Report` carries the same observation object in up to four\n * places, so encoding it directly repeated every reason and every `meta` that\n * many times.\n *\n * `samples` and `inconclusive` are absent by construction: both are filters of\n * `run.observations` by `kind`, so a reader recomputes them rather than\n * downloading a third copy.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport interface Data {\n  readonly reportVersion: typeof version\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run:\n    & Omit<RunResult, \"cases\">\n    & {\n      readonly cases: ReadonlyArray<\n        & Omit<CaseResult, \"observations\">\n        & { readonly observations: ReadonlyArray<ObservationRef> }\n      >\n    }\n  readonly regressions: ReadonlyArray<Omit<Regression, \"actual\"> & { readonly actual: ObservationRef }>\n  readonly nondeterminism: ReadonlyArray<Omit<Nondeterminism, \"actual\"> & { readonly actual: ObservationRef }>\n  readonly missing: ReadonlyArray<MissingObservation>\n}\n\n/**\n * Projects a comparison into its serialized form.\n *\n * The observation table starts as `run.observations` and grows to hold any\n * observation reached only through a case or a comparison entry, so a\n * hand-built report whose case lists are not flattened still serializes every\n * reference. Observations are pooled by identity, not by value: two distinct\n * objects that happen to be equal stay two rows, because collapsing them would\n * claim one grading where there were two.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const data = (report: RegressionReport): Data => {\n  const positions = new Map<Observation, ObservationRef>()\n  const observations: Array<Observation> = []\n  const refer = (observation: Observation): ObservationRef => {\n    const known = positions.get(observation)\n    if (known !== undefined) return known\n    const position = observations.length\n    observations.push(observation)\n    positions.set(observation, position)\n    return position\n  }\n  for (const observation of report.run.observations) refer(observation)\n  const cases = report.run.cases.map((result) => ({ ...result, observations: result.observations.map(refer) }))\n  return {\n    reportVersion: version,\n    suite: report.suite,\n    baseline: report.baseline,\n    run: { ...report.run, cases, observations },\n    regressions: report.regressions.map((entry) => ({ ...entry, actual: refer(entry.actual) })),\n    nondeterminism: report.nondeterminism.map((entry) => ({ ...entry, actual: refer(entry.actual) })),\n    missing: report.missing\n  }\n}\n\n/**\n * Serializes a regression report as stable, sorted-key JSON.\n *\n * The wire shape is {@link Data}, stamped with {@link version}: each\n * observation is encoded once and referred to by index everywhere else. The\n * report embeds each case's raw `execution.output`, which comes from an\n * arbitrary target flow, so the encoding is total rather than trusting: keys\n * are sorted by code unit, embedded strings are capped, total traversal is\n * bounded, and anything JSON cannot express becomes a named marker\n * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a\n * `RangeError` or a silent `null`. Two identical runs therefore produce\n * byte-identical JSON.\n *\n * Nothing redacts the embedded output. A suite whose cases carry secrets must\n * not print this report where the log is readable.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const json = (report: RegressionReport): string =>\n  CanonicalJson.stringify(data(report), { maxStringLength: CanonicalJson.maxStringLength })\n\n/** Maximum escaped UTF-16 code units per value, before a truncation ellipsis. */\nconst maxCellLength = 240\n\nconst cell = (value: unknown): string => {\n  let text = \"\"\n  if (value !== undefined && value !== null) {\n    if (typeof value === \"string\") text = value\n    else {\n      try {\n        text = String(value)\n      } catch {\n        text = \"[unreadable]\"\n      }\n    }\n  }\n  let escaped = \"\"\n  for (const character of text) {\n    const code = character.codePointAt(0)!\n    // Escape each input character once, including backslashes before pipes.\n    // Ampersands prevent entities; dots, colons, and @ prevent GFM autolinks.\n    const token = isControlCodePoint(code)\n      ? \" \"\n      : /[\\\\|`*_[\\]<>!#~&.:@]/u.test(character)\n      ? `\\\\${character}`\n      : character\n    if (escaped.length + token.length > maxCellLength) return `${escaped}\u2026`\n    escaped += token\n  }\n  return escaped\n}\n\nconst scorerCell = (scorer: string, scorerName: string | undefined): string => cell(scorerLabel({ scorer, scorerName }))\n\nconst section = (\n  heading: string,\n  header: ReadonlyArray<string>,\n  alignment: ReadonlyArray<string>,\n  rows: ReadonlyArray<ReadonlyArray<string>>\n): ReadonlyArray<string> =>\n  rows.length === 0 ? [] : [\n    `## ${heading}`,\n    \"\",\n    `| ${header.join(\" | \")} |`,\n    `| ${alignment.join(\" | \")} |`,\n    ...rows.map((row) => `| ${row.join(\" | \")} |`),\n    \"\"\n  ]\n\n/**\n * Renders a concise stable Markdown regression report.\n *\n * Every count in the summary that is not zero has a section naming its rows:\n * the regressions and the nondeterminism a gate reads as red, and the case\n * failures, missing observations, and inconclusive observations that leave a\n * gate undecided. A report that only counted them named nothing an operator\n * could act on, which is exactly the run that needs debugging.\n *\n * Every cell and the suite heading value replace C0 controls and DEL with\n * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,\n * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,\n * plus an ellipsis when truncated, without splitting escapes or code points.\n *\n * @category rendering\n * @since 0.1.0\n */\nexport const markdown = (report: RegressionReport): string => {\n  const lines = [\n    `# Evaluation report: ${cell(report.suite)}`,\n    \"\",\n    `- Regressions: ${report.regressions.length}`,\n    `- Nondeterminism: ${report.nondeterminism.length}`,\n    `- Missing observations: ${report.missing.length}`,\n    `- Inconclusive observations: ${report.inconclusive.length}`,\n    `- Failed cases: ${report.run.cases.filter((result) => result.error !== undefined).length}`,\n    ...(report.run.trials === undefined ? [] : [\n      `- pass@1: ${report.run.trials.passAt1.toFixed(6)}`,\n      `- pass@${report.run.k ?? 1}: ${report.run.trials.passAtK.toFixed(6)}`,\n      `- pass^${report.run.k ?? 1}: ${report.run.trials.passHatK.toFixed(6)}`,\n      `- stderr: ${\n        (Object.values(report.run.trials.perCase).reduce((sum, item) => sum + item.stderr, 0) / report.run.trials.cases)\n          .toFixed(6)\n      }`\n    ]),\n    \"\",\n    ...section(\n      \"Regressions\",\n      [\"Case\", \"Scorer\", \"Baseline\", \"Actual\", \"Drop\"],\n      [\"---\", \"---\", \"---:\", \"---:\", \"---:\"],\n      report.regressions.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.actual.scorerName),\n        item.baseline.score.toFixed(6),\n        item.actual.score.toFixed(6),\n        item.drop.toFixed(6)\n      ])\n    ),\n    ...section(\n      \"Nondeterminism\",\n      [\"Case\", \"Scorer\", \"Step key\", \"Baseline\", \"Actual\", \"Delta\"],\n      [\"---\", \"---\", \"---\", \"---:\", \"---:\", \"---:\"],\n      report.nondeterminism.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.actual.scorerName),\n        cell(item.actual.stepKey),\n        item.baseline.score.toFixed(6),\n        item.actual.score.toFixed(6),\n        item.delta.toFixed(6)\n      ])\n    ),\n    ...section(\n      \"Case failures\",\n      [\"Case\", \"Code\", \"Message\"],\n      [\"---\", \"---\", \"---\"],\n      report.run.cases.flatMap((result) =>\n        result.error === undefined ? [] : [[cell(result.case), cell(result.error.code), cell(result.error.message)]]\n      )\n    ),\n    ...section(\n      \"Missing observations\",\n      [\"Side\", \"Case\", \"Scorer\", \"Step key\"],\n      [\"---\", \"---\", \"---\", \"---\"],\n      report.missing.map((item) => [\n        cell(item.side),\n        cell(item.case),\n        scorerCell(item.scorer, item.scorerName),\n        cell(item.stepKey)\n      ])\n    ),\n    ...section(\n      \"Inconclusive\",\n      [\"Case\", \"Scorer\", \"Step key\", \"Reason\"],\n      [\"---\", \"---\", \"---\", \"---\"],\n      report.inconclusive.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.scorerName),\n        cell(item.stepKey),\n        cell(item.reason)\n      ])\n    )\n  ]\n  return `${lines.join(\"\\n\").trim()}\\n`\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,2BAA8B;AAC9B,+BAAmC;AACnC,yBAA4B;AAcrB,MAAM,UAAU;AAqDhB,MAAM,OAAO,CAAC,WAAmC;AACtD,QAAM,YAAY,oBAAI,IAAiC;AACvD,QAAM,eAAmC,CAAC;AAC1C,QAAM,QAAQ,CAAC,gBAA6C;AAC1D,UAAM,QAAQ,UAAU,IAAI,WAAW;AACvC,QAAI,UAAU,OAAW,QAAO;AAChC,UAAM,WAAW,aAAa;AAC9B,iBAAa,KAAK,WAAW;AAC7B,cAAU,IAAI,aAAa,QAAQ;AACnC,WAAO;AAAA,EACT;AACA,aAAW,eAAe,OAAO,IAAI,aAAc,OAAM,WAAW;AACpE,QAAM,QAAQ,OAAO,IAAI,MAAM,IAAI,CAAC,YAAY,EAAE,GAAG,QAAQ,cAAc,OAAO,aAAa,IAAI,KAAK,EAAE,EAAE;AAC5G,SAAO;AAAA,IACL,eAAe;AAAA,IACf,OAAO,OAAO;AAAA,IACd,UAAU,OAAO;AAAA,IACjB,KAAK,EAAE,GAAG,OAAO,KAAK,OAAO,aAAa;AAAA,IAC1C,aAAa,OAAO,YAAY,IAAI,CAAC,WAAW,EAAE,GAAG,OAAO,QAAQ,MAAM,MAAM,MAAM,EAAE,EAAE;AAAA,IAC1F,gBAAgB,OAAO,eAAe,IAAI,CAAC,WAAW,EAAE,GAAG,OAAO,QAAQ,MAAM,MAAM,MAAM,EAAE,EAAE;AAAA,IAChG,SAAS,OAAO;AAAA,EAClB;AACF;AAqBO,MAAM,OAAO,CAAC,WACnB,mCAAc,UAAU,KAAK,MAAM,GAAG,EAAE,iBAAiB,mCAAc,gBAAgB,CAAC;AAG1F,MAAM,gBAAgB;AAEtB,MAAM,OAAO,CAAC,UAA2B;AACvC,MAAI,OAAO;AACX,MAAI,UAAU,UAAa,UAAU,MAAM;AACzC,QAAI,OAAO,UAAU,SAAU,QAAO;AAAA,SACjC;AACH,UAAI;AACF,eAAO,OAAO,KAAK;AAAA,MACrB,QAAQ;AACN,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU;AACd,aAAW,aAAa,MAAM;AAC5B,UAAM,OAAO,UAAU,YAAY,CAAC;AAGpC,UAAM,YAAQ,6CAAmB,IAAI,IACjC,MACA,wBAAwB,KAAK,SAAS,IACtC,KAAK,SAAS,KACd;AACJ,QAAI,QAAQ,SAAS,MAAM,SAAS,cAAe,QAAO,GAAG,OAAO;AACpE,eAAW;AAAA,EACb;AACA,SAAO;AACT;AAEA,MAAM,aAAa,CAAC,QAAgB,eAA2C,SAAK,gCAAY,EAAE,QAAQ,WAAW,CAAC,CAAC;AAEvH,MAAM,UAAU,CACd,SACA,QACA,WACA,SAEA,KAAK,WAAW,IAAI,CAAC,IAAI;AAAA,EACvB,MAAM,OAAO;AAAA,EACb;AAAA,EACA,KAAK,OAAO,KAAK,KAAK,CAAC;AAAA,EACvB,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,EAC1B,GAAG,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,KAAK,KAAK,CAAC,IAAI;AAAA,EAC7C;AACF;AAmBK,MAAM,WAAW,CAAC,WAAqC;AAC5D,QAAM,QAAQ;AAAA,IACZ,wBAAwB,KAAK,OAAO,KAAK,CAAC;AAAA,IAC1C;AAAA,IACA,kBAAkB,OAAO,YAAY,MAAM;AAAA,IAC3C,qBAAqB,OAAO,eAAe,MAAM;AAAA,IACjD,2BAA2B,OAAO,QAAQ,MAAM;AAAA,IAChD,gCAAgC,OAAO,aAAa,MAAM;AAAA,IAC1D,mBAAmB,OAAO,IAAI,MAAM,OAAO,CAAC,WAAW,OAAO,UAAU,MAAS,EAAE,MAAM;AAAA,IACzF,GAAI,OAAO,IAAI,WAAW,SAAY,CAAC,IAAI;AAAA,MACzC,aAAa,OAAO,IAAI,OAAO,QAAQ,QAAQ,CAAC,CAAC;AAAA,MACjD,UAAU,OAAO,IAAI,KAAK,CAAC,KAAK,OAAO,IAAI,OAAO,QAAQ,QAAQ,CAAC,CAAC;AAAA,MACpE,UAAU,OAAO,IAAI,KAAK,CAAC,KAAK,OAAO,IAAI,OAAO,SAAS,QAAQ,CAAC,CAAC;AAAA,MACrE,cACG,OAAO,OAAO,OAAO,IAAI,OAAO,OAAO,EAAE,OAAO,CAAC,KAAK,SAAS,MAAM,KAAK,QAAQ,CAAC,IAAI,OAAO,IAAI,OAAO,OACvG,QAAQ,CAAC,CACd;AAAA,IACF;AAAA,IACA;AAAA,IACA,GAAG;AAAA,MACD;AAAA,MACA,CAAC,QAAQ,UAAU,YAAY,UAAU,MAAM;AAAA,MAC/C,CAAC,OAAO,OAAO,QAAQ,QAAQ,MAAM;AAAA,MACrC,OAAO,YAAY,IAAI,CAAC,SAAS;AAAA,QAC/B,KAAK,KAAK,IAAI;AAAA,QACd,WAAW,KAAK,QAAQ,KAAK,OAAO,UAAU;AAAA,QAC9C,KAAK,SAAS,MAAM,QAAQ,CAAC;AAAA,QAC7B,KAAK,OAAO,MAAM,QAAQ,CAAC;AAAA,QAC3B,KAAK,KAAK,QAAQ,CAAC;AAAA,MACrB,CAAC;AAAA,IACH;AAAA,IACA,GAAG;AAAA,MACD;AAAA,MACA,CAAC,QAAQ,UAAU,YAAY,YAAY,UAAU,OAAO;AAAA,MAC5D,CAAC,OAAO,OAAO,OAAO,QAAQ,QAAQ,MAAM;AAAA,MAC5C,OAAO,eAAe,IAAI,CAAC,SAAS;AAAA,QAClC,KAAK,KAAK,IAAI;AAAA,QACd,WAAW,KAAK,QAAQ,KAAK,OAAO,UAAU;AAAA,QAC9C,KAAK,KAAK,OAAO,OAAO;AAAA,QACxB,KAAK,SAAS,MAAM,QAAQ,CAAC;AAAA,QAC7B,KAAK,OAAO,MAAM,QAAQ,CAAC;AAAA,QAC3B,KAAK,MAAM,QAAQ,CAAC;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,IACA,GAAG;AAAA,MACD;AAAA,MACA,CAAC,QAAQ,QAAQ,SAAS;AAAA,MAC1B,CAAC,OAAO,OAAO,KAAK;AAAA,MACpB,OAAO,IAAI,MAAM;AAAA,QAAQ,CAAC,WACxB,OAAO,UAAU,SAAY,CAAC,IAAI,CAAC,CAAC,KAAK,OAAO,IAAI,GAAG,KAAK,OAAO,MAAM,IAAI,GAAG,KAAK,OAAO,MAAM,OAAO,CAAC,CAAC;AAAA,MAC7G;AAAA,IACF;AAAA,IACA,GAAG;AAAA,MACD;AAAA,MACA,CAAC,QAAQ,QAAQ,UAAU,UAAU;AAAA,MACrC,CAAC,OAAO,OAAO,OAAO,KAAK;AAAA,MAC3B,OAAO,QAAQ,IAAI,CAAC,SAAS;AAAA,QAC3B,KAAK,KAAK,IAAI;AAAA,QACd,KAAK,KAAK,IAAI;AAAA,QACd,WAAW,KAAK,QAAQ,KAAK,UAAU;AAAA,QACvC,KAAK,KAAK,OAAO;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,IACA,GAAG;AAAA,MACD;AAAA,MACA,CAAC,QAAQ,UAAU,YAAY,QAAQ;AAAA,MACvC,CAAC,OAAO,OAAO,OAAO,KAAK;AAAA,MAC3B,OAAO,aAAa,IAAI,CAAC,SAAS;AAAA,QAChC,KAAK,KAAK,IAAI;AAAA,QACd,WAAW,KAAK,QAAQ,KAAK,UAAU;AAAA,QACvC,KAAK,KAAK,OAAO;AAAA,QACjB,KAAK,KAAK,MAAM;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO,GAAG,MAAM,KAAK,IAAI,EAAE,KAAK,CAAC;AAAA;AACnC;",
+  "names": []
+}