@smthrs/evals 1.0.0-rc.7 → 1.0.0-rc.8

package/dist/cjs/Trials.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/Trials.ts"],
-  "sourcesContent": ["/**\n * Repeated-run statistics for live model evaluations.\n *\n * A live case runs `n` times and passes `c` of them. {@link passAtK} is the\n * chance that at least one of `k` runs passes, the capability a best-of-k\n * caller sees; {@link passHatK} is the chance that all `k` pass, the\n * consistency a user sees on every attempt. Both are exact for sampling `k` of\n * the `n` recorded runs without replacement.\n *\n * Every function is pure and throws an `EvalError` with code `invalid_trials`\n * for counts outside the documented bounds.\n *\n * @since 0.1.0\n */\n\nimport { EvalError } from \"./EvalError.ts\"\n\n/**\n * Repeated-run statistics for one case.\n *\n * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,\n * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Summary {\n  readonly n: number\n  readonly passes: number\n  readonly rate: number\n  readonly passAt1: number\n  readonly passAtK: number\n  readonly passHatK: number\n  readonly stderr: number\n}\n\n/**\n * Repeated-run statistics across cases.\n *\n * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is\n * the fraction of cases whose every run passed.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Aggregate {\n  readonly cases: number\n  readonly passAt1: number\n  readonly passAtK: number\n  readonly passHatK: number\n  readonly allPass: number\n  readonly perCase: Readonly<Record<string, Summary>>\n}\n\nconst invalid = (message: string, path: string): EvalError => new EvalError({ code: \"invalid_trials\", message, path })\n\nconst validate = (n: number, c: number, k: number): void => {\n  if (!Number.isInteger(n) || n < 1) throw invalid(`Run count must be a positive integer, received ${n}`, \"n\")\n  if (!Number.isInteger(c) || c < 0 || c > n) {\n    throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, \"c\")\n  }\n  if (!Number.isInteger(k) || k < 1 || k > n) throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, \"k\")\n}\n\n/**\n * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,\n * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is\n * computed as a product, so large `n` neither overflows nor loses precision.\n *\n * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with\n * `0 <= c <= n` and `1 <= k <= n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passAtK = (n: number, c: number, k: number): number => {\n  validate(n, c, k)\n  if (n - c < k) return 1\n  let fail = 1\n  for (let i = n - c + 1; i <= n; i++) fail *= 1 - k / i\n  return 1 - fail\n}\n\n/**\n * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`\n * without replacement pass, the consistency measure of tau-bench.\n *\n * Throws `invalid_trials` under the same bounds as {@link passAtK}.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passHatK = (n: number, c: number, k: number): number => {\n  validate(n, c, k)\n  let all = 1\n  for (let i = 0; i < k; i++) all *= (c - i) / (n - i)\n  return Math.max(0, all)\n}\n\n/**\n * Summarizes one case's run outcomes at `k`.\n *\n * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const summarize = (outcomes: ReadonlyArray<boolean>, k: number): Summary => {\n  const n = outcomes.length\n  const passes = outcomes.filter(Boolean).length\n  validate(n, passes, k)\n  const rate = passes / n\n  return {\n    n,\n    passes,\n    rate,\n    passAt1: rate,\n    passAtK: passAtK(n, passes, k),\n    passHatK: passHatK(n, passes, k),\n    stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)\n  }\n}\n\nconst mean = (values: ReadonlyArray<number>): number =>\n  values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length\n\n/**\n * Summarizes every case at `k` and averages over cases.\n *\n * A case with fewer than `k` runs is summarized at `k = n`, its own run\n * count, so a short case still contributes rather than failing the whole\n * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case\n * with no runs or a `k` that is not a positive integer.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {\n  if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, \"k\")\n  const perCase: Record<string, Summary> = {}\n  for (const [name, outcomes] of Object.entries(cases)) {\n    if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)\n    perCase[name] = summarize(outcomes, Math.min(k, outcomes.length))\n  }\n  const summaries = Object.values(perCase)\n  return {\n    cases: summaries.length,\n    passAt1: mean(summaries.map((summary) => summary.passAt1)),\n    passAtK: mean(summaries.map((summary) => summary.passAtK)),\n    passHatK: mean(summaries.map((summary) => summary.passHatK)),\n    allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),\n    perCase\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAeA,uBAA0B;AAuC1B,MAAM,UAAU,CAAC,SAAiB,SAA4B,IAAI,2BAAU,EAAE,MAAM,kBAAkB,SAAS,KAAK,CAAC;AAErH,MAAM,WAAW,CAAC,GAAW,GAAW,MAAoB;AAC1D,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,EAAG,OAAM,QAAQ,kDAAkD,CAAC,IAAI,GAAG;AAC3G,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AAC1C,UAAM,QAAQ,2CAA2C,CAAC,cAAc,CAAC,IAAI,GAAG;AAAA,EAClF;AACA,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,KAAK,IAAI,EAAG,OAAM,QAAQ,kCAAkC,CAAC,cAAc,CAAC,IAAI,GAAG;AACrH;AAaO,MAAM,UAAU,CAAC,GAAW,GAAW,MAAsB;AAClE,WAAS,GAAG,GAAG,CAAC;AAChB,MAAI,IAAI,IAAI,EAAG,QAAO;AACtB,MAAI,OAAO;AACX,WAAS,IAAI,IAAI,IAAI,GAAG,KAAK,GAAG,IAAK,SAAQ,IAAI,IAAI;AACrD,SAAO,IAAI;AACb;AAWO,MAAM,WAAW,CAAC,GAAW,GAAW,MAAsB;AACnE,WAAS,GAAG,GAAG,CAAC;AAChB,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,SAAQ,IAAI,MAAM,IAAI;AAClD,SAAO,KAAK,IAAI,GAAG,GAAG;AACxB;AAUO,MAAM,YAAY,CAAC,UAAkC,MAAuB;AACjF,QAAM,IAAI,SAAS;AACnB,QAAM,SAAS,SAAS,OAAO,OAAO,EAAE;AACxC,WAAS,GAAG,QAAQ,CAAC;AACrB,QAAM,OAAO,SAAS;AACtB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,SAAS,QAAQ,GAAG,QAAQ,CAAC;AAAA,IAC7B,UAAU,SAAS,GAAG,QAAQ,CAAC;AAAA,IAC/B,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC;AAAA,EACtD;AACF;AAEA,MAAM,OAAO,CAAC,WACZ,OAAO,WAAW,IAAI,IAAI,OAAO,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IAAI,OAAO;AAahF,MAAM,YAAY,CAAC,OAAyD,MAAyB;AAC1G,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,EAAG,OAAM,QAAQ,0CAA0C,CAAC,IAAI,GAAG;AACnG,QAAM,UAAmC,CAAC;AAC1C,aAAW,CAAC,MAAM,QAAQ,KAAK,OAAO,QAAQ,KAAK,GAAG;AACpD,QAAI,SAAS,WAAW,EAAG,OAAM,QAAQ,SAAS,IAAI,iBAAiB,UAAU,IAAI,IAAI;AACzF,YAAQ,IAAI,IAAI,UAAU,UAAU,KAAK,IAAI,GAAG,SAAS,MAAM,CAAC;AAAA,EAClE;AACA,QAAM,YAAY,OAAO,OAAO,OAAO;AACvC,SAAO;AAAA,IACL,OAAO,UAAU;AAAA,IACjB,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,OAAO,CAAC;AAAA,IACzD,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,OAAO,CAAC;AAAA,IACzD,UAAU,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,QAAQ,CAAC;AAAA,IAC3D,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,WAAW,QAAQ,IAAI,IAAI,CAAC,CAAC;AAAA,IAC9E;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * Repeated-run statistics for live model evaluations.\n *\n * A live case runs `n` times and passes `c` of them. {@link passAtK} is the\n * chance that at least one of `k` runs passes, the capability a best-of-k\n * caller sees; {@link passHatK} is the chance that all `k` pass, the\n * consistency a user sees on every attempt. Both are exact for sampling `k` of\n * the `n` recorded runs without replacement.\n *\n * Every function is pure and throws an `EvalError` with code `invalid_trials`\n * for counts outside the documented bounds.\n *\n * @since 0.1.0\n */\n\nimport { EvalError } from \"./EvalError.ts\"\n\n/**\n * Repeated-run statistics for one case.\n *\n * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,\n * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Summary {\n  readonly n: number\n  readonly passes: number\n  readonly rate: number\n  readonly passAt1: number\n  readonly passAtK: number\n  readonly passHatK: number\n  readonly stderr: number\n}\n\n/**\n * Repeated-run statistics across cases.\n *\n * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is\n * the fraction of cases whose every run passed.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Aggregate {\n  readonly cases: number\n  readonly passAt1: number\n  readonly passAtK: number\n  readonly passHatK: number\n  readonly allPass: number\n  readonly perCase: Readonly<Record<string, Summary>>\n}\n\nconst invalid = (message: string, path: string): EvalError => new EvalError({ code: \"invalid_trials\", message, path })\n\nconst validate = (n: number, c: number, k: number): void => {\n  if (!Number.isInteger(n) || n < 1) throw invalid(`Run count must be a positive integer, received ${n}`, \"n\")\n  if (!Number.isInteger(c) || c < 0 || c > n) {\n    throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, \"c\")\n  }\n  if (!Number.isInteger(k) || k < 1 || k > n) throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, \"k\")\n}\n\n/**\n * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,\n * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is\n * computed as a product, so large `n` neither overflows nor loses precision.\n *\n * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with\n * `0 <= c <= n` and `1 <= k <= n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passAtK = (n: number, c: number, k: number): number => {\n  validate(n, c, k)\n  if (n - c < k) return 1\n  let fail = 1\n  for (let i = n - c + 1; i <= n; i++) fail *= 1 - k / i\n  return 1 - fail\n}\n\n/**\n * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`\n * without replacement pass, the consistency measure of tau-bench.\n *\n * Throws `invalid_trials` under the same bounds as {@link passAtK}.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const passHatK = (n: number, c: number, k: number): number => {\n  validate(n, c, k)\n  let all = 1\n  for (let i = 0; i < k; i++) all *= (c - i) / (n - i)\n  return Math.max(0, all)\n}\n\n/**\n * Summarizes one case's run outcomes at `k`.\n *\n * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const summarize = (outcomes: ReadonlyArray<boolean>, k: number): Summary => {\n  const n = outcomes.length\n  const passes = outcomes.filter(Boolean).length\n  validate(n, passes, k)\n  const rate = passes / n\n  return {\n    n,\n    passes,\n    rate,\n    passAt1: rate,\n    passAtK: passAtK(n, passes, k),\n    passHatK: passHatK(n, passes, k),\n    stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)\n  }\n}\n\nconst mean = (values: ReadonlyArray<number>): number =>\n  values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length\n\n/**\n * Summarizes every case at `k` and averages over cases.\n *\n * A case with fewer than `k` runs is summarized at `k = n`, its own run\n * count, so a short case still contributes rather than failing the whole\n * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case\n * with no runs or a `k` that is not a positive integer.\n *\n * @since 0.1.0\n * @category statistics\n */\nexport const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {\n  if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, \"k\")\n  const entries = Object.entries(cases).map(([name, outcomes]): [string, Summary] => {\n    if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)\n    return [name, summarize(outcomes, Math.min(k, outcomes.length))]\n  })\n  // fromEntries defines own properties, so a case named `__proto__` stays a case.\n  const perCase: Record<string, Summary> = Object.fromEntries(entries)\n  const summaries = Object.values(perCase)\n  return {\n    cases: summaries.length,\n    passAt1: mean(summaries.map((summary) => summary.passAt1)),\n    passAtK: mean(summaries.map((summary) => summary.passAtK)),\n    passHatK: mean(summaries.map((summary) => summary.passHatK)),\n    allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),\n    perCase\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAeA,uBAA0B;AAuC1B,MAAM,UAAU,CAAC,SAAiB,SAA4B,IAAI,2BAAU,EAAE,MAAM,kBAAkB,SAAS,KAAK,CAAC;AAErH,MAAM,WAAW,CAAC,GAAW,GAAW,MAAoB;AAC1D,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,EAAG,OAAM,QAAQ,kDAAkD,CAAC,IAAI,GAAG;AAC3G,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AAC1C,UAAM,QAAQ,2CAA2C,CAAC,cAAc,CAAC,IAAI,GAAG;AAAA,EAClF;AACA,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,KAAK,IAAI,EAAG,OAAM,QAAQ,kCAAkC,CAAC,cAAc,CAAC,IAAI,GAAG;AACrH;AAaO,MAAM,UAAU,CAAC,GAAW,GAAW,MAAsB;AAClE,WAAS,GAAG,GAAG,CAAC;AAChB,MAAI,IAAI,IAAI,EAAG,QAAO;AACtB,MAAI,OAAO;AACX,WAAS,IAAI,IAAI,IAAI,GAAG,KAAK,GAAG,IAAK,SAAQ,IAAI,IAAI;AACrD,SAAO,IAAI;AACb;AAWO,MAAM,WAAW,CAAC,GAAW,GAAW,MAAsB;AACnE,WAAS,GAAG,GAAG,CAAC;AAChB,MAAI,MAAM;AACV,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,SAAQ,IAAI,MAAM,IAAI;AAClD,SAAO,KAAK,IAAI,GAAG,GAAG;AACxB;AAUO,MAAM,YAAY,CAAC,UAAkC,MAAuB;AACjF,QAAM,IAAI,SAAS;AACnB,QAAM,SAAS,SAAS,OAAO,OAAO,EAAE;AACxC,WAAS,GAAG,QAAQ,CAAC;AACrB,QAAM,OAAO,SAAS;AACtB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,SAAS,QAAQ,GAAG,QAAQ,CAAC;AAAA,IAC7B,UAAU,SAAS,GAAG,QAAQ,CAAC;AAAA,IAC/B,QAAQ,KAAK,IAAI,IAAI,KAAK,KAAK,QAAQ,IAAI,QAAQ,CAAC;AAAA,EACtD;AACF;AAEA,MAAM,OAAO,CAAC,WACZ,OAAO,WAAW,IAAI,IAAI,OAAO,OAAO,CAAC,OAAO,UAAU,QAAQ,OAAO,CAAC,IAAI,OAAO;AAahF,MAAM,YAAY,CAAC,OAAyD,MAAyB;AAC1G,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,IAAI,EAAG,OAAM,QAAQ,0CAA0C,CAAC,IAAI,GAAG;AACnG,QAAM,UAAU,OAAO,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC,MAAM,QAAQ,MAAyB;AACjF,QAAI,SAAS,WAAW,EAAG,OAAM,QAAQ,SAAS,IAAI,iBAAiB,UAAU,IAAI,IAAI;AACzF,WAAO,CAAC,MAAM,UAAU,UAAU,KAAK,IAAI,GAAG,SAAS,MAAM,CAAC,CAAC;AAAA,EACjE,CAAC;AAED,QAAM,UAAmC,OAAO,YAAY,OAAO;AACnE,QAAM,YAAY,OAAO,OAAO,OAAO;AACvC,SAAO;AAAA,IACL,OAAO,UAAU;AAAA,IACjB,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,OAAO,CAAC;AAAA,IACzD,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,OAAO,CAAC;AAAA,IACzD,UAAU,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,QAAQ,CAAC;AAAA,IAC3D,SAAS,KAAK,UAAU,IAAI,CAAC,YAAY,QAAQ,WAAW,QAAQ,IAAI,IAAI,CAAC,CAAC;AAAA,IAC9E;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/esm/Baseline.d.ts

@@ -95,7 +95,9 @@ export declare const make: (baseline: Omit<Baseline, "version"> & {
  *
  * Records are ordered by an injective encoding of `(suite, case, scorer,
  * stepKey)`, so two records whose names differ only in where a separator falls
- * still sort apart. The result ends with a newline and never throws: a
+ * still sort apart. Records that share that tuple, one per trial of a repeated
+ * run, are ordered by score and then by `scorerName`, so the bytes never
+ * depend on the order trials finished in. The result ends with a newline and never throws: a
  * validated baseline holds only strings and finite numbers.
  *
  * @category serialization

package/dist/esm/Baseline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Baseline.d.ts","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,OAAO,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM;IACjB,wDAAwD;;CAEhD,CAAA;AAiGV;;;;;;;;GAQG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,SAAS,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAczE,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,IAAI,GACf,UAAU,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,OAAO,CAAA;CAAE,KAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAInC,CAAA;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAAI,UAAU,QAAQ,KAAG,MAW1C,CAAA;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAMhC,CAAA"}
+{"version":3,"file":"Baseline.d.ts","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,OAAO,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM;IACjB,wDAAwD;;CAEhD,CAAA;AAiGV;;;;;;;;GAQG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,SAAS,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAczE,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,IAAI,GACf,UAAU,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,OAAO,CAAA;CAAE,KAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAInC,CAAA;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,KAAK,GAAI,UAAU,QAAQ,KAAG,MAgB1C,CAAA;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAMhC,CAAA"}

package/dist/esm/Baseline.js

@@ -164,7 +164,9 @@ export const make = (baseline) => {
  *
  * Records are ordered by an injective encoding of `(suite, case, scorer,
  * stepKey)`, so two records whose names differ only in where a separator falls
- * still sort apart. The result ends with a newline and never throws: a
+ * still sort apart. Records that share that tuple, one per trial of a repeated
+ * run, are ordered by score and then by `scorerName`, so the bytes never
+ * depend on the order trials finished in. The result ends with a newline and never throws: a
  * validated baseline holds only strings and finite numbers.
  *
  * @category serialization
@@ -174,7 +176,10 @@ export const write = (baseline) => {
     const sortKey = (record) => tupleKey(record.suite, record.case, record.scorer, record.stepKey);
     const records = baseline.records
         .map((record) => [sortKey(record), record])
-        .sort(([left], [right]) => CanonicalJson.compareText(left, right))
+        .sort(([leftKey, left], [rightKey, right]) => CanonicalJson.compareText(leftKey, rightKey) ||
+        left.score - right.score ||
+        CanonicalJson.compareText(left.scorerName ?? "", right.scorerName ?? "") ||
+        Number(left.scorerName !== undefined) - Number(right.scorerName !== undefined))
         .map(([, record]) => record);
     return CanonicalJson.stringify({
         version: baseline.version,

package/dist/esm/Baseline.js.map

@@ -1 +1 @@
-{"version":3,"file":"Baseline.js","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAGjD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAU,CAAA;AAkCjC;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,wDAAwD;IACxD,MAAM,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;CAChB,CAAA;AAEV,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,IAAa,EAAmC,EAAE,CAC/E,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAC9F,CAAA;AAEH,6EAA6E;AAC7E,6EAA6E;AAC7E,0CAA0C;AAC1C,MAAM,WAAW,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,IAAY,EAAoC,EAAE;IACpG,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,0BAA0B,KAAK,2BAA2B,OAAO,KAAK,EAAE,EAAE,IAAI,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;IACvC,OAAO,OAAO,KAAK,SAAS;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;QACvB,CAAC,CAAC,IAAI,CAAC,0BAA0B,KAAK,4CAA4C,OAAO,EAAE,EAAE,IAAI,CAAC,CAAA;AACtG,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAA4C,EAAE,CAC/F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,GAAG,WAAW,KAAK,GAAG,CAAA;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,0CAA0C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,CAAC,CAAA;IAC5G,CAAC;IACD,MAAM,MAAM,GAAG,KAA4C,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAA;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAA;IAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAA;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAA;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,CAAC,CAAA;IAC1E,MAAM,KAAK,GAAG,QAAQ,CAAA;IACtB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC,CAAC,IAAI,CAChB,wEAAwE,MAAM,CAAC,KAAK,CAAC,EAAE,EACvF,GAAG,EAAE,QAAQ,CACd,CAAA;IACH,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,KAAK,SAAS;QAC5C,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,CAAC,CAAA;IACvE,4EAA4E;IAC5E,2EAA2E;IAC3E,YAAY;IACZ,OAAO;QACL,KAAK;QACL,IAAI,EAAE,QAAQ;QACd,MAAM;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,OAAO;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;KACxC,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsC,EAAE,CACtE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,QAAQ,GAAG,KAA6F,CAAA;IAC9G,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAA;IAC/B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,OAAO,SAAS,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;IACjG,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,gDAAgD,OAAO,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACpF,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,iEAAiE,YAAY,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9G,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,SAAS,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAC/D,IAAI,KAAK,GAAG,QAAQ,CAAA;IACpB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACxB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,6DAA6D,EAAE,OAAO,CAAC,CAAA;QAC5F,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,kEAAkE,EAAE,OAAO,CAAC,CAAA;QACjG,CAAC;QACD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;IACrB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;AAC5D,CAAC,CAAC,CAAA;AAEJ;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAc,EAAsC,EAAE;IAC5E,MAAM,OAAO,GAA0B,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,WAAwB,EAAE,EAAE,CAC3F,WAAW,CAAC,IAAI,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC;gBACD,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,GAAG,CAAC,WAAW,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,EAAE,CAAC;gBACvF,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,KAAK,EAAE,WAAW,CAAC,KAAK;aACzB,CAAC;QACF,CAAC,CAAC,EAAE,CACP,CAAA;IACD,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AACzD,CAAC,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,QAA2E,EACvC,EAAE;IACtC,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAA;IACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;IAChC,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,QAAkB,EAAU,EAAE;IAClD,MAAM,OAAO,GAAG,CAAC,MAAsB,EAAU,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO;SAC7B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC;SACnD,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SACjE,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAA;IAC9B,OAAO,aAAa,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,OAAO;KACR,CAAC,CAAA;AACJ,CAAC,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAsC,EAAE,CACvE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;IACzB,CAAC,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC;IAC1F,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACX,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY;QACtC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,4BAA4B,EAAE,KAAK,EAAE,CAAC;KAC5G,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Canonical committed evaluation baselines.\n *\n * A baseline is the record of what a suite used to score, committed beside the\n * suite it belongs to. It holds only the fields a comparison reads, and\n * validation rebuilds every record from those fields, so nothing a caller\n * happened to attach to an object travels into a committed artifact.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current committed baseline artifact version.\n *\n * @category models\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/**\n * One successful score retained by a baseline.\n *\n * `scorer` is the scorer key a comparison matches on. `scorerName` is the\n * optional readable name of the same scorer.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BaselineRecord {\n  readonly suite: string\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n  readonly score: number\n}\n\n/**\n * Canonical committed evaluation baseline.\n *\n * `suite` records artifact ownership even when `records` is empty.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Baseline {\n  readonly version: typeof version\n  readonly suite: string\n  readonly records: ReadonlyArray<BaselineRecord>\n}\n\n/**\n * The declared ceilings a committed baseline is validated against.\n *\n * `length` bounds the JSON text `load` will parse, so an oversized committed\n * file fails with a sentence instead of exhausting a CI runner's memory.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  /** Longest baseline JSON text, in UTF-16 code units. */\n  length: 16 * 1024 * 1024\n} as const\n\nconst fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>\n  Effect.fail(\n    new EvalError({ code: \"invalid_baseline\", message, ...(path === undefined ? {} : { path }) })\n  )\n\n// A committed record is read by a gate whose summary goes to a CI log, so it\n// follows the same control-character rule as a suite name: rejected where it\n// enters the system, not flattened later.\nconst stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {\n  if (typeof value !== \"string\") {\n    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)\n  }\n  const control = controlCharacter(value)\n  return control === undefined\n    ? Effect.succeed(value)\n    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)\n}\n\nconst decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>\n  Effect.gen(function*() {\n    const at = `records[${index}]`\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(`Baseline record must be an object, got ${value === null ? \"null\" : typeof value}`, at)\n    }\n    const source = value as { readonly [key: string]: unknown }\n    const rawSuite = source.suite\n    const rawCase = source.case\n    const rawScorer = source.scorer\n    const rawScorerName = source.scorerName\n    const rawStepKey = source.stepKey\n    const rawScore = source.score\n    const suite = yield* stringField(rawSuite, \"suite\", `${at}.suite`)\n    const caseName = yield* stringField(rawCase, \"case\", `${at}.case`)\n    const scorer = yield* stringField(rawScorer, \"scorer\", `${at}.scorer`)\n    const stepKey = yield* stringField(rawStepKey, \"stepKey\", `${at}.stepKey`)\n    const score = rawScore\n    if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n      return yield* fail(\n        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,\n        `${at}.score`\n      )\n    }\n    const scorerName = rawScorerName === undefined\n      ? undefined\n      : yield* stringField(rawScorerName, \"scorerName\", `${at}.scorerName`)\n    // Every record is rebuilt field by field. Keeping the caller's object would\n    // carry unknown keys, and any getter among them, straight into a committed\n    // artifact.\n    return {\n      suite,\n      case: caseName,\n      scorer,\n      ...(scorerName === undefined ? {} : { scorerName }),\n      stepKey,\n      score: Object.is(score, -0) ? 0 : score\n    }\n  })\n\nconst validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>\n  Effect.gen(function*() {\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(\"Baseline must be an object\")\n    }\n    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }\n    const rawVersion = artifact.version\n    const rawSuite = artifact.suite\n    const rawRecords = artifact.records\n    if (rawVersion !== version) {\n      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, \"version\")\n    }\n    if (rawSuite !== undefined && typeof rawSuite !== \"string\") {\n      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, \"suite\")\n    }\n    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)\n    if (suiteControl !== undefined) {\n      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, \"suite\")\n    }\n    if (!Array.isArray(rawRecords)) {\n      return yield* fail(\"Baseline records must be an array\", \"records\")\n    }\n    const records = yield* Effect.forEach(rawRecords, decodeRecord)\n    let suite = rawSuite\n    if (suite === undefined) {\n      const first = records[0]\n      if (first === undefined) {\n        return yield* fail(\"Cannot infer baseline suite: legacy artifact has no records\", \"suite\")\n      }\n      if (records.some((record) => record.suite !== first.suite)) {\n        return yield* fail(\"Cannot infer baseline suite: legacy records name multiple suites\", \"suite\")\n      }\n      suite = first.suite\n    }\n    return { version, suite, records: Object.freeze(records) }\n  })\n\n/**\n * Builds and validates a baseline from a run's successful observations.\n *\n * Inconclusive observations are dropped: a baseline records what was measured,\n * and an inconclusive observation measured nothing.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {\n  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>\n    observation.kind === \"score\"\n      ? [{\n        suite: run.suite,\n        case: observation.case,\n        scorer: observation.scorer,\n        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),\n        stepKey: observation.stepKey,\n        score: observation.score\n      }]\n      : []\n  )\n  return validate({ version, suite: run.suite, records })\n}\n\n/**\n * Validates an in-memory baseline.\n *\n * The result is a snapshot: every known field is read once, then records are\n * rebuilt from those validated values and the array is frozen. Mutating the\n * array or the objects that were passed in cannot change the validated\n * baseline, and a stateful getter cannot substitute a value after validation.\n *\n * Fails with `invalid_baseline` carrying the record index and field name in\n * `path` for a wrong version, a non-array `records`, a record that is not an\n * object, an identity field that is not a string or holds a control\n * character, or a score that is not finite in `[0, 1]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  baseline: Omit<Baseline, \"version\"> & { readonly version?: typeof version }\n): Effect.Effect<Baseline, EvalError> => {\n  const artifactVersion = baseline.version ?? version\n  const records = baseline.records\n  return validate({ version: artifactVersion, suite: baseline.suite, records })\n}\n\n/**\n * Serializes a baseline with recursively sorted keys and stable numbers.\n *\n * Records are ordered by an injective encoding of `(suite, case, scorer,\n * stepKey)`, so two records whose names differ only in where a separator falls\n * still sort apart. The result ends with a newline and never throws: a\n * validated baseline holds only strings and finite numbers.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const write = (baseline: Baseline): string => {\n  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)\n  const records = baseline.records\n    .map((record) => [sortKey(record), record] as const)\n    .sort(([left], [right]) => CanonicalJson.compareText(left, right))\n    .map(([, record]) => record)\n  return CanonicalJson.stringify({\n    version: baseline.version,\n    suite: baseline.suite,\n    records\n  })\n}\n\n/**\n * Loads and validates baseline JSON.\n *\n * A version-1 artifact without a top-level `suite` is accepted only when its\n * nonempty records all name the same suite. That suite becomes the artifact's\n * owner, so writing the result includes it. Empty or ambiguous legacy artifacts\n * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`\n * fails with `invalid_baseline` at `text` before any of it is parsed.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const load = (text: string): Effect.Effect<Baseline, EvalError> =>\n  text.length > limits.length\n    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, \"text\")\n    : Effect.try({\n      try: () => JSON.parse(text) as unknown,\n      catch: (cause) => new EvalError({ code: \"invalid_baseline\", message: \"Baseline is not valid JSON\", cause })\n    }).pipe(Effect.flatMap(validate))\n"]}
+{"version":3,"file":"Baseline.js","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAGjD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAU,CAAA;AAkCjC;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,wDAAwD;IACxD,MAAM,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;CAChB,CAAA;AAEV,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,IAAa,EAAmC,EAAE,CAC/E,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAC9F,CAAA;AAEH,6EAA6E;AAC7E,6EAA6E;AAC7E,0CAA0C;AAC1C,MAAM,WAAW,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,IAAY,EAAoC,EAAE;IACpG,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,0BAA0B,KAAK,2BAA2B,OAAO,KAAK,EAAE,EAAE,IAAI,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;IACvC,OAAO,OAAO,KAAK,SAAS;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;QACvB,CAAC,CAAC,IAAI,CAAC,0BAA0B,KAAK,4CAA4C,OAAO,EAAE,EAAE,IAAI,CAAC,CAAA;AACtG,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAA4C,EAAE,CAC/F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,GAAG,WAAW,KAAK,GAAG,CAAA;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,0CAA0C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,CAAC,CAAA;IAC5G,CAAC;IACD,MAAM,MAAM,GAAG,KAA4C,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAA;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAA;IAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAA;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAA;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,CAAC,CAAA;IAC1E,MAAM,KAAK,GAAG,QAAQ,CAAA;IACtB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC,CAAC,IAAI,CAChB,wEAAwE,MAAM,CAAC,KAAK,CAAC,EAAE,EACvF,GAAG,EAAE,QAAQ,CACd,CAAA;IACH,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,KAAK,SAAS;QAC5C,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,CAAC,CAAA;IACvE,4EAA4E;IAC5E,2EAA2E;IAC3E,YAAY;IACZ,OAAO;QACL,KAAK;QACL,IAAI,EAAE,QAAQ;QACd,MAAM;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,OAAO;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;KACxC,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsC,EAAE,CACtE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,QAAQ,GAAG,KAA6F,CAAA;IAC9G,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAA;IAC/B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,OAAO,SAAS,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;IACjG,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,gDAAgD,OAAO,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACpF,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,iEAAiE,YAAY,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9G,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,SAAS,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAC/D,IAAI,KAAK,GAAG,QAAQ,CAAA;IACpB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACxB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,6DAA6D,EAAE,OAAO,CAAC,CAAA;QAC5F,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,kEAAkE,EAAE,OAAO,CAAC,CAAA;QACjG,CAAC;QACD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;IACrB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;AAC5D,CAAC,CAAC,CAAA;AAEJ;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAc,EAAsC,EAAE;IAC5E,MAAM,OAAO,GAA0B,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,WAAwB,EAAE,EAAE,CAC3F,WAAW,CAAC,IAAI,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC;gBACD,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,GAAG,CAAC,WAAW,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,EAAE,CAAC;gBACvF,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,KAAK,EAAE,WAAW,CAAC,KAAK;aACzB,CAAC;QACF,CAAC,CAAC,EAAE,CACP,CAAA;IACD,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AACzD,CAAC,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,QAA2E,EACvC,EAAE;IACtC,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAA;IACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;IAChC,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,QAAkB,EAAU,EAAE;IAClD,MAAM,OAAO,GAAG,CAAC,MAAsB,EAAU,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO;SAC7B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC;SACnD,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,CAC3C,aAAa,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC;QAC5C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;QACxB,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;QACxE,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAC/E;SACA,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAA;IAC9B,OAAO,aAAa,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,OAAO;KACR,CAAC,CAAA;AACJ,CAAC,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAsC,EAAE,CACvE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;IACzB,CAAC,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC;IAC1F,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACX,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY;QACtC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,4BAA4B,EAAE,KAAK,EAAE,CAAC;KAC5G,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Canonical committed evaluation baselines.\n *\n * A baseline is the record of what a suite used to score, committed beside the\n * suite it belongs to. It holds only the fields a comparison reads, and\n * validation rebuilds every record from those fields, so nothing a caller\n * happened to attach to an object travels into a committed artifact.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current committed baseline artifact version.\n *\n * @category models\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/**\n * One successful score retained by a baseline.\n *\n * `scorer` is the scorer key a comparison matches on. `scorerName` is the\n * optional readable name of the same scorer.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BaselineRecord {\n  readonly suite: string\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n  readonly score: number\n}\n\n/**\n * Canonical committed evaluation baseline.\n *\n * `suite` records artifact ownership even when `records` is empty.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Baseline {\n  readonly version: typeof version\n  readonly suite: string\n  readonly records: ReadonlyArray<BaselineRecord>\n}\n\n/**\n * The declared ceilings a committed baseline is validated against.\n *\n * `length` bounds the JSON text `load` will parse, so an oversized committed\n * file fails with a sentence instead of exhausting a CI runner's memory.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  /** Longest baseline JSON text, in UTF-16 code units. */\n  length: 16 * 1024 * 1024\n} as const\n\nconst fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>\n  Effect.fail(\n    new EvalError({ code: \"invalid_baseline\", message, ...(path === undefined ? {} : { path }) })\n  )\n\n// A committed record is read by a gate whose summary goes to a CI log, so it\n// follows the same control-character rule as a suite name: rejected where it\n// enters the system, not flattened later.\nconst stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {\n  if (typeof value !== \"string\") {\n    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)\n  }\n  const control = controlCharacter(value)\n  return control === undefined\n    ? Effect.succeed(value)\n    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)\n}\n\nconst decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>\n  Effect.gen(function*() {\n    const at = `records[${index}]`\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(`Baseline record must be an object, got ${value === null ? \"null\" : typeof value}`, at)\n    }\n    const source = value as { readonly [key: string]: unknown }\n    const rawSuite = source.suite\n    const rawCase = source.case\n    const rawScorer = source.scorer\n    const rawScorerName = source.scorerName\n    const rawStepKey = source.stepKey\n    const rawScore = source.score\n    const suite = yield* stringField(rawSuite, \"suite\", `${at}.suite`)\n    const caseName = yield* stringField(rawCase, \"case\", `${at}.case`)\n    const scorer = yield* stringField(rawScorer, \"scorer\", `${at}.scorer`)\n    const stepKey = yield* stringField(rawStepKey, \"stepKey\", `${at}.stepKey`)\n    const score = rawScore\n    if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n      return yield* fail(\n        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,\n        `${at}.score`\n      )\n    }\n    const scorerName = rawScorerName === undefined\n      ? undefined\n      : yield* stringField(rawScorerName, \"scorerName\", `${at}.scorerName`)\n    // Every record is rebuilt field by field. Keeping the caller's object would\n    // carry unknown keys, and any getter among them, straight into a committed\n    // artifact.\n    return {\n      suite,\n      case: caseName,\n      scorer,\n      ...(scorerName === undefined ? {} : { scorerName }),\n      stepKey,\n      score: Object.is(score, -0) ? 0 : score\n    }\n  })\n\nconst validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>\n  Effect.gen(function*() {\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(\"Baseline must be an object\")\n    }\n    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }\n    const rawVersion = artifact.version\n    const rawSuite = artifact.suite\n    const rawRecords = artifact.records\n    if (rawVersion !== version) {\n      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, \"version\")\n    }\n    if (rawSuite !== undefined && typeof rawSuite !== \"string\") {\n      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, \"suite\")\n    }\n    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)\n    if (suiteControl !== undefined) {\n      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, \"suite\")\n    }\n    if (!Array.isArray(rawRecords)) {\n      return yield* fail(\"Baseline records must be an array\", \"records\")\n    }\n    const records = yield* Effect.forEach(rawRecords, decodeRecord)\n    let suite = rawSuite\n    if (suite === undefined) {\n      const first = records[0]\n      if (first === undefined) {\n        return yield* fail(\"Cannot infer baseline suite: legacy artifact has no records\", \"suite\")\n      }\n      if (records.some((record) => record.suite !== first.suite)) {\n        return yield* fail(\"Cannot infer baseline suite: legacy records name multiple suites\", \"suite\")\n      }\n      suite = first.suite\n    }\n    return { version, suite, records: Object.freeze(records) }\n  })\n\n/**\n * Builds and validates a baseline from a run's successful observations.\n *\n * Inconclusive observations are dropped: a baseline records what was measured,\n * and an inconclusive observation measured nothing.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {\n  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>\n    observation.kind === \"score\"\n      ? [{\n        suite: run.suite,\n        case: observation.case,\n        scorer: observation.scorer,\n        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),\n        stepKey: observation.stepKey,\n        score: observation.score\n      }]\n      : []\n  )\n  return validate({ version, suite: run.suite, records })\n}\n\n/**\n * Validates an in-memory baseline.\n *\n * The result is a snapshot: every known field is read once, then records are\n * rebuilt from those validated values and the array is frozen. Mutating the\n * array or the objects that were passed in cannot change the validated\n * baseline, and a stateful getter cannot substitute a value after validation.\n *\n * Fails with `invalid_baseline` carrying the record index and field name in\n * `path` for a wrong version, a non-array `records`, a record that is not an\n * object, an identity field that is not a string or holds a control\n * character, or a score that is not finite in `[0, 1]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  baseline: Omit<Baseline, \"version\"> & { readonly version?: typeof version }\n): Effect.Effect<Baseline, EvalError> => {\n  const artifactVersion = baseline.version ?? version\n  const records = baseline.records\n  return validate({ version: artifactVersion, suite: baseline.suite, records })\n}\n\n/**\n * Serializes a baseline with recursively sorted keys and stable numbers.\n *\n * Records are ordered by an injective encoding of `(suite, case, scorer,\n * stepKey)`, so two records whose names differ only in where a separator falls\n * still sort apart. Records that share that tuple, one per trial of a repeated\n * run, are ordered by score and then by `scorerName`, so the bytes never\n * depend on the order trials finished in. The result ends with a newline and never throws: a\n * validated baseline holds only strings and finite numbers.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const write = (baseline: Baseline): string => {\n  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)\n  const records = baseline.records\n    .map((record) => [sortKey(record), record] as const)\n    .sort(([leftKey, left], [rightKey, right]) =>\n      CanonicalJson.compareText(leftKey, rightKey) ||\n      left.score - right.score ||\n      CanonicalJson.compareText(left.scorerName ?? \"\", right.scorerName ?? \"\") ||\n      Number(left.scorerName !== undefined) - Number(right.scorerName !== undefined)\n    )\n    .map(([, record]) => record)\n  return CanonicalJson.stringify({\n    version: baseline.version,\n    suite: baseline.suite,\n    records\n  })\n}\n\n/**\n * Loads and validates baseline JSON.\n *\n * A version-1 artifact without a top-level `suite` is accepted only when its\n * nonempty records all name the same suite. That suite becomes the artifact's\n * owner, so writing the result includes it. Empty or ambiguous legacy artifacts\n * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`\n * fails with `invalid_baseline` at `text` before any of it is parsed.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const load = (text: string): Effect.Effect<Baseline, EvalError> =>\n  text.length > limits.length\n    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, \"text\")\n    : Effect.try({\n      try: () => JSON.parse(text) as unknown,\n      catch: (cause) => new EvalError({ code: \"invalid_baseline\", message: \"Baseline is not valid JSON\", cause })\n    }).pipe(Effect.flatMap(validate))\n"]}

package/dist/esm/Regression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AA2ED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAwHjC,CAAA"}
+{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AAiFD;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAyHjC,CAAA"}

package/dist/esm/Regression.js

@@ -19,6 +19,10 @@ import { tupleKey } from "./internal/tupleKey.js";
 // delimiter.
 const key = (caseName, scorer) => tupleKey(caseName, scorer);
 const validTolerance = (value) => Number.isFinite(value) && value >= 0;
+// Decimal scores are not exactly representable, so 0.8 - 0.7 is
+// 0.10000000000000009. A move counts only when it clears a positive tolerance
+// by more than a few ulps; a zero tolerance still reports every move.
+const exceeds = (move, tolerance) => tolerance === 0 ? move > 0 : move - tolerance > 8 * Number.EPSILON * Math.max(1, tolerance);
 const groupBy = (items, of) => {
     const groups = new Map();
     for (const item of items) {
@@ -160,7 +164,8 @@ export const compare = (baseline, run, tolerances = {}) => {
             // exactly 0 used to be special-cased in the regression branch, which was
             // dead: a score is validated into [0, 1], so nothing can drop below 0.
             if (observation.stepKey === baselineRecord.stepKey) {
-                if (Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative) {
+                if (exceeds(Math.abs(delta), absolute) &&
+                    exceeds(Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12), relative)) {
                     nondeterminism.push({
                         case: observation.case,
                         scorer: observation.scorer,
@@ -170,7 +175,7 @@ export const compare = (baseline, run, tolerances = {}) => {
                     });
                 }
             }
-            else if (delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative) {
+            else if (delta < 0 && exceeds(-delta, absolute) && exceeds(-delta / Math.max(baselineRecord.score, 1e-12), relative)) {
                 regressions.push({
                     case: observation.case,
                     scorer: observation.scorer,

package/dist/esm/Regression.js.map

@@ -1 +1 @@
-{"version":3,"file":"Regression.js","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAmFjD,4EAA4E;AAC5E,uEAAuE;AACvE,aAAa;AACb,MAAM,GAAG,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAU,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;AACpF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAEvF,MAAM,OAAO,GAAG,CAAI,KAAuB,EAAE,EAAuB,EAAyB,EAAE;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAA;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;;YACjD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAuC,KAAuB,EAAY,EAAE,CAC1F,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAE5D,MAAM,eAAe,GAAG,CACtB,KAAuB,EACb,EAAE,CACZ,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAQtH;;;;;;;GAOG;AACH,MAAM,QAAQ,GAAG,CACf,QAAuC,EACvC,QAA8B,EACT,EAAE;IACvB,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,gBAAgB,GAA0B,EAAE,CAAA;IAClD,MAAM,gBAAgB,GAAiB,EAAE,CAAA;IACzC,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QACjF,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;QACjE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC5D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;QAC7F,CAAC;QACD,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7C,gBAAgB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;QAC/C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAC5E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;IAC3G,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC5G,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjH,OAAO,KAAK,CAAA;AACd,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,QAAkB,EAClB,GAAc,EACd,aAAyB,EAAE,EACO,EAAE;IACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAA;IACpC,IAAI,aAAa,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,8BAA8B,aAAa,4BAA4B,GAAG,CAAC,KAAK,GAAG,CACpF;YACD,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,oCACE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAChD,2BAA2B,GAAG,CAAC,KAAK,GAAG,CACxC;YACD,IAAI,EAAE,kBAAkB;SACzB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAwB,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;IAC3G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAC3C,WAAW,EAC6D,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,CAAC,CAAA;IACjH,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/F,MAAM,WAAW,GAAsB,EAAE,CAAA;IACzC,MAAM,cAAc,GAA0B,EAAE,CAAA;IAChD,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,KAAK,MAAM,UAAU,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;QAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAA;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAA;YACjC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAA;YACtD,0EAA0E;YAC1E,yEAAyE;YACzE,uEAAuE;YACvE,IAAI,WAAW,CAAC,OAAO,KAAK,cAAc,CAAC,OAAO,EAAE,CAAC;gBACnD,IACE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,QAAQ,EAC1G,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC;wBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;wBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;wBAC1B,QAAQ,EAAE,cAAc;wBACxB,MAAM,EAAE,WAAW;wBACnB,KAAK;qBACN,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;iBAAM,IACL,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,QAAQ,EAC3F,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,QAAQ,EAAE,cAAc;oBACxB,MAAM,EAAE,WAAW;oBACnB,IAAI,EAAE,CAAC,KAAK;iBACb,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;QACR,GAAG;QACH,WAAW;QACX,cAAc;QACd,OAAO;QACP,OAAO,EAAE,MAAM;QACf,YAAY;KACb,CAAC,CAAA;AACJ,CAAC,CAAA","sourcesContent":["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"]}
+{"version":3,"file":"Regression.js","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAmFjD,4EAA4E;AAC5E,uEAAuE;AACvE,aAAa;AACb,MAAM,GAAG,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAU,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;AACpF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAEvF,gEAAgE;AAChE,8EAA8E;AAC9E,sEAAsE;AACtE,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,SAAiB,EAAW,EAAE,CAC3D,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;AAE7F,MAAM,OAAO,GAAG,CAAI,KAAuB,EAAE,EAAuB,EAAyB,EAAE;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAA;IAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;QACnC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;;YACjD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAuC,KAAuB,EAAY,EAAE,CAC1F,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAE5D,MAAM,eAAe,GAAG,CACtB,KAAuB,EACb,EAAE,CACZ,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;AAQtH;;;;;;;GAOG;AACH,MAAM,QAAQ,GAAG,CACf,QAAuC,EACvC,QAA8B,EACT,EAAE;IACvB,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACpE,MAAM,gBAAgB,GAA0B,EAAE,CAAA;IAClD,MAAM,gBAAgB,GAAiB,EAAE,CAAA;IACzC,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QACjF,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;QACjE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC5D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;QAC7F,CAAC;QACD,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;QAC7C,gBAAgB,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;QAC/C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,iBAAiB,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAC5E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,KAAK,CAAE,EAAE,CAAC,CAAA;IAC3G,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC5G,KAAK,MAAM,MAAM,IAAI,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjH,OAAO,KAAK,CAAA;AACd,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,QAAkB,EAClB,GAAc,EACd,aAAyB,EAAE,EACO,EAAE;IACpC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,IAAI,CAAC,CAAA;IACzC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,mBAAmB;YACzB,OAAO,EAAE,wEAAwE,MAAM,CAAC,QAAQ,CAAC,EAAE;YACnG,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAA;IACpC,IAAI,aAAa,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC;QAChC,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,8BAA8B,aAAa,4BAA4B,GAAG,CAAC,KAAK,GAAG,CACpF;YACD,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACjH,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,IAAI,CAChB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,kBAAkB;YACxB,OAAO,EAAE,wBAAwB,CAC/B,oCACE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAChD,2BAA2B,GAAG,CAAC,KAAK,GAAG,CACxC;YACD,IAAI,EAAE,kBAAkB;SACzB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAwB,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;IAC3G,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAC3C,WAAW,EAC6D,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,CAAC,CAAA;IACjH,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAA;IAC/F,MAAM,WAAW,GAAsB,EAAE,CAAA;IACzC,MAAM,cAAc,GAA0B,EAAE,CAAA;IAChD,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,KAAK,MAAM,UAAU,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;QAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,UAAU;oBAChB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;oBAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3F,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;iBAC/B,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YACD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAA;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAA;YACjC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAA;YACtD,0EAA0E;YAC1E,yEAAyE;YACzE,uEAAuE;YACvE,IAAI,WAAW,CAAC,OAAO,KAAK,cAAc,CAAC,OAAO,EAAE,CAAC;gBACnD,IACE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,EACpF,CAAC;oBACD,cAAc,CAAC,IAAI,CAAC;wBAClB,IAAI,EAAE,WAAW,CAAC,IAAI;wBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;wBAC1B,QAAQ,EAAE,cAAc;wBACxB,MAAM,EAAE,WAAW;wBACnB,KAAK;qBACN,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;iBAAM,IACL,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,EAC3G,CAAC;gBACD,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,WAAW,CAAC,IAAI;oBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,QAAQ,EAAE,cAAc;oBACxB,MAAM,EAAE,WAAW;oBACnB,IAAI,EAAE,CAAC,KAAK;iBACb,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC;QACpB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ;QACR,GAAG;QACH,WAAW;QACX,cAAc;QACd,OAAO;QACP,OAAO,EAAE,MAAM;QACf,YAAY;KACb,CAAC,CAAA;AACJ,CAAC,CAAA","sourcesContent":["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\n// Decimal scores are not exactly representable, so 0.8 - 0.7 is\n// 0.10000000000000009. A move counts only when it clears a positive tolerance\n// by more than a few ulps; a zero tolerance still reports every move.\nconst exceeds = (move: number, tolerance: number): boolean =>\n  tolerance === 0 ? move > 0 : move - tolerance > 8 * Number.EPSILON * Math.max(1, tolerance)\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          exceeds(Math.abs(delta), absolute) &&\n          exceeds(Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12), relative)\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && exceeds(-delta, absolute) && exceeds(-delta / Math.max(baselineRecord.score, 1e-12), relative)\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"]}

package/dist/esm/Runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAyKhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAoFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}
+{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAsMhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAqFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}

package/dist/esm/Runner.js

@@ -82,7 +82,31 @@ const caseError = (suiteCase, cause) => {
 // A step key is a runtime value the target returns, so unlike a suite name it
 // is flattened rather than rejected: it reaches baselines, reports and the
 // gate summary, and a control character in any of them corrupts a CI log line.
-const runCase = (executor, suiteCase) => Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(Effect.map((execution) => ({
+// An executor written in plain JavaScript can return any shape, so the two
+// fields every scorer and baseline reads are checked here, where the failure
+// can still name the case and the field.
+const malformedExecution = (suiteCase, execution) => {
+    const shape = typeof execution === "object" && execution !== null ? execution : {};
+    if (typeof shape.stepKey !== "string") {
+        return new EvalError({
+            code: "executor",
+            message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,
+            path: `${casePath(suiteCase.name)}.stepKey`
+        });
+    }
+    if (typeof shape.latencyMs !== "number" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {
+        return new EvalError({
+            code: "executor",
+            message: `executor returned a latencyMs that is not a finite non-negative number (${typeof shape.latencyMs === "number" ? String(shape.latencyMs) : typeof shape.latencyMs})`,
+            path: `${casePath(suiteCase.name)}.latencyMs`
+        });
+    }
+    return undefined;
+};
+const runCase = (executor, suiteCase) => Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(Effect.flatMap((execution) => {
+    const malformed = malformedExecution(suiteCase, execution);
+    return malformed === undefined ? Effect.succeed(execution) : Effect.fail(malformed);
+}), Effect.map((execution) => ({
     case: suiteCase.name,
     execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },
     observations: []
@@ -385,12 +409,12 @@ export const run = (suite, options) => Effect.gen(function* () {
             sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))
         });
     }), { concurrency: 1 });
-    const outcomes = {};
+    const outcomes = [];
     const cases = suite.cases.map((suiteCase, index) => {
         const attempts = batches.map((batch) => batch[index]);
         const passed = attempts.map((attempt) => attempt.error === undefined && attempt.observations.length > 0 &&
             attempt.observations.every((observation) => observation.kind === "score" && observation.score === 1));
-        outcomes[suiteCase.name] = passed;
+        outcomes.push([suiteCase.name, passed]);
         return {
             ...attempts[0],
             error: attempts.find((attempt) => attempt.error !== undefined)?.error,
@@ -403,7 +427,8 @@ export const run = (suite, options) => Effect.gen(function* () {
         suite: suite.name,
         cases,
         observations: cases.flatMap((caseResult) => caseResult.observations),
-        trials: Trials.aggregate(outcomes, k),
+        // fromEntries defines own properties, so a case named `__proto__` stays a case.
+        trials: Trials.aggregate(Object.fromEntries(outcomes), k),
         k
     };
 }).pipe(Effect.withSpan("Eval.run", { attributes: { suite: suite.name, runId: options.runId } }));