@smthrs/evals 1.0.0-rc.6 → 1.0.0-rc.8

package/README.md

@@ -20,9 +20,13 @@ runner. Runtime evaluation code imports scorers directly.
 
 ## Install
 
-The package is at 1.0.0-rc.0 and is not on the npm registry yet. It is a
-workspace package of https://github.com/smithersai/smithers, so you use it from
-a package in a clone of that repository. The steps are at
+Prereleases are published to npm under the `next` tag:
+
+```bash
+pnpm add @smthrs/evals@next effect@^4.0.4
+```
+
+Requirements and entry points are at
 https://evals.smithers.sh/installation/.
 
 ## Example

package/dist/cjs/Baseline.d.ts

@@ -95,7 +95,9 @@ export declare const make: (baseline: Omit<Baseline, "version"> & {
  *
  * Records are ordered by an injective encoding of `(suite, case, scorer,
  * stepKey)`, so two records whose names differ only in where a separator falls
- * still sort apart. The result ends with a newline and never throws: a
+ * still sort apart. Records that share that tuple, one per trial of a repeated
+ * run, are ordered by score and then by `scorerName`, so the bytes never
+ * depend on the order trials finished in. The result ends with a newline and never throws: a
  * validated baseline holds only strings and finite numbers.
  *
  * @category serialization

package/dist/cjs/Baseline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Baseline.d.ts","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,OAAO,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM;IACjB,wDAAwD;;CAEhD,CAAA;AAiGV;;;;;;;;GAQG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,SAAS,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAczE,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,IAAI,GACf,UAAU,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,OAAO,CAAA;CAAE,KAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAInC,CAAA;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAAI,UAAU,QAAQ,KAAG,MAW1C,CAAA;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAMhC,CAAA"}
+{"version":3,"file":"Baseline.d.ts","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,OAAO,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM;IACjB,wDAAwD;;CAEhD,CAAA;AAiGV;;;;;;;;GAQG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,SAAS,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAczE,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,IAAI,GACf,UAAU,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,OAAO,CAAA;CAAE,KAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAInC,CAAA;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,KAAK,GAAI,UAAU,QAAQ,KAAG,MAgB1C,CAAA;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAMhC,CAAA"}

package/dist/cjs/Baseline.js

@@ -144,7 +144,9 @@ const make = (baseline) => {
 };
 const write = (baseline) => {
   const sortKey = (record) => (0, import_tupleKey.tupleKey)(record.suite, record.case, record.scorer, record.stepKey);
-  const records = baseline.records.map((record) => [sortKey(record), record]).sort(([left], [right]) => import_CanonicalJson.CanonicalJson.compareText(left, right)).map(([, record]) => record);
+  const records = baseline.records.map((record) => [sortKey(record), record]).sort(
+    ([leftKey, left], [rightKey, right]) => import_CanonicalJson.CanonicalJson.compareText(leftKey, rightKey) || left.score - right.score || import_CanonicalJson.CanonicalJson.compareText(left.scorerName ?? "", right.scorerName ?? "") || Number(left.scorerName !== void 0) - Number(right.scorerName !== void 0)
+  ).map(([, record]) => record);
   return import_CanonicalJson.CanonicalJson.stringify({
     version: baseline.version,
     suite: baseline.suite,

package/dist/cjs/Baseline.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/Baseline.ts"],
-  "sourcesContent": ["/**\n * Canonical committed evaluation baselines.\n *\n * A baseline is the record of what a suite used to score, committed beside the\n * suite it belongs to. It holds only the fields a comparison reads, and\n * validation rebuilds every record from those fields, so nothing a caller\n * happened to attach to an object travels into a committed artifact.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current committed baseline artifact version.\n *\n * @category models\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/**\n * One successful score retained by a baseline.\n *\n * `scorer` is the scorer key a comparison matches on. `scorerName` is the\n * optional readable name of the same scorer.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BaselineRecord {\n  readonly suite: string\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n  readonly score: number\n}\n\n/**\n * Canonical committed evaluation baseline.\n *\n * `suite` records artifact ownership even when `records` is empty.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Baseline {\n  readonly version: typeof version\n  readonly suite: string\n  readonly records: ReadonlyArray<BaselineRecord>\n}\n\n/**\n * The declared ceilings a committed baseline is validated against.\n *\n * `length` bounds the JSON text `load` will parse, so an oversized committed\n * file fails with a sentence instead of exhausting a CI runner's memory.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  /** Longest baseline JSON text, in UTF-16 code units. */\n  length: 16 * 1024 * 1024\n} as const\n\nconst fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>\n  Effect.fail(\n    new EvalError({ code: \"invalid_baseline\", message, ...(path === undefined ? {} : { path }) })\n  )\n\n// A committed record is read by a gate whose summary goes to a CI log, so it\n// follows the same control-character rule as a suite name: rejected where it\n// enters the system, not flattened later.\nconst stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {\n  if (typeof value !== \"string\") {\n    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)\n  }\n  const control = controlCharacter(value)\n  return control === undefined\n    ? Effect.succeed(value)\n    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)\n}\n\nconst decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>\n  Effect.gen(function*() {\n    const at = `records[${index}]`\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(`Baseline record must be an object, got ${value === null ? \"null\" : typeof value}`, at)\n    }\n    const source = value as { readonly [key: string]: unknown }\n    const rawSuite = source.suite\n    const rawCase = source.case\n    const rawScorer = source.scorer\n    const rawScorerName = source.scorerName\n    const rawStepKey = source.stepKey\n    const rawScore = source.score\n    const suite = yield* stringField(rawSuite, \"suite\", `${at}.suite`)\n    const caseName = yield* stringField(rawCase, \"case\", `${at}.case`)\n    const scorer = yield* stringField(rawScorer, \"scorer\", `${at}.scorer`)\n    const stepKey = yield* stringField(rawStepKey, \"stepKey\", `${at}.stepKey`)\n    const score = rawScore\n    if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n      return yield* fail(\n        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,\n        `${at}.score`\n      )\n    }\n    const scorerName = rawScorerName === undefined\n      ? undefined\n      : yield* stringField(rawScorerName, \"scorerName\", `${at}.scorerName`)\n    // Every record is rebuilt field by field. Keeping the caller's object would\n    // carry unknown keys, and any getter among them, straight into a committed\n    // artifact.\n    return {\n      suite,\n      case: caseName,\n      scorer,\n      ...(scorerName === undefined ? {} : { scorerName }),\n      stepKey,\n      score: Object.is(score, -0) ? 0 : score\n    }\n  })\n\nconst validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>\n  Effect.gen(function*() {\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(\"Baseline must be an object\")\n    }\n    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }\n    const rawVersion = artifact.version\n    const rawSuite = artifact.suite\n    const rawRecords = artifact.records\n    if (rawVersion !== version) {\n      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, \"version\")\n    }\n    if (rawSuite !== undefined && typeof rawSuite !== \"string\") {\n      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, \"suite\")\n    }\n    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)\n    if (suiteControl !== undefined) {\n      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, \"suite\")\n    }\n    if (!Array.isArray(rawRecords)) {\n      return yield* fail(\"Baseline records must be an array\", \"records\")\n    }\n    const records = yield* Effect.forEach(rawRecords, decodeRecord)\n    let suite = rawSuite\n    if (suite === undefined) {\n      const first = records[0]\n      if (first === undefined) {\n        return yield* fail(\"Cannot infer baseline suite: legacy artifact has no records\", \"suite\")\n      }\n      if (records.some((record) => record.suite !== first.suite)) {\n        return yield* fail(\"Cannot infer baseline suite: legacy records name multiple suites\", \"suite\")\n      }\n      suite = first.suite\n    }\n    return { version, suite, records: Object.freeze(records) }\n  })\n\n/**\n * Builds and validates a baseline from a run's successful observations.\n *\n * Inconclusive observations are dropped: a baseline records what was measured,\n * and an inconclusive observation measured nothing.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {\n  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>\n    observation.kind === \"score\"\n      ? [{\n        suite: run.suite,\n        case: observation.case,\n        scorer: observation.scorer,\n        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),\n        stepKey: observation.stepKey,\n        score: observation.score\n      }]\n      : []\n  )\n  return validate({ version, suite: run.suite, records })\n}\n\n/**\n * Validates an in-memory baseline.\n *\n * The result is a snapshot: every known field is read once, then records are\n * rebuilt from those validated values and the array is frozen. Mutating the\n * array or the objects that were passed in cannot change the validated\n * baseline, and a stateful getter cannot substitute a value after validation.\n *\n * Fails with `invalid_baseline` carrying the record index and field name in\n * `path` for a wrong version, a non-array `records`, a record that is not an\n * object, an identity field that is not a string or holds a control\n * character, or a score that is not finite in `[0, 1]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  baseline: Omit<Baseline, \"version\"> & { readonly version?: typeof version }\n): Effect.Effect<Baseline, EvalError> => {\n  const artifactVersion = baseline.version ?? version\n  const records = baseline.records\n  return validate({ version: artifactVersion, suite: baseline.suite, records })\n}\n\n/**\n * Serializes a baseline with recursively sorted keys and stable numbers.\n *\n * Records are ordered by an injective encoding of `(suite, case, scorer,\n * stepKey)`, so two records whose names differ only in where a separator falls\n * still sort apart. The result ends with a newline and never throws: a\n * validated baseline holds only strings and finite numbers.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const write = (baseline: Baseline): string => {\n  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)\n  const records = baseline.records\n    .map((record) => [sortKey(record), record] as const)\n    .sort(([left], [right]) => CanonicalJson.compareText(left, right))\n    .map(([, record]) => record)\n  return CanonicalJson.stringify({\n    version: baseline.version,\n    suite: baseline.suite,\n    records\n  })\n}\n\n/**\n * Loads and validates baseline JSON.\n *\n * A version-1 artifact without a top-level `suite` is accepted only when its\n * nonempty records all name the same suite. That suite becomes the artifact's\n * owner, so writing the result includes it. Empty or ambiguous legacy artifacts\n * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`\n * fails with `invalid_baseline` at `text` before any of it is parsed.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const load = (text: string): Effect.Effect<Baseline, EvalError> =>\n  text.length > limits.length\n    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, \"text\")\n    : Effect.try({\n      try: () => JSON.parse(text) as unknown,\n      catch: (cause) => new EvalError({ code: \"invalid_baseline\", message: \"Baseline is not valid JSON\", cause })\n    }).pipe(Effect.flatMap(validate))\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,aAAwB;AACxB,uBAA0B;AAC1B,2BAA8B;AAC9B,+BAAiC;AACjC,sBAAyB;AASlB,MAAM,UAAU;AA2ChB,MAAM,SAAS;AAAA;AAAA,EAEpB,QAAQ,KAAK,OAAO;AACtB;AAEA,MAAM,OAAO,CAAC,SAAiB,SAC7B,OAAO;AAAA,EACL,IAAI,2BAAU,EAAE,MAAM,oBAAoB,SAAS,GAAI,SAAS,SAAY,CAAC,IAAI,EAAE,KAAK,EAAG,CAAC;AAC9F;AAKF,MAAM,cAAc,CAAC,OAAgB,OAAe,SAAmD;AACrG,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,KAAK,0BAA0B,KAAK,2BAA2B,OAAO,KAAK,IAAI,IAAI;AAAA,EAC5F;AACA,QAAM,cAAU,2CAAiB,KAAK;AACtC,SAAO,YAAY,SACf,OAAO,QAAQ,KAAK,IACpB,KAAK,0BAA0B,KAAK,4CAA4C,OAAO,IAAI,IAAI;AACrG;AAEA,MAAM,eAAe,CAAC,OAAgB,UACpC,OAAO,IAAI,aAAY;AACrB,QAAM,KAAK,WAAW,KAAK;AAC3B,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AACvE,WAAO,OAAO,KAAK,0CAA0C,UAAU,OAAO,SAAS,OAAO,KAAK,IAAI,EAAE;AAAA,EAC3G;AACA,QAAM,SAAS;AACf,QAAM,WAAW,OAAO;AACxB,QAAM,UAAU,OAAO;AACvB,QAAM,YAAY,OAAO;AACzB,QAAM,gBAAgB,OAAO;AAC7B,QAAM,aAAa,OAAO;AAC1B,QAAM,WAAW,OAAO;AACxB,QAAM,QAAQ,OAAO,YAAY,UAAU,SAAS,GAAG,EAAE,QAAQ;AACjE,QAAM,WAAW,OAAO,YAAY,SAAS,QAAQ,GAAG,EAAE,OAAO;AACjE,QAAM,SAAS,OAAO,YAAY,WAAW,UAAU,GAAG,EAAE,SAAS;AACrE,QAAM,UAAU,OAAO,YAAY,YAAY,WAAW,GAAG,EAAE,UAAU;AACzE,QAAM,QAAQ;AACd,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AAClF,WAAO,OAAO;AAAA,MACZ,wEAAwE,OAAO,KAAK,CAAC;AAAA,MACrF,GAAG,EAAE;AAAA,IACP;AAAA,EACF;AACA,QAAM,aAAa,kBAAkB,SACjC,SACA,OAAO,YAAY,eAAe,cAAc,GAAG,EAAE,aAAa;AAItE,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN;AAAA,IACA,GAAI,eAAe,SAAY,CAAC,IAAI,EAAE,WAAW;AAAA,IACjD;AAAA,IACA,OAAO,OAAO,GAAG,OAAO,EAAE,IAAI,IAAI;AAAA,EACpC;AACF,CAAC;AAEH,MAAM,WAAW,CAAC,UAChB,OAAO,IAAI,aAAY;AACrB,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AACvE,WAAO,OAAO,KAAK,4BAA4B;AAAA,EACjD;AACA,QAAM,WAAW;AACjB,QAAM,aAAa,SAAS;AAC5B,QAAM,WAAW,SAAS;AAC1B,QAAM,aAAa,SAAS;AAC5B,MAAI,eAAe,SAAS;AAC1B,WAAO,OAAO,KAAK,4BAA4B,OAAO,SAAS,OAAO,UAAU,CAAC,IAAI,SAAS;AAAA,EAChG;AACA,MAAI,aAAa,UAAa,OAAO,aAAa,UAAU;AAC1D,WAAO,OAAO,KAAK,gDAAgD,OAAO,QAAQ,IAAI,OAAO;AAAA,EAC/F;AACA,QAAM,eAAe,aAAa,SAAY,aAAY,2CAAiB,QAAQ;AACnF,MAAI,iBAAiB,QAAW;AAC9B,WAAO,OAAO,KAAK,iEAAiE,YAAY,IAAI,OAAO;AAAA,EAC7G;AACA,MAAI,CAAC,MAAM,QAAQ,UAAU,GAAG;AAC9B,WAAO,OAAO,KAAK,qCAAqC,SAAS;AAAA,EACnE;AACA,QAAM,UAAU,OAAO,OAAO,QAAQ,YAAY,YAAY;AAC9D,MAAI,QAAQ;AACZ,MAAI,UAAU,QAAW;AACvB,UAAM,QAAQ,QAAQ,CAAC;AACvB,QAAI,UAAU,QAAW;AACvB,aAAO,OAAO,KAAK,+DAA+D,OAAO;AAAA,IAC3F;AACA,QAAI,QAAQ,KAAK,CAAC,WAAW,OAAO,UAAU,MAAM,KAAK,GAAG;AAC1D,aAAO,OAAO,KAAK,oEAAoE,OAAO;AAAA,IAChG;AACA,YAAQ,MAAM;AAAA,EAChB;AACA,SAAO,EAAE,SAAS,OAAO,SAAS,OAAO,OAAO,OAAO,EAAE;AAC3D,CAAC;AAWI,MAAM,UAAU,CAAC,QAAuD;AAC7E,QAAM,UAAiC,IAAI,aAAa;AAAA,IAAQ,CAAC,gBAC/D,YAAY,SAAS,UACjB,CAAC;AAAA,MACD,OAAO,IAAI;AAAA,MACX,MAAM,YAAY;AAAA,MAClB,QAAQ,YAAY;AAAA,MACpB,GAAI,YAAY,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,YAAY,WAAW;AAAA,MACrF,SAAS,YAAY;AAAA,MACrB,OAAO,YAAY;AAAA,IACrB,CAAC,IACC,CAAC;AAAA,EACP;AACA,SAAO,SAAS,EAAE,SAAS,OAAO,IAAI,OAAO,QAAQ,CAAC;AACxD;AAkBO,MAAM,OAAO,CAClB,aACuC;AACvC,QAAM,kBAAkB,SAAS,WAAW;AAC5C,QAAM,UAAU,SAAS;AACzB,SAAO,SAAS,EAAE,SAAS,iBAAiB,OAAO,SAAS,OAAO,QAAQ,CAAC;AAC9E;AAaO,MAAM,QAAQ,CAAC,aAA+B;AACnD,QAAM,UAAU,CAAC,eAAmC,0BAAS,OAAO,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO,OAAO;AACrH,QAAM,UAAU,SAAS,QACtB,IAAI,CAAC,WAAW,CAAC,QAAQ,MAAM,GAAG,MAAM,CAAU,EAClD,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,MAAM,mCAAc,YAAY,MAAM,KAAK,CAAC,EAChE,IAAI,CAAC,CAAC,EAAE,MAAM,MAAM,MAAM;AAC7B,SAAO,mCAAc,UAAU;AAAA,IAC7B,SAAS,SAAS;AAAA,IAClB,OAAO,SAAS;AAAA,IAChB;AAAA,EACF,CAAC;AACH;AAcO,MAAM,OAAO,CAAC,SACnB,KAAK,SAAS,OAAO,SACjB,KAAK,4BAA4B,OAAO,MAAM,oBAAoB,KAAK,MAAM,IAAI,MAAM,IACvF,OAAO,IAAI;AAAA,EACX,KAAK,MAAM,KAAK,MAAM,IAAI;AAAA,EAC1B,OAAO,CAAC,UAAU,IAAI,2BAAU,EAAE,MAAM,oBAAoB,SAAS,8BAA8B,MAAM,CAAC;AAC5G,CAAC,EAAE,KAAK,OAAO,QAAQ,QAAQ,CAAC;",
+  "sourcesContent": ["/**\n * Canonical committed evaluation baselines.\n *\n * A baseline is the record of what a suite used to score, committed beside the\n * suite it belongs to. It holds only the fields a comparison reads, and\n * validation rebuilds every record from those fields, so nothing a caller\n * happened to attach to an object travels into a committed artifact.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current committed baseline artifact version.\n *\n * @category models\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/**\n * One successful score retained by a baseline.\n *\n * `scorer` is the scorer key a comparison matches on. `scorerName` is the\n * optional readable name of the same scorer.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BaselineRecord {\n  readonly suite: string\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n  readonly score: number\n}\n\n/**\n * Canonical committed evaluation baseline.\n *\n * `suite` records artifact ownership even when `records` is empty.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Baseline {\n  readonly version: typeof version\n  readonly suite: string\n  readonly records: ReadonlyArray<BaselineRecord>\n}\n\n/**\n * The declared ceilings a committed baseline is validated against.\n *\n * `length` bounds the JSON text `load` will parse, so an oversized committed\n * file fails with a sentence instead of exhausting a CI runner's memory.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  /** Longest baseline JSON text, in UTF-16 code units. */\n  length: 16 * 1024 * 1024\n} as const\n\nconst fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>\n  Effect.fail(\n    new EvalError({ code: \"invalid_baseline\", message, ...(path === undefined ? {} : { path }) })\n  )\n\n// A committed record is read by a gate whose summary goes to a CI log, so it\n// follows the same control-character rule as a suite name: rejected where it\n// enters the system, not flattened later.\nconst stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {\n  if (typeof value !== \"string\") {\n    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)\n  }\n  const control = controlCharacter(value)\n  return control === undefined\n    ? Effect.succeed(value)\n    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)\n}\n\nconst decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>\n  Effect.gen(function*() {\n    const at = `records[${index}]`\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(`Baseline record must be an object, got ${value === null ? \"null\" : typeof value}`, at)\n    }\n    const source = value as { readonly [key: string]: unknown }\n    const rawSuite = source.suite\n    const rawCase = source.case\n    const rawScorer = source.scorer\n    const rawScorerName = source.scorerName\n    const rawStepKey = source.stepKey\n    const rawScore = source.score\n    const suite = yield* stringField(rawSuite, \"suite\", `${at}.suite`)\n    const caseName = yield* stringField(rawCase, \"case\", `${at}.case`)\n    const scorer = yield* stringField(rawScorer, \"scorer\", `${at}.scorer`)\n    const stepKey = yield* stringField(rawStepKey, \"stepKey\", `${at}.stepKey`)\n    const score = rawScore\n    if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n      return yield* fail(\n        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,\n        `${at}.score`\n      )\n    }\n    const scorerName = rawScorerName === undefined\n      ? undefined\n      : yield* stringField(rawScorerName, \"scorerName\", `${at}.scorerName`)\n    // Every record is rebuilt field by field. Keeping the caller's object would\n    // carry unknown keys, and any getter among them, straight into a committed\n    // artifact.\n    return {\n      suite,\n      case: caseName,\n      scorer,\n      ...(scorerName === undefined ? {} : { scorerName }),\n      stepKey,\n      score: Object.is(score, -0) ? 0 : score\n    }\n  })\n\nconst validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>\n  Effect.gen(function*() {\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(\"Baseline must be an object\")\n    }\n    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }\n    const rawVersion = artifact.version\n    const rawSuite = artifact.suite\n    const rawRecords = artifact.records\n    if (rawVersion !== version) {\n      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, \"version\")\n    }\n    if (rawSuite !== undefined && typeof rawSuite !== \"string\") {\n      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, \"suite\")\n    }\n    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)\n    if (suiteControl !== undefined) {\n      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, \"suite\")\n    }\n    if (!Array.isArray(rawRecords)) {\n      return yield* fail(\"Baseline records must be an array\", \"records\")\n    }\n    const records = yield* Effect.forEach(rawRecords, decodeRecord)\n    let suite = rawSuite\n    if (suite === undefined) {\n      const first = records[0]\n      if (first === undefined) {\n        return yield* fail(\"Cannot infer baseline suite: legacy artifact has no records\", \"suite\")\n      }\n      if (records.some((record) => record.suite !== first.suite)) {\n        return yield* fail(\"Cannot infer baseline suite: legacy records name multiple suites\", \"suite\")\n      }\n      suite = first.suite\n    }\n    return { version, suite, records: Object.freeze(records) }\n  })\n\n/**\n * Builds and validates a baseline from a run's successful observations.\n *\n * Inconclusive observations are dropped: a baseline records what was measured,\n * and an inconclusive observation measured nothing.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {\n  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>\n    observation.kind === \"score\"\n      ? [{\n        suite: run.suite,\n        case: observation.case,\n        scorer: observation.scorer,\n        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),\n        stepKey: observation.stepKey,\n        score: observation.score\n      }]\n      : []\n  )\n  return validate({ version, suite: run.suite, records })\n}\n\n/**\n * Validates an in-memory baseline.\n *\n * The result is a snapshot: every known field is read once, then records are\n * rebuilt from those validated values and the array is frozen. Mutating the\n * array or the objects that were passed in cannot change the validated\n * baseline, and a stateful getter cannot substitute a value after validation.\n *\n * Fails with `invalid_baseline` carrying the record index and field name in\n * `path` for a wrong version, a non-array `records`, a record that is not an\n * object, an identity field that is not a string or holds a control\n * character, or a score that is not finite in `[0, 1]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  baseline: Omit<Baseline, \"version\"> & { readonly version?: typeof version }\n): Effect.Effect<Baseline, EvalError> => {\n  const artifactVersion = baseline.version ?? version\n  const records = baseline.records\n  return validate({ version: artifactVersion, suite: baseline.suite, records })\n}\n\n/**\n * Serializes a baseline with recursively sorted keys and stable numbers.\n *\n * Records are ordered by an injective encoding of `(suite, case, scorer,\n * stepKey)`, so two records whose names differ only in where a separator falls\n * still sort apart. Records that share that tuple, one per trial of a repeated\n * run, are ordered by score and then by `scorerName`, so the bytes never\n * depend on the order trials finished in. The result ends with a newline and never throws: a\n * validated baseline holds only strings and finite numbers.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const write = (baseline: Baseline): string => {\n  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)\n  const records = baseline.records\n    .map((record) => [sortKey(record), record] as const)\n    .sort(([leftKey, left], [rightKey, right]) =>\n      CanonicalJson.compareText(leftKey, rightKey) ||\n      left.score - right.score ||\n      CanonicalJson.compareText(left.scorerName ?? \"\", right.scorerName ?? \"\") ||\n      Number(left.scorerName !== undefined) - Number(right.scorerName !== undefined)\n    )\n    .map(([, record]) => record)\n  return CanonicalJson.stringify({\n    version: baseline.version,\n    suite: baseline.suite,\n    records\n  })\n}\n\n/**\n * Loads and validates baseline JSON.\n *\n * A version-1 artifact without a top-level `suite` is accepted only when its\n * nonempty records all name the same suite. That suite becomes the artifact's\n * owner, so writing the result includes it. Empty or ambiguous legacy artifacts\n * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`\n * fails with `invalid_baseline` at `text` before any of it is parsed.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const load = (text: string): Effect.Effect<Baseline, EvalError> =>\n  text.length > limits.length\n    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, \"text\")\n    : Effect.try({\n      try: () => JSON.parse(text) as unknown,\n      catch: (cause) => new EvalError({ code: \"invalid_baseline\", message: \"Baseline is not valid JSON\", cause })\n    }).pipe(Effect.flatMap(validate))\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,aAAwB;AACxB,uBAA0B;AAC1B,2BAA8B;AAC9B,+BAAiC;AACjC,sBAAyB;AASlB,MAAM,UAAU;AA2ChB,MAAM,SAAS;AAAA;AAAA,EAEpB,QAAQ,KAAK,OAAO;AACtB;AAEA,MAAM,OAAO,CAAC,SAAiB,SAC7B,OAAO;AAAA,EACL,IAAI,2BAAU,EAAE,MAAM,oBAAoB,SAAS,GAAI,SAAS,SAAY,CAAC,IAAI,EAAE,KAAK,EAAG,CAAC;AAC9F;AAKF,MAAM,cAAc,CAAC,OAAgB,OAAe,SAAmD;AACrG,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO,KAAK,0BAA0B,KAAK,2BAA2B,OAAO,KAAK,IAAI,IAAI;AAAA,EAC5F;AACA,QAAM,cAAU,2CAAiB,KAAK;AACtC,SAAO,YAAY,SACf,OAAO,QAAQ,KAAK,IACpB,KAAK,0BAA0B,KAAK,4CAA4C,OAAO,IAAI,IAAI;AACrG;AAEA,MAAM,eAAe,CAAC,OAAgB,UACpC,OAAO,IAAI,aAAY;AACrB,QAAM,KAAK,WAAW,KAAK;AAC3B,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AACvE,WAAO,OAAO,KAAK,0CAA0C,UAAU,OAAO,SAAS,OAAO,KAAK,IAAI,EAAE;AAAA,EAC3G;AACA,QAAM,SAAS;AACf,QAAM,WAAW,OAAO;AACxB,QAAM,UAAU,OAAO;AACvB,QAAM,YAAY,OAAO;AACzB,QAAM,gBAAgB,OAAO;AAC7B,QAAM,aAAa,OAAO;AAC1B,QAAM,WAAW,OAAO;AACxB,QAAM,QAAQ,OAAO,YAAY,UAAU,SAAS,GAAG,EAAE,QAAQ;AACjE,QAAM,WAAW,OAAO,YAAY,SAAS,QAAQ,GAAG,EAAE,OAAO;AACjE,QAAM,SAAS,OAAO,YAAY,WAAW,UAAU,GAAG,EAAE,SAAS;AACrE,QAAM,UAAU,OAAO,YAAY,YAAY,WAAW,GAAG,EAAE,UAAU;AACzE,QAAM,QAAQ;AACd,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AAClF,WAAO,OAAO;AAAA,MACZ,wEAAwE,OAAO,KAAK,CAAC;AAAA,MACrF,GAAG,EAAE;AAAA,IACP;AAAA,EACF;AACA,QAAM,aAAa,kBAAkB,SACjC,SACA,OAAO,YAAY,eAAe,cAAc,GAAG,EAAE,aAAa;AAItE,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN;AAAA,IACA,GAAI,eAAe,SAAY,CAAC,IAAI,EAAE,WAAW;AAAA,IACjD;AAAA,IACA,OAAO,OAAO,GAAG,OAAO,EAAE,IAAI,IAAI;AAAA,EACpC;AACF,CAAC;AAEH,MAAM,WAAW,CAAC,UAChB,OAAO,IAAI,aAAY;AACrB,MAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AACvE,WAAO,OAAO,KAAK,4BAA4B;AAAA,EACjD;AACA,QAAM,WAAW;AACjB,QAAM,aAAa,SAAS;AAC5B,QAAM,WAAW,SAAS;AAC1B,QAAM,aAAa,SAAS;AAC5B,MAAI,eAAe,SAAS;AAC1B,WAAO,OAAO,KAAK,4BAA4B,OAAO,SAAS,OAAO,UAAU,CAAC,IAAI,SAAS;AAAA,EAChG;AACA,MAAI,aAAa,UAAa,OAAO,aAAa,UAAU;AAC1D,WAAO,OAAO,KAAK,gDAAgD,OAAO,QAAQ,IAAI,OAAO;AAAA,EAC/F;AACA,QAAM,eAAe,aAAa,SAAY,aAAY,2CAAiB,QAAQ;AACnF,MAAI,iBAAiB,QAAW;AAC9B,WAAO,OAAO,KAAK,iEAAiE,YAAY,IAAI,OAAO;AAAA,EAC7G;AACA,MAAI,CAAC,MAAM,QAAQ,UAAU,GAAG;AAC9B,WAAO,OAAO,KAAK,qCAAqC,SAAS;AAAA,EACnE;AACA,QAAM,UAAU,OAAO,OAAO,QAAQ,YAAY,YAAY;AAC9D,MAAI,QAAQ;AACZ,MAAI,UAAU,QAAW;AACvB,UAAM,QAAQ,QAAQ,CAAC;AACvB,QAAI,UAAU,QAAW;AACvB,aAAO,OAAO,KAAK,+DAA+D,OAAO;AAAA,IAC3F;AACA,QAAI,QAAQ,KAAK,CAAC,WAAW,OAAO,UAAU,MAAM,KAAK,GAAG;AAC1D,aAAO,OAAO,KAAK,oEAAoE,OAAO;AAAA,IAChG;AACA,YAAQ,MAAM;AAAA,EAChB;AACA,SAAO,EAAE,SAAS,OAAO,SAAS,OAAO,OAAO,OAAO,EAAE;AAC3D,CAAC;AAWI,MAAM,UAAU,CAAC,QAAuD;AAC7E,QAAM,UAAiC,IAAI,aAAa;AAAA,IAAQ,CAAC,gBAC/D,YAAY,SAAS,UACjB,CAAC;AAAA,MACD,OAAO,IAAI;AAAA,MACX,MAAM,YAAY;AAAA,MAClB,QAAQ,YAAY;AAAA,MACpB,GAAI,YAAY,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,YAAY,WAAW;AAAA,MACrF,SAAS,YAAY;AAAA,MACrB,OAAO,YAAY;AAAA,IACrB,CAAC,IACC,CAAC;AAAA,EACP;AACA,SAAO,SAAS,EAAE,SAAS,OAAO,IAAI,OAAO,QAAQ,CAAC;AACxD;AAkBO,MAAM,OAAO,CAClB,aACuC;AACvC,QAAM,kBAAkB,SAAS,WAAW;AAC5C,QAAM,UAAU,SAAS;AACzB,SAAO,SAAS,EAAE,SAAS,iBAAiB,OAAO,SAAS,OAAO,QAAQ,CAAC;AAC9E;AAeO,MAAM,QAAQ,CAAC,aAA+B;AACnD,QAAM,UAAU,CAAC,eAAmC,0BAAS,OAAO,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO,OAAO;AACrH,QAAM,UAAU,SAAS,QACtB,IAAI,CAAC,WAAW,CAAC,QAAQ,MAAM,GAAG,MAAM,CAAU,EAClD;AAAA,IAAK,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,MACtC,mCAAc,YAAY,SAAS,QAAQ,KAC3C,KAAK,QAAQ,MAAM,SACnB,mCAAc,YAAY,KAAK,cAAc,IAAI,MAAM,cAAc,EAAE,KACvE,OAAO,KAAK,eAAe,MAAS,IAAI,OAAO,MAAM,eAAe,MAAS;AAAA,EAC/E,EACC,IAAI,CAAC,CAAC,EAAE,MAAM,MAAM,MAAM;AAC7B,SAAO,mCAAc,UAAU;AAAA,IAC7B,SAAS,SAAS;AAAA,IAClB,OAAO,SAAS;AAAA,IAChB;AAAA,EACF,CAAC;AACH;AAcO,MAAM,OAAO,CAAC,SACnB,KAAK,SAAS,OAAO,SACjB,KAAK,4BAA4B,OAAO,MAAM,oBAAoB,KAAK,MAAM,IAAI,MAAM,IACvF,OAAO,IAAI;AAAA,EACX,KAAK,MAAM,KAAK,MAAM,IAAI;AAAA,EAC1B,OAAO,CAAC,UAAU,IAAI,2BAAU,EAAE,MAAM,oBAAoB,SAAS,8BAA8B,MAAM,CAAC;AAC5G,CAAC,EAAE,KAAK,OAAO,QAAQ,QAAQ,CAAC;",
   "names": []
 }

package/dist/cjs/Regression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AA2ED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAwHjC,CAAA"}
+{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AAiFD;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAyHjC,CAAA"}

package/dist/cjs/Regression.js

@@ -38,6 +38,7 @@ var import_controlCharacters = require("./internal/controlCharacters.js");
 var import_tupleKey = require("./internal/tupleKey.js");
 const key = (caseName, scorer) => (0, import_tupleKey.tupleKey)(caseName, scorer);
 const validTolerance = (value) => Number.isFinite(value) && value >= 0;
+const exceeds = (move, tolerance) => tolerance === 0 ? move > 0 : move - tolerance > 8 * Number.EPSILON * Math.max(1, tolerance);
 const groupBy = (items, of) => {
   const groups = /* @__PURE__ */ new Map();
   for (const item of items) {
@@ -157,7 +158,7 @@ const compare = (baseline, run, tolerances = {}) => {
       const observation = pair.observed;
       const delta = observation.score - baselineRecord.score;
       if (observation.stepKey === baselineRecord.stepKey) {
-        if (Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative) {
+        if (exceeds(Math.abs(delta), absolute) && exceeds(Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12), relative)) {
           nondeterminism.push({
             case: observation.case,
             scorer: observation.scorer,
@@ -166,7 +167,7 @@ const compare = (baseline, run, tolerances = {}) => {
             delta
           });
         }
-      } else if (delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative) {
+      } else if (delta < 0 && exceeds(-delta, absolute) && exceeds(-delta / Math.max(baselineRecord.score, 1e-12), relative)) {
         regressions.push({
           case: observation.case,
           scorer: observation.scorer,

package/dist/cjs/Regression.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/Regression.ts"],
-  "sourcesContent": ["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAYA,aAAwB;AAExB,uBAA0B;AAC1B,2BAA8B;AAC9B,+BAAyC;AACzC,sBAAyB;AAsFzB,MAAM,MAAM,CAAC,UAAkB,eAA2B,0BAAS,UAAU,MAAM;AACnF,MAAM,iBAAiB,CAAC,UAA2B,OAAO,SAAS,KAAK,KAAK,SAAS;AAEtF,MAAM,UAAU,CAAI,OAAyB,OAAmD;AAC9F,QAAM,SAAS,oBAAI,IAAsB;AACzC,aAAW,QAAQ,OAAO;AACxB,UAAM,SAAS,OAAO,IAAI,GAAG,IAAI,CAAC;AAClC,QAAI,WAAW,OAAW,QAAO,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC;AAAA,QAChD,QAAO,KAAK,IAAI;AAAA,EACvB;AACA,SAAO;AACT;AAEA,MAAM,UAAU,CAAuC,UACrD,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,KAAK,QAAQ,MAAM,KAAK;AAE3D,MAAM,kBAAkB,CACtB,UAEA,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,mCAAc,YAAY,KAAK,SAAS,MAAM,OAAO,KAAK,KAAK,QAAQ,MAAM,KAAK;AAgBrH,MAAM,WAAW,CACf,UACA,aACwB;AACxB,QAAM,QAAqB,CAAC;AAC5B,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,mBAA0C,CAAC;AACjD,QAAM,mBAAiC,CAAC;AACxC,aAAW,WAAW,CAAC,GAAG,eAAe,KAAK,CAAC,EAAE,KAAK,mCAAc,WAAW,GAAG;AAChF,UAAM,SAAS,QAAQ,eAAe,IAAI,OAAO,CAAE;AACnD,UAAM,iBAAiB,QAAQ,eAAe,IAAI,OAAO,KAAK,CAAC,CAAC;AAChE,UAAM,QAAQ,KAAK,IAAI,OAAO,QAAQ,eAAe,MAAM;AAC3D,aAAS,QAAQ,GAAG,QAAQ,OAAO,SAAS;AAC1C,YAAM,KAAK,EAAE,MAAM,WAAW,UAAU,OAAO,KAAK,GAAI,UAAU,eAAe,KAAK,EAAG,CAAC;AAAA,IAC5F;AACA,qBAAiB,KAAK,GAAG,OAAO,MAAM,KAAK,CAAC;AAC5C,qBAAiB,KAAK,GAAG,eAAe,MAAM,KAAK,CAAC;AAAA,EACtD;AACA,aAAW,CAAC,SAAS,MAAM,KAAK,gBAAgB;AAC9C,QAAI,CAAC,eAAe,IAAI,OAAO,EAAG,kBAAiB,KAAK,GAAG,MAAM;AAAA,EACnE;AACA,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,UAAU,KAAK,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AAC3E,WAAS,QAAQ,GAAG,QAAQ,SAAS,SAAS;AAC5C,UAAM,KAAK,EAAE,MAAM,WAAW,UAAU,kBAAkB,KAAK,GAAI,UAAU,kBAAkB,KAAK,EAAG,CAAC;AAAA,EAC1G;AACA,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,eAAe,UAAU,OAAO,CAAC;AAC3G,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,oBAAoB,UAAU,OAAO,CAAC;AAChH,SAAO;AACT;AAoBO,MAAM,UAAU,CACrB,UACA,KACA,aAAyB,CAAC,MACW;AACrC,QAAM,WAAW,WAAW,YAAY;AACxC,QAAM,WAAW,WAAW,YAAY;AACxC,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,gBAAgB,SAAS;AAC/B,MAAI,kBAAkB,IAAI,OAAO;AAC/B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,8BAA8B,aAAa,4BAA4B,IAAI,KAAK;AAAA,QAClF;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,SAAS,QAAQ,OAAO,CAAC,WAAW,OAAO,UAAU,IAAI,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAChH,MAAI,QAAQ,SAAS,GAAG;AACtB,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,oCACE,QAAQ,IAAI,CAAC,UAAU,IAAI,KAAK,GAAG,EAAE,KAAK,IAAI,CAChD,2BAA2B,IAAI,KAAK;AAAA,QACtC;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,SAAS,IAAI,aAAa,OAAO,CAAC,gBAAsC,YAAY,SAAS,OAAO;AAC1G,QAAM,eAAe,IAAI,aAAa,OAAO,CAC3C,gBAC2E,YAAY,SAAS,cAAc;AAChH,QAAM,gBAAgB,QAAQ,SAAS,SAAS,CAAC,WAAW,IAAI,OAAO,MAAM,OAAO,MAAM,CAAC;AAC3F,QAAM,cAAc,QAAQ,QAAQ,CAAC,gBAAgB,IAAI,YAAY,MAAM,YAAY,MAAM,CAAC;AAC9F,QAAM,cAAiC,CAAC;AACxC,QAAM,iBAAwC,CAAC;AAC/C,QAAM,UAAqC,CAAC;AAC5C,aAAW,cAAc,oBAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,GAAG;AAClF,UAAM,QAAQ,SAAS,cAAc,IAAI,UAAU,KAAK,CAAC,GAAG,YAAY,IAAI,UAAU,KAAK,CAAC,CAAC;AAC7F,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,SAAS,oBAAoB;AACpC,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,UAAI,KAAK,SAAS,eAAe;AAC/B,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,YAAM,iBAAiB,KAAK;AAC5B,YAAM,cAAc,KAAK;AACzB,YAAM,QAAQ,YAAY,QAAQ,eAAe;AAIjD,UAAI,YAAY,YAAY,eAAe,SAAS;AAClD,YACE,KAAK,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,eAAe,KAAK,GAAG,KAAK,IAAI,UAClG;AACA,yBAAe,KAAK;AAAA,YAClB,MAAM,YAAY;AAAA,YAClB,QAAQ,YAAY;AAAA,YACpB,UAAU;AAAA,YACV,QAAQ;AAAA,YACR;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,WACE,QAAQ,KAAK,CAAC,QAAQ,YAAY,CAAC,QAAQ,KAAK,IAAI,eAAe,OAAO,KAAK,IAAI,UACnF;AACA,oBAAY,KAAK;AAAA,UACf,MAAM,YAAY;AAAA,UAClB,QAAQ,YAAY;AAAA,UACpB,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,MAAM,CAAC;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,OAAO,QAAQ;AAAA,IACpB,OAAO,IAAI;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;",
+  "sourcesContent": ["/**\n * Step-key-aware evaluation regression comparison.\n *\n * The step key is what separates the two findings this module reports. A score\n * that moved at a *changed* step key is a regression: the target produced\n * different work and it graded worse. A score that moved at an *unchanged* step\n * key is nondeterminism: the same work graded differently twice. Both are\n * results, and a gate reads both as red.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport type { Baseline, BaselineRecord } from \"./Baseline.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Tolerances used for score comparisons.\n *\n * A move is reported only when it exceeds both tolerances, so either one alone\n * is enough to silence it. Both default to `0`, which reports every move.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Tolerances {\n  readonly absolute?: number | undefined\n  readonly relative?: number | undefined\n}\n\n/**\n * A score drop at a changed step key.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Regression {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly drop: number\n}\n\n/**\n * A changed score at the same step key, indicating nondeterminism.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Nondeterminism {\n  readonly case: string\n  readonly scorer: string\n  readonly baseline: BaselineRecord\n  readonly actual: Extract<Observation, { readonly kind: \"score\" }>\n  readonly delta: number\n}\n\n/**\n * An observation present on only one side of a comparison.\n *\n * `side` names the side it is missing from: `\"run\"` for a baseline record the\n * run never reproduced, `\"baseline\"` for a score the run produced that no\n * baseline record accounts for. `scorerName` comes from the observation that\n * is present, when that artifact carried one.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MissingObservation {\n  readonly side: \"baseline\" | \"run\"\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n}\n\n/**\n * Complete regression comparison.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Report {\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run: RunResult\n  readonly regressions: ReadonlyArray<Regression>\n  readonly nondeterminism: ReadonlyArray<Nondeterminism>\n  readonly missing: ReadonlyArray<MissingObservation>\n  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: \"score\" }>>\n  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: \"inconclusive\" }>>\n}\n\ntype Score = Extract<Observation, { readonly kind: \"score\" }>\n\n// Injective: two distinct (case, scorer) pairs can never encode to one key,\n// which a delimiter join could not promise once a name may contain the\n// delimiter.\nconst key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)\nconst validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0\n\n// Decimal scores are not exactly representable, so 0.8 - 0.7 is\n// 0.10000000000000009. A move counts only when it clears a positive tolerance\n// by more than a few ulps; a zero tolerance still reports every move.\nconst exceeds = (move: number, tolerance: number): boolean =>\n  tolerance === 0 ? move > 0 : move - tolerance > 8 * Number.EPSILON * Math.max(1, tolerance)\n\nconst groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {\n  const groups = new Map<string, Array<A>>()\n  for (const item of items) {\n    const bucket = groups.get(of(item))\n    if (bucket === undefined) groups.set(of(item), [item])\n    else bucket.push(item)\n  }\n  return groups\n}\n\nconst byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>\n  [...items].sort((left, right) => left.score - right.score)\n\nconst byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(\n  items: ReadonlyArray<A>\n): Array<A> =>\n  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)\n\n/** One matched pair, or one record that nothing on the other side accounts for. */\ntype Pair =\n  | { readonly kind: \"matched\"; readonly expected: BaselineRecord; readonly observed: Score }\n  | { readonly kind: \"missing-run\"; readonly expected: BaselineRecord }\n  | { readonly kind: \"missing-baseline\"; readonly observed: Score }\n\n/**\n * Pairs one case's baseline records with one case's observations.\n *\n * Same-step-key pairs are matched first, lowest score to lowest score, so a\n * repeated scorer does not report a move that never happened. Whatever is left\n * over is paired in stable order, and an unpaired record on either side is a\n * missing observation rather than a silent drop.\n */\nconst pairsFor = (\n  expected: ReadonlyArray<BaselineRecord>,\n  observed: ReadonlyArray<Score>\n): ReadonlyArray<Pair> => {\n  const pairs: Array<Pair> = []\n  const expectedByStep = groupBy(expected, (record) => record.stepKey)\n  const observedByStep = groupBy(observed, (record) => record.stepKey)\n  const leftoverExpected: Array<BaselineRecord> = []\n  const leftoverObserved: Array<Score> = []\n  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {\n    const atStep = byScore(expectedByStep.get(stepKey)!)\n    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])\n    const count = Math.min(atStep.length, observedAtStep.length)\n    for (let index = 0; index < count; index++) {\n      pairs.push({ kind: \"matched\", expected: atStep[index]!, observed: observedAtStep[index]! })\n    }\n    leftoverExpected.push(...atStep.slice(count))\n    leftoverObserved.push(...observedAtStep.slice(count))\n  }\n  for (const [stepKey, atStep] of observedByStep) {\n    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)\n  }\n  const remainingExpected = byStepThenScore(leftoverExpected)\n  const remainingObserved = byStepThenScore(leftoverObserved)\n  const matched = Math.min(remainingExpected.length, remainingObserved.length)\n  for (let index = 0; index < matched; index++) {\n    pairs.push({ kind: \"matched\", expected: remainingExpected[index]!, observed: remainingObserved[index]! })\n  }\n  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: \"missing-run\", expected: record })\n  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: \"missing-baseline\", observed: record })\n  return pairs\n}\n\n/**\n * Compares a run to a baseline, preserving missing and inconclusive observations.\n *\n * Records and observations are grouped by `(case, scorer)` and then paired by\n * step key, so a scorer that ran several times against one case is compared\n * pairwise instead of by array position. A pair whose step key is unchanged and\n * whose score moved is nondeterminism; a pair whose step key changed and whose\n * score dropped is a regression; an unpaired record on either side is a missing\n * observation. Inconclusive observations are carried through untouched, because\n * they decide nothing and a gate has to see them.\n *\n * Fails with `invalid_tolerance` when a tolerance is not finite and\n * non-negative, and with `invalid_baseline` when the artifact or any of its\n * records belongs to a suite other than the one the run reports.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const compare = (\n  baseline: Baseline,\n  run: RunResult,\n  tolerances: Tolerances = {}\n): Effect.Effect<Report, EvalError> => {\n  const absolute = tolerances.absolute ?? 0\n  const relative = tolerances.relative ?? 0\n  if (!validTolerance(absolute)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,\n        path: \"tolerances.absolute\"\n      })\n    )\n  }\n  if (!validTolerance(relative)) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_tolerance\",\n        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,\n        path: \"tolerances.relative\"\n      })\n    )\n  }\n  const baselineSuite = baseline.suite\n  if (baselineSuite !== run.suite) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.suite\"\n      })\n    )\n  }\n  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]\n  if (foreign.length > 0) {\n    return Effect.fail(\n      new EvalError({\n        code: \"invalid_baseline\",\n        message: flattenControlCharacters(\n          `Baseline holds records for suite ${\n            foreign.map((suite) => `'${suite}'`).join(\", \")\n          }, but the run is suite '${run.suite}'`\n        ),\n        path: \"baseline.records\"\n      })\n    )\n  }\n  const actual = run.observations.filter((observation): observation is Score => observation.kind === \"score\")\n  const inconclusive = run.observations.filter((\n    observation\n  ): observation is Extract<Observation, { readonly kind: \"inconclusive\" }> => observation.kind === \"inconclusive\")\n  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))\n  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))\n  const regressions: Array<Regression> = []\n  const nondeterminism: Array<Nondeterminism> = []\n  const missing: Array<MissingObservation> = []\n  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {\n    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])\n    for (const pair of pairs) {\n      if (pair.kind === \"missing-baseline\") {\n        missing.push({\n          side: \"baseline\",\n          case: pair.observed.case,\n          scorer: pair.observed.scorer,\n          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),\n          stepKey: pair.observed.stepKey\n        })\n        continue\n      }\n      if (pair.kind === \"missing-run\") {\n        missing.push({\n          side: \"run\",\n          case: pair.expected.case,\n          scorer: pair.expected.scorer,\n          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),\n          stepKey: pair.expected.stepKey\n        })\n        continue\n      }\n      const baselineRecord = pair.expected\n      const observation = pair.observed\n      const delta = observation.score - baselineRecord.score\n      // Both branches divide by the same guarded magnitude. A baseline score of\n      // exactly 0 used to be special-cased in the regression branch, which was\n      // dead: a score is validated into [0, 1], so nothing can drop below 0.\n      if (observation.stepKey === baselineRecord.stepKey) {\n        if (\n          exceeds(Math.abs(delta), absolute) &&\n          exceeds(Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12), relative)\n        ) {\n          nondeterminism.push({\n            case: observation.case,\n            scorer: observation.scorer,\n            baseline: baselineRecord,\n            actual: observation,\n            delta\n          })\n        }\n      } else if (\n        delta < 0 && exceeds(-delta, absolute) && exceeds(-delta / Math.max(baselineRecord.score, 1e-12), relative)\n      ) {\n        regressions.push({\n          case: observation.case,\n          scorer: observation.scorer,\n          baseline: baselineRecord,\n          actual: observation,\n          drop: -delta\n        })\n      }\n    }\n  }\n  return Effect.succeed({\n    suite: run.suite,\n    baseline,\n    run,\n    regressions,\n    nondeterminism,\n    missing,\n    samples: actual,\n    inconclusive\n  })\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAYA,aAAwB;AAExB,uBAA0B;AAC1B,2BAA8B;AAC9B,+BAAyC;AACzC,sBAAyB;AAsFzB,MAAM,MAAM,CAAC,UAAkB,eAA2B,0BAAS,UAAU,MAAM;AACnF,MAAM,iBAAiB,CAAC,UAA2B,OAAO,SAAS,KAAK,KAAK,SAAS;AAKtF,MAAM,UAAU,CAAC,MAAc,cAC7B,cAAc,IAAI,OAAO,IAAI,OAAO,YAAY,IAAI,OAAO,UAAU,KAAK,IAAI,GAAG,SAAS;AAE5F,MAAM,UAAU,CAAI,OAAyB,OAAmD;AAC9F,QAAM,SAAS,oBAAI,IAAsB;AACzC,aAAW,QAAQ,OAAO;AACxB,UAAM,SAAS,OAAO,IAAI,GAAG,IAAI,CAAC;AAClC,QAAI,WAAW,OAAW,QAAO,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC;AAAA,QAChD,QAAO,KAAK,IAAI;AAAA,EACvB;AACA,SAAO;AACT;AAEA,MAAM,UAAU,CAAuC,UACrD,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,KAAK,QAAQ,MAAM,KAAK;AAE3D,MAAM,kBAAkB,CACtB,UAEA,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,MAAM,UAAU,mCAAc,YAAY,KAAK,SAAS,MAAM,OAAO,KAAK,KAAK,QAAQ,MAAM,KAAK;AAgBrH,MAAM,WAAW,CACf,UACA,aACwB;AACxB,QAAM,QAAqB,CAAC;AAC5B,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,iBAAiB,QAAQ,UAAU,CAAC,WAAW,OAAO,OAAO;AACnE,QAAM,mBAA0C,CAAC;AACjD,QAAM,mBAAiC,CAAC;AACxC,aAAW,WAAW,CAAC,GAAG,eAAe,KAAK,CAAC,EAAE,KAAK,mCAAc,WAAW,GAAG;AAChF,UAAM,SAAS,QAAQ,eAAe,IAAI,OAAO,CAAE;AACnD,UAAM,iBAAiB,QAAQ,eAAe,IAAI,OAAO,KAAK,CAAC,CAAC;AAChE,UAAM,QAAQ,KAAK,IAAI,OAAO,QAAQ,eAAe,MAAM;AAC3D,aAAS,QAAQ,GAAG,QAAQ,OAAO,SAAS;AAC1C,YAAM,KAAK,EAAE,MAAM,WAAW,UAAU,OAAO,KAAK,GAAI,UAAU,eAAe,KAAK,EAAG,CAAC;AAAA,IAC5F;AACA,qBAAiB,KAAK,GAAG,OAAO,MAAM,KAAK,CAAC;AAC5C,qBAAiB,KAAK,GAAG,eAAe,MAAM,KAAK,CAAC;AAAA,EACtD;AACA,aAAW,CAAC,SAAS,MAAM,KAAK,gBAAgB;AAC9C,QAAI,CAAC,eAAe,IAAI,OAAO,EAAG,kBAAiB,KAAK,GAAG,MAAM;AAAA,EACnE;AACA,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,oBAAoB,gBAAgB,gBAAgB;AAC1D,QAAM,UAAU,KAAK,IAAI,kBAAkB,QAAQ,kBAAkB,MAAM;AAC3E,WAAS,QAAQ,GAAG,QAAQ,SAAS,SAAS;AAC5C,UAAM,KAAK,EAAE,MAAM,WAAW,UAAU,kBAAkB,KAAK,GAAI,UAAU,kBAAkB,KAAK,EAAG,CAAC;AAAA,EAC1G;AACA,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,eAAe,UAAU,OAAO,CAAC;AAC3G,aAAW,UAAU,kBAAkB,MAAM,OAAO,EAAG,OAAM,KAAK,EAAE,MAAM,oBAAoB,UAAU,OAAO,CAAC;AAChH,SAAO;AACT;AAoBO,MAAM,UAAU,CACrB,UACA,KACA,aAAyB,CAAC,MACW;AACrC,QAAM,WAAW,WAAW,YAAY;AACxC,QAAM,WAAW,WAAW,YAAY;AACxC,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,CAAC,eAAe,QAAQ,GAAG;AAC7B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS,wEAAwE,OAAO,QAAQ,CAAC;AAAA,QACjG,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,gBAAgB,SAAS;AAC/B,MAAI,kBAAkB,IAAI,OAAO;AAC/B,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,8BAA8B,aAAa,4BAA4B,IAAI,KAAK;AAAA,QAClF;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,UAAU,CAAC,GAAG,IAAI,IAAI,SAAS,QAAQ,OAAO,CAAC,WAAW,OAAO,UAAU,IAAI,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAChH,MAAI,QAAQ,SAAS,GAAG;AACtB,WAAO,OAAO;AAAA,MACZ,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,aAAS;AAAA,UACP,oCACE,QAAQ,IAAI,CAAC,UAAU,IAAI,KAAK,GAAG,EAAE,KAAK,IAAI,CAChD,2BAA2B,IAAI,KAAK;AAAA,QACtC;AAAA,QACA,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,SAAS,IAAI,aAAa,OAAO,CAAC,gBAAsC,YAAY,SAAS,OAAO;AAC1G,QAAM,eAAe,IAAI,aAAa,OAAO,CAC3C,gBAC2E,YAAY,SAAS,cAAc;AAChH,QAAM,gBAAgB,QAAQ,SAAS,SAAS,CAAC,WAAW,IAAI,OAAO,MAAM,OAAO,MAAM,CAAC;AAC3F,QAAM,cAAc,QAAQ,QAAQ,CAAC,gBAAgB,IAAI,YAAY,MAAM,YAAY,MAAM,CAAC;AAC9F,QAAM,cAAiC,CAAC;AACxC,QAAM,iBAAwC,CAAC;AAC/C,QAAM,UAAqC,CAAC;AAC5C,aAAW,cAAc,oBAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,GAAG;AAClF,UAAM,QAAQ,SAAS,cAAc,IAAI,UAAU,KAAK,CAAC,GAAG,YAAY,IAAI,UAAU,KAAK,CAAC,CAAC;AAC7F,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,SAAS,oBAAoB;AACpC,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,UAAI,KAAK,SAAS,eAAe;AAC/B,gBAAQ,KAAK;AAAA,UACX,MAAM;AAAA,UACN,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,GAAI,KAAK,SAAS,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,KAAK,SAAS,WAAW;AAAA,UACzF,SAAS,KAAK,SAAS;AAAA,QACzB,CAAC;AACD;AAAA,MACF;AACA,YAAM,iBAAiB,KAAK;AAC5B,YAAM,cAAc,KAAK;AACzB,YAAM,QAAQ,YAAY,QAAQ,eAAe;AAIjD,UAAI,YAAY,YAAY,eAAe,SAAS;AAClD,YACE,QAAQ,KAAK,IAAI,KAAK,GAAG,QAAQ,KACjC,QAAQ,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,eAAe,KAAK,GAAG,KAAK,GAAG,QAAQ,GACnF;AACA,yBAAe,KAAK;AAAA,YAClB,MAAM,YAAY;AAAA,YAClB,QAAQ,YAAY;AAAA,YACpB,UAAU;AAAA,YACV,QAAQ;AAAA,YACR;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,WACE,QAAQ,KAAK,QAAQ,CAAC,OAAO,QAAQ,KAAK,QAAQ,CAAC,QAAQ,KAAK,IAAI,eAAe,OAAO,KAAK,GAAG,QAAQ,GAC1G;AACA,oBAAY,KAAK;AAAA,UACf,MAAM,YAAY;AAAA,UAClB,QAAQ,YAAY;AAAA,UACpB,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,MAAM,CAAC;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO,OAAO,QAAQ;AAAA,IACpB,OAAO,IAAI;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;",
   "names": []
 }

package/dist/cjs/Runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAyKhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAoFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}
+{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAsMhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAqFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}

package/dist/cjs/Runner.js

@@ -81,7 +81,29 @@ const caseError = (suiteCase, cause) => {
     cause: reason
   });
 };
+const malformedExecution = (suiteCase, execution) => {
+  const shape = typeof execution === "object" && execution !== null ? execution : {};
+  if (typeof shape.stepKey !== "string") {
+    return new import_EvalError.EvalError({
+      code: "executor",
+      message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,
+      path: `${casePath(suiteCase.name)}.stepKey`
+    });
+  }
+  if (typeof shape.latencyMs !== "number" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {
+    return new import_EvalError.EvalError({
+      code: "executor",
+      message: `executor returned a latencyMs that is not a finite non-negative number (${typeof shape.latencyMs === "number" ? String(shape.latencyMs) : typeof shape.latencyMs})`,
+      path: `${casePath(suiteCase.name)}.latencyMs`
+    });
+  }
+  return void 0;
+};
 const runCase = (executor, suiteCase) => Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(
+  Effect.flatMap((execution) => {
+    const malformed = malformedExecution(suiteCase, execution);
+    return malformed === void 0 ? Effect.succeed(execution) : Effect.fail(malformed);
+  }),
   Effect.map((execution) => ({
     case: suiteCase.name,
     execution: { ...execution, stepKey: (0, import_controlCharacters.flattenControlCharacters)(execution.stepKey) },
@@ -392,7 +414,7 @@ const run = (suite, options) => Effect.gen(function* () {
       sampleId: n === 1 ? sampleId : (0, import_tupleKey.tupleKey)(sampleId, String(index))
     });
   }), { concurrency: 1 });
-  const outcomes = {};
+  const outcomes = [];
   const cases = suite.cases.map((suiteCase, index) => {
     const attempts = batches.map(
       (batch) => batch[index]
@@ -400,7 +422,7 @@ const run = (suite, options) => Effect.gen(function* () {
     const passed = attempts.map(
       (attempt) => attempt.error === void 0 && attempt.observations.length > 0 && attempt.observations.every((observation) => observation.kind === "score" && observation.score === 1)
     );
-    outcomes[suiteCase.name] = passed;
+    outcomes.push([suiteCase.name, passed]);
     return {
       ...attempts[0],
       error: attempts.find((attempt) => attempt.error !== void 0)?.error,
@@ -413,7 +435,8 @@ const run = (suite, options) => Effect.gen(function* () {
     suite: suite.name,
     cases,
     observations: cases.flatMap((caseResult) => caseResult.observations),
-    trials: Trials.aggregate(outcomes, k),
+    // fromEntries defines own properties, so a case named `__proto__` stays a case.
+    trials: Trials.aggregate(Object.fromEntries(outcomes), k),
     k
   };
 }).pipe(Effect.withSpan("Eval.run", { attributes: { suite: suite.name, runId: options.runId } }));

package/dist/cjs/Runner.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/Runner.ts"],
-  "sourcesContent": ["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n  & {\n    readonly case: string\n    readonly scorer: string\n    readonly scorerName?: string | undefined\n    readonly stepKey: string\n    readonly at: string\n  }\n  & (\n    | {\n      readonly kind: \"score\"\n      readonly score: number\n      readonly reason?: string | undefined\n      readonly meta?: unknown\n    }\n    | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n  )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n  readonly case: string\n  readonly stepKey: string\n  readonly binding: Binding\n  readonly input: {\n    readonly input: unknown\n    readonly output: unknown\n    readonly groundTruth?: unknown\n    readonly context?: unknown\n    readonly latencyMs?: number\n  }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n  readonly runBatch: (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n  readonly runBatchCorrelated?:\n    | ((\n      jobs: ReadonlyArray<ScoreJob>,\n      options?: { readonly concurrency?: number | undefined }\n    ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n    | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n  readonly identity: string\n  readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n  readonly case: string\n  readonly execution?: Execution | undefined\n  readonly error?: EvalError | undefined\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n  readonly runId: string\n  readonly suite: string\n  readonly cases: ReadonlyArray<CaseResult>\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Aggregate | undefined\n  readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n  readonly scorer?: ScoreBatchRunner | undefined\n  readonly runId: string\n  readonly sampleId?: string | undefined\n  readonly trials?: number | undefined\n  readonly k?: number | undefined\n  readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n  reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n  scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"inconclusive\",\n    reason,\n    at\n  }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n  const reason = Cause.squash(cause)\n  return reason instanceof EvalError\n    ? new EvalError({\n      code: reason.code,\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n      path: reason.path ?? casePath(suiteCase.name),\n      cause: reason\n    })\n    : new EvalError({\n      code: \"executor\",\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n      path: casePath(suiteCase.name),\n      cause: reason\n    })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n  Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n    Effect.map((execution): CaseResult => ({\n      case: suiteCase.name,\n      execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n      observations: []\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n    ),\n    Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n  )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n  const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n  return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n  case: suiteCase.name,\n  stepKey: execution.stepKey,\n  binding: {\n    ...structuredClone({\n      sampling: binding.sampling,\n      ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n      ...(binding.context === undefined ? {} : { context: binding.context })\n    }),\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo\n  },\n  input: {\n    input: structuredClone(suiteCase.input),\n    output: execution.output,\n    ...(groundTruthFor(suiteCase, binding)),\n    ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n    latencyMs: execution.latencyMs\n  }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n  const source = result as {\n    readonly kind: unknown\n    readonly score?: unknown\n    readonly reason?: unknown\n    readonly meta?: unknown\n  }\n  const kind = source.kind\n  if (kind !== \"score\" && kind !== \"inconclusive\") {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n      at\n    )\n  }\n  const reason = source.reason\n  if (kind === \"inconclusive\") {\n    return inconclusive(\n      request,\n      typeof reason === \"string\" && reason.length > 0\n        ? boundedReason(reason)\n        : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n      at\n    )\n  }\n  const score = source.score\n  if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n      at\n    )\n  }\n  const meta = source.meta\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"score\",\n    score,\n    ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n    ...(meta === undefined ? {} : { meta }),\n    at\n  }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n  job.score.pipe(\n    Effect.flatMap(Scorer.validate),\n    Effect.map((result): ScoreStore.ScoreObservation => ({\n      ...job.observation,\n      kind: \"score\",\n      score: result.score,\n      ...(result.reason === undefined ? {} : { reason: result.reason }),\n      ...(result.meta === undefined ? {} : { meta: result.meta }),\n      at: job.at\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n    )\n  )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n  const runBatchCorrelated = (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n    Effect.forEach(\n      jobs,\n      (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n      { concurrency: options?.concurrency ?? 1 }\n    )\n  return {\n    runBatchCorrelated,\n    runBatch: (jobs, options) =>\n      runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n  }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n  readonly at: string\n  readonly runId: string\n  readonly sampleId: string\n  readonly concurrency: number\n}\n\nconst buildJobs = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n  { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n  EvalError\n> =>\n  Effect.gen(function*() {\n    // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n    // two arrays are index-aligned by construction; nothing here has to look a\n    // case back up by name.\n    const candidates = cases.flatMap((caseResult, index) => {\n      const execution = caseResult.execution\n      if (execution === undefined) return []\n      const suiteCase = suite.cases[index]!\n      return suite.bindings.flatMap((binding, bindingIndex) =>\n        binding.appliesTo === execution.target\n          ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n          : []\n      )\n    })\n    const sampled = yield* Effect.forEach(\n      candidates,\n      ({ request, bindingIndex }) =>\n        Sampling.decide(\n          request.binding.sampling,\n          request.stepKey,\n          scorerKeyOf(request.binding)\n        ).pipe(\n          Effect.map((selected) => selected ? request : undefined),\n          Effect.mapError((cause) =>\n            new EvalError({\n              code: \"invalid_suite\",\n              message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n              path: `bindings[${bindingIndex}].sampling`,\n              cause\n            })\n          )\n        ),\n      { concurrency }\n    )\n    const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n    const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n      identity: tupleKey(\n        suite.name,\n        runId,\n        sampleId,\n        request.case,\n        request.stepKey,\n        scorerKeyOf(request.binding),\n        `${index}`\n      ),\n      observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n      score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n      at: Date.parse(at)\n    }))\n    return { requests, jobs }\n  })\n\nconst batchFallback = (\n  jobs: ReadonlyArray<ScoreJob>,\n  cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n  Cause.hasInterrupts(cause)\n    ? Effect.interrupt\n    : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n  jobs: ReadonlyArray<ScoreJob>,\n  scorer: ScoreBatchRunner,\n  runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) =>\n        batchFallback(jobs, cause).pipe(\n          Effect.map((observations) =>\n            observations.map((observation, index) => ({\n              identity: jobs[index]!.identity,\n              observation\n            }))\n          )\n        )\n      )\n    )\n    if (batchResults.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n          path: \"runBatchCorrelated\"\n        })\n      )\n    }\n    const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n    const byIdentity = new Map<string, ScoreObservation>()\n    let unknown: { readonly identity: string; readonly index: number } | undefined\n    for (const [index, result] of batchResults.entries()) {\n      const jobIndex = jobIndexByIdentity.get(result.identity)\n      if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message:\n              `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n            path: `runBatchCorrelated[${index}]`\n          })\n        )\n      }\n      if (jobIndex === undefined) unknown = { identity: result.identity, index }\n      byIdentity.set(result.identity, result.observation)\n    }\n    if (unknown !== undefined) {\n      const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n      const missingIdentity = jobs[missingJobIndex]!.identity\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n          path: `runBatchCorrelated[${unknown.index}]`\n        })\n      )\n    }\n    return jobs.map((job) => byIdentity.get(job.identity)!)\n  })\n\nconst scoreOrderOnly = (\n  jobs: ReadonlyArray<ScoreJob>,\n  requests: ReadonlyArray<ScoreRequest>,\n  scorer: ScoreBatchRunner,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const pairIndexes = new Map<string, number>()\n    for (const [index, job] of jobs.entries()) {\n      const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n      const previous = pairIndexes.get(pair)\n      if (previous !== undefined) {\n        const first = requests[previous]!\n        const second = requests[index]!\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"ambiguous_score_job\",\n            message:\n              `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n                label(second.binding)\n              }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n            path: \"runBatch\"\n          })\n        )\n      }\n      pairIndexes.set(pair, index)\n    }\n    const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) => batchFallback(jobs, cause))\n    )\n    if (results.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n          path: \"runBatch\"\n        })\n      )\n    }\n    return results\n  })\n\nconst score = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  scorer: ScoreBatchRunner,\n  context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n  Effect.gen(function*() {\n    const { requests, jobs } = yield* buildJobs(cases, suite, context)\n    if (requests.length === 0) return cases\n    const runBatchCorrelated = scorer.runBatchCorrelated\n    const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n    const results = yield* runBatchCorrelated === undefined\n      ? scoreOrderOnly(jobs, requests, scorer, context)\n      : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n    const { at } = context\n    const observations: Array<Observation> = []\n    for (const [index, request] of requests.entries()) {\n      const result = results[index]!\n      if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message: protocol === \"runBatchCorrelated\"\n              ? `Correlated scorer batch identity '${\n                jobs[index]!.identity\n              }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'`\n              : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n            path: `${protocol}[${index}]`\n          })\n        )\n      }\n      observations.push(observationFor(request, result, at))\n    }\n    const byCase = new Map<string, Array<Observation>>()\n    for (const observation of observations) {\n      const bucket = byCase.get(observation.case)\n      if (bucket === undefined) byCase.set(observation.case, [observation])\n      else bucket.push(observation)\n    }\n    return cases.map((caseResult) => ({\n      ...caseResult,\n      observations: byCase.get(caseResult.case) ?? []\n    }))\n  })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n  suite: Suite,\n  options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n  Effect.gen(function*() {\n    const executor = yield* CaseExecutor\n    const injected = yield* Effect.serviceOption(Runner)\n    const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n    const atMillis = Date.parse(options.at)\n    if (options.runId.trim().length === 0) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message: \"Deterministic runs require a non-empty runId\",\n          path: \"options.runId\"\n        })\n      )\n    }\n    if (\n      !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n      !Number.isFinite(atMillis) ||\n      new Date(atMillis).toISOString() !== options.at\n    ) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message:\n            `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n          path: \"options.at\"\n        })\n      )\n    }\n    const n = options.trials ?? suite.trials ?? 1\n    const k = options.k ?? 1\n    if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_trials\",\n          message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n          path: \"options.trials\"\n        })\n      )\n    }\n    const at = options.at\n    const runId = options.runId\n    const sampleId = options.sampleId ?? \"default\"\n    // Await each batch before starting the next: concurrency within a trial is\n    // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n    const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n      Effect.gen(function*() {\n        const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n          runCase(executor, suiteCase), {\n          concurrency: suite.concurrency,\n          discard: false\n        })\n        return yield* score(cases, suite, scorer, {\n          concurrency: suite.concurrency,\n          at,\n          runId,\n          sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n        })\n      }), { concurrency: 1 })\n    const outcomes: Record<string, Array<boolean>> = {}\n    const cases = suite.cases.map((suiteCase, index) => {\n      const attempts = batches.map((batch) =>\n        batch[index]!\n      )\n      const passed = attempts.map((attempt) =>\n        attempt.error === undefined && attempt.observations.length > 0 &&\n        attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n      )\n      outcomes[suiteCase.name] = passed\n      return {\n        ...attempts[0]!,\n        error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n        observations: attempts.flatMap((attempt) => attempt.observations),\n        trials: Trials.summarize(passed, k)\n      }\n    })\n    return {\n      runId,\n      suite: suite.name,\n      cases,\n      observations: cases.flatMap((caseResult) => caseResult.observations),\n      trials: Trials.aggregate(outcomes, k),\n      k\n    }\n  }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n  const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n  return {\n    runBatch: () => Effect.fail(unavailable),\n    runBatchCorrelated: (jobs) =>\n      Effect.succeed(jobs.map((job) => ({\n        identity: job.identity,\n        observation: ScorerRunner.inconclusive(job, unavailable)\n      })))\n  }\n})\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,mBAA8B;AAC9B,eAA0B;AAC1B,aAAwB;AAExB,YAAuB;AACvB,cAAyB;AACzB,aAAwB;AACxB,YAAuB;AACvB,0BAAkF;AAClF,uBAA0B;AAC1B,+BAAyC;AACzC,yBAA4B;AAC5B,sBAAyB;AAEzB,aAAwB;AAoKjB,MAAM,eAAe,QAAQ,QAAkC,EAAE,oBAAoB,EAAE;AAAC;AAG/F,MAAM,kBAAkB;AAExB,MAAM,gBAAgB,CAAC,WACrB,OAAO,UAAU,kBAAkB,SAAS,GAAG,OAAO,MAAM,GAAG,eAAe,CAAC;AAEjF,MAAM,cAAc,CAAC,YAA6B,QAAQ,OAAO;AAKjE,MAAM,eAAe,CAAC,YAA6B,QAAQ,OAAO;AAElE,MAAM,QAAQ,CAAC,gBACb,gCAAY,EAAE,QAAQ,YAAY,OAAO,GAAG,YAAY,aAAa,OAAO,EAAE,CAAC;AAEjF,MAAM,eAAe,CAAC,SAAuB,QAAgB,OAA4B;AACvF,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN;AAAA,IACA;AAAA,EACF;AACF;AAKA,MAAM,WAAW,CAAC,SAAyB,UAAU,IAAI;AAQzD,MAAM,YAAY,CAAC,WAAiB,UAA2C;AAC7E,QAAM,SAAS,MAAM,OAAO,KAAK;AACjC,SAAO,kBAAkB,6BACrB,IAAI,2BAAU;AAAA,IACd,MAAM,OAAO;AAAA,IACb,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,OAAO,EAAE;AAAA,IACjG,MAAM,OAAO,QAAQ,SAAS,UAAU,IAAI;AAAA,IAC5C,OAAO;AAAA,EACT,CAAC,IACC,IAAI,2BAAU;AAAA,IACd,MAAM;AAAA,IACN,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,MAAM,CAAC,EAAE;AAAA,IACjG,MAAM,SAAS,UAAU,IAAI;AAAA,IAC7B,OAAO;AAAA,EACT,CAAC;AACL;AAKA,MAAM,UAAU,CAAC,UAA+B,cAC9C,OAAO,QAAQ,MAAM,SAAS,IAAI,gBAAgB,SAAS,CAAC,CAAC,EAAE;AAAA,EAC7D,OAAO,IAAI,CAAC,eAA2B;AAAA,IACrC,MAAM,UAAU;AAAA,IAChB,WAAW,EAAE,GAAG,WAAW,aAAS,mDAAyB,UAAU,OAAO,EAAE;AAAA,IAChF,cAAc,CAAC;AAAA,EACjB,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAoB,EAAE,MAAM,UAAU,MAAM,OAAO,UAAU,WAAW,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC;AAAA,EAC/G;AAAA,EACA,OAAO,SAAS,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,KAAK,EAAE,CAAC;AACvE;AAIF,MAAM,iBAAiB,CAAC,WAAiB,YAAyD;AAChG,QAAM,cAAc,UAAU,aAAa,SAAY,UAAU,WAAW,QAAQ;AACpF,SAAO,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,gBAAgB,WAAW,EAAE;AACtF;AAEA,MAAM,aAAa,CAAC,WAAiB,WAAsB,aAAoC;AAAA,EAC7F,MAAM,UAAU;AAAA,EAChB,SAAS,UAAU;AAAA,EACnB,SAAS;AAAA,IACP,GAAG,gBAAgB;AAAA,MACjB,UAAU,QAAQ;AAAA,MAClB,GAAI,QAAQ,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,YAAY;AAAA,MAChF,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,IACtE,CAAC;AAAA,IACD,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,EACrB;AAAA,EACA,OAAO;AAAA,IACL,OAAO,gBAAgB,UAAU,KAAK;AAAA,IACtC,QAAQ,UAAU;AAAA,IAClB,GAAI,eAAe,WAAW,OAAO;AAAA,IACrC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,gBAAgB,QAAQ,OAAO,EAAE;AAAA,IACrF,WAAW,UAAU;AAAA,EACvB;AACF;AAEA,MAAM,iBAAiB,CAAC,SAAuB,QAA0B,OAA4B;AACnG,QAAM,SAAS;AAMf,QAAM,OAAO,OAAO;AACpB,MAAI,SAAS,WAAW,SAAS,gBAAgB;AAC/C,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,2CAA2C,OAAO,IAAI,CAAC,GAAG;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAM,SAAS,OAAO;AACtB,MAAI,SAAS,gBAAgB;AAC3B,WAAO;AAAA,MACL;AAAA,MACA,OAAO,WAAW,YAAY,OAAO,SAAS,IAC1C,cAAc,MAAM,IACpB,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,sDAAsD;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAMA,SAAQ,OAAO;AACrB,MAAI,OAAOA,WAAU,YAAY,CAAC,OAAO,SAASA,MAAK,KAAKA,SAAQ,KAAKA,SAAQ,GAAG;AAClF,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,qCAAqC,OAAOA,MAAK,CAAC,EAAE;AAAA,MAClG;AAAA,IACF;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN,OAAAA;AAAA,IACA,GAAI,OAAO,WAAW,WAAW,EAAE,QAAQ,cAAc,MAAM,EAAE,IAAI,CAAC;AAAA,IACtE,GAAI,SAAS,SAAY,CAAC,IAAI,EAAE,KAAK;AAAA,IACrC;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,CAAC,QACrB,IAAI,MAAM;AAAA,EACR,OAAO,QAAQ,OAAO,QAAQ;AAAA,EAC9B,OAAO,IAAI,CAAC,YAAyC;AAAA,IACnD,GAAG,IAAI;AAAA,IACP,MAAM;AAAA,IACN,OAAO,OAAO;AAAA,IACd,GAAI,OAAO,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,OAAO,KAAK;AAAA,IACzD,IAAI,IAAI;AAAA,EACV,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC;AAAA,EAC1D;AACF;AAcK,MAAM,aAAa,MAAwB;AAChD,QAAM,qBAAqB,CACzB,MACA,YAEA,OAAO;AAAA,IACL;AAAA,IACA,CAAC,QAAQ,cAAc,GAAG,EAAE,KAAK,OAAO,IAAI,CAAC,iBAAiB,EAAE,UAAU,IAAI,UAAU,YAAY,EAAE,CAAC;AAAA,IACvG,EAAE,aAAa,SAAS,eAAe,EAAE;AAAA,EAC3C;AACF,SAAO;AAAA,IACL;AAAA,IACA,UAAU,CAAC,MAAM,YACf,mBAAmB,MAAM,OAAO,EAAE,KAAK,OAAO,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,WAAW,OAAO,WAAW,CAAC,CAAC;AAAA,EAC/G;AACF;AAQO,MAAM,cAAmC,MAAM,QAAQ,MAAM,EAAE,WAAW,CAAC;AASlF,MAAM,YAAY,CAChB,OACA,OACA,EAAE,IAAI,OAAO,UAAU,YAAY,MAKnC,OAAO,IAAI,aAAY;AAIrB,QAAM,aAAa,MAAM,QAAQ,CAAC,YAAY,UAAU;AACtD,UAAM,YAAY,WAAW;AAC7B,QAAI,cAAc,OAAW,QAAO,CAAC;AACrC,UAAM,YAAY,MAAM,MAAM,KAAK;AACnC,WAAO,MAAM,SAAS;AAAA,MAAQ,CAAC,SAAS,iBACtC,QAAQ,cAAc,UAAU,SAC5B,CAAC,EAAE,SAAS,WAAW,WAAW,WAAW,OAAO,GAAG,aAAa,CAAC,IACrE,CAAC;AAAA,IACP;AAAA,EACF,CAAC;AACD,QAAM,UAAU,OAAO,OAAO;AAAA,IAC5B;AAAA,IACA,CAAC,EAAE,SAAS,aAAa,MACvB,SAAS;AAAA,MACP,QAAQ,QAAQ;AAAA,MAChB,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,IAC7B,EAAE;AAAA,MACA,OAAO,IAAI,CAAC,aAAa,WAAW,UAAU,MAAS;AAAA,MACvD,OAAO;AAAA,QAAS,CAAC,UACf,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,sCAAsC,MAAM,QAAQ,OAAO,CAAC;AAAA,UACrE,MAAM,YAAY,YAAY;AAAA,UAC9B;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IACF,EAAE,YAAY;AAAA,EAChB;AACA,QAAM,WAAW,QAAQ,OAAO,CAAC,YAAqC,YAAY,MAAS;AAC3F,QAAM,OAAwB,SAAS,IAAI,CAAC,SAAS,WAAW;AAAA,IAC9D,cAAU;AAAA,MACR,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,MAC3B,GAAG,KAAK;AAAA,IACV;AAAA,IACA,aAAa,EAAE,eAAe,QAAQ,SAAS,WAAW,YAAY,QAAQ,OAAO,EAAE;AAAA,IACvF,OAAO,OAAO,QAAQ,MAAM,QAAQ,QAAQ,OAAO,MAAM,QAAQ,KAAK,CAAC;AAAA,IACvE,IAAI,KAAK,MAAM,EAAE;AAAA,EACnB,EAAE;AACF,SAAO,EAAE,UAAU,KAAK;AAC1B,CAAC;AAEH,MAAM,gBAAgB,CACpB,MACA,UAEA,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,KAAK,IAAI,CAAC,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC,CAAC;AAE7E,MAAM,kBAAkB,CACtB,MACA,QACA,oBACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,eAAe,OAAO,OAAO,QAAQ,MAAM,mBAAmB,KAAK,QAAQ,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IACvG,OAAO;AAAA,MAAW,CAAC,UACjB,cAAc,MAAM,KAAK,EAAE;AAAA,QACzB,OAAO;AAAA,UAAI,CAAC,iBACV,aAAa,IAAI,CAAC,aAAa,WAAW;AAAA,YACxC,UAAU,KAAK,KAAK,EAAG;AAAA,YACvB;AAAA,UACF,EAAE;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,aAAa,WAAW,KAAK,QAAQ;AACvC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,oCAAoC,aAAa,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACpF,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,qBAAqB,IAAI,IAAI,KAAK,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,UAAU,KAAK,CAAU,CAAC;AAC3F,QAAM,aAAa,oBAAI,IAA8B;AACrD,MAAI;AACJ,aAAW,CAAC,OAAO,MAAM,KAAK,aAAa,QAAQ,GAAG;AACpD,UAAM,WAAW,mBAAmB,IAAI,OAAO,QAAQ;AACvD,QAAI,aAAa,UAAa,WAAW,IAAI,OAAO,QAAQ,GAAG;AAC7D,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,wDAAwD,OAAO,QAAQ,aAAa,QAAQ,oBAAoB,KAAK;AAAA,UACvH,MAAM,sBAAsB,KAAK;AAAA,QACnC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,aAAa,OAAW,WAAU,EAAE,UAAU,OAAO,UAAU,MAAM;AACzE,eAAW,IAAI,OAAO,UAAU,OAAO,WAAW;AAAA,EACpD;AACA,MAAI,YAAY,QAAW;AACzB,UAAM,kBAAkB,KAAK,UAAU,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI,QAAQ,CAAC;AAC7E,UAAM,kBAAkB,KAAK,eAAe,EAAG;AAC/C,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,sDAAsD,QAAQ,QAAQ,qBAAqB,QAAQ,KAAK,SAAS,eAAe,cAAc,eAAe;AAAA,QAC/J,MAAM,sBAAsB,QAAQ,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO,KAAK,IAAI,CAAC,QAAQ,WAAW,IAAI,IAAI,QAAQ,CAAE;AACxD,CAAC;AAEH,MAAM,iBAAiB,CACrB,MACA,UACA,QACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,WAAO,0BAAS,IAAI,YAAY,eAAe,IAAI,YAAY,SAAS;AAC9E,UAAM,WAAW,YAAY,IAAI,IAAI;AACrC,QAAI,aAAa,QAAW;AAC1B,YAAM,QAAQ,SAAS,QAAQ;AAC/B,YAAM,SAAS,SAAS,KAAK;AAC7B,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,0CAA0C,MAAM,IAAI,UAAU,OAAO,IAAI,yBAAyB,IAAI,YAAY,aAAa,gBAC7H,MAAM,OAAO,OAAO,CACtB;AAAA,UACF,MAAM;AAAA,QACR,CAAC;AAAA,MACH;AAAA,IACF;AACA,gBAAY,IAAI,MAAM,KAAK;AAAA,EAC7B;AACA,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,OAAO,SAAS,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IAClF,OAAO,WAAW,CAAC,UAAU,cAAc,MAAM,KAAK,CAAC;AAAA,EACzD;AACA,MAAI,QAAQ,WAAW,KAAK,QAAQ;AAClC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,yBAAyB,QAAQ,MAAM,qBAAqB,KAAK,MAAM;AAAA,QACzE,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT,CAAC;AAEH,MAAM,QAAQ,CACZ,OACA,OACA,QACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,EAAE,UAAU,KAAK,IAAI,OAAO,UAAU,OAAO,OAAO,OAAO;AACjE,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,qBAAqB,OAAO;AAClC,QAAM,WAAW,uBAAuB,SAAY,aAAa;AACjE,QAAM,UAAU,OAAO,uBAAuB,SAC1C,eAAe,MAAM,UAAU,QAAQ,OAAO,IAC9C,gBAAgB,MAAM,QAAQ,oBAAoB,OAAO;AAC7D,QAAM,EAAE,GAAG,IAAI;AACf,QAAM,eAAmC,CAAC;AAC1C,aAAW,CAAC,OAAO,OAAO,KAAK,SAAS,QAAQ,GAAG;AACjD,UAAM,SAAS,QAAQ,KAAK;AAC5B,QAAI,OAAO,kBAAkB,QAAQ,WAAW,OAAO,cAAc,YAAY,QAAQ,OAAO,GAAG;AACjG,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,aAAa,uBAClB,qCACA,KAAK,KAAK,EAAG,QACf,oBAAoB,OAAO,aAAa,iBAAiB,OAAO,SAAS,eAAe,KAAK,sBAC3F,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO,MAC3B,6CAA6C,OAAO,SAAS,cAAc,OAAO,aAAa,eAAe,KAAK,eACnH,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO;AAAA,UAC/B,MAAM,GAAG,QAAQ,IAAI,KAAK;AAAA,QAC5B,CAAC;AAAA,MACH;AAAA,IACF;AACA,iBAAa,KAAK,eAAe,SAAS,QAAQ,EAAE,CAAC;AAAA,EACvD;AACA,QAAM,SAAS,oBAAI,IAAgC;AACnD,aAAW,eAAe,cAAc;AACtC,UAAM,SAAS,OAAO,IAAI,YAAY,IAAI;AAC1C,QAAI,WAAW,OAAW,QAAO,IAAI,YAAY,MAAM,CAAC,WAAW,CAAC;AAAA,QAC/D,QAAO,KAAK,WAAW;AAAA,EAC9B;AACA,SAAO,MAAM,IAAI,CAAC,gBAAgB;AAAA,IAChC,GAAG;AAAA,IACH,cAAc,OAAO,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EAChD,EAAE;AACJ,CAAC;AA2BI,MAAM,MAAM,CACjB,OACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,OAAO,cAAc,MAAM;AACnD,QAAM,SAAS,QAAQ,WAAW,SAAS,SAAS,SAAS,SAAS,QAAQ,WAAW;AACzF,QAAM,WAAW,KAAK,MAAM,QAAQ,EAAE;AACtC,MAAI,QAAQ,MAAM,KAAK,EAAE,WAAW,GAAG;AACrC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MACE,CAAC,iDAAiD,KAAK,QAAQ,EAAE,KACjE,CAAC,OAAO,SAAS,QAAQ,KACzB,IAAI,KAAK,QAAQ,EAAE,YAAY,MAAM,QAAQ,IAC7C;AACA,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,+FAA+F,QAAQ,EAAE;AAAA,QAC3G,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,IAAI,QAAQ,UAAU,MAAM,UAAU;AAC5C,QAAM,IAAI,QAAQ,KAAK;AACvB,MAAI,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,OAAQ,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AAC/F,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,KAAK,QAAQ;AACnB,QAAM,QAAQ,QAAQ;AACtB,QAAM,WAAW,QAAQ,YAAY;AAGrC,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,UAAU,KAAK,GAAG,CAAC,UACrF,OAAO,IAAI,aAAY;AACrB,UAAMC,SAAQ,OAAO,OAAO,QAAQ,MAAM,OAAO,CAAC,cAChD,QAAQ,UAAU,SAAS,GAAG;AAAA,MAC9B,aAAa,MAAM;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AACD,WAAO,OAAO,MAAMA,QAAO,OAAO,QAAQ;AAAA,MACxC,aAAa,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,UAAU,MAAM,IAAI,eAAW,0BAAS,UAAU,OAAO,KAAK,CAAC;AAAA,IACjE,CAAC;AAAA,EACH,CAAC,GAAG,EAAE,aAAa,EAAE,CAAC;AACxB,QAAM,WAA2C,CAAC;AAClD,QAAM,QAAQ,MAAM,MAAM,IAAI,CAAC,WAAW,UAAU;AAClD,UAAM,WAAW,QAAQ;AAAA,MAAI,CAAC,UAC5B,MAAM,KAAK;AAAA,IACb;AACA,UAAM,SAAS,SAAS;AAAA,MAAI,CAAC,YAC3B,QAAQ,UAAU,UAAa,QAAQ,aAAa,SAAS,KAC7D,QAAQ,aAAa,MAAM,CAAC,gBAAgB,YAAY,SAAS,WAAW,YAAY,UAAU,CAAC;AAAA,IACrG;AACA,aAAS,UAAU,IAAI,IAAI;AAC3B,WAAO;AAAA,MACL,GAAG,SAAS,CAAC;AAAA,MACb,OAAO,SAAS,KAAK,CAAC,YAAY,QAAQ,UAAU,MAAS,GAAG;AAAA,MAChE,cAAc,SAAS,QAAQ,CAAC,YAAY,QAAQ,YAAY;AAAA,MAChE,QAAQ,OAAO,UAAU,QAAQ,CAAC;AAAA,IACpC;AAAA,EACF,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb;AAAA,IACA,cAAc,MAAM,QAAQ,CAAC,eAAe,WAAW,YAAY;AAAA,IACnE,QAAQ,OAAO,UAAU,UAAU,CAAC;AAAA,IACpC;AAAA,EACF;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,YAAY,EAAE,YAAY,EAAE,OAAO,MAAM,MAAM,OAAO,QAAQ,MAAM,EAAE,CAAC,CAAC;AAY3F,MAAM,YAAiC,MAAM,KAAK,MAAM,EAAE,MAAM;AACrE,QAAM,cAAc,IAAI,2BAAU,EAAE,MAAM,sBAAsB,SAAS,sCAAsC,CAAC;AAChH,SAAO;AAAA,IACL,UAAU,MAAM,OAAO,KAAK,WAAW;AAAA,IACvC,oBAAoB,CAAC,SACnB,OAAO,QAAQ,KAAK,IAAI,CAAC,SAAS;AAAA,MAChC,UAAU,IAAI;AAAA,MACd,aAAa,aAAa,aAAa,KAAK,WAAW;AAAA,IACzD,EAAE,CAAC;AAAA,EACP;AACF,CAAC;",
+  "sourcesContent": ["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n  & {\n    readonly case: string\n    readonly scorer: string\n    readonly scorerName?: string | undefined\n    readonly stepKey: string\n    readonly at: string\n  }\n  & (\n    | {\n      readonly kind: \"score\"\n      readonly score: number\n      readonly reason?: string | undefined\n      readonly meta?: unknown\n    }\n    | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n  )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n  readonly case: string\n  readonly stepKey: string\n  readonly binding: Binding\n  readonly input: {\n    readonly input: unknown\n    readonly output: unknown\n    readonly groundTruth?: unknown\n    readonly context?: unknown\n    readonly latencyMs?: number\n  }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n  readonly runBatch: (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n  readonly runBatchCorrelated?:\n    | ((\n      jobs: ReadonlyArray<ScoreJob>,\n      options?: { readonly concurrency?: number | undefined }\n    ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n    | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n  readonly identity: string\n  readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n  readonly case: string\n  readonly execution?: Execution | undefined\n  readonly error?: EvalError | undefined\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n  readonly runId: string\n  readonly suite: string\n  readonly cases: ReadonlyArray<CaseResult>\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Aggregate | undefined\n  readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n  readonly scorer?: ScoreBatchRunner | undefined\n  readonly runId: string\n  readonly sampleId?: string | undefined\n  readonly trials?: number | undefined\n  readonly k?: number | undefined\n  readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n  reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n  scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"inconclusive\",\n    reason,\n    at\n  }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n  const reason = Cause.squash(cause)\n  return reason instanceof EvalError\n    ? new EvalError({\n      code: reason.code,\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n      path: reason.path ?? casePath(suiteCase.name),\n      cause: reason\n    })\n    : new EvalError({\n      code: \"executor\",\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n      path: casePath(suiteCase.name),\n      cause: reason\n    })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\n// An executor written in plain JavaScript can return any shape, so the two\n// fields every scorer and baseline reads are checked here, where the failure\n// can still name the case and the field.\nconst malformedExecution = (suiteCase: Case, execution: Execution): EvalError | undefined => {\n  const shape: { readonly stepKey?: unknown; readonly latencyMs?: unknown } =\n    typeof execution === \"object\" && execution !== null ? execution : {}\n  if (typeof shape.stepKey !== \"string\") {\n    return new EvalError({\n      code: \"executor\",\n      message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,\n      path: `${casePath(suiteCase.name)}.stepKey`\n    })\n  }\n  if (typeof shape.latencyMs !== \"number\" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {\n    return new EvalError({\n      code: \"executor\",\n      message: `executor returned a latencyMs that is not a finite non-negative number (${\n        typeof shape.latencyMs === \"number\" ? String(shape.latencyMs) : typeof shape.latencyMs\n      })`,\n      path: `${casePath(suiteCase.name)}.latencyMs`\n    })\n  }\n  return undefined\n}\n\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n  Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n    Effect.flatMap((execution) => {\n      const malformed = malformedExecution(suiteCase, execution)\n      return malformed === undefined ? Effect.succeed(execution) : Effect.fail(malformed)\n    }),\n    Effect.map((execution): CaseResult => ({\n      case: suiteCase.name,\n      execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n      observations: []\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n    ),\n    Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n  )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n  const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n  return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n  case: suiteCase.name,\n  stepKey: execution.stepKey,\n  binding: {\n    ...structuredClone({\n      sampling: binding.sampling,\n      ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n      ...(binding.context === undefined ? {} : { context: binding.context })\n    }),\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo\n  },\n  input: {\n    input: structuredClone(suiteCase.input),\n    output: execution.output,\n    ...(groundTruthFor(suiteCase, binding)),\n    ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n    latencyMs: execution.latencyMs\n  }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n  const source = result as {\n    readonly kind: unknown\n    readonly score?: unknown\n    readonly reason?: unknown\n    readonly meta?: unknown\n  }\n  const kind = source.kind\n  if (kind !== \"score\" && kind !== \"inconclusive\") {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n      at\n    )\n  }\n  const reason = source.reason\n  if (kind === \"inconclusive\") {\n    return inconclusive(\n      request,\n      typeof reason === \"string\" && reason.length > 0\n        ? boundedReason(reason)\n        : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n      at\n    )\n  }\n  const score = source.score\n  if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n      at\n    )\n  }\n  const meta = source.meta\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"score\",\n    score,\n    ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n    ...(meta === undefined ? {} : { meta }),\n    at\n  }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n  job.score.pipe(\n    Effect.flatMap(Scorer.validate),\n    Effect.map((result): ScoreStore.ScoreObservation => ({\n      ...job.observation,\n      kind: \"score\",\n      score: result.score,\n      ...(result.reason === undefined ? {} : { reason: result.reason }),\n      ...(result.meta === undefined ? {} : { meta: result.meta }),\n      at: job.at\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n    )\n  )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n  const runBatchCorrelated = (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n    Effect.forEach(\n      jobs,\n      (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n      { concurrency: options?.concurrency ?? 1 }\n    )\n  return {\n    runBatchCorrelated,\n    runBatch: (jobs, options) =>\n      runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n  }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n  readonly at: string\n  readonly runId: string\n  readonly sampleId: string\n  readonly concurrency: number\n}\n\nconst buildJobs = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n  { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n  EvalError\n> =>\n  Effect.gen(function*() {\n    // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n    // two arrays are index-aligned by construction; nothing here has to look a\n    // case back up by name.\n    const candidates = cases.flatMap((caseResult, index) => {\n      const execution = caseResult.execution\n      if (execution === undefined) return []\n      const suiteCase = suite.cases[index]!\n      return suite.bindings.flatMap((binding, bindingIndex) =>\n        binding.appliesTo === execution.target\n          ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n          : []\n      )\n    })\n    const sampled = yield* Effect.forEach(\n      candidates,\n      ({ request, bindingIndex }) =>\n        Sampling.decide(\n          request.binding.sampling,\n          request.stepKey,\n          scorerKeyOf(request.binding)\n        ).pipe(\n          Effect.map((selected) => selected ? request : undefined),\n          Effect.mapError((cause) =>\n            new EvalError({\n              code: \"invalid_suite\",\n              message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n              path: `bindings[${bindingIndex}].sampling`,\n              cause\n            })\n          )\n        ),\n      { concurrency }\n    )\n    const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n    const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n      identity: tupleKey(\n        suite.name,\n        runId,\n        sampleId,\n        request.case,\n        request.stepKey,\n        scorerKeyOf(request.binding),\n        `${index}`\n      ),\n      observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n      score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n      at: Date.parse(at)\n    }))\n    return { requests, jobs }\n  })\n\nconst batchFallback = (\n  jobs: ReadonlyArray<ScoreJob>,\n  cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n  Cause.hasInterrupts(cause)\n    ? Effect.interrupt\n    : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n  jobs: ReadonlyArray<ScoreJob>,\n  scorer: ScoreBatchRunner,\n  runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) =>\n        batchFallback(jobs, cause).pipe(\n          Effect.map((observations) =>\n            observations.map((observation, index) => ({\n              identity: jobs[index]!.identity,\n              observation\n            }))\n          )\n        )\n      )\n    )\n    if (batchResults.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n          path: \"runBatchCorrelated\"\n        })\n      )\n    }\n    const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n    const byIdentity = new Map<string, ScoreObservation>()\n    let unknown: { readonly identity: string; readonly index: number } | undefined\n    for (const [index, result] of batchResults.entries()) {\n      const jobIndex = jobIndexByIdentity.get(result.identity)\n      if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message:\n              `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n            path: `runBatchCorrelated[${index}]`\n          })\n        )\n      }\n      if (jobIndex === undefined) unknown = { identity: result.identity, index }\n      byIdentity.set(result.identity, result.observation)\n    }\n    if (unknown !== undefined) {\n      const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n      const missingIdentity = jobs[missingJobIndex]!.identity\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n          path: `runBatchCorrelated[${unknown.index}]`\n        })\n      )\n    }\n    return jobs.map((job) => byIdentity.get(job.identity)!)\n  })\n\nconst scoreOrderOnly = (\n  jobs: ReadonlyArray<ScoreJob>,\n  requests: ReadonlyArray<ScoreRequest>,\n  scorer: ScoreBatchRunner,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const pairIndexes = new Map<string, number>()\n    for (const [index, job] of jobs.entries()) {\n      const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n      const previous = pairIndexes.get(pair)\n      if (previous !== undefined) {\n        const first = requests[previous]!\n        const second = requests[index]!\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"ambiguous_score_job\",\n            message:\n              `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n                label(second.binding)\n              }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n            path: \"runBatch\"\n          })\n        )\n      }\n      pairIndexes.set(pair, index)\n    }\n    const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) => batchFallback(jobs, cause))\n    )\n    if (results.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n          path: \"runBatch\"\n        })\n      )\n    }\n    return results\n  })\n\nconst score = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  scorer: ScoreBatchRunner,\n  context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n  Effect.gen(function*() {\n    const { requests, jobs } = yield* buildJobs(cases, suite, context)\n    if (requests.length === 0) return cases\n    const runBatchCorrelated = scorer.runBatchCorrelated\n    const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n    const results = yield* runBatchCorrelated === undefined\n      ? scoreOrderOnly(jobs, requests, scorer, context)\n      : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n    const { at } = context\n    const observations: Array<Observation> = []\n    for (const [index, request] of requests.entries()) {\n      const result = results[index]!\n      if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message: protocol === \"runBatchCorrelated\"\n              ? `Correlated scorer batch identity '${\n                jobs[index]!.identity\n              }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'`\n              : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n            path: `${protocol}[${index}]`\n          })\n        )\n      }\n      observations.push(observationFor(request, result, at))\n    }\n    const byCase = new Map<string, Array<Observation>>()\n    for (const observation of observations) {\n      const bucket = byCase.get(observation.case)\n      if (bucket === undefined) byCase.set(observation.case, [observation])\n      else bucket.push(observation)\n    }\n    return cases.map((caseResult) => ({\n      ...caseResult,\n      observations: byCase.get(caseResult.case) ?? []\n    }))\n  })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n  suite: Suite,\n  options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n  Effect.gen(function*() {\n    const executor = yield* CaseExecutor\n    const injected = yield* Effect.serviceOption(Runner)\n    const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n    const atMillis = Date.parse(options.at)\n    if (options.runId.trim().length === 0) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message: \"Deterministic runs require a non-empty runId\",\n          path: \"options.runId\"\n        })\n      )\n    }\n    if (\n      !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n      !Number.isFinite(atMillis) ||\n      new Date(atMillis).toISOString() !== options.at\n    ) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message:\n            `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n          path: \"options.at\"\n        })\n      )\n    }\n    const n = options.trials ?? suite.trials ?? 1\n    const k = options.k ?? 1\n    if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_trials\",\n          message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n          path: \"options.trials\"\n        })\n      )\n    }\n    const at = options.at\n    const runId = options.runId\n    const sampleId = options.sampleId ?? \"default\"\n    // Await each batch before starting the next: concurrency within a trial is\n    // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n    const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n      Effect.gen(function*() {\n        const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n          runCase(executor, suiteCase), {\n          concurrency: suite.concurrency,\n          discard: false\n        })\n        return yield* score(cases, suite, scorer, {\n          concurrency: suite.concurrency,\n          at,\n          runId,\n          sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n        })\n      }), { concurrency: 1 })\n    const outcomes: Array<[string, ReadonlyArray<boolean>]> = []\n    const cases = suite.cases.map((suiteCase, index) => {\n      const attempts = batches.map((batch) =>\n        batch[index]!\n      )\n      const passed = attempts.map((attempt) =>\n        attempt.error === undefined && attempt.observations.length > 0 &&\n        attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n      )\n      outcomes.push([suiteCase.name, passed])\n      return {\n        ...attempts[0]!,\n        error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n        observations: attempts.flatMap((attempt) => attempt.observations),\n        trials: Trials.summarize(passed, k)\n      }\n    })\n    return {\n      runId,\n      suite: suite.name,\n      cases,\n      observations: cases.flatMap((caseResult) => caseResult.observations),\n      // fromEntries defines own properties, so a case named `__proto__` stays a case.\n      trials: Trials.aggregate(Object.fromEntries(outcomes), k),\n      k\n    }\n  }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n  const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n  return {\n    runBatch: () => Effect.fail(unavailable),\n    runBatchCorrelated: (jobs) =>\n      Effect.succeed(jobs.map((job) => ({\n        identity: job.identity,\n        observation: ScorerRunner.inconclusive(job, unavailable)\n      })))\n  }\n})\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,mBAA8B;AAC9B,eAA0B;AAC1B,aAAwB;AAExB,YAAuB;AACvB,cAAyB;AACzB,aAAwB;AACxB,YAAuB;AACvB,0BAAkF;AAClF,uBAA0B;AAC1B,+BAAyC;AACzC,yBAA4B;AAC5B,sBAAyB;AAEzB,aAAwB;AAoKjB,MAAM,eAAe,QAAQ,QAAkC,EAAE,oBAAoB,EAAE;AAAC;AAG/F,MAAM,kBAAkB;AAExB,MAAM,gBAAgB,CAAC,WACrB,OAAO,UAAU,kBAAkB,SAAS,GAAG,OAAO,MAAM,GAAG,eAAe,CAAC;AAEjF,MAAM,cAAc,CAAC,YAA6B,QAAQ,OAAO;AAKjE,MAAM,eAAe,CAAC,YAA6B,QAAQ,OAAO;AAElE,MAAM,QAAQ,CAAC,gBACb,gCAAY,EAAE,QAAQ,YAAY,OAAO,GAAG,YAAY,aAAa,OAAO,EAAE,CAAC;AAEjF,MAAM,eAAe,CAAC,SAAuB,QAAgB,OAA4B;AACvF,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN;AAAA,IACA;AAAA,EACF;AACF;AAKA,MAAM,WAAW,CAAC,SAAyB,UAAU,IAAI;AAQzD,MAAM,YAAY,CAAC,WAAiB,UAA2C;AAC7E,QAAM,SAAS,MAAM,OAAO,KAAK;AACjC,SAAO,kBAAkB,6BACrB,IAAI,2BAAU;AAAA,IACd,MAAM,OAAO;AAAA,IACb,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,OAAO,EAAE;AAAA,IACjG,MAAM,OAAO,QAAQ,SAAS,UAAU,IAAI;AAAA,IAC5C,OAAO;AAAA,EACT,CAAC,IACC,IAAI,2BAAU;AAAA,IACd,MAAM;AAAA,IACN,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,MAAM,CAAC,EAAE;AAAA,IACjG,MAAM,SAAS,UAAU,IAAI;AAAA,IAC7B,OAAO;AAAA,EACT,CAAC;AACL;AAQA,MAAM,qBAAqB,CAAC,WAAiB,cAAgD;AAC3F,QAAM,QACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;AACrE,MAAI,OAAO,MAAM,YAAY,UAAU;AACrC,WAAO,IAAI,2BAAU;AAAA,MACnB,MAAM;AAAA,MACN,SAAS,qDAAqD,OAAO,MAAM,OAAO;AAAA,MAClF,MAAM,GAAG,SAAS,UAAU,IAAI,CAAC;AAAA,IACnC,CAAC;AAAA,EACH;AACA,MAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG;AACnG,WAAO,IAAI,2BAAU;AAAA,MACnB,MAAM;AAAA,MACN,SAAS,2EACP,OAAO,MAAM,cAAc,WAAW,OAAO,MAAM,SAAS,IAAI,OAAO,MAAM,SAC/E;AAAA,MACA,MAAM,GAAG,SAAS,UAAU,IAAI,CAAC;AAAA,IACnC,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,UAAU,CAAC,UAA+B,cAC9C,OAAO,QAAQ,MAAM,SAAS,IAAI,gBAAgB,SAAS,CAAC,CAAC,EAAE;AAAA,EAC7D,OAAO,QAAQ,CAAC,cAAc;AAC5B,UAAM,YAAY,mBAAmB,WAAW,SAAS;AACzD,WAAO,cAAc,SAAY,OAAO,QAAQ,SAAS,IAAI,OAAO,KAAK,SAAS;AAAA,EACpF,CAAC;AAAA,EACD,OAAO,IAAI,CAAC,eAA2B;AAAA,IACrC,MAAM,UAAU;AAAA,IAChB,WAAW,EAAE,GAAG,WAAW,aAAS,mDAAyB,UAAU,OAAO,EAAE;AAAA,IAChF,cAAc,CAAC;AAAA,EACjB,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAoB,EAAE,MAAM,UAAU,MAAM,OAAO,UAAU,WAAW,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC;AAAA,EAC/G;AAAA,EACA,OAAO,SAAS,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,KAAK,EAAE,CAAC;AACvE;AAIF,MAAM,iBAAiB,CAAC,WAAiB,YAAyD;AAChG,QAAM,cAAc,UAAU,aAAa,SAAY,UAAU,WAAW,QAAQ;AACpF,SAAO,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,gBAAgB,WAAW,EAAE;AACtF;AAEA,MAAM,aAAa,CAAC,WAAiB,WAAsB,aAAoC;AAAA,EAC7F,MAAM,UAAU;AAAA,EAChB,SAAS,UAAU;AAAA,EACnB,SAAS;AAAA,IACP,GAAG,gBAAgB;AAAA,MACjB,UAAU,QAAQ;AAAA,MAClB,GAAI,QAAQ,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,YAAY;AAAA,MAChF,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,IACtE,CAAC;AAAA,IACD,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,EACrB;AAAA,EACA,OAAO;AAAA,IACL,OAAO,gBAAgB,UAAU,KAAK;AAAA,IACtC,QAAQ,UAAU;AAAA,IAClB,GAAI,eAAe,WAAW,OAAO;AAAA,IACrC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,gBAAgB,QAAQ,OAAO,EAAE;AAAA,IACrF,WAAW,UAAU;AAAA,EACvB;AACF;AAEA,MAAM,iBAAiB,CAAC,SAAuB,QAA0B,OAA4B;AACnG,QAAM,SAAS;AAMf,QAAM,OAAO,OAAO;AACpB,MAAI,SAAS,WAAW,SAAS,gBAAgB;AAC/C,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,2CAA2C,OAAO,IAAI,CAAC,GAAG;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAM,SAAS,OAAO;AACtB,MAAI,SAAS,gBAAgB;AAC3B,WAAO;AAAA,MACL;AAAA,MACA,OAAO,WAAW,YAAY,OAAO,SAAS,IAC1C,cAAc,MAAM,IACpB,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,sDAAsD;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAMA,SAAQ,OAAO;AACrB,MAAI,OAAOA,WAAU,YAAY,CAAC,OAAO,SAASA,MAAK,KAAKA,SAAQ,KAAKA,SAAQ,GAAG;AAClF,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,qCAAqC,OAAOA,MAAK,CAAC,EAAE;AAAA,MAClG;AAAA,IACF;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN,OAAAA;AAAA,IACA,GAAI,OAAO,WAAW,WAAW,EAAE,QAAQ,cAAc,MAAM,EAAE,IAAI,CAAC;AAAA,IACtE,GAAI,SAAS,SAAY,CAAC,IAAI,EAAE,KAAK;AAAA,IACrC;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,CAAC,QACrB,IAAI,MAAM;AAAA,EACR,OAAO,QAAQ,OAAO,QAAQ;AAAA,EAC9B,OAAO,IAAI,CAAC,YAAyC;AAAA,IACnD,GAAG,IAAI;AAAA,IACP,MAAM;AAAA,IACN,OAAO,OAAO;AAAA,IACd,GAAI,OAAO,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,OAAO,KAAK;AAAA,IACzD,IAAI,IAAI;AAAA,EACV,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC;AAAA,EAC1D;AACF;AAcK,MAAM,aAAa,MAAwB;AAChD,QAAM,qBAAqB,CACzB,MACA,YAEA,OAAO;AAAA,IACL;AAAA,IACA,CAAC,QAAQ,cAAc,GAAG,EAAE,KAAK,OAAO,IAAI,CAAC,iBAAiB,EAAE,UAAU,IAAI,UAAU,YAAY,EAAE,CAAC;AAAA,IACvG,EAAE,aAAa,SAAS,eAAe,EAAE;AAAA,EAC3C;AACF,SAAO;AAAA,IACL;AAAA,IACA,UAAU,CAAC,MAAM,YACf,mBAAmB,MAAM,OAAO,EAAE,KAAK,OAAO,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,WAAW,OAAO,WAAW,CAAC,CAAC;AAAA,EAC/G;AACF;AAQO,MAAM,cAAmC,MAAM,QAAQ,MAAM,EAAE,WAAW,CAAC;AASlF,MAAM,YAAY,CAChB,OACA,OACA,EAAE,IAAI,OAAO,UAAU,YAAY,MAKnC,OAAO,IAAI,aAAY;AAIrB,QAAM,aAAa,MAAM,QAAQ,CAAC,YAAY,UAAU;AACtD,UAAM,YAAY,WAAW;AAC7B,QAAI,cAAc,OAAW,QAAO,CAAC;AACrC,UAAM,YAAY,MAAM,MAAM,KAAK;AACnC,WAAO,MAAM,SAAS;AAAA,MAAQ,CAAC,SAAS,iBACtC,QAAQ,cAAc,UAAU,SAC5B,CAAC,EAAE,SAAS,WAAW,WAAW,WAAW,OAAO,GAAG,aAAa,CAAC,IACrE,CAAC;AAAA,IACP;AAAA,EACF,CAAC;AACD,QAAM,UAAU,OAAO,OAAO;AAAA,IAC5B;AAAA,IACA,CAAC,EAAE,SAAS,aAAa,MACvB,SAAS;AAAA,MACP,QAAQ,QAAQ;AAAA,MAChB,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,IAC7B,EAAE;AAAA,MACA,OAAO,IAAI,CAAC,aAAa,WAAW,UAAU,MAAS;AAAA,MACvD,OAAO;AAAA,QAAS,CAAC,UACf,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,sCAAsC,MAAM,QAAQ,OAAO,CAAC;AAAA,UACrE,MAAM,YAAY,YAAY;AAAA,UAC9B;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IACF,EAAE,YAAY;AAAA,EAChB;AACA,QAAM,WAAW,QAAQ,OAAO,CAAC,YAAqC,YAAY,MAAS;AAC3F,QAAM,OAAwB,SAAS,IAAI,CAAC,SAAS,WAAW;AAAA,IAC9D,cAAU;AAAA,MACR,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,MAC3B,GAAG,KAAK;AAAA,IACV;AAAA,IACA,aAAa,EAAE,eAAe,QAAQ,SAAS,WAAW,YAAY,QAAQ,OAAO,EAAE;AAAA,IACvF,OAAO,OAAO,QAAQ,MAAM,QAAQ,QAAQ,OAAO,MAAM,QAAQ,KAAK,CAAC;AAAA,IACvE,IAAI,KAAK,MAAM,EAAE;AAAA,EACnB,EAAE;AACF,SAAO,EAAE,UAAU,KAAK;AAC1B,CAAC;AAEH,MAAM,gBAAgB,CACpB,MACA,UAEA,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,KAAK,IAAI,CAAC,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC,CAAC;AAE7E,MAAM,kBAAkB,CACtB,MACA,QACA,oBACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,eAAe,OAAO,OAAO,QAAQ,MAAM,mBAAmB,KAAK,QAAQ,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IACvG,OAAO;AAAA,MAAW,CAAC,UACjB,cAAc,MAAM,KAAK,EAAE;AAAA,QACzB,OAAO;AAAA,UAAI,CAAC,iBACV,aAAa,IAAI,CAAC,aAAa,WAAW;AAAA,YACxC,UAAU,KAAK,KAAK,EAAG;AAAA,YACvB;AAAA,UACF,EAAE;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,aAAa,WAAW,KAAK,QAAQ;AACvC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,oCAAoC,aAAa,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACpF,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,qBAAqB,IAAI,IAAI,KAAK,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,UAAU,KAAK,CAAU,CAAC;AAC3F,QAAM,aAAa,oBAAI,IAA8B;AACrD,MAAI;AACJ,aAAW,CAAC,OAAO,MAAM,KAAK,aAAa,QAAQ,GAAG;AACpD,UAAM,WAAW,mBAAmB,IAAI,OAAO,QAAQ;AACvD,QAAI,aAAa,UAAa,WAAW,IAAI,OAAO,QAAQ,GAAG;AAC7D,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,wDAAwD,OAAO,QAAQ,aAAa,QAAQ,oBAAoB,KAAK;AAAA,UACvH,MAAM,sBAAsB,KAAK;AAAA,QACnC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,aAAa,OAAW,WAAU,EAAE,UAAU,OAAO,UAAU,MAAM;AACzE,eAAW,IAAI,OAAO,UAAU,OAAO,WAAW;AAAA,EACpD;AACA,MAAI,YAAY,QAAW;AACzB,UAAM,kBAAkB,KAAK,UAAU,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI,QAAQ,CAAC;AAC7E,UAAM,kBAAkB,KAAK,eAAe,EAAG;AAC/C,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,sDAAsD,QAAQ,QAAQ,qBAAqB,QAAQ,KAAK,SAAS,eAAe,cAAc,eAAe;AAAA,QAC/J,MAAM,sBAAsB,QAAQ,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO,KAAK,IAAI,CAAC,QAAQ,WAAW,IAAI,IAAI,QAAQ,CAAE;AACxD,CAAC;AAEH,MAAM,iBAAiB,CACrB,MACA,UACA,QACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,WAAO,0BAAS,IAAI,YAAY,eAAe,IAAI,YAAY,SAAS;AAC9E,UAAM,WAAW,YAAY,IAAI,IAAI;AACrC,QAAI,aAAa,QAAW;AAC1B,YAAM,QAAQ,SAAS,QAAQ;AAC/B,YAAM,SAAS,SAAS,KAAK;AAC7B,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,0CAA0C,MAAM,IAAI,UAAU,OAAO,IAAI,yBAAyB,IAAI,YAAY,aAAa,gBAC7H,MAAM,OAAO,OAAO,CACtB;AAAA,UACF,MAAM;AAAA,QACR,CAAC;AAAA,MACH;AAAA,IACF;AACA,gBAAY,IAAI,MAAM,KAAK;AAAA,EAC7B;AACA,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,OAAO,SAAS,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IAClF,OAAO,WAAW,CAAC,UAAU,cAAc,MAAM,KAAK,CAAC;AAAA,EACzD;AACA,MAAI,QAAQ,WAAW,KAAK,QAAQ;AAClC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,yBAAyB,QAAQ,MAAM,qBAAqB,KAAK,MAAM;AAAA,QACzE,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT,CAAC;AAEH,MAAM,QAAQ,CACZ,OACA,OACA,QACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,EAAE,UAAU,KAAK,IAAI,OAAO,UAAU,OAAO,OAAO,OAAO;AACjE,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,qBAAqB,OAAO;AAClC,QAAM,WAAW,uBAAuB,SAAY,aAAa;AACjE,QAAM,UAAU,OAAO,uBAAuB,SAC1C,eAAe,MAAM,UAAU,QAAQ,OAAO,IAC9C,gBAAgB,MAAM,QAAQ,oBAAoB,OAAO;AAC7D,QAAM,EAAE,GAAG,IAAI;AACf,QAAM,eAAmC,CAAC;AAC1C,aAAW,CAAC,OAAO,OAAO,KAAK,SAAS,QAAQ,GAAG;AACjD,UAAM,SAAS,QAAQ,KAAK;AAC5B,QAAI,OAAO,kBAAkB,QAAQ,WAAW,OAAO,cAAc,YAAY,QAAQ,OAAO,GAAG;AACjG,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,aAAa,uBAClB,qCACA,KAAK,KAAK,EAAG,QACf,oBAAoB,OAAO,aAAa,iBAAiB,OAAO,SAAS,eAAe,KAAK,sBAC3F,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO,MAC3B,6CAA6C,OAAO,SAAS,cAAc,OAAO,aAAa,eAAe,KAAK,eACnH,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO;AAAA,UAC/B,MAAM,GAAG,QAAQ,IAAI,KAAK;AAAA,QAC5B,CAAC;AAAA,MACH;AAAA,IACF;AACA,iBAAa,KAAK,eAAe,SAAS,QAAQ,EAAE,CAAC;AAAA,EACvD;AACA,QAAM,SAAS,oBAAI,IAAgC;AACnD,aAAW,eAAe,cAAc;AACtC,UAAM,SAAS,OAAO,IAAI,YAAY,IAAI;AAC1C,QAAI,WAAW,OAAW,QAAO,IAAI,YAAY,MAAM,CAAC,WAAW,CAAC;AAAA,QAC/D,QAAO,KAAK,WAAW;AAAA,EAC9B;AACA,SAAO,MAAM,IAAI,CAAC,gBAAgB;AAAA,IAChC,GAAG;AAAA,IACH,cAAc,OAAO,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EAChD,EAAE;AACJ,CAAC;AA2BI,MAAM,MAAM,CACjB,OACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,OAAO,cAAc,MAAM;AACnD,QAAM,SAAS,QAAQ,WAAW,SAAS,SAAS,SAAS,SAAS,QAAQ,WAAW;AACzF,QAAM,WAAW,KAAK,MAAM,QAAQ,EAAE;AACtC,MAAI,QAAQ,MAAM,KAAK,EAAE,WAAW,GAAG;AACrC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MACE,CAAC,iDAAiD,KAAK,QAAQ,EAAE,KACjE,CAAC,OAAO,SAAS,QAAQ,KACzB,IAAI,KAAK,QAAQ,EAAE,YAAY,MAAM,QAAQ,IAC7C;AACA,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,+FAA+F,QAAQ,EAAE;AAAA,QAC3G,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,IAAI,QAAQ,UAAU,MAAM,UAAU;AAC5C,QAAM,IAAI,QAAQ,KAAK;AACvB,MAAI,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,OAAQ,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AAC/F,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,KAAK,QAAQ;AACnB,QAAM,QAAQ,QAAQ;AACtB,QAAM,WAAW,QAAQ,YAAY;AAGrC,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,UAAU,KAAK,GAAG,CAAC,UACrF,OAAO,IAAI,aAAY;AACrB,UAAMC,SAAQ,OAAO,OAAO,QAAQ,MAAM,OAAO,CAAC,cAChD,QAAQ,UAAU,SAAS,GAAG;AAAA,MAC9B,aAAa,MAAM;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AACD,WAAO,OAAO,MAAMA,QAAO,OAAO,QAAQ;AAAA,MACxC,aAAa,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,UAAU,MAAM,IAAI,eAAW,0BAAS,UAAU,OAAO,KAAK,CAAC;AAAA,IACjE,CAAC;AAAA,EACH,CAAC,GAAG,EAAE,aAAa,EAAE,CAAC;AACxB,QAAM,WAAoD,CAAC;AAC3D,QAAM,QAAQ,MAAM,MAAM,IAAI,CAAC,WAAW,UAAU;AAClD,UAAM,WAAW,QAAQ;AAAA,MAAI,CAAC,UAC5B,MAAM,KAAK;AAAA,IACb;AACA,UAAM,SAAS,SAAS;AAAA,MAAI,CAAC,YAC3B,QAAQ,UAAU,UAAa,QAAQ,aAAa,SAAS,KAC7D,QAAQ,aAAa,MAAM,CAAC,gBAAgB,YAAY,SAAS,WAAW,YAAY,UAAU,CAAC;AAAA,IACrG;AACA,aAAS,KAAK,CAAC,UAAU,MAAM,MAAM,CAAC;AACtC,WAAO;AAAA,MACL,GAAG,SAAS,CAAC;AAAA,MACb,OAAO,SAAS,KAAK,CAAC,YAAY,QAAQ,UAAU,MAAS,GAAG;AAAA,MAChE,cAAc,SAAS,QAAQ,CAAC,YAAY,QAAQ,YAAY;AAAA,MAChE,QAAQ,OAAO,UAAU,QAAQ,CAAC;AAAA,IACpC;AAAA,EACF,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb;AAAA,IACA,cAAc,MAAM,QAAQ,CAAC,eAAe,WAAW,YAAY;AAAA;AAAA,IAEnE,QAAQ,OAAO,UAAU,OAAO,YAAY,QAAQ,GAAG,CAAC;AAAA,IACxD;AAAA,EACF;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,YAAY,EAAE,YAAY,EAAE,OAAO,MAAM,MAAM,OAAO,QAAQ,MAAM,EAAE,CAAC,CAAC;AAY3F,MAAM,YAAiC,MAAM,KAAK,MAAM,EAAE,MAAM;AACrE,QAAM,cAAc,IAAI,2BAAU,EAAE,MAAM,sBAAsB,SAAS,sCAAsC,CAAC;AAChH,SAAO;AAAA,IACL,UAAU,MAAM,OAAO,KAAK,WAAW;AAAA,IACvC,oBAAoB,CAAC,SACnB,OAAO,QAAQ,KAAK,IAAI,CAAC,SAAS;AAAA,MAChC,UAAU,IAAI;AAAA,MACd,aAAa,aAAa,aAAa,KAAK,WAAW;AAAA,IACzD,EAAE,CAAC;AAAA,EACP;AACF,CAAC;",
   "names": ["score", "cases"]
 }

package/dist/cjs/Trials.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAgB9F,CAAA"}
+{"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAiB9F,CAAA"}

package/dist/cjs/Trials.js

@@ -64,11 +64,11 @@ const summarize = (outcomes, k) => {
 const mean = (values) => values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length;
 const aggregate = (cases, k) => {
   if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, "k");
-  const perCase = {};
-  for (const [name, outcomes] of Object.entries(cases)) {
+  const entries = Object.entries(cases).map(([name, outcomes]) => {
     if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`);
-    perCase[name] = summarize(outcomes, Math.min(k, outcomes.length));
-  }
+    return [name, summarize(outcomes, Math.min(k, outcomes.length))];
+  });
+  const perCase = Object.fromEntries(entries);
   const summaries = Object.values(perCase);
   return {
     cases: summaries.length,