@smthrs/evals 0.0.0-stage → 1.0.0-rc.13

package/dist/cjs/Runner.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/Runner.ts"],
+  "sourcesContent": ["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n  & {\n    readonly case: string\n    readonly scorer: string\n    readonly scorerName?: string | undefined\n    readonly stepKey: string\n    readonly at: string\n  }\n  & (\n    | {\n      readonly kind: \"score\"\n      readonly score: number\n      readonly reason?: string | undefined\n      readonly meta?: unknown\n    }\n    | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n  )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n  readonly case: string\n  readonly stepKey: string\n  readonly binding: Binding\n  readonly input: {\n    readonly input: unknown\n    readonly output: unknown\n    readonly groundTruth?: unknown\n    readonly context?: unknown\n    readonly latencyMs?: number\n  }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n  readonly runBatch: (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n  readonly runBatchCorrelated?:\n    | ((\n      jobs: ReadonlyArray<ScoreJob>,\n      options?: { readonly concurrency?: number | undefined }\n    ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n    | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n  readonly identity: string\n  readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n  readonly case: string\n  readonly execution?: Execution | undefined\n  readonly error?: EvalError | undefined\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n  readonly runId: string\n  readonly suite: string\n  readonly cases: ReadonlyArray<CaseResult>\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Aggregate | undefined\n  readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n  readonly scorer?: ScoreBatchRunner | undefined\n  readonly runId: string\n  readonly sampleId?: string | undefined\n  readonly trials?: number | undefined\n  readonly k?: number | undefined\n  readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n  reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n  scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"inconclusive\",\n    reason,\n    at\n  }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n  const reason = Cause.squash(cause)\n  return reason instanceof EvalError\n    ? new EvalError({\n      code: reason.code,\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n      path: reason.path ?? casePath(suiteCase.name),\n      cause: reason\n    })\n    : new EvalError({\n      code: \"executor\",\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n      path: casePath(suiteCase.name),\n      cause: reason\n    })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\n// An executor written in plain JavaScript can return any shape, so the two\n// fields every scorer and baseline reads are checked here, where the failure\n// can still name the case and the field.\nconst malformedExecution = (suiteCase: Case, execution: Execution): EvalError | undefined => {\n  const shape: { readonly stepKey?: unknown; readonly latencyMs?: unknown } =\n    typeof execution === \"object\" && execution !== null ? execution : {}\n  if (typeof shape.stepKey !== \"string\") {\n    return new EvalError({\n      code: \"executor\",\n      message: `executor returned a stepKey that is not a string (${typeof shape.stepKey})`,\n      path: `${casePath(suiteCase.name)}.stepKey`\n    })\n  }\n  if (typeof shape.latencyMs !== \"number\" || !Number.isFinite(shape.latencyMs) || shape.latencyMs < 0) {\n    return new EvalError({\n      code: \"executor\",\n      message: `executor returned a latencyMs that is not a finite non-negative number (${\n        typeof shape.latencyMs === \"number\" ? String(shape.latencyMs) : typeof shape.latencyMs\n      })`,\n      path: `${casePath(suiteCase.name)}.latencyMs`\n    })\n  }\n  return undefined\n}\n\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n  Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n    Effect.flatMap((execution) => {\n      const malformed = malformedExecution(suiteCase, execution)\n      return malformed === undefined ? Effect.succeed(execution) : Effect.fail(malformed)\n    }),\n    Effect.map((execution): CaseResult => ({\n      case: suiteCase.name,\n      execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n      observations: []\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n    ),\n    Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n  )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n  const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n  return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n  case: suiteCase.name,\n  stepKey: execution.stepKey,\n  binding: {\n    ...structuredClone({\n      sampling: binding.sampling,\n      ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n      ...(binding.context === undefined ? {} : { context: binding.context })\n    }),\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo\n  },\n  input: {\n    input: structuredClone(suiteCase.input),\n    output: execution.output,\n    ...(groundTruthFor(suiteCase, binding)),\n    ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n    latencyMs: execution.latencyMs\n  }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n  const source = result as {\n    readonly kind: unknown\n    readonly score?: unknown\n    readonly reason?: unknown\n    readonly meta?: unknown\n  }\n  const kind = source.kind\n  if (kind !== \"score\" && kind !== \"inconclusive\") {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n      at\n    )\n  }\n  const reason = source.reason\n  if (kind === \"inconclusive\") {\n    return inconclusive(\n      request,\n      typeof reason === \"string\" && reason.length > 0\n        ? boundedReason(reason)\n        : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n      at\n    )\n  }\n  const score = source.score\n  if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n      at\n    )\n  }\n  const meta = source.meta\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"score\",\n    score,\n    ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n    ...(meta === undefined ? {} : { meta }),\n    at\n  }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n  job.score.pipe(\n    Effect.flatMap(Scorer.validate),\n    Effect.map((result): ScoreStore.ScoreObservation => ({\n      ...job.observation,\n      kind: \"score\",\n      score: result.score,\n      ...(result.reason === undefined ? {} : { reason: result.reason }),\n      ...(result.meta === undefined ? {} : { meta: result.meta }),\n      at: job.at\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n    )\n  )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n  const runBatchCorrelated = (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n    Effect.forEach(\n      jobs,\n      (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n      { concurrency: options?.concurrency ?? 1 }\n    )\n  return {\n    runBatchCorrelated,\n    runBatch: (jobs, options) =>\n      runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n  }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n  readonly at: string\n  readonly runId: string\n  readonly sampleId: string\n  readonly concurrency: number\n}\n\nconst buildJobs = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n  { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n  EvalError\n> =>\n  Effect.gen(function*() {\n    // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n    // two arrays are index-aligned by construction; nothing here has to look a\n    // case back up by name.\n    const candidates = cases.flatMap((caseResult, index) => {\n      const execution = caseResult.execution\n      if (execution === undefined) return []\n      const suiteCase = suite.cases[index]!\n      return suite.bindings.flatMap((binding, bindingIndex) =>\n        binding.appliesTo === execution.target\n          ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n          : []\n      )\n    })\n    const sampled = yield* Effect.forEach(\n      candidates,\n      ({ request, bindingIndex }) =>\n        Sampling.decide(\n          request.binding.sampling,\n          request.stepKey,\n          scorerKeyOf(request.binding)\n        ).pipe(\n          Effect.map((selected) => selected ? request : undefined),\n          Effect.mapError((cause) =>\n            new EvalError({\n              code: \"invalid_suite\",\n              message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n              path: `bindings[${bindingIndex}].sampling`,\n              cause\n            })\n          )\n        ),\n      { concurrency }\n    )\n    const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n    const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n      identity: tupleKey(\n        suite.name,\n        runId,\n        sampleId,\n        request.case,\n        request.stepKey,\n        scorerKeyOf(request.binding),\n        `${index}`\n      ),\n      observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n      score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n      at: Date.parse(at)\n    }))\n    return { requests, jobs }\n  })\n\nconst batchFallback = (\n  jobs: ReadonlyArray<ScoreJob>,\n  cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n  Cause.hasInterrupts(cause)\n    ? Effect.interrupt\n    : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n  jobs: ReadonlyArray<ScoreJob>,\n  scorer: ScoreBatchRunner,\n  runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) =>\n        batchFallback(jobs, cause).pipe(\n          Effect.map((observations) =>\n            observations.map((observation, index) => ({\n              identity: jobs[index]!.identity,\n              observation\n            }))\n          )\n        )\n      )\n    )\n    if (batchResults.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n          path: \"runBatchCorrelated\"\n        })\n      )\n    }\n    const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n    const byIdentity = new Map<string, ScoreObservation>()\n    let unknown: { readonly identity: string; readonly index: number } | undefined\n    for (const [index, result] of batchResults.entries()) {\n      const jobIndex = jobIndexByIdentity.get(result.identity)\n      if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message:\n              `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n            path: `runBatchCorrelated[${index}]`\n          })\n        )\n      }\n      if (jobIndex === undefined) unknown = { identity: result.identity, index }\n      byIdentity.set(result.identity, result.observation)\n    }\n    if (unknown !== undefined) {\n      const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n      const missingIdentity = jobs[missingJobIndex]!.identity\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n          path: `runBatchCorrelated[${unknown.index}]`\n        })\n      )\n    }\n    return jobs.map((job) => byIdentity.get(job.identity)!)\n  })\n\nconst scoreOrderOnly = (\n  jobs: ReadonlyArray<ScoreJob>,\n  requests: ReadonlyArray<ScoreRequest>,\n  scorer: ScoreBatchRunner,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const pairIndexes = new Map<string, number>()\n    for (const [index, job] of jobs.entries()) {\n      const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n      const previous = pairIndexes.get(pair)\n      if (previous !== undefined) {\n        const first = requests[previous]!\n        const second = requests[index]!\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"ambiguous_score_job\",\n            message:\n              `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n                label(second.binding)\n              }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n            path: \"runBatch\"\n          })\n        )\n      }\n      pairIndexes.set(pair, index)\n    }\n    const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) => batchFallback(jobs, cause))\n    )\n    if (results.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n          path: \"runBatch\"\n        })\n      )\n    }\n    return results\n  })\n\nconst score = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  scorer: ScoreBatchRunner,\n  context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n  Effect.gen(function*() {\n    const { requests, jobs } = yield* buildJobs(cases, suite, context)\n    if (requests.length === 0) return cases\n    const runBatchCorrelated = scorer.runBatchCorrelated\n    const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n    const results = yield* runBatchCorrelated === undefined\n      ? scoreOrderOnly(jobs, requests, scorer, context)\n      : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n    const { at } = context\n    const observations: Array<Observation> = []\n    for (const [index, request] of requests.entries()) {\n      const result = results[index]!\n      if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message: protocol === \"runBatchCorrelated\"\n              ? `Correlated scorer batch identity '${\n                jobs[index]!.identity\n              }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'`\n              : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n            path: `${protocol}[${index}]`\n          })\n        )\n      }\n      observations.push(observationFor(request, result, at))\n    }\n    const byCase = new Map<string, Array<Observation>>()\n    for (const observation of observations) {\n      const bucket = byCase.get(observation.case)\n      if (bucket === undefined) byCase.set(observation.case, [observation])\n      else bucket.push(observation)\n    }\n    return cases.map((caseResult) => ({\n      ...caseResult,\n      observations: byCase.get(caseResult.case) ?? []\n    }))\n  })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n  suite: Suite,\n  options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n  Effect.gen(function*() {\n    const executor = yield* CaseExecutor\n    const injected = yield* Effect.serviceOption(Runner)\n    const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n    const atMillis = Date.parse(options.at)\n    if (options.runId.trim().length === 0) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message: \"Deterministic runs require a non-empty runId\",\n          path: \"options.runId\"\n        })\n      )\n    }\n    if (\n      !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n      !Number.isFinite(atMillis) ||\n      new Date(atMillis).toISOString() !== options.at\n    ) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message:\n            `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n          path: \"options.at\"\n        })\n      )\n    }\n    const n = options.trials ?? suite.trials ?? 1\n    const k = options.k ?? 1\n    if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_trials\",\n          message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n          path: \"options.trials\"\n        })\n      )\n    }\n    const at = options.at\n    const runId = options.runId\n    const sampleId = options.sampleId ?? \"default\"\n    // Await each batch before starting the next: concurrency within a trial is\n    // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n    const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n      Effect.gen(function*() {\n        const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n          runCase(executor, suiteCase), {\n          concurrency: suite.concurrency,\n          discard: false\n        })\n        return yield* score(cases, suite, scorer, {\n          concurrency: suite.concurrency,\n          at,\n          runId,\n          sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n        })\n      }), { concurrency: 1 })\n    const outcomes: Array<[string, ReadonlyArray<boolean>]> = []\n    const cases = suite.cases.map((suiteCase, index) => {\n      const attempts = batches.map((batch) =>\n        batch[index]!\n      )\n      const passed = attempts.map((attempt) =>\n        attempt.error === undefined && attempt.observations.length > 0 &&\n        attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n      )\n      outcomes.push([suiteCase.name, passed])\n      return {\n        ...attempts[0]!,\n        error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n        observations: attempts.flatMap((attempt) => attempt.observations),\n        trials: Trials.summarize(passed, k)\n      }\n    })\n    return {\n      runId,\n      suite: suite.name,\n      cases,\n      observations: cases.flatMap((caseResult) => caseResult.observations),\n      // fromEntries defines own properties, so a case named `__proto__` stays a case.\n      trials: Trials.aggregate(Object.fromEntries(outcomes), k),\n      k\n    }\n  }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n  const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n  return {\n    runBatch: () => Effect.fail(unavailable),\n    runBatchCorrelated: (jobs) =>\n      Effect.succeed(jobs.map((job) => ({\n        identity: job.identity,\n        observation: ScorerRunner.inconclusive(job, unavailable)\n      })))\n  }\n})\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,mBAA8B;AAC9B,eAA0B;AAC1B,aAAwB;AAExB,YAAuB;AACvB,cAAyB;AACzB,aAAwB;AACxB,YAAuB;AACvB,0BAAkF;AAClF,uBAA0B;AAC1B,+BAAyC;AACzC,yBAA4B;AAC5B,sBAAyB;AAEzB,aAAwB;AAoKjB,MAAM,eAAe,QAAQ,QAAkC,EAAE,oBAAoB,EAAE;AAAC;AAG/F,MAAM,kBAAkB;AAExB,MAAM,gBAAgB,CAAC,WACrB,OAAO,UAAU,kBAAkB,SAAS,GAAG,OAAO,MAAM,GAAG,eAAe,CAAC;AAEjF,MAAM,cAAc,CAAC,YAA6B,QAAQ,OAAO;AAKjE,MAAM,eAAe,CAAC,YAA6B,QAAQ,OAAO;AAElE,MAAM,QAAQ,CAAC,gBACb,gCAAY,EAAE,QAAQ,YAAY,OAAO,GAAG,YAAY,aAAa,OAAO,EAAE,CAAC;AAEjF,MAAM,eAAe,CAAC,SAAuB,QAAgB,OAA4B;AACvF,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN;AAAA,IACA;AAAA,EACF;AACF;AAKA,MAAM,WAAW,CAAC,SAAyB,UAAU,IAAI;AAQzD,MAAM,YAAY,CAAC,WAAiB,UAA2C;AAC7E,QAAM,SAAS,MAAM,OAAO,KAAK;AACjC,SAAO,kBAAkB,6BACrB,IAAI,2BAAU;AAAA,IACd,MAAM,OAAO;AAAA,IACb,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,OAAO,EAAE;AAAA,IACjG,MAAM,OAAO,QAAQ,SAAS,UAAU,IAAI;AAAA,IAC5C,OAAO;AAAA,EACT,CAAC,IACC,IAAI,2BAAU;AAAA,IACd,MAAM;AAAA,IACN,aAAS,mDAAyB,2BAA2B,UAAU,IAAI,MAAM,OAAO,MAAM,CAAC,EAAE;AAAA,IACjG,MAAM,SAAS,UAAU,IAAI;AAAA,IAC7B,OAAO;AAAA,EACT,CAAC;AACL;AAQA,MAAM,qBAAqB,CAAC,WAAiB,cAAgD;AAC3F,QAAM,QACJ,OAAO,cAAc,YAAY,cAAc,OAAO,YAAY,CAAC;AACrE,MAAI,OAAO,MAAM,YAAY,UAAU;AACrC,WAAO,IAAI,2BAAU;AAAA,MACnB,MAAM;AAAA,MACN,SAAS,qDAAqD,OAAO,MAAM,OAAO;AAAA,MAClF,MAAM,GAAG,SAAS,UAAU,IAAI,CAAC;AAAA,IACnC,CAAC;AAAA,EACH;AACA,MAAI,OAAO,MAAM,cAAc,YAAY,CAAC,OAAO,SAAS,MAAM,SAAS,KAAK,MAAM,YAAY,GAAG;AACnG,WAAO,IAAI,2BAAU;AAAA,MACnB,MAAM;AAAA,MACN,SAAS,2EACP,OAAO,MAAM,cAAc,WAAW,OAAO,MAAM,SAAS,IAAI,OAAO,MAAM,SAC/E;AAAA,MACA,MAAM,GAAG,SAAS,UAAU,IAAI,CAAC;AAAA,IACnC,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,MAAM,UAAU,CAAC,UAA+B,cAC9C,OAAO,QAAQ,MAAM,SAAS,IAAI,gBAAgB,SAAS,CAAC,CAAC,EAAE;AAAA,EAC7D,OAAO,QAAQ,CAAC,cAAc;AAC5B,UAAM,YAAY,mBAAmB,WAAW,SAAS;AACzD,WAAO,cAAc,SAAY,OAAO,QAAQ,SAAS,IAAI,OAAO,KAAK,SAAS;AAAA,EACpF,CAAC;AAAA,EACD,OAAO,IAAI,CAAC,eAA2B;AAAA,IACrC,MAAM,UAAU;AAAA,IAChB,WAAW,EAAE,GAAG,WAAW,aAAS,mDAAyB,UAAU,OAAO,EAAE;AAAA,IAChF,cAAc,CAAC;AAAA,EACjB,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAoB,EAAE,MAAM,UAAU,MAAM,OAAO,UAAU,WAAW,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC;AAAA,EAC/G;AAAA,EACA,OAAO,SAAS,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,KAAK,EAAE,CAAC;AACvE;AAIF,MAAM,iBAAiB,CAAC,WAAiB,YAAyD;AAChG,QAAM,cAAc,UAAU,aAAa,SAAY,UAAU,WAAW,QAAQ;AACpF,SAAO,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,gBAAgB,WAAW,EAAE;AACtF;AAEA,MAAM,aAAa,CAAC,WAAiB,WAAsB,aAAoC;AAAA,EAC7F,MAAM,UAAU;AAAA,EAChB,SAAS,UAAU;AAAA,EACnB,SAAS;AAAA,IACP,GAAG,gBAAgB;AAAA,MACjB,UAAU,QAAQ;AAAA,MAClB,GAAI,QAAQ,gBAAgB,SAAY,CAAC,IAAI,EAAE,aAAa,QAAQ,YAAY;AAAA,MAChF,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,QAAQ,QAAQ;AAAA,IACtE,CAAC;AAAA,IACD,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,EACrB;AAAA,EACA,OAAO;AAAA,IACL,OAAO,gBAAgB,UAAU,KAAK;AAAA,IACtC,QAAQ,UAAU;AAAA,IAClB,GAAI,eAAe,WAAW,OAAO;AAAA,IACrC,GAAI,QAAQ,YAAY,SAAY,CAAC,IAAI,EAAE,SAAS,gBAAgB,QAAQ,OAAO,EAAE;AAAA,IACrF,WAAW,UAAU;AAAA,EACvB;AACF;AAEA,MAAM,iBAAiB,CAAC,SAAuB,QAA0B,OAA4B;AACnG,QAAM,SAAS;AAMf,QAAM,OAAO,OAAO;AACpB,MAAI,SAAS,WAAW,SAAS,gBAAgB;AAC/C,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,2CAA2C,OAAO,IAAI,CAAC,GAAG;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAM,SAAS,OAAO;AACtB,MAAI,SAAS,gBAAgB;AAC3B,WAAO;AAAA,MACL;AAAA,MACA,OAAO,WAAW,YAAY,OAAO,SAAS,IAC1C,cAAc,MAAM,IACpB,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,sDAAsD;AAAA,MACxG;AAAA,IACF;AAAA,EACF;AACA,QAAMA,SAAQ,OAAO;AACrB,MAAI,OAAOA,WAAU,YAAY,CAAC,OAAO,SAASA,MAAK,KAAKA,SAAQ,KAAKA,SAAQ,GAAG;AAClF,WAAO;AAAA,MACL;AAAA,MACA,cAAc,UAAU,MAAM,QAAQ,OAAO,CAAC,qCAAqC,OAAOA,MAAK,CAAC,EAAE;AAAA,MAClG;AAAA,IACF;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,SAAO;AAAA,IACL,MAAM,QAAQ;AAAA,IACd,QAAQ,YAAY,QAAQ,OAAO;AAAA,IACnC,YAAY,aAAa,QAAQ,OAAO;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB,MAAM;AAAA,IACN,OAAAA;AAAA,IACA,GAAI,OAAO,WAAW,WAAW,EAAE,QAAQ,cAAc,MAAM,EAAE,IAAI,CAAC;AAAA,IACtE,GAAI,SAAS,SAAY,CAAC,IAAI,EAAE,KAAK;AAAA,IACrC;AAAA,EACF;AACF;AAEA,MAAM,gBAAgB,CAAC,QACrB,IAAI,MAAM;AAAA,EACR,OAAO,QAAQ,OAAO,QAAQ;AAAA,EAC9B,OAAO,IAAI,CAAC,YAAyC;AAAA,IACnD,GAAG,IAAI;AAAA,IACP,MAAM;AAAA,IACN,OAAO,OAAO;AAAA,IACd,GAAI,OAAO,WAAW,SAAY,CAAC,IAAI,EAAE,QAAQ,OAAO,OAAO;AAAA,IAC/D,GAAI,OAAO,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,OAAO,KAAK;AAAA,IACzD,IAAI,IAAI;AAAA,EACV,EAAE;AAAA,EACF,OAAO;AAAA,IAAW,CAAC,UACjB,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC;AAAA,EAC1D;AACF;AAcK,MAAM,aAAa,MAAwB;AAChD,QAAM,qBAAqB,CACzB,MACA,YAEA,OAAO;AAAA,IACL;AAAA,IACA,CAAC,QAAQ,cAAc,GAAG,EAAE,KAAK,OAAO,IAAI,CAAC,iBAAiB,EAAE,UAAU,IAAI,UAAU,YAAY,EAAE,CAAC;AAAA,IACvG,EAAE,aAAa,SAAS,eAAe,EAAE;AAAA,EAC3C;AACF,SAAO;AAAA,IACL;AAAA,IACA,UAAU,CAAC,MAAM,YACf,mBAAmB,MAAM,OAAO,EAAE,KAAK,OAAO,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,WAAW,OAAO,WAAW,CAAC,CAAC;AAAA,EAC/G;AACF;AAQO,MAAM,cAAmC,MAAM,QAAQ,MAAM,EAAE,WAAW,CAAC;AASlF,MAAM,YAAY,CAChB,OACA,OACA,EAAE,IAAI,OAAO,UAAU,YAAY,MAKnC,OAAO,IAAI,aAAY;AAIrB,QAAM,aAAa,MAAM,QAAQ,CAAC,YAAY,UAAU;AACtD,UAAM,YAAY,WAAW;AAC7B,QAAI,cAAc,OAAW,QAAO,CAAC;AACrC,UAAM,YAAY,MAAM,MAAM,KAAK;AACnC,WAAO,MAAM,SAAS;AAAA,MAAQ,CAAC,SAAS,iBACtC,QAAQ,cAAc,UAAU,SAC5B,CAAC,EAAE,SAAS,WAAW,WAAW,WAAW,OAAO,GAAG,aAAa,CAAC,IACrE,CAAC;AAAA,IACP;AAAA,EACF,CAAC;AACD,QAAM,UAAU,OAAO,OAAO;AAAA,IAC5B;AAAA,IACA,CAAC,EAAE,SAAS,aAAa,MACvB,SAAS;AAAA,MACP,QAAQ,QAAQ;AAAA,MAChB,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,IAC7B,EAAE;AAAA,MACA,OAAO,IAAI,CAAC,aAAa,WAAW,UAAU,MAAS;AAAA,MACvD,OAAO;AAAA,QAAS,CAAC,UACf,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,sCAAsC,MAAM,QAAQ,OAAO,CAAC;AAAA,UACrE,MAAM,YAAY,YAAY;AAAA,UAC9B;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,IACF,EAAE,YAAY;AAAA,EAChB;AACA,QAAM,WAAW,QAAQ,OAAO,CAAC,YAAqC,YAAY,MAAS;AAC3F,QAAM,OAAwB,SAAS,IAAI,CAAC,SAAS,WAAW;AAAA,IAC9D,cAAU;AAAA,MACR,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,YAAY,QAAQ,OAAO;AAAA,MAC3B,GAAG,KAAK;AAAA,IACV;AAAA,IACA,aAAa,EAAE,eAAe,QAAQ,SAAS,WAAW,YAAY,QAAQ,OAAO,EAAE;AAAA,IACvF,OAAO,OAAO,QAAQ,MAAM,QAAQ,QAAQ,OAAO,MAAM,QAAQ,KAAK,CAAC;AAAA,IACvE,IAAI,KAAK,MAAM,EAAE;AAAA,EACnB,EAAE;AACF,SAAO,EAAE,UAAU,KAAK;AAC1B,CAAC;AAEH,MAAM,gBAAgB,CACpB,MACA,UAEA,MAAM,cAAc,KAAK,IACrB,OAAO,YACP,OAAO,QAAQ,KAAK,IAAI,CAAC,QAAQ,aAAa,aAAa,KAAK,KAAK,CAAC,CAAC;AAE7E,MAAM,kBAAkB,CACtB,MACA,QACA,oBACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,eAAe,OAAO,OAAO,QAAQ,MAAM,mBAAmB,KAAK,QAAQ,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IACvG,OAAO;AAAA,MAAW,CAAC,UACjB,cAAc,MAAM,KAAK,EAAE;AAAA,QACzB,OAAO;AAAA,UAAI,CAAC,iBACV,aAAa,IAAI,CAAC,aAAa,WAAW;AAAA,YACxC,UAAU,KAAK,KAAK,EAAG;AAAA,YACvB;AAAA,UACF,EAAE;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,aAAa,WAAW,KAAK,QAAQ;AACvC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,oCAAoC,aAAa,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACpF,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,qBAAqB,IAAI,IAAI,KAAK,IAAI,CAAC,KAAK,UAAU,CAAC,IAAI,UAAU,KAAK,CAAU,CAAC;AAC3F,QAAM,aAAa,oBAAI,IAA8B;AACrD,MAAI;AACJ,aAAW,CAAC,OAAO,MAAM,KAAK,aAAa,QAAQ,GAAG;AACpD,UAAM,WAAW,mBAAmB,IAAI,OAAO,QAAQ;AACvD,QAAI,aAAa,UAAa,WAAW,IAAI,OAAO,QAAQ,GAAG;AAC7D,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,wDAAwD,OAAO,QAAQ,aAAa,QAAQ,oBAAoB,KAAK;AAAA,UACvH,MAAM,sBAAsB,KAAK;AAAA,QACnC,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,aAAa,OAAW,WAAU,EAAE,UAAU,OAAO,UAAU,MAAM;AACzE,eAAW,IAAI,OAAO,UAAU,OAAO,WAAW;AAAA,EACpD;AACA,MAAI,YAAY,QAAW;AACzB,UAAM,kBAAkB,KAAK,UAAU,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI,QAAQ,CAAC;AAC7E,UAAM,kBAAkB,KAAK,eAAe,EAAG;AAC/C,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,sDAAsD,QAAQ,QAAQ,qBAAqB,QAAQ,KAAK,SAAS,eAAe,cAAc,eAAe;AAAA,QAC/J,MAAM,sBAAsB,QAAQ,KAAK;AAAA,MAC3C,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO,KAAK,IAAI,CAAC,QAAQ,WAAW,IAAI,IAAI,QAAQ,CAAE;AACxD,CAAC;AAEH,MAAM,iBAAiB,CACrB,MACA,UACA,QACA,EAAE,YAAY,MAEd,OAAO,IAAI,aAAY;AACrB,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACzC,UAAM,WAAO,0BAAS,IAAI,YAAY,eAAe,IAAI,YAAY,SAAS;AAC9E,UAAM,WAAW,YAAY,IAAI,IAAI;AACrC,QAAI,aAAa,QAAW;AAC1B,YAAM,QAAQ,SAAS,QAAQ;AAC/B,YAAM,SAAS,SAAS,KAAK;AAC7B,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SACE,0CAA0C,MAAM,IAAI,UAAU,OAAO,IAAI,yBAAyB,IAAI,YAAY,aAAa,gBAC7H,MAAM,OAAO,OAAO,CACtB;AAAA,UACF,MAAM;AAAA,QACR,CAAC;AAAA,MACH;AAAA,IACF;AACA,gBAAY,IAAI,MAAM,KAAK;AAAA,EAC7B;AACA,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,OAAO,SAAS,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE;AAAA,IAClF,OAAO,WAAW,CAAC,UAAU,cAAc,MAAM,KAAK,CAAC;AAAA,EACzD;AACA,MAAI,QAAQ,WAAW,KAAK,QAAQ;AAClC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,yBAAyB,QAAQ,MAAM,qBAAqB,KAAK,MAAM;AAAA,QACzE,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT,CAAC;AAEH,MAAM,QAAQ,CACZ,OACA,OACA,QACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,EAAE,UAAU,KAAK,IAAI,OAAO,UAAU,OAAO,OAAO,OAAO;AACjE,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,QAAM,qBAAqB,OAAO;AAClC,QAAM,WAAW,uBAAuB,SAAY,aAAa;AACjE,QAAM,UAAU,OAAO,uBAAuB,SAC1C,eAAe,MAAM,UAAU,QAAQ,OAAO,IAC9C,gBAAgB,MAAM,QAAQ,oBAAoB,OAAO;AAC7D,QAAM,EAAE,GAAG,IAAI;AACf,QAAM,eAAmC,CAAC;AAC1C,aAAW,CAAC,OAAO,OAAO,KAAK,SAAS,QAAQ,GAAG;AACjD,UAAM,SAAS,QAAQ,KAAK;AAC5B,QAAI,OAAO,kBAAkB,QAAQ,WAAW,OAAO,cAAc,YAAY,QAAQ,OAAO,GAAG;AACjG,aAAO,OAAO,OAAO;AAAA,QACnB,IAAI,2BAAU;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,aAAa,uBAClB,qCACA,KAAK,KAAK,EAAG,QACf,oBAAoB,OAAO,aAAa,iBAAiB,OAAO,SAAS,eAAe,KAAK,sBAC3F,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO,MAC3B,6CAA6C,OAAO,SAAS,cAAc,OAAO,aAAa,eAAe,KAAK,eACnH,YAAY,QAAQ,OAAO,CAC7B,cAAc,QAAQ,OAAO;AAAA,UAC/B,MAAM,GAAG,QAAQ,IAAI,KAAK;AAAA,QAC5B,CAAC;AAAA,MACH;AAAA,IACF;AACA,iBAAa,KAAK,eAAe,SAAS,QAAQ,EAAE,CAAC;AAAA,EACvD;AACA,QAAM,SAAS,oBAAI,IAAgC;AACnD,aAAW,eAAe,cAAc;AACtC,UAAM,SAAS,OAAO,IAAI,YAAY,IAAI;AAC1C,QAAI,WAAW,OAAW,QAAO,IAAI,YAAY,MAAM,CAAC,WAAW,CAAC;AAAA,QAC/D,QAAO,KAAK,WAAW;AAAA,EAC9B;AACA,SAAO,MAAM,IAAI,CAAC,gBAAgB;AAAA,IAChC,GAAG;AAAA,IACH,cAAc,OAAO,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EAChD,EAAE;AACJ,CAAC;AA2BI,MAAM,MAAM,CACjB,OACA,YAEA,OAAO,IAAI,aAAY;AACrB,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,OAAO,cAAc,MAAM;AACnD,QAAM,SAAS,QAAQ,WAAW,SAAS,SAAS,SAAS,SAAS,QAAQ,WAAW;AACzF,QAAM,WAAW,KAAK,MAAM,QAAQ,EAAE;AACtC,MAAI,QAAQ,MAAM,KAAK,EAAE,WAAW,GAAG;AACrC,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,MACE,CAAC,iDAAiD,KAAK,QAAQ,EAAE,KACjE,CAAC,OAAO,SAAS,QAAQ,KACzB,IAAI,KAAK,QAAQ,EAAE,YAAY,MAAM,QAAQ,IAC7C;AACA,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SACE,+FAA+F,QAAQ,EAAE;AAAA,QAC3G,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,IAAI,QAAQ,UAAU,MAAM,UAAU;AAC5C,QAAM,IAAI,QAAQ,KAAK;AACvB,MAAI,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,OAAQ,CAAC,OAAO,cAAc,CAAC,KAAK,IAAI,KAAK,IAAI,GAAG;AAC/F,WAAO,OAAO,OAAO;AAAA,MACnB,IAAI,2BAAU;AAAA,QACZ,MAAM;AAAA,QACN,SAAS;AAAA,QACT,MAAM;AAAA,MACR,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,KAAK,QAAQ;AACnB,QAAM,QAAQ,QAAQ;AACtB,QAAM,WAAW,QAAQ,YAAY;AAGrC,QAAM,UAAU,OAAO,OAAO,QAAQ,MAAM,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,UAAU,KAAK,GAAG,CAAC,UACrF,OAAO,IAAI,aAAY;AACrB,UAAMC,SAAQ,OAAO,OAAO,QAAQ,MAAM,OAAO,CAAC,cAChD,QAAQ,UAAU,SAAS,GAAG;AAAA,MAC9B,aAAa,MAAM;AAAA,MACnB,SAAS;AAAA,IACX,CAAC;AACD,WAAO,OAAO,MAAMA,QAAO,OAAO,QAAQ;AAAA,MACxC,aAAa,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,UAAU,MAAM,IAAI,eAAW,0BAAS,UAAU,OAAO,KAAK,CAAC;AAAA,IACjE,CAAC;AAAA,EACH,CAAC,GAAG,EAAE,aAAa,EAAE,CAAC;AACxB,QAAM,WAAoD,CAAC;AAC3D,QAAM,QAAQ,MAAM,MAAM,IAAI,CAAC,WAAW,UAAU;AAClD,UAAM,WAAW,QAAQ;AAAA,MAAI,CAAC,UAC5B,MAAM,KAAK;AAAA,IACb;AACA,UAAM,SAAS,SAAS;AAAA,MAAI,CAAC,YAC3B,QAAQ,UAAU,UAAa,QAAQ,aAAa,SAAS,KAC7D,QAAQ,aAAa,MAAM,CAAC,gBAAgB,YAAY,SAAS,WAAW,YAAY,UAAU,CAAC;AAAA,IACrG;AACA,aAAS,KAAK,CAAC,UAAU,MAAM,MAAM,CAAC;AACtC,WAAO;AAAA,MACL,GAAG,SAAS,CAAC;AAAA,MACb,OAAO,SAAS,KAAK,CAAC,YAAY,QAAQ,UAAU,MAAS,GAAG;AAAA,MAChE,cAAc,SAAS,QAAQ,CAAC,YAAY,QAAQ,YAAY;AAAA,MAChE,QAAQ,OAAO,UAAU,QAAQ,CAAC;AAAA,IACpC;AAAA,EACF,CAAC;AACD,SAAO;AAAA,IACL;AAAA,IACA,OAAO,MAAM;AAAA,IACb;AAAA,IACA,cAAc,MAAM,QAAQ,CAAC,eAAe,WAAW,YAAY;AAAA;AAAA,IAEnE,QAAQ,OAAO,UAAU,OAAO,YAAY,QAAQ,GAAG,CAAC;AAAA,IACxD;AAAA,EACF;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,YAAY,EAAE,YAAY,EAAE,OAAO,MAAM,MAAM,OAAO,QAAQ,MAAM,EAAE,CAAC,CAAC;AAY3F,MAAM,YAAiC,MAAM,KAAK,MAAM,EAAE,MAAM;AACrE,QAAM,cAAc,IAAI,2BAAU,EAAE,MAAM,sBAAsB,SAAS,sCAAsC,CAAC;AAChH,SAAO;AAAA,IACL,UAAU,MAAM,OAAO,KAAK,WAAW;AAAA,IACvC,oBAAoB,CAAC,SACnB,OAAO,QAAQ,KAAK,IAAI,CAAC,SAAS;AAAA,MAChC,UAAU,IAAI;AAAA,MACd,aAAa,aAAa,aAAa,KAAK,WAAW;AAAA,IACzD,EAAE,CAAC;AAAA,EACP;AACF,CAAC;",
+  "names": ["score", "cases"]
+}

package/dist/cjs/Suite.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Fixed evaluation suite declarations.
+ *
+ * A suite is the fixed input half of an evaluation: named cases, the scorer
+ * bindings that grade them, and the concurrency the runner is allowed. It is
+ * validated once and then never changes, because the run that produced a
+ * committed baseline has to be reproducible from the suite value that was
+ * validated.
+ *
+ * @since 0.1.0
+ */
+import type * as ScorerBinding from "@smthrs/scorers/Binding";
+import * as Effect from "effect/Effect";
+import { EvalError } from "./EvalError.ts";
+/**
+ * A scorer binding accepted from `@smthrs/scorers`.
+ *
+ * The binding's `appliesTo` flow is matched against an execution's `target` by
+ * reference identity, so a binding only ever grades the exact flow value it was
+ * declared against.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Binding = ScorerBinding.Binding;
+/**
+ * One immutable fixed-suite case.
+ *
+ * `input` is handed to the executor and `expected` is offered to a bound scorer
+ * as ground truth. A declared `expected`, including `null`, wins over the
+ * binding's `groundTruth`; only an absent one defers to the binding. Both are
+ * snapshots taken by {@link make}: mutating the object a caller passed in never
+ * changes the suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Case {
+    readonly name: string;
+    readonly input: unknown;
+    readonly expected?: unknown | undefined;
+}
+/**
+ * Options for constructing a suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MakeOptions {
+    readonly name: string;
+    readonly cases: ReadonlyArray<Case>;
+    readonly bindings?: ReadonlyArray<Binding> | undefined;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * A validated, named collection of fixed cases and scorer bindings.
+ *
+ * The suite, cases, bindings, and their inert data are deeply frozen snapshots.
+ * Executable scorer and target identities remain unchanged.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Suite {
+    readonly name: string;
+    readonly cases: ReadonlyArray<Case>;
+    readonly bindings: ReadonlyArray<Binding>;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * The declared ceilings a suite is validated against.
+ *
+ * They exist so a mistake fails at construction with a sentence instead of
+ * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold
+ * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines
+ * fixture.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const limits: {
+    readonly concurrency: 1024;
+    readonly cases: 10000;
+    /** Longest JSON Lines fixture, in UTF-16 code units. */
+    readonly fixtureLength: number;
+};
+/**
+ * Builds and validates a fixed suite.
+ *
+ * When the effect runs, it reads every option, case field, and binding field
+ * exactly once, then validates and copies only what it read. Validating one
+ * value and copying another let a getter hand the suite something validation
+ * never saw. Every case and binding is then copied, including a binding's
+ * sampling policy, so the suite is a snapshot the caller can no longer reach:
+ * mutating an array, case input, or ratio policy afterwards leaves the
+ * validated suite unchanged. Data is checked before `structuredClone` and
+ * deeply frozen afterwards. Only plain objects, arrays, and cloneable
+ * primitives are admitted, with enumerable string-keyed data properties.
+ * Functions, class instances, mutable built-ins, and accessors fail with
+ * `invalid_suite` naming the offending path. Cycles and shared references are
+ * supported; null-prototype records become ordinary objects in the clone.
+ *
+ * Fails with `invalid_suite` for an empty or control-character name, no cases,
+ * more than `limits.cases` cases, a duplicate case name, or a concurrency that
+ * is not a safe integer in `[1, limits.concurrency]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const make: (options: MakeOptions) => Effect.Effect<Suite, EvalError>;
+/**
+ * Options used when decoding JSON Lines.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface JsonLinesOptions {
+    readonly name: string;
+    readonly bindings?: ReadonlyArray<Binding> | undefined;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * Loads the `{ name, input, expected? }` JSON Lines fixture format.
+ *
+ * Blank lines are skipped, a leading byte-order mark is stripped, and both LF
+ * and CRLF terminate a line. A malformed line fails with `invalid_suite`
+ * carrying the 1-based line number in both the message and the path; a fixture
+ * larger than `limits.fixtureLength` is rejected before any of it is parsed.
+ * The first non-blank line after `limits.cases` cases fails with
+ * `invalid_suite` at `cases` before it or any later line is parsed or decoded.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const fromJsonLines: (text: string, options: JsonLinesOptions) => Effect.Effect<Suite, EvalError>;
+//# sourceMappingURL=Suite.d.ts.map

package/dist/cjs/Suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Suite.d.ts","sourceRoot":"","sources":["../../src/Suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,KAAK,aAAa,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAG1C;;;;;;;;;GASG;AACH,MAAM,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAA;AAE3C;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;CACxC;AAED;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,SAAS,CAAA;IACtD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,MAAM;;;IAGjB,wDAAwD;;CAEhD,CAAA;AAkJV;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,IAAI,GAAI,SAAS,WAAW,KAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAqBtE,CAAA;AAEJ;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,SAAS,CAAA;IACtD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAQD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,SAAS,gBAAgB,KAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAsClG,CAAA"}

package/dist/cjs/Suite.js

@@ -0,0 +1,221 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var Suite_exports = {};
+__export(Suite_exports, {
+  fromJsonLines: () => fromJsonLines,
+  limits: () => limits,
+  make: () => make
+});
+module.exports = __toCommonJS(Suite_exports);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Schema = __toESM(require("effect/Schema"), 1);
+var import_EvalError = require("./EvalError.js");
+var import_controlCharacters = require("./internal/controlCharacters.js");
+const limits = {
+  concurrency: 1024,
+  cases: 1e4,
+  /** Longest JSON Lines fixture, in UTF-16 code units. */
+  fixtureLength: 8 * 1024 * 1024
+};
+const invalid = (message, path, cause) => new import_EvalError.EvalError({ code: "invalid_suite", message, path, ...cause === void 0 ? {} : { cause } });
+const cloneMessage = "Suite data must be structured-cloneable so the suite cannot change after it is validated";
+const validateData = (value, path, seen = /* @__PURE__ */ new WeakSet()) => {
+  if (typeof value === "function" || typeof value === "symbol") throw invalid(cloneMessage, path);
+  if (value === null || typeof value !== "object") return;
+  if (seen.has(value)) return;
+  seen.add(value);
+  const array = Array.isArray(value);
+  const prototype = Object.getPrototypeOf(value);
+  if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) {
+    throw invalid("Suite data must contain only plain objects, arrays, and structured-cloneable primitives", path);
+  }
+  for (const key of Reflect.ownKeys(value)) {
+    if (array && key === "length") continue;
+    const childPath = typeof key === "string" && array && /^(0|[1-9]\d*)$/.test(key) ? `${path}[${key}]` : `${path}.${String(key)}`;
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    if (typeof key === "symbol" || !descriptor.enumerable || !("value" in descriptor)) {
+      throw invalid("Suite data properties must be enumerable string-keyed data fields", childPath);
+    }
+    validateData(descriptor.value, childPath, seen);
+  }
+};
+const freezeData = (value, seen = /* @__PURE__ */ new WeakSet()) => {
+  if (value === null || typeof value !== "object" || seen.has(value)) return value;
+  seen.add(value);
+  for (const child of Object.values(value)) freezeData(child, seen);
+  return Object.freeze(value);
+};
+const clone = (value, path) => Effect.try({
+  try: () => {
+    validateData(value, path);
+    return freezeData(structuredClone(value));
+  },
+  catch: (cause) => cause instanceof import_EvalError.EvalError ? cause : invalid(cloneMessage, path, cause)
+});
+const validate = (options) => {
+  if (options.name.trim().length === 0) return invalid("Suite name must not be empty", "name");
+  const nameControl = (0, import_controlCharacters.controlCharacter)(options.name);
+  if (nameControl !== void 0) {
+    return invalid(`Suite name must not contain the control character ${nameControl}`, "name");
+  }
+  if (!Number.isSafeInteger(options.trials) || options.trials < 1 || options.trials > 1e3) {
+    return invalid("Suite trials must be an integer from 1 to 1000", "trials");
+  }
+  if (options.cases.length === 0) return invalid("Suite must contain at least one case", "cases");
+  if (options.cases.length > limits.cases) {
+    return invalid(`Suite must contain at most ${limits.cases} cases, got ${options.cases.length}`, "cases");
+  }
+  if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {
+    return invalid(
+      `Suite concurrency must be a positive safe integer, got ${String(options.concurrency)}`,
+      "concurrency"
+    );
+  }
+  if (options.concurrency > limits.concurrency) {
+    return invalid(
+      `Suite concurrency must be at most ${limits.concurrency}, got ${options.concurrency}`,
+      "concurrency"
+    );
+  }
+  const names = /* @__PURE__ */ new Set();
+  for (const [index, suiteCase] of options.cases.entries()) {
+    if (suiteCase.name.trim().length === 0) {
+      return invalid("Suite case name must not be empty", `cases[${index}].name`);
+    }
+    const caseControl = (0, import_controlCharacters.controlCharacter)(suiteCase.name);
+    if (caseControl !== void 0) {
+      return invalid(
+        `Suite case name must not contain the control character ${caseControl}`,
+        `cases[${index}].name`
+      );
+    }
+    if (names.has(suiteCase.name)) {
+      return invalid(`Duplicate suite case: ${suiteCase.name}`, `cases[${index}].name`);
+    }
+    names.add(suiteCase.name);
+  }
+  return void 0;
+};
+const readCase = (suiteCase) => ({
+  name: suiteCase.name,
+  input: suiteCase.input,
+  expected: suiteCase.expected
+});
+const readBinding = (binding) => {
+  const read = {
+    scorer: binding.scorer,
+    appliesTo: binding.appliesTo,
+    sampling: binding.sampling
+  };
+  const groundTruth = binding.groundTruth;
+  if (groundTruth !== void 0) read.groundTruth = groundTruth;
+  const context = binding.context;
+  if (context !== void 0) read.context = context;
+  return read;
+};
+const copyCase = (suiteCase, index) => Effect.gen(function* () {
+  const input = yield* clone(suiteCase.input, `cases[${index}].input`);
+  if (suiteCase.expected === void 0) return Object.freeze({ name: suiteCase.name, input });
+  const expected = yield* clone(suiteCase.expected, `cases[${index}].expected`);
+  return Object.freeze({ name: suiteCase.name, input, expected });
+});
+const copyBinding = (binding, index) => Effect.gen(function* () {
+  const sampling = binding.sampling === void 0 ? void 0 : yield* clone(binding.sampling, `bindings[${index}].sampling`);
+  const copied = {
+    scorer: binding.scorer,
+    appliesTo: binding.appliesTo,
+    sampling
+  };
+  if (binding.groundTruth !== void 0) {
+    copied.groundTruth = yield* clone(binding.groundTruth, `bindings[${index}].groundTruth`);
+  }
+  if (binding.context !== void 0) {
+    copied.context = yield* clone(binding.context, `bindings[${index}].context`);
+  }
+  return Object.freeze(copied);
+});
+const make = (options) => Effect.suspend(() => {
+  const name = options.name;
+  const concurrency = options.concurrency;
+  const cases = [...options.cases].map(readCase);
+  const bindings = [...options.bindings ?? []].map(readBinding);
+  const trials = options.trials ?? 1;
+  const snapshot = { name, concurrency, cases, bindings, trials };
+  const error = validate(snapshot);
+  if (error !== void 0) return Effect.fail(error);
+  return Effect.gen(function* () {
+    const copiedCases = yield* Effect.forEach(cases, copyCase);
+    const copiedBindings = yield* Effect.forEach(bindings, copyBinding);
+    return Object.freeze({
+      name,
+      cases: Object.freeze(copiedCases),
+      bindings: Object.freeze(copiedBindings),
+      concurrency,
+      trials
+    });
+  });
+});
+const jsonCase = Schema.Struct({
+  name: Schema.String,
+  input: Schema.Unknown,
+  expected: Schema.optional(Schema.Unknown)
+});
+const fromJsonLines = (text, options) => Effect.gen(function* () {
+  if (text.length > limits.fixtureLength) {
+    return yield* Effect.fail(
+      invalid(
+        `JSON Lines fixture must be at most ${limits.fixtureLength} characters, got ${text.length}`,
+        "text"
+      )
+    );
+  }
+  const cases = [];
+  const body = text.startsWith("\uFEFF") ? text.slice(1) : text;
+  for (const [index, line] of body.split(/\r?\n/).entries()) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) continue;
+    if (cases.length === limits.cases) {
+      return yield* Effect.fail(
+        invalid(`Suite must contain at most ${limits.cases} cases, got ${cases.length + 1}`, "cases")
+      );
+    }
+    const decoded = yield* Effect.try({
+      try: () => JSON.parse(trimmed),
+      catch: (cause) => invalid(`Invalid JSON on line ${index + 1}`, `line[${index + 1}]`, cause)
+    }).pipe(
+      Effect.flatMap(Schema.decodeUnknownEffect(jsonCase)),
+      Effect.mapError(
+        (cause) => cause instanceof import_EvalError.EvalError ? cause : invalid(`Invalid suite case on line ${index + 1}`, `line[${index + 1}]`, cause)
+      )
+    );
+    cases.push(decoded);
+  }
+  return yield* make({ ...options, cases });
+});
+//# sourceMappingURL=Suite.js.map

package/dist/cjs/Suite.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/Suite.ts"],
+  "sourcesContent": ["/**\n * Fixed evaluation suite declarations.\n *\n * A suite is the fixed input half of an evaluation: named cases, the scorer\n * bindings that grade them, and the concurrency the runner is allowed. It is\n * validated once and then never changes, because the run that produced a\n * committed baseline has to be reproducible from the suite value that was\n * validated.\n *\n * @since 0.1.0\n */\n\nimport type * as ScorerBinding from \"@smthrs/scorers/Binding\"\nimport * as Effect from \"effect/Effect\"\nimport * as Schema from \"effect/Schema\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\n\n/**\n * A scorer binding accepted from `@smthrs/scorers`.\n *\n * The binding's `appliesTo` flow is matched against an execution's `target` by\n * reference identity, so a binding only ever grades the exact flow value it was\n * declared against.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Binding = ScorerBinding.Binding\n\n/**\n * One immutable fixed-suite case.\n *\n * `input` is handed to the executor and `expected` is offered to a bound scorer\n * as ground truth. A declared `expected`, including `null`, wins over the\n * binding's `groundTruth`; only an absent one defers to the binding. Both are\n * snapshots taken by {@link make}: mutating the object a caller passed in never\n * changes the suite.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Case {\n  readonly name: string\n  readonly input: unknown\n  readonly expected?: unknown | undefined\n}\n\n/**\n * Options for constructing a suite.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MakeOptions {\n  readonly name: string\n  readonly cases: ReadonlyArray<Case>\n  readonly bindings?: ReadonlyArray<Binding> | undefined\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\n/**\n * A validated, named collection of fixed cases and scorer bindings.\n *\n * The suite, cases, bindings, and their inert data are deeply frozen snapshots.\n * Executable scorer and target identities remain unchanged.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Suite {\n  readonly name: string\n  readonly cases: ReadonlyArray<Case>\n  readonly bindings: ReadonlyArray<Binding>\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\n/**\n * The declared ceilings a suite is validated against.\n *\n * They exist so a mistake fails at construction with a sentence instead of\n * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold\n * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines\n * fixture.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  concurrency: 1024,\n  cases: 10_000,\n  /** Longest JSON Lines fixture, in UTF-16 code units. */\n  fixtureLength: 8 * 1024 * 1024\n} as const\n\nconst invalid = (message: string, path: string, cause?: unknown): EvalError =>\n  new EvalError({ code: \"invalid_suite\", message, path, ...(cause === undefined ? {} : { cause }) })\n\nconst cloneMessage = \"Suite data must be structured-cloneable so the suite cannot change after it is validated\"\n\n// Validate before structuredClone can erase a prototype or invoke a getter.\n// Only records and arrays can be protected completely with Object.freeze.\nconst validateData = (value: unknown, path: string, seen = new WeakSet<object>()): void => {\n  if (typeof value === \"function\" || typeof value === \"symbol\") throw invalid(cloneMessage, path)\n  if (value === null || typeof value !== \"object\") return\n  if (seen.has(value)) return\n  seen.add(value)\n  const array = Array.isArray(value)\n  const prototype = Object.getPrototypeOf(value)\n  if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) {\n    throw invalid(\"Suite data must contain only plain objects, arrays, and structured-cloneable primitives\", path)\n  }\n  for (const key of Reflect.ownKeys(value)) {\n    if (array && key === \"length\") continue\n    const childPath = typeof key === \"string\" && array && /^(0|[1-9]\\d*)$/.test(key)\n      ? `${path}[${key}]`\n      : `${path}.${String(key)}`\n    const descriptor = Object.getOwnPropertyDescriptor(value, key)!\n    if (typeof key === \"symbol\" || !descriptor.enumerable || !(\"value\" in descriptor)) {\n      throw invalid(\"Suite data properties must be enumerable string-keyed data fields\", childPath)\n    }\n    validateData(descriptor.value, childPath, seen)\n  }\n}\n\nconst freezeData = (value: unknown, seen = new WeakSet<object>()): unknown => {\n  if (value === null || typeof value !== \"object\" || seen.has(value)) return value\n  seen.add(value)\n  for (const child of Object.values(value)) freezeData(child, seen)\n  return Object.freeze(value)\n}\n\nconst clone = (value: unknown, path: string): Effect.Effect<unknown, EvalError> =>\n  Effect.try({\n    try: () => {\n      validateData(value, path)\n      return freezeData(structuredClone(value))\n    },\n    catch: (cause) => cause instanceof EvalError ? cause : invalid(cloneMessage, path, cause)\n  })\n\nconst validate = (options: MakeOptions & { readonly trials: number }): EvalError | undefined => {\n  if (options.name.trim().length === 0) return invalid(\"Suite name must not be empty\", \"name\")\n  const nameControl = controlCharacter(options.name)\n  if (nameControl !== undefined) {\n    return invalid(`Suite name must not contain the control character ${nameControl}`, \"name\")\n  }\n  if (!Number.isSafeInteger(options.trials) || options.trials < 1 || options.trials > 1000) {\n    return invalid(\"Suite trials must be an integer from 1 to 1000\", \"trials\")\n  }\n  if (options.cases.length === 0) return invalid(\"Suite must contain at least one case\", \"cases\")\n  if (options.cases.length > limits.cases) {\n    return invalid(`Suite must contain at most ${limits.cases} cases, got ${options.cases.length}`, \"cases\")\n  }\n  if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {\n    return invalid(\n      `Suite concurrency must be a positive safe integer, got ${String(options.concurrency)}`,\n      \"concurrency\"\n    )\n  }\n  if (options.concurrency > limits.concurrency) {\n    return invalid(\n      `Suite concurrency must be at most ${limits.concurrency}, got ${options.concurrency}`,\n      \"concurrency\"\n    )\n  }\n  const names = new Set<string>()\n  for (const [index, suiteCase] of options.cases.entries()) {\n    if (suiteCase.name.trim().length === 0) {\n      return invalid(\"Suite case name must not be empty\", `cases[${index}].name`)\n    }\n    const caseControl = controlCharacter(suiteCase.name)\n    if (caseControl !== undefined) {\n      return invalid(\n        `Suite case name must not contain the control character ${caseControl}`,\n        `cases[${index}].name`\n      )\n    }\n    if (names.has(suiteCase.name)) {\n      return invalid(`Duplicate suite case: ${suiteCase.name}`, `cases[${index}].name`)\n    }\n    names.add(suiteCase.name)\n  }\n  return undefined\n}\n\n// Every field is read exactly once, into these two shapes, before anything is\n// validated or copied. Reading a field twice let a getter return the name that\n// passed validation and then a different one into the suite, which is the same\n// hole a baseline record had.\nconst readCase = (suiteCase: Case): Case => ({\n  name: suiteCase.name,\n  input: suiteCase.input,\n  expected: suiteCase.expected\n})\n\nconst readBinding = (binding: Binding): Binding => {\n  const read: { -readonly [K in keyof Binding]: Binding[K] } = {\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo,\n    sampling: binding.sampling\n  }\n  const groundTruth = binding.groundTruth\n  if (groundTruth !== undefined) read.groundTruth = groundTruth\n  const context = binding.context\n  if (context !== undefined) read.context = context\n  return read\n}\n\nconst copyCase = (suiteCase: Case, index: number): Effect.Effect<Case, EvalError> =>\n  Effect.gen(function*() {\n    const input = yield* clone(suiteCase.input, `cases[${index}].input`)\n    if (suiteCase.expected === undefined) return Object.freeze({ name: suiteCase.name, input })\n    const expected = yield* clone(suiteCase.expected, `cases[${index}].expected`)\n    return Object.freeze({ name: suiteCase.name, input, expected })\n  })\n\nconst copyBinding = (binding: Binding, index: number): Effect.Effect<Binding, EvalError> =>\n  Effect.gen(function*() {\n    // `scorer` and `appliesTo` are executable identities matched by reference,\n    // so they are carried over unchanged; sampling, ground truth, and context\n    // are inert data that must not remain caller-owned.\n    const sampling = binding.sampling === undefined\n      ? undefined\n      : yield* clone(binding.sampling, `bindings[${index}].sampling`)\n    const copied: { -readonly [K in keyof Binding]: Binding[K] } = {\n      scorer: binding.scorer,\n      appliesTo: binding.appliesTo,\n      sampling: sampling as Binding[\"sampling\"]\n    }\n    if (binding.groundTruth !== undefined) {\n      copied.groundTruth = yield* clone(binding.groundTruth, `bindings[${index}].groundTruth`)\n    }\n    if (binding.context !== undefined) {\n      copied.context = yield* clone(binding.context, `bindings[${index}].context`)\n    }\n    return Object.freeze(copied)\n  })\n\n/**\n * Builds and validates a fixed suite.\n *\n * When the effect runs, it reads every option, case field, and binding field\n * exactly once, then validates and copies only what it read. Validating one\n * value and copying another let a getter hand the suite something validation\n * never saw. Every case and binding is then copied, including a binding's\n * sampling policy, so the suite is a snapshot the caller can no longer reach:\n * mutating an array, case input, or ratio policy afterwards leaves the\n * validated suite unchanged. Data is checked before `structuredClone` and\n * deeply frozen afterwards. Only plain objects, arrays, and cloneable\n * primitives are admitted, with enumerable string-keyed data properties.\n * Functions, class instances, mutable built-ins, and accessors fail with\n * `invalid_suite` naming the offending path. Cycles and shared references are\n * supported; null-prototype records become ordinary objects in the clone.\n *\n * Fails with `invalid_suite` for an empty or control-character name, no cases,\n * more than `limits.cases` cases, a duplicate case name, or a concurrency that\n * is not a safe integer in `[1, limits.concurrency]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (options: MakeOptions): Effect.Effect<Suite, EvalError> =>\n  Effect.suspend(() => {\n    const name = options.name\n    const concurrency = options.concurrency\n    const cases = [...options.cases].map(readCase)\n    const bindings = [...(options.bindings ?? [])].map(readBinding)\n    const trials = options.trials ?? 1\n    const snapshot = { name, concurrency, cases, bindings, trials }\n    const error = validate(snapshot)\n    if (error !== undefined) return Effect.fail(error)\n    return Effect.gen(function*() {\n      const copiedCases = yield* Effect.forEach(cases, copyCase)\n      const copiedBindings = yield* Effect.forEach(bindings, copyBinding)\n      return Object.freeze({\n        name,\n        cases: Object.freeze(copiedCases),\n        bindings: Object.freeze(copiedBindings),\n        concurrency,\n        trials\n      })\n    })\n  })\n\n/**\n * Options used when decoding JSON Lines.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface JsonLinesOptions {\n  readonly name: string\n  readonly bindings?: ReadonlyArray<Binding> | undefined\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\nconst jsonCase = Schema.Struct({\n  name: Schema.String,\n  input: Schema.Unknown,\n  expected: Schema.optional(Schema.Unknown)\n})\n\n/**\n * Loads the `{ name, input, expected? }` JSON Lines fixture format.\n *\n * Blank lines are skipped, a leading byte-order mark is stripped, and both LF\n * and CRLF terminate a line. A malformed line fails with `invalid_suite`\n * carrying the 1-based line number in both the message and the path; a fixture\n * larger than `limits.fixtureLength` is rejected before any of it is parsed.\n * The first non-blank line after `limits.cases` cases fails with\n * `invalid_suite` at `cases` before it or any later line is parsed or decoded.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromJsonLines = (text: string, options: JsonLinesOptions): Effect.Effect<Suite, EvalError> =>\n  Effect.gen(function*() {\n    if (text.length > limits.fixtureLength) {\n      return yield* Effect.fail(\n        invalid(\n          `JSON Lines fixture must be at most ${limits.fixtureLength} characters, got ${text.length}`,\n          \"text\"\n        )\n      )\n    }\n    const cases: Array<Case> = []\n    const body = text.startsWith(\"\\uFEFF\") ? text.slice(1) : text\n    for (const [index, line] of body.split(/\\r?\\n/).entries()) {\n      const trimmed = line.trim()\n      if (trimmed.length === 0) continue\n      // Stop at the ceiling before parsing or decoding the excess line: a\n      // fixture under the length limit can still hold hundreds of thousands\n      // of tiny lines, and reporting the count only in make() would parse and\n      // materialize every one of them first.\n      if (cases.length === limits.cases) {\n        return yield* Effect.fail(\n          invalid(`Suite must contain at most ${limits.cases} cases, got ${cases.length + 1}`, \"cases\")\n        )\n      }\n      const decoded = yield* Effect.try({\n        try: () => JSON.parse(trimmed) as unknown,\n        catch: (cause) => invalid(`Invalid JSON on line ${index + 1}`, `line[${index + 1}]`, cause)\n      }).pipe(\n        Effect.flatMap(Schema.decodeUnknownEffect(jsonCase)),\n        Effect.mapError((cause) =>\n          cause instanceof EvalError\n            ? cause\n            : invalid(`Invalid suite case on line ${index + 1}`, `line[${index + 1}]`, cause)\n        )\n      )\n      cases.push(decoded)\n    }\n    return yield* make({ ...options, cases })\n  })\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAaA,aAAwB;AACxB,aAAwB;AACxB,uBAA0B;AAC1B,+BAAiC;AA0E1B,MAAM,SAAS;AAAA,EACpB,aAAa;AAAA,EACb,OAAO;AAAA;AAAA,EAEP,eAAe,IAAI,OAAO;AAC5B;AAEA,MAAM,UAAU,CAAC,SAAiB,MAAc,UAC9C,IAAI,2BAAU,EAAE,MAAM,iBAAiB,SAAS,MAAM,GAAI,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM,EAAG,CAAC;AAEnG,MAAM,eAAe;AAIrB,MAAM,eAAe,CAAC,OAAgB,MAAc,OAAO,oBAAI,QAAgB,MAAY;AACzF,MAAI,OAAO,UAAU,cAAc,OAAO,UAAU,SAAU,OAAM,QAAQ,cAAc,IAAI;AAC9F,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU;AACjD,MAAI,KAAK,IAAI,KAAK,EAAG;AACrB,OAAK,IAAI,KAAK;AACd,QAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,QAAM,YAAY,OAAO,eAAe,KAAK;AAC7C,MAAI,QAAQ,cAAc,MAAM,YAAY,cAAc,OAAO,aAAa,cAAc,MAAM;AAChG,UAAM,QAAQ,2FAA2F,IAAI;AAAA,EAC/G;AACA,aAAW,OAAO,QAAQ,QAAQ,KAAK,GAAG;AACxC,QAAI,SAAS,QAAQ,SAAU;AAC/B,UAAM,YAAY,OAAO,QAAQ,YAAY,SAAS,iBAAiB,KAAK,GAAG,IAC3E,GAAG,IAAI,IAAI,GAAG,MACd,GAAG,IAAI,IAAI,OAAO,GAAG,CAAC;AAC1B,UAAM,aAAa,OAAO,yBAAyB,OAAO,GAAG;AAC7D,QAAI,OAAO,QAAQ,YAAY,CAAC,WAAW,cAAc,EAAE,WAAW,aAAa;AACjF,YAAM,QAAQ,qEAAqE,SAAS;AAAA,IAC9F;AACA,iBAAa,WAAW,OAAO,WAAW,IAAI;AAAA,EAChD;AACF;AAEA,MAAM,aAAa,CAAC,OAAgB,OAAO,oBAAI,QAAgB,MAAe;AAC5E,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,KAAK,IAAI,KAAK,EAAG,QAAO;AAC3E,OAAK,IAAI,KAAK;AACd,aAAW,SAAS,OAAO,OAAO,KAAK,EAAG,YAAW,OAAO,IAAI;AAChE,SAAO,OAAO,OAAO,KAAK;AAC5B;AAEA,MAAM,QAAQ,CAAC,OAAgB,SAC7B,OAAO,IAAI;AAAA,EACT,KAAK,MAAM;AACT,iBAAa,OAAO,IAAI;AACxB,WAAO,WAAW,gBAAgB,KAAK,CAAC;AAAA,EAC1C;AAAA,EACA,OAAO,CAAC,UAAU,iBAAiB,6BAAY,QAAQ,QAAQ,cAAc,MAAM,KAAK;AAC1F,CAAC;AAEH,MAAM,WAAW,CAAC,YAA8E;AAC9F,MAAI,QAAQ,KAAK,KAAK,EAAE,WAAW,EAAG,QAAO,QAAQ,gCAAgC,MAAM;AAC3F,QAAM,kBAAc,2CAAiB,QAAQ,IAAI;AACjD,MAAI,gBAAgB,QAAW;AAC7B,WAAO,QAAQ,qDAAqD,WAAW,IAAI,MAAM;AAAA,EAC3F;AACA,MAAI,CAAC,OAAO,cAAc,QAAQ,MAAM,KAAK,QAAQ,SAAS,KAAK,QAAQ,SAAS,KAAM;AACxF,WAAO,QAAQ,kDAAkD,QAAQ;AAAA,EAC3E;AACA,MAAI,QAAQ,MAAM,WAAW,EAAG,QAAO,QAAQ,wCAAwC,OAAO;AAC9F,MAAI,QAAQ,MAAM,SAAS,OAAO,OAAO;AACvC,WAAO,QAAQ,8BAA8B,OAAO,KAAK,eAAe,QAAQ,MAAM,MAAM,IAAI,OAAO;AAAA,EACzG;AACA,MAAI,CAAC,OAAO,cAAc,QAAQ,WAAW,KAAK,QAAQ,cAAc,GAAG;AACzE,WAAO;AAAA,MACL,0DAA0D,OAAO,QAAQ,WAAW,CAAC;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AACA,MAAI,QAAQ,cAAc,OAAO,aAAa;AAC5C,WAAO;AAAA,MACL,qCAAqC,OAAO,WAAW,SAAS,QAAQ,WAAW;AAAA,MACnF;AAAA,IACF;AAAA,EACF;AACA,QAAM,QAAQ,oBAAI,IAAY;AAC9B,aAAW,CAAC,OAAO,SAAS,KAAK,QAAQ,MAAM,QAAQ,GAAG;AACxD,QAAI,UAAU,KAAK,KAAK,EAAE,WAAW,GAAG;AACtC,aAAO,QAAQ,qCAAqC,SAAS,KAAK,QAAQ;AAAA,IAC5E;AACA,UAAM,kBAAc,2CAAiB,UAAU,IAAI;AACnD,QAAI,gBAAgB,QAAW;AAC7B,aAAO;AAAA,QACL,0DAA0D,WAAW;AAAA,QACrE,SAAS,KAAK;AAAA,MAChB;AAAA,IACF;AACA,QAAI,MAAM,IAAI,UAAU,IAAI,GAAG;AAC7B,aAAO,QAAQ,yBAAyB,UAAU,IAAI,IAAI,SAAS,KAAK,QAAQ;AAAA,IAClF;AACA,UAAM,IAAI,UAAU,IAAI;AAAA,EAC1B;AACA,SAAO;AACT;AAMA,MAAM,WAAW,CAAC,eAA2B;AAAA,EAC3C,MAAM,UAAU;AAAA,EAChB,OAAO,UAAU;AAAA,EACjB,UAAU,UAAU;AACtB;AAEA,MAAM,cAAc,CAAC,YAA8B;AACjD,QAAM,OAAuD;AAAA,IAC3D,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB,UAAU,QAAQ;AAAA,EACpB;AACA,QAAM,cAAc,QAAQ;AAC5B,MAAI,gBAAgB,OAAW,MAAK,cAAc;AAClD,QAAM,UAAU,QAAQ;AACxB,MAAI,YAAY,OAAW,MAAK,UAAU;AAC1C,SAAO;AACT;AAEA,MAAM,WAAW,CAAC,WAAiB,UACjC,OAAO,IAAI,aAAY;AACrB,QAAM,QAAQ,OAAO,MAAM,UAAU,OAAO,SAAS,KAAK,SAAS;AACnE,MAAI,UAAU,aAAa,OAAW,QAAO,OAAO,OAAO,EAAE,MAAM,UAAU,MAAM,MAAM,CAAC;AAC1F,QAAM,WAAW,OAAO,MAAM,UAAU,UAAU,SAAS,KAAK,YAAY;AAC5E,SAAO,OAAO,OAAO,EAAE,MAAM,UAAU,MAAM,OAAO,SAAS,CAAC;AAChE,CAAC;AAEH,MAAM,cAAc,CAAC,SAAkB,UACrC,OAAO,IAAI,aAAY;AAIrB,QAAM,WAAW,QAAQ,aAAa,SAClC,SACA,OAAO,MAAM,QAAQ,UAAU,YAAY,KAAK,YAAY;AAChE,QAAM,SAAyD;AAAA,IAC7D,QAAQ,QAAQ;AAAA,IAChB,WAAW,QAAQ;AAAA,IACnB;AAAA,EACF;AACA,MAAI,QAAQ,gBAAgB,QAAW;AACrC,WAAO,cAAc,OAAO,MAAM,QAAQ,aAAa,YAAY,KAAK,eAAe;AAAA,EACzF;AACA,MAAI,QAAQ,YAAY,QAAW;AACjC,WAAO,UAAU,OAAO,MAAM,QAAQ,SAAS,YAAY,KAAK,WAAW;AAAA,EAC7E;AACA,SAAO,OAAO,OAAO,MAAM;AAC7B,CAAC;AAyBI,MAAM,OAAO,CAAC,YACnB,OAAO,QAAQ,MAAM;AACnB,QAAM,OAAO,QAAQ;AACrB,QAAM,cAAc,QAAQ;AAC5B,QAAM,QAAQ,CAAC,GAAG,QAAQ,KAAK,EAAE,IAAI,QAAQ;AAC7C,QAAM,WAAW,CAAC,GAAI,QAAQ,YAAY,CAAC,CAAE,EAAE,IAAI,WAAW;AAC9D,QAAM,SAAS,QAAQ,UAAU;AACjC,QAAM,WAAW,EAAE,MAAM,aAAa,OAAO,UAAU,OAAO;AAC9D,QAAM,QAAQ,SAAS,QAAQ;AAC/B,MAAI,UAAU,OAAW,QAAO,OAAO,KAAK,KAAK;AACjD,SAAO,OAAO,IAAI,aAAY;AAC5B,UAAM,cAAc,OAAO,OAAO,QAAQ,OAAO,QAAQ;AACzD,UAAM,iBAAiB,OAAO,OAAO,QAAQ,UAAU,WAAW;AAClE,WAAO,OAAO,OAAO;AAAA,MACnB;AAAA,MACA,OAAO,OAAO,OAAO,WAAW;AAAA,MAChC,UAAU,OAAO,OAAO,cAAc;AAAA,MACtC;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACH,CAAC;AAeH,MAAM,WAAW,OAAO,OAAO;AAAA,EAC7B,MAAM,OAAO;AAAA,EACb,OAAO,OAAO;AAAA,EACd,UAAU,OAAO,SAAS,OAAO,OAAO;AAC1C,CAAC;AAeM,MAAM,gBAAgB,CAAC,MAAc,YAC1C,OAAO,IAAI,aAAY;AACrB,MAAI,KAAK,SAAS,OAAO,eAAe;AACtC,WAAO,OAAO,OAAO;AAAA,MACnB;AAAA,QACE,sCAAsC,OAAO,aAAa,oBAAoB,KAAK,MAAM;AAAA,QACzF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,QAAM,QAAqB,CAAC;AAC5B,QAAM,OAAO,KAAK,WAAW,QAAQ,IAAI,KAAK,MAAM,CAAC,IAAI;AACzD,aAAW,CAAC,OAAO,IAAI,KAAK,KAAK,MAAM,OAAO,EAAE,QAAQ,GAAG;AACzD,UAAM,UAAU,KAAK,KAAK;AAC1B,QAAI,QAAQ,WAAW,EAAG;AAK1B,QAAI,MAAM,WAAW,OAAO,OAAO;AACjC,aAAO,OAAO,OAAO;AAAA,QACnB,QAAQ,8BAA8B,OAAO,KAAK,eAAe,MAAM,SAAS,CAAC,IAAI,OAAO;AAAA,MAC9F;AAAA,IACF;AACA,UAAM,UAAU,OAAO,OAAO,IAAI;AAAA,MAChC,KAAK,MAAM,KAAK,MAAM,OAAO;AAAA,MAC7B,OAAO,CAAC,UAAU,QAAQ,wBAAwB,QAAQ,CAAC,IAAI,QAAQ,QAAQ,CAAC,KAAK,KAAK;AAAA,IAC5F,CAAC,EAAE;AAAA,MACD,OAAO,QAAQ,OAAO,oBAAoB,QAAQ,CAAC;AAAA,MACnD,OAAO;AAAA,QAAS,CAAC,UACf,iBAAiB,6BACb,QACA,QAAQ,8BAA8B,QAAQ,CAAC,IAAI,QAAQ,QAAQ,CAAC,KAAK,KAAK;AAAA,MACpF;AAAA,IACF;AACA,UAAM,KAAK,OAAO;AAAA,EACpB;AACA,SAAO,OAAO,KAAK,EAAE,GAAG,SAAS,MAAM,CAAC;AAC1C,CAAC;",
+  "names": []
+}

package/dist/cjs/Trials.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Repeated-run statistics for live model evaluations.
+ *
+ * A live case runs `n` times and passes `c` of them. {@link passAtK} is the
+ * chance that at least one of `k` runs passes, the capability a best-of-k
+ * caller sees; {@link passHatK} is the chance that all `k` pass, the
+ * consistency a user sees on every attempt. Both are exact for sampling `k` of
+ * the `n` recorded runs without replacement.
+ *
+ * Every function is pure and throws an `EvalError` with code `invalid_trials`
+ * for counts outside the documented bounds.
+ *
+ * @since 0.1.0
+ */
+/**
+ * Repeated-run statistics for one case.
+ *
+ * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,
+ * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Summary {
+    readonly n: number;
+    readonly passes: number;
+    readonly rate: number;
+    readonly passAt1: number;
+    readonly passAtK: number;
+    readonly passHatK: number;
+    readonly stderr: number;
+}
+/**
+ * Repeated-run statistics across cases.
+ *
+ * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is
+ * the fraction of cases whose every run passed.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Aggregate {
+    readonly cases: number;
+    readonly passAt1: number;
+    readonly passAtK: number;
+    readonly passHatK: number;
+    readonly allPass: number;
+    readonly perCase: Readonly<Record<string, Summary>>;
+}
+/**
+ * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,
+ * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is
+ * computed as a product, so large `n` neither overflows nor loses precision.
+ *
+ * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with
+ * `0 <= c <= n` and `1 <= k <= n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const passAtK: (n: number, c: number, k: number) => number;
+/**
+ * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`
+ * without replacement pass, the consistency measure of tau-bench.
+ *
+ * Throws `invalid_trials` under the same bounds as {@link passAtK}.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const passHatK: (n: number, c: number, k: number) => number;
+/**
+ * Summarizes one case's run outcomes at `k`.
+ *
+ * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const summarize: (outcomes: ReadonlyArray<boolean>, k: number) => Summary;
+/**
+ * Summarizes every case at `k` and averages over cases.
+ *
+ * A case with fewer than `k` runs is summarized at `k = n`, its own run
+ * count, so a short case still contributes rather than failing the whole
+ * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case
+ * with no runs or a `k` that is not a positive integer.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const aggregate: (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number) => Aggregate;
+//# sourceMappingURL=Trials.d.ts.map

package/dist/cjs/Trials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAiB9F,CAAA"}