@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/Runner.d.ts

@@ -0,0 +1,229 @@
+/**
+ * Deterministic fixed-suite execution and bound scorer evaluation.
+ *
+ * A run executes every case through the injected {@link CaseExecutor}, then
+ * grades each execution with the scorers bound to the flow it executed. The
+ * timestamp and the run identity come from the caller, so two runs of the same
+ * suite over the same inputs produce byte-identical observations.
+ *
+ * @since 0.1.0
+ */
+import * as ScorerRunner from "@smthrs/scorers/Runner";
+import type * as ScoreStore from "@smthrs/scorers/ScoreStore";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { CaseExecutor, type Execution } from "./CaseExecutor.ts";
+import { EvalError } from "./EvalError.ts";
+import type { Binding, Suite } from "./Suite.ts";
+import * as Trials from "./Trials.ts";
+/**
+ * One score observation emitted by a suite run.
+ *
+ * `scorer` is the scorer key, a digest of the scorer's own declaration, which
+ * is what a baseline matches on. `scorerName` is the human name from the same
+ * declaration, carried alongside so a report can be read without grepping for
+ * the digest. `at` is the run's timestamp, not the scorer's: a run's
+ * observations all carry one instant so a baseline stays reproducible.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Observation = {
+    readonly case: string;
+    readonly scorer: string;
+    readonly scorerName?: string | undefined;
+    readonly stepKey: string;
+    readonly at: string;
+} & ({
+    readonly kind: "score";
+    readonly score: number;
+    readonly reason?: string | undefined;
+    readonly meta?: unknown;
+} | {
+    readonly kind: "inconclusive";
+    readonly reason: string;
+    readonly meta?: unknown;
+});
+/**
+ * A request sent to the scorers batch runner.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface ScoreRequest {
+    readonly case: string;
+    readonly stepKey: string;
+    readonly binding: Binding;
+    readonly input: {
+        readonly input: unknown;
+        readonly output: unknown;
+        readonly groundTruth?: unknown;
+        readonly context?: unknown;
+        readonly latencyMs?: number;
+    };
+}
+/**
+ * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ScoreJob = ScorerRunner.Job;
+/**
+ * Structural adapter for `@smthrs/scorers`' blocking batch runner.
+ *
+ * A runner that implements `runBatchCorrelated` tags every observation with
+ * its job identity and may return results in any order. A runner that implements
+ * only `runBatch` is correlated positionally: it must return exactly one
+ * observation per job, in order, and each observation must echo its job's
+ * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before
+ * scoring when two jobs share that pair, because their results could not be
+ * attributed to a case.
+ *
+ * A run fails with `scorer_protocol` when either protocol is broken.
+ * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its
+ * service value can be used directly and `ambiguous_score_job` never applies
+ * to it.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export interface ScoreBatchRunner {
+    readonly runBatch: (jobs: ReadonlyArray<ScoreJob>, options?: {
+        readonly concurrency?: number | undefined;
+    }) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>;
+    readonly runBatchCorrelated?: ((jobs: ReadonlyArray<ScoreJob>, options?: {
+        readonly concurrency?: number | undefined;
+    }) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>) | undefined;
+}
+/**
+ * A score result aligned with a {@link ScoreRequest}.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ScoreObservation = ScoreStore.Observation;
+/**
+ * A batch result tagged with the identity of the job that produced it.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface BatchResult {
+    readonly identity: string;
+    readonly observation: ScoreObservation;
+}
+/**
+ * Per-case result retained by the deterministic runner.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface CaseResult {
+    readonly case: string;
+    readonly execution?: Execution | undefined;
+    readonly error?: EvalError | undefined;
+    readonly observations: ReadonlyArray<Observation>;
+    readonly trials?: Trials.Summary | undefined;
+}
+/**
+ * Stable result of a suite run.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface RunResult {
+    readonly runId: string;
+    readonly suite: string;
+    readonly cases: ReadonlyArray<CaseResult>;
+    readonly observations: ReadonlyArray<Observation>;
+    readonly trials?: Trials.Aggregate | undefined;
+    readonly k?: number | undefined;
+}
+/**
+ * Options for a deterministic suite run.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface RunOptions {
+    readonly scorer?: ScoreBatchRunner | undefined;
+    readonly runId: string;
+    readonly sampleId?: string | undefined;
+    readonly trials?: number | undefined;
+    readonly k?: number | undefined;
+    readonly at: string;
+}
+declare const Runner_base: Context.ServiceClass<Runner, "flows/evals/Runner", ScoreBatchRunner>;
+/**
+ * Injectable batch-runner service used when a caller wants a reusable adapter.
+ *
+ * {@link run} does not require it: a run scores with `options.scorer` when one
+ * is passed, otherwise with this service when one is provided, otherwise
+ * in process through {@link makeInline}.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export declare class Runner extends Runner_base {
+}
+/**
+ * Builds the in-process batch runner a run scores with by default.
+ *
+ * Each job's scorer runs in the current process, its result is checked by
+ * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an
+ * inconclusive observation naming its cause rather than failing the run. It
+ * honours the {@link ScoreBatchRunner} protocol, so it is also the reference
+ * a custom adapter can be compared against.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const makeInline: () => ScoreBatchRunner;
+/**
+ * Provides the in-process batch runner built by {@link makeInline}.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layerInline: Layer.Layer<Runner>;
+/**
+ * Runs a fixed suite with bounded execution and declaration-order results.
+ *
+ * Cases run through the provided {@link CaseExecutor} at the suite's
+ * concurrency and are returned in declaration order. Each executor receives
+ * its own mutable case copy. Each scorer receives independent copies of the
+ * original case input, expected value, and binding data. Every execution is then
+ * graded by the bindings whose `appliesTo` is the flow the execution reports as
+ * its target, matched by reference identity, so a binding attached to another
+ * flow contributes no observations.
+ *
+ * Scoring goes through `options.scorer` when the caller passes one, the
+ * {@link Runner} service when one is provided, and {@link makeInline}
+ * otherwise. A case whose target failed keeps its typed error, locates itself
+ * with the executor's own `path` or with `cases['<name>']`, and produces no
+ * observations; a scorer that failed produces an inconclusive observation. The
+ * run itself fails only with `invalid_run_options` for a non-canonical run
+ * identity or timestamp, `invalid_suite` for an unusable sampling policy,
+ * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,
+ * or `scorer_protocol` for a batch runner that broke the
+ * {@link ScoreBatchRunner} contract.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const run: (suite: Suite, options: RunOptions) => Effect.Effect<RunResult, EvalError, CaseExecutor>;
+/**
+ * Provides a batch runner that is never available.
+ *
+ * Every bound score under this layer becomes an inconclusive observation, which
+ * a gate grades as an undecidable run rather than a red. Provide it to state
+ * that a suite must not score, and {@link layerInline} to score in process.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layerNoop: Layer.Layer<Runner>;
+export {};
+//# sourceMappingURL=Runner.d.ts.map

package/dist/cjs/Runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAyKhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAoFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}

package/dist/cjs/Runner.js

@@ -0,0 +1,430 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var Runner_exports = {};
+__export(Runner_exports, {
+  Runner: () => Runner,
+  layerInline: () => layerInline,
+  layerNoop: () => layerNoop,
+  makeInline: () => makeInline,
+  run: () => run
+});
+module.exports = __toCommonJS(Runner_exports);
+var ScorerRunner = __toESM(require("@smthrs/scorers/Runner"), 1);
+var Sampling = __toESM(require("@smthrs/scorers/Sampling"), 1);
+var Scorer = __toESM(require("@smthrs/scorers/Scorer"), 1);
+var Cause = __toESM(require("effect/Cause"), 1);
+var Context = __toESM(require("effect/Context"), 1);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Layer = __toESM(require("effect/Layer"), 1);
+var import_CaseExecutor = require("./CaseExecutor.js");
+var import_EvalError = require("./EvalError.js");
+var import_controlCharacters = require("./internal/controlCharacters.js");
+var import_scorerLabel = require("./internal/scorerLabel.js");
+var import_tupleKey = require("./internal/tupleKey.js");
+var Trials = __toESM(require("./Trials.js"), 1);
+class Runner extends Context.Service()("flows/evals/Runner") {
+}
+const maxReasonLength = 2048;
+const boundedReason = (reason) => reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`;
+const scorerKeyOf = (binding) => binding.scorer.scorerKey;
+const scorerNameOf = (binding) => binding.scorer.name;
+const label = (binding) => (0, import_scorerLabel.scorerLabel)({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) });
+const inconclusive = (request, reason, at) => {
+  return {
+    case: request.case,
+    scorer: scorerKeyOf(request.binding),
+    scorerName: scorerNameOf(request.binding),
+    stepKey: request.stepKey,
+    kind: "inconclusive",
+    reason,
+    at
+  };
+};
+const casePath = (name) => `cases['${name}']`;
+const caseError = (suiteCase, cause) => {
+  const reason = Cause.squash(cause);
+  return reason instanceof import_EvalError.EvalError ? new import_EvalError.EvalError({
+    code: reason.code,
+    message: (0, import_controlCharacters.flattenControlCharacters)(`Target failed for case '${suiteCase.name}': ${reason.message}`),
+    path: reason.path ?? casePath(suiteCase.name),
+    cause: reason
+  }) : new import_EvalError.EvalError({
+    code: "executor",
+    message: (0, import_controlCharacters.flattenControlCharacters)(`Target failed for case '${suiteCase.name}': ${String(reason)}`),
+    path: casePath(suiteCase.name),
+    cause: reason
+  });
+};
+const runCase = (executor, suiteCase) => Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(
+  Effect.map((execution) => ({
+    case: suiteCase.name,
+    execution: { ...execution, stepKey: (0, import_controlCharacters.flattenControlCharacters)(execution.stepKey) },
+    observations: []
+  })),
+  Effect.catchCause(
+    (cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })
+  ),
+  Effect.withSpan("Eval.case", { attributes: { case: suiteCase.name } })
+);
+const groundTruthFor = (suiteCase, binding) => {
+  const groundTruth = suiteCase.expected !== void 0 ? suiteCase.expected : binding.groundTruth;
+  return groundTruth === void 0 ? {} : { groundTruth: structuredClone(groundTruth) };
+};
+const requestFor = (suiteCase, execution, binding) => ({
+  case: suiteCase.name,
+  stepKey: execution.stepKey,
+  binding: {
+    ...structuredClone({
+      sampling: binding.sampling,
+      ...binding.groundTruth === void 0 ? {} : { groundTruth: binding.groundTruth },
+      ...binding.context === void 0 ? {} : { context: binding.context }
+    }),
+    scorer: binding.scorer,
+    appliesTo: binding.appliesTo
+  },
+  input: {
+    input: structuredClone(suiteCase.input),
+    output: execution.output,
+    ...groundTruthFor(suiteCase, binding),
+    ...binding.context === void 0 ? {} : { context: structuredClone(binding.context) },
+    latencyMs: execution.latencyMs
+  }
+});
+const observationFor = (request, result, at) => {
+  const source = result;
+  const kind = source.kind;
+  if (kind !== "score" && kind !== "inconclusive") {
+    return inconclusive(
+      request,
+      boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),
+      at
+    );
+  }
+  const reason = source.reason;
+  if (kind === "inconclusive") {
+    return inconclusive(
+      request,
+      typeof reason === "string" && reason.length > 0 ? boundedReason(reason) : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),
+      at
+    );
+  }
+  const score2 = source.score;
+  if (typeof score2 !== "number" || !Number.isFinite(score2) || score2 < 0 || score2 > 1) {
+    return inconclusive(
+      request,
+      boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score2)}`),
+      at
+    );
+  }
+  const meta = source.meta;
+  return {
+    case: request.case,
+    scorer: scorerKeyOf(request.binding),
+    scorerName: scorerNameOf(request.binding),
+    stepKey: request.stepKey,
+    kind: "score",
+    score: score2,
+    ...typeof reason === "string" ? { reason: boundedReason(reason) } : {},
+    ...meta === void 0 ? {} : { meta },
+    at
+  };
+};
+const executeInline = (job) => job.score.pipe(
+  Effect.flatMap(Scorer.validate),
+  Effect.map((result) => ({
+    ...job.observation,
+    kind: "score",
+    score: result.score,
+    ...result.reason === void 0 ? {} : { reason: result.reason },
+    ...result.meta === void 0 ? {} : { meta: result.meta },
+    at: job.at
+  })),
+  Effect.catchCause(
+    (cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(ScorerRunner.inconclusive(job, cause))
+  )
+);
+const makeInline = () => {
+  const runBatchCorrelated = (jobs, options) => Effect.forEach(
+    jobs,
+    (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),
+    { concurrency: options?.concurrency ?? 1 }
+  );
+  return {
+    runBatchCorrelated,
+    runBatch: (jobs, options) => runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))
+  };
+};
+const layerInline = Layer.succeed(Runner)(makeInline());
+const buildJobs = (cases, suite, { at, runId, sampleId, concurrency }) => Effect.gen(function* () {
+  const candidates = cases.flatMap((caseResult, index) => {
+    const execution = caseResult.execution;
+    if (execution === void 0) return [];
+    const suiteCase = suite.cases[index];
+    return suite.bindings.flatMap(
+      (binding, bindingIndex) => binding.appliesTo === execution.target ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }] : []
+    );
+  });
+  const sampled = yield* Effect.forEach(
+    candidates,
+    ({ request, bindingIndex }) => Sampling.decide(
+      request.binding.sampling,
+      request.stepKey,
+      scorerKeyOf(request.binding)
+    ).pipe(
+      Effect.map((selected) => selected ? request : void 0),
+      Effect.mapError(
+        (cause) => new import_EvalError.EvalError({
+          code: "invalid_suite",
+          message: `Invalid sampling policy for scorer ${label(request.binding)}`,
+          path: `bindings[${bindingIndex}].sampling`,
+          cause
+        })
+      )
+    ),
+    { concurrency }
+  );
+  const requests = sampled.filter((request) => request !== void 0);
+  const jobs = requests.map((request, index) => ({
+    identity: (0, import_tupleKey.tupleKey)(
+      suite.name,
+      runId,
+      sampleId,
+      request.case,
+      request.stepKey,
+      scorerKeyOf(request.binding),
+      `${index}`
+    ),
+    observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },
+    score: Effect.suspend(() => request.binding.scorer.score(request.input)),
+    at: Date.parse(at)
+  }));
+  return { requests, jobs };
+});
+const batchFallback = (jobs, cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)));
+const scoreCorrelated = (jobs, scorer, runBatchCorrelated, { concurrency }) => Effect.gen(function* () {
+  const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(
+    Effect.catchCause(
+      (cause) => batchFallback(jobs, cause).pipe(
+        Effect.map(
+          (observations) => observations.map((observation, index) => ({
+            identity: jobs[index].identity,
+            observation
+          }))
+        )
+      )
+    )
+  );
+  if (batchResults.length !== jobs.length) {
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "scorer_protocol",
+        message: `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,
+        path: "runBatchCorrelated"
+      })
+    );
+  }
+  const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index]));
+  const byIdentity = /* @__PURE__ */ new Map();
+  let unknown;
+  for (const [index, result] of batchResults.entries()) {
+    const jobIndex = jobIndexByIdentity.get(result.identity);
+    if (jobIndex !== void 0 && byIdentity.has(result.identity)) {
+      return yield* Effect.fail(
+        new import_EvalError.EvalError({
+          code: "scorer_protocol",
+          message: `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,
+          path: `runBatchCorrelated[${index}]`
+        })
+      );
+    }
+    if (jobIndex === void 0) unknown = { identity: result.identity, index };
+    byIdentity.set(result.identity, result.observation);
+  }
+  if (unknown !== void 0) {
+    const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity));
+    const missingIdentity = jobs[missingJobIndex].identity;
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "scorer_protocol",
+        message: `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,
+        path: `runBatchCorrelated[${unknown.index}]`
+      })
+    );
+  }
+  return jobs.map((job) => byIdentity.get(job.identity));
+});
+const scoreOrderOnly = (jobs, requests, scorer, { concurrency }) => Effect.gen(function* () {
+  const pairIndexes = /* @__PURE__ */ new Map();
+  for (const [index, job] of jobs.entries()) {
+    const pair = (0, import_tupleKey.tupleKey)(job.observation.targetStepKey, job.observation.scorerKey);
+    const previous = pairIndexes.get(pair);
+    if (previous !== void 0) {
+      const first = requests[previous];
+      const second = requests[index];
+      return yield* Effect.fail(
+        new import_EvalError.EvalError({
+          code: "ambiguous_score_job",
+          message: `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${label(second.binding)}. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,
+          path: "runBatch"
+        })
+      );
+    }
+    pairIndexes.set(pair, index);
+  }
+  const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(
+    Effect.catchCause((cause) => batchFallback(jobs, cause))
+  );
+  if (results.length !== jobs.length) {
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "scorer_protocol",
+        message: `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,
+        path: "runBatch"
+      })
+    );
+  }
+  return results;
+});
+const score = (cases, suite, scorer, context) => Effect.gen(function* () {
+  const { requests, jobs } = yield* buildJobs(cases, suite, context);
+  if (requests.length === 0) return cases;
+  const runBatchCorrelated = scorer.runBatchCorrelated;
+  const protocol = runBatchCorrelated === void 0 ? "runBatch" : "runBatchCorrelated";
+  const results = yield* runBatchCorrelated === void 0 ? scoreOrderOnly(jobs, requests, scorer, context) : scoreCorrelated(jobs, scorer, runBatchCorrelated, context);
+  const { at } = context;
+  const observations = [];
+  for (const [index, request] of requests.entries()) {
+    const result = results[index];
+    if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {
+      return yield* Effect.fail(
+        new import_EvalError.EvalError({
+          code: "scorer_protocol",
+          message: protocol === "runBatchCorrelated" ? `Correlated scorer batch identity '${jobs[index].identity}' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${scorerKeyOf(request.binding)}' at step '${request.stepKey}'` : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${scorerKeyOf(request.binding)}' at step '${request.stepKey}'; results must stay aligned with their jobs`,
+          path: `${protocol}[${index}]`
+        })
+      );
+    }
+    observations.push(observationFor(request, result, at));
+  }
+  const byCase = /* @__PURE__ */ new Map();
+  for (const observation of observations) {
+    const bucket = byCase.get(observation.case);
+    if (bucket === void 0) byCase.set(observation.case, [observation]);
+    else bucket.push(observation);
+  }
+  return cases.map((caseResult) => ({
+    ...caseResult,
+    observations: byCase.get(caseResult.case) ?? []
+  }));
+});
+const run = (suite, options) => Effect.gen(function* () {
+  const executor = yield* import_CaseExecutor.CaseExecutor;
+  const injected = yield* Effect.serviceOption(Runner);
+  const scorer = options.scorer ?? (injected._tag === "Some" ? injected.value : makeInline());
+  const atMillis = Date.parse(options.at);
+  if (options.runId.trim().length === 0) {
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_run_options",
+        message: "Deterministic runs require a non-empty runId",
+        path: "options.runId"
+      })
+    );
+  }
+  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(options.at) || !Number.isFinite(atMillis) || new Date(atMillis).toISOString() !== options.at) {
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_run_options",
+        message: `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,
+        path: "options.at"
+      })
+    );
+  }
+  const n = options.trials ?? suite.trials ?? 1;
+  const k = options.k ?? 1;
+  if (!Number.isSafeInteger(n) || n < 1 || n > 1e3 || !Number.isSafeInteger(k) || k < 1 || k > n) {
+    return yield* Effect.fail(
+      new import_EvalError.EvalError({
+        code: "invalid_trials",
+        message: "trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials",
+        path: "options.trials"
+      })
+    );
+  }
+  const at = options.at;
+  const runId = options.runId;
+  const sampleId = options.sampleId ?? "default";
+  const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) => Effect.gen(function* () {
+    const cases2 = yield* Effect.forEach(suite.cases, (suiteCase) => runCase(executor, suiteCase), {
+      concurrency: suite.concurrency,
+      discard: false
+    });
+    return yield* score(cases2, suite, scorer, {
+      concurrency: suite.concurrency,
+      at,
+      runId,
+      sampleId: n === 1 ? sampleId : (0, import_tupleKey.tupleKey)(sampleId, String(index))
+    });
+  }), { concurrency: 1 });
+  const outcomes = {};
+  const cases = suite.cases.map((suiteCase, index) => {
+    const attempts = batches.map(
+      (batch) => batch[index]
+    );
+    const passed = attempts.map(
+      (attempt) => attempt.error === void 0 && attempt.observations.length > 0 && attempt.observations.every((observation) => observation.kind === "score" && observation.score === 1)
+    );
+    outcomes[suiteCase.name] = passed;
+    return {
+      ...attempts[0],
+      error: attempts.find((attempt) => attempt.error !== void 0)?.error,
+      observations: attempts.flatMap((attempt) => attempt.observations),
+      trials: Trials.summarize(passed, k)
+    };
+  });
+  return {
+    runId,
+    suite: suite.name,
+    cases,
+    observations: cases.flatMap((caseResult) => caseResult.observations),
+    trials: Trials.aggregate(outcomes, k),
+    k
+  };
+}).pipe(Effect.withSpan("Eval.run", { attributes: { suite: suite.name, runId: options.runId } }));
+const layerNoop = Layer.sync(Runner)(() => {
+  const unavailable = new import_EvalError.EvalError({ code: "scorer_unavailable", message: "No scorer batch runner is available" });
+  return {
+    runBatch: () => Effect.fail(unavailable),
+    runBatchCorrelated: (jobs) => Effect.succeed(jobs.map((job) => ({
+      identity: job.identity,
+      observation: ScorerRunner.inconclusive(job, unavailable)
+    })))
+  };
+});
+//# sourceMappingURL=Runner.js.map