@smthrs/evals 0.0.0-stage → 1.0.0-rc.4

package/dist/esm/Runner.js

@@ -0,0 +1,430 @@
+/**
+ * Deterministic fixed-suite execution and bound scorer evaluation.
+ *
+ * A run executes every case through the injected {@link CaseExecutor}, then
+ * grades each execution with the scorers bound to the flow it executed. The
+ * timestamp and the run identity come from the caller, so two runs of the same
+ * suite over the same inputs produce byte-identical observations.
+ *
+ * @since 0.1.0
+ */
+import * as ScorerRunner from "@smthrs/scorers/Runner";
+import * as Sampling from "@smthrs/scorers/Sampling";
+import * as Scorer from "@smthrs/scorers/Scorer";
+import * as Cause from "effect/Cause";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { CaseExecutor } from "./CaseExecutor.js";
+import { EvalError } from "./EvalError.js";
+import { flattenControlCharacters } from "./internal/controlCharacters.js";
+import { scorerLabel } from "./internal/scorerLabel.js";
+import { tupleKey } from "./internal/tupleKey.js";
+import * as Trials from "./Trials.js";
+/**
+ * Injectable batch-runner service used when a caller wants a reusable adapter.
+ *
+ * {@link run} does not require it: a run scores with `options.scorer` when one
+ * is passed, otherwise with this service when one is provided, otherwise
+ * in process through {@link makeInline}.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class Runner extends Context.Service()("flows/evals/Runner") {
+}
+/** The longest cause summary a public reason carries. */
+const maxReasonLength = 2048;
+const boundedReason = (reason) => reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`;
+const scorerKeyOf = (binding) => binding.scorer.scorerKey;
+// A scorer is a flow and every flow carries a name, so a run always has one to
+// report. A stored observation may still lack it, which is why `scorerLabel`
+// and the `Observation` field both keep the absent case.
+const scorerNameOf = (binding) => binding.scorer.name;
+const label = (binding) => scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) });
+const inconclusive = (request, reason, at) => {
+    return {
+        case: request.case,
+        scorer: scorerKeyOf(request.binding),
+        scorerName: scorerNameOf(request.binding),
+        stepKey: request.stepKey,
+        kind: "inconclusive",
+        reason,
+        at
+    };
+};
+// A case failure always locates itself. An executor that named the offending
+// value keeps its own path; one that named nothing still gets the case, because
+// a failure with no path leaves a caller nothing to branch on but prose.
+const casePath = (name) => `cases['${name}']`;
+// The executor boundary handles every non-interruption cause, not only typed
+// failures: an executor that wraps a parser in `Effect.sync` dies with a
+// defect, and one case's defect must not abort its siblings or the report.
+// The wrapped message is flattened because a gate prints it to a CI log, where
+// a newline from the target would let it forge a workflow command; the
+// original message survives untouched on `cause`.
+const caseError = (suiteCase, cause) => {
+    const reason = Cause.squash(cause);
+    return reason instanceof EvalError
+        ? new EvalError({
+            code: reason.code,
+            message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),
+            path: reason.path ?? casePath(suiteCase.name),
+            cause: reason
+        })
+        : new EvalError({
+            code: "executor",
+            message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),
+            path: casePath(suiteCase.name),
+            cause: reason
+        });
+};
+// A step key is a runtime value the target returns, so unlike a suite name it
+// is flattened rather than rejected: it reaches baselines, reports and the
+// gate summary, and a control character in any of them corrupts a CI log line.
+const runCase = (executor, suiteCase) => Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(Effect.map((execution) => ({
+    case: suiteCase.name,
+    execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },
+    observations: []
+})), Effect.catchCause((cause) => Cause.hasInterrupts(cause)
+    ? Effect.interrupt
+    : Effect.succeed({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })), Effect.withSpan("Eval.case", { attributes: { case: suiteCase.name } }));
+// A declared case value is ground truth even when it is `null`, `false`, `0`
+// or the empty string; only an absent `expected` defers to the binding.
+const groundTruthFor = (suiteCase, binding) => {
+    const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth;
+    return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) };
+};
+const requestFor = (suiteCase, execution, binding) => ({
+    case: suiteCase.name,
+    stepKey: execution.stepKey,
+    binding: {
+        ...structuredClone({
+            sampling: binding.sampling,
+            ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),
+            ...(binding.context === undefined ? {} : { context: binding.context })
+        }),
+        scorer: binding.scorer,
+        appliesTo: binding.appliesTo
+    },
+    input: {
+        input: structuredClone(suiteCase.input),
+        output: execution.output,
+        ...(groundTruthFor(suiteCase, binding)),
+        ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),
+        latencyMs: execution.latencyMs
+    }
+});
+const observationFor = (request, result, at) => {
+    const source = result;
+    const kind = source.kind;
+    if (kind !== "score" && kind !== "inconclusive") {
+        return inconclusive(request, boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`), at);
+    }
+    const reason = source.reason;
+    if (kind === "inconclusive") {
+        return inconclusive(request, typeof reason === "string" && reason.length > 0
+            ? boundedReason(reason)
+            : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`), at);
+    }
+    const score = source.score;
+    if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) {
+        return inconclusive(request, boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`), at);
+    }
+    const meta = source.meta;
+    return {
+        case: request.case,
+        scorer: scorerKeyOf(request.binding),
+        scorerName: scorerNameOf(request.binding),
+        stepKey: request.stepKey,
+        kind: "score",
+        score,
+        ...(typeof reason === "string" ? { reason: boundedReason(reason) } : {}),
+        ...(meta === undefined ? {} : { meta }),
+        at
+    };
+};
+const executeInline = (job) => job.score.pipe(Effect.flatMap(Scorer.validate), Effect.map((result) => ({
+    ...job.observation,
+    kind: "score",
+    score: result.score,
+    ...(result.reason === undefined ? {} : { reason: result.reason }),
+    ...(result.meta === undefined ? {} : { meta: result.meta }),
+    at: job.at
+})), Effect.catchCause((cause) => Cause.hasInterrupts(cause)
+    ? Effect.interrupt
+    : Effect.succeed(ScorerRunner.inconclusive(job, cause))));
+/**
+ * Builds the in-process batch runner a run scores with by default.
+ *
+ * Each job's scorer runs in the current process, its result is checked by
+ * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an
+ * inconclusive observation naming its cause rather than failing the run. It
+ * honours the {@link ScoreBatchRunner} protocol, so it is also the reference
+ * a custom adapter can be compared against.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeInline = () => {
+    const runBatchCorrelated = (jobs, options) => Effect.forEach(jobs, (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))), { concurrency: options?.concurrency ?? 1 });
+    return {
+        runBatchCorrelated,
+        runBatch: (jobs, options) => runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))
+    };
+};
+/**
+ * Provides the in-process batch runner built by {@link makeInline}.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerInline = Layer.succeed(Runner)(makeInline());
+const buildJobs = (cases, suite, { at, runId, sampleId, concurrency }) => Effect.gen(function* () {
+    // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the
+    // two arrays are index-aligned by construction; nothing here has to look a
+    // case back up by name.
+    const candidates = cases.flatMap((caseResult, index) => {
+        const execution = caseResult.execution;
+        if (execution === undefined)
+            return [];
+        const suiteCase = suite.cases[index];
+        return suite.bindings.flatMap((binding, bindingIndex) => binding.appliesTo === execution.target
+            ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]
+            : []);
+    });
+    const sampled = yield* Effect.forEach(candidates, ({ request, bindingIndex }) => Sampling.decide(request.binding.sampling, request.stepKey, scorerKeyOf(request.binding)).pipe(Effect.map((selected) => selected ? request : undefined), Effect.mapError((cause) => new EvalError({
+        code: "invalid_suite",
+        message: `Invalid sampling policy for scorer ${label(request.binding)}`,
+        path: `bindings[${bindingIndex}].sampling`,
+        cause
+    }))), { concurrency });
+    const requests = sampled.filter((request) => request !== undefined);
+    const jobs = requests.map((request, index) => ({
+        identity: tupleKey(suite.name, runId, sampleId, request.case, request.stepKey, scorerKeyOf(request.binding), `${index}`),
+        observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },
+        score: Effect.suspend(() => request.binding.scorer.score(request.input)),
+        at: Date.parse(at)
+    }));
+    return { requests, jobs };
+});
+const batchFallback = (jobs, cause) => Cause.hasInterrupts(cause)
+    ? Effect.interrupt
+    : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)));
+const scoreCorrelated = (jobs, scorer, runBatchCorrelated, { concurrency }) => Effect.gen(function* () {
+    const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(Effect.catchCause((cause) => batchFallback(jobs, cause).pipe(Effect.map((observations) => observations.map((observation, index) => ({
+        identity: jobs[index].identity,
+        observation
+    }))))));
+    if (batchResults.length !== jobs.length) {
+        return yield* Effect.fail(new EvalError({
+            code: "scorer_protocol",
+            message: `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,
+            path: "runBatchCorrelated"
+        }));
+    }
+    const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index]));
+    const byIdentity = new Map();
+    let unknown;
+    for (const [index, result] of batchResults.entries()) {
+        const jobIndex = jobIndexByIdentity.get(result.identity);
+        if (jobIndex !== undefined && byIdentity.has(result.identity)) {
+            return yield* Effect.fail(new EvalError({
+                code: "scorer_protocol",
+                message: `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,
+                path: `runBatchCorrelated[${index}]`
+            }));
+        }
+        if (jobIndex === undefined)
+            unknown = { identity: result.identity, index };
+        byIdentity.set(result.identity, result.observation);
+    }
+    if (unknown !== undefined) {
+        const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity));
+        const missingIdentity = jobs[missingJobIndex].identity;
+        return yield* Effect.fail(new EvalError({
+            code: "scorer_protocol",
+            message: `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,
+            path: `runBatchCorrelated[${unknown.index}]`
+        }));
+    }
+    return jobs.map((job) => byIdentity.get(job.identity));
+});
+const scoreOrderOnly = (jobs, requests, scorer, { concurrency }) => Effect.gen(function* () {
+    const pairIndexes = new Map();
+    for (const [index, job] of jobs.entries()) {
+        const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey);
+        const previous = pairIndexes.get(pair);
+        if (previous !== undefined) {
+            const first = requests[previous];
+            const second = requests[index];
+            return yield* Effect.fail(new EvalError({
+                code: "ambiguous_score_job",
+                message: `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${label(second.binding)}. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,
+                path: "runBatch"
+            }));
+        }
+        pairIndexes.set(pair, index);
+    }
+    const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(Effect.catchCause((cause) => batchFallback(jobs, cause)));
+    if (results.length !== jobs.length) {
+        return yield* Effect.fail(new EvalError({
+            code: "scorer_protocol",
+            message: `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,
+            path: "runBatch"
+        }));
+    }
+    return results;
+});
+const score = (cases, suite, scorer, context) => Effect.gen(function* () {
+    const { requests, jobs } = yield* buildJobs(cases, suite, context);
+    if (requests.length === 0)
+        return cases;
+    const runBatchCorrelated = scorer.runBatchCorrelated;
+    const protocol = runBatchCorrelated === undefined ? "runBatch" : "runBatchCorrelated";
+    const results = yield* runBatchCorrelated === undefined
+        ? scoreOrderOnly(jobs, requests, scorer, context)
+        : scoreCorrelated(jobs, scorer, runBatchCorrelated, context);
+    const { at } = context;
+    const observations = [];
+    for (const [index, request] of requests.entries()) {
+        const result = results[index];
+        if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {
+            return yield* Effect.fail(new EvalError({
+                code: "scorer_protocol",
+                message: protocol === "runBatchCorrelated"
+                    ? `Correlated scorer batch identity '${jobs[index].identity}' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${scorerKeyOf(request.binding)}' at step '${request.stepKey}'`
+                    : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${scorerKeyOf(request.binding)}' at step '${request.stepKey}'; results must stay aligned with their jobs`,
+                path: `${protocol}[${index}]`
+            }));
+        }
+        observations.push(observationFor(request, result, at));
+    }
+    const byCase = new Map();
+    for (const observation of observations) {
+        const bucket = byCase.get(observation.case);
+        if (bucket === undefined)
+            byCase.set(observation.case, [observation]);
+        else
+            bucket.push(observation);
+    }
+    return cases.map((caseResult) => ({
+        ...caseResult,
+        observations: byCase.get(caseResult.case) ?? []
+    }));
+});
+/**
+ * Runs a fixed suite with bounded execution and declaration-order results.
+ *
+ * Cases run through the provided {@link CaseExecutor} at the suite's
+ * concurrency and are returned in declaration order. Each executor receives
+ * its own mutable case copy. Each scorer receives independent copies of the
+ * original case input, expected value, and binding data. Every execution is then
+ * graded by the bindings whose `appliesTo` is the flow the execution reports as
+ * its target, matched by reference identity, so a binding attached to another
+ * flow contributes no observations.
+ *
+ * Scoring goes through `options.scorer` when the caller passes one, the
+ * {@link Runner} service when one is provided, and {@link makeInline}
+ * otherwise. A case whose target failed keeps its typed error, locates itself
+ * with the executor's own `path` or with `cases['<name>']`, and produces no
+ * observations; a scorer that failed produces an inconclusive observation. The
+ * run itself fails only with `invalid_run_options` for a non-canonical run
+ * identity or timestamp, `invalid_suite` for an unusable sampling policy,
+ * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,
+ * or `scorer_protocol` for a batch runner that broke the
+ * {@link ScoreBatchRunner} contract.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const run = (suite, options) => Effect.gen(function* () {
+    const executor = yield* CaseExecutor;
+    const injected = yield* Effect.serviceOption(Runner);
+    const scorer = options.scorer ?? (injected._tag === "Some" ? injected.value : makeInline());
+    const atMillis = Date.parse(options.at);
+    if (options.runId.trim().length === 0) {
+        return yield* Effect.fail(new EvalError({
+            code: "invalid_run_options",
+            message: "Deterministic runs require a non-empty runId",
+            path: "options.runId"
+        }));
+    }
+    if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(options.at) ||
+        !Number.isFinite(atMillis) ||
+        new Date(atMillis).toISOString() !== options.at) {
+        return yield* Effect.fail(new EvalError({
+            code: "invalid_run_options",
+            message: `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,
+            path: "options.at"
+        }));
+    }
+    const n = options.trials ?? suite.trials ?? 1;
+    const k = options.k ?? 1;
+    if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {
+        return yield* Effect.fail(new EvalError({
+            code: "invalid_trials",
+            message: "trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials",
+            path: "options.trials"
+        }));
+    }
+    const at = options.at;
+    const runId = options.runId;
+    const sampleId = options.sampleId ?? "default";
+    // Await each batch before starting the next: concurrency within a trial is
+    // bounded by the suite, and neither scheduling nor scorer latency reorders trials.
+    const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) => Effect.gen(function* () {
+        const cases = yield* Effect.forEach(suite.cases, (suiteCase) => runCase(executor, suiteCase), {
+            concurrency: suite.concurrency,
+            discard: false
+        });
+        return yield* score(cases, suite, scorer, {
+            concurrency: suite.concurrency,
+            at,
+            runId,
+            sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))
+        });
+    }), { concurrency: 1 });
+    const outcomes = {};
+    const cases = suite.cases.map((suiteCase, index) => {
+        const attempts = batches.map((batch) => batch[index]);
+        const passed = attempts.map((attempt) => attempt.error === undefined && attempt.observations.length > 0 &&
+            attempt.observations.every((observation) => observation.kind === "score" && observation.score === 1));
+        outcomes[suiteCase.name] = passed;
+        return {
+            ...attempts[0],
+            error: attempts.find((attempt) => attempt.error !== undefined)?.error,
+            observations: attempts.flatMap((attempt) => attempt.observations),
+            trials: Trials.summarize(passed, k)
+        };
+    });
+    return {
+        runId,
+        suite: suite.name,
+        cases,
+        observations: cases.flatMap((caseResult) => caseResult.observations),
+        trials: Trials.aggregate(outcomes, k),
+        k
+    };
+}).pipe(Effect.withSpan("Eval.run", { attributes: { suite: suite.name, runId: options.runId } }));
+/**
+ * Provides a batch runner that is never available.
+ *
+ * Every bound score under this layer becomes an inconclusive observation, which
+ * a gate grades as an undecidable run rather than a red. Provide it to state
+ * that a suite must not score, and {@link layerInline} to score in process.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop = Layer.sync(Runner)(() => {
+    const unavailable = new EvalError({ code: "scorer_unavailable", message: "No scorer batch runner is available" });
+    return {
+        runBatch: () => Effect.fail(unavailable),
+        runBatchCorrelated: (jobs) => Effect.succeed(jobs.map((job) => ({
+            identity: job.identity,
+            observation: ScorerRunner.inconclusive(job, unavailable)
+        })))
+    };
+});
+//# sourceMappingURL=Runner.js.map

package/dist/esm/Runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Runner.js","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AACtD,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,MAAM,MAAM,wBAAwB,CAAA;AAEhD,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAuD,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAA;AAC1E,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAEjD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AA0JrC;;;;;;;;;GASG;AACH,MAAM,OAAO,MAAO,SAAQ,OAAO,CAAC,OAAO,EAA4B,CAAC,oBAAoB,CAAC;CAAG;AAEhG,yDAAyD;AACzD,MAAM,eAAe,GAAG,IAAI,CAAA;AAE5B,MAAM,aAAa,GAAG,CAAC,MAAc,EAAU,EAAE,CAC/C,MAAM,CAAC,MAAM,IAAI,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,aAAa,CAAA;AAE9F,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAA;AAE1E,+EAA+E;AAC/E,6EAA6E;AAC7E,yDAAyD;AACzD,MAAM,YAAY,GAAG,CAAC,OAAgB,EAAU,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA;AAEtE,MAAM,KAAK,GAAG,CAAC,OAAgB,EAAU,EAAE,CACzC,WAAW,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;AAElF,MAAM,YAAY,GAAG,CAAC,OAAqB,EAAE,MAAc,EAAE,EAAU,EAAe,EAAE;IACtF,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,cAAc;QACpB,MAAM;QACN,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,6EAA6E;AAC7E,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,UAAU,IAAI,IAAI,CAAA;AAE7D,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,+EAA+E;AAC/E,uEAAuE;AACvE,kDAAkD;AAClD,MAAM,SAAS,GAAG,CAAC,SAAe,EAAE,KAA2B,EAAa,EAAE;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAClC,OAAO,MAAM,YAAY,SAAS;QAChC,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;YAClG,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC7C,KAAK,EAAE,MAAM;SACd,CAAC;QACF,CAAC,CAAC,IAAI,SAAS,CAAC;YACd,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,wBAAwB,CAAC,2BAA2B,SAAS,CAAC,IAAI,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAClG,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,KAAK,EAAE,MAAM;SACd,CAAC,CAAA;AACN,CAAC,CAAA;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,MAAM,OAAO,GAAG,CAAC,QAA6B,EAAE,SAAe,EAA6B,EAAE,CAC5F,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CACjE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAc,EAAE,CAAC,CAAC;IACrC,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,SAAS,EAAE,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE;IACjF,YAAY,EAAE,EAAE;CACjB,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAC/G,EACD,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CACvE,CAAA;AAEH,6EAA6E;AAC7E,wEAAwE;AACxE,MAAM,cAAc,GAAG,CAAC,SAAe,EAAE,OAAgB,EAAsC,EAAE;IAC/F,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;IAC/F,OAAO,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,CAAC,WAAW,CAAC,EAAE,CAAA;AACvF,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,SAAe,EAAE,SAAoB,EAAE,OAAgB,EAAgB,EAAE,CAAC,CAAC;IAC7F,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,OAAO,EAAE,SAAS,CAAC,OAAO;IAC1B,OAAO,EAAE;QACP,GAAG,eAAe,CAAC;YACjB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;SACvE,CAAC;QACF,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B;IACD,KAAK,EAAE;QACL,KAAK,EAAE,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,GAAG,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACvC,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACvF,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B;CACF,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,CAAC,OAAqB,EAAE,MAAwB,EAAE,EAAU,EAAe,EAAE;IAClG,MAAM,MAAM,GAAG,MAKd,CAAA;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAChD,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,2CAA2C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5B,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;QAC5B,OAAO,YAAY,CACjB,OAAO,EACP,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC;YACvB,CAAC,CAAC,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,sDAAsD,CAAC,EACzG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;IAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,YAAY,CACjB,OAAO,EACP,aAAa,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,qCAAqC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EACnG,EAAE,CACH,CAAA;IACH,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO;QACb,KAAK;QACL,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,EAAE;KACH,CAAA;AACH,CAAC,CAAA;AAED,MAAM,aAAa,GAAG,CAAC,GAAa,EAAmC,EAAE,CACvE,GAAG,CAAC,KAAK,CAAC,IAAI,CACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAA+B,EAAE,CAAC,CAAC;IACnD,GAAG,GAAG,CAAC,WAAW;IAClB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE,MAAM,CAAC,KAAK;IACnB,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACjE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAC3D,EAAE,EAAE,GAAG,CAAC,EAAE;CACX,CAAC,CAAC,EACH,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAC1D,CACF,CAAA;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAqB,EAAE;IAC/C,MAAM,kBAAkB,GAAG,CACzB,IAA6B,EAC7B,OAAuD,EACZ,EAAE,CAC7C,MAAM,CAAC,OAAO,CACZ,IAAI,EACJ,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EACxG,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,EAAE,CAC3C,CAAA;IACH,OAAO;QACL,kBAAkB;QAClB,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAC1B,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;KAC/G,CAAA;AACH,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAwB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;AASnF,MAAM,SAAS,GAAG,CAChB,KAAgC,EAChC,KAAY,EACZ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAgB,EAIlD,EAAE,CACF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2EAA2E;IAC3E,2EAA2E;IAC3E,wBAAwB;IACxB,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QACrD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAA;QACtC,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAE,CAAA;QACrC,OAAO,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,CACtD,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,MAAM;YACpC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;YACxE,CAAC,CAAC,EAAE,CACP,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CACnC,UAAU,EACV,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,EAAE,CAC5B,QAAQ,CAAC,MAAM,CACb,OAAO,CAAC,OAAO,CAAC,QAAQ,EACxB,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAC7B,CAAC,IAAI,CACJ,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EACxD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,IAAI,SAAS,CAAC;QACZ,IAAI,EAAE,eAAe;QACrB,OAAO,EAAE,sCAAsC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACvE,IAAI,EAAE,YAAY,YAAY,YAAY;QAC1C,KAAK;KACN,CAAC,CACH,CACF,EACH,EAAE,WAAW,EAAE,CAChB,CAAA;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAA2B,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAoB,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,QAAQ,CAChB,KAAK,CAAC,IAAI,EACV,KAAK,EACL,QAAQ,EACR,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,OAAO,EACf,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAC5B,GAAG,KAAK,EAAE,CACX;QACD,WAAW,EAAE,EAAE,aAAa,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACxF,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxE,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;KACnB,CAAC,CAAC,CAAA;IACH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;AAC3B,CAAC,CAAC,CAAA;AAEJ,MAAM,aAAa,GAAG,CACpB,IAA6B,EAC7B,KAA2B,EACqB,EAAE,CAClD,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;IACxB,CAAC,CAAC,MAAM,CAAC,SAAS;IAClB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;AAE9E,MAAM,eAAe,GAAG,CACtB,IAA6B,EAC7B,MAAwB,EACxB,kBAAuE,EACvE,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAC3G,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAC1B,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACxC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,QAAQ;QAC/B,WAAW;KACZ,CAAC,CAAC,CACJ,CACF,CACF,CACF,CAAA;IACD,IAAI,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,oCAAoC,YAAY,CAAC,MAAM,gBAAgB,IAAI,CAAC,MAAM,uEAAuE;YAC3J,IAAI,EAAE,oBAAoB;SAC3B,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAU,CAAC,CAAC,CAAA;IAC5F,MAAM,UAAU,GAAG,IAAI,GAAG,EAA4B,CAAA;IACtD,IAAI,OAA0E,CAAA;IAC9E,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACxD,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EACL,wDAAwD,MAAM,CAAC,QAAQ,aAAa,QAAQ,oBAAoB,KAAK,EAAE;gBACzH,IAAI,EAAE,sBAAsB,KAAK,GAAG;aACrC,CAAC,CACH,CAAA;QACH,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAA;QAC1E,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAA;IACrD,CAAC;IACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC9E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAE,CAAC,QAAQ,CAAA;QACvD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,sDAAsD,OAAO,CAAC,QAAQ,qBAAqB,OAAO,CAAC,KAAK,SAAS,eAAe,cAAc,eAAe,aAAa;YAC5K,IAAI,EAAE,sBAAsB,OAAO,CAAC,KAAK,GAAG;SAC7C,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,CAAA;AACzD,CAAC,CAAC,CAAA;AAEJ,MAAM,cAAc,GAAG,CACrB,IAA6B,EAC7B,QAAqC,EACrC,MAAwB,EACxB,EAAE,WAAW,EAAgB,EAC8B,EAAE,CAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,aAAa,EAAE,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QAC/E,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAE,CAAA;YACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAE,CAAA;YAC/B,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EACL,0CAA0C,KAAK,CAAC,IAAI,UAAU,MAAM,CAAC,IAAI,yBAAyB,GAAG,CAAC,WAAW,CAAC,aAAa,gBAC7H,KAAK,CAAC,MAAM,CAAC,OAAO,CACtB,iGAAiG;gBACnG,IAAI,EAAE,UAAU;aACjB,CAAC,CACH,CAAA;QACH,CAAC;QACD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CACtF,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CACzD,CAAA;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,iBAAiB;YACvB,OAAO,EACL,yBAAyB,OAAO,CAAC,MAAM,qBAAqB,IAAI,CAAC,MAAM,6EAA6E;YACtJ,IAAI,EAAE,UAAU;SACjB,CAAC,CACH,CAAA;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAC,CAAA;AAEJ,MAAM,KAAK,GAAG,CACZ,KAAgC,EAChC,KAAY,EACZ,MAAwB,EACxB,OAAqB,EACgC,EAAE,CACvD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACvC,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAA;IACpD,MAAM,QAAQ,GAAG,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAA;IACrF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,kBAAkB,KAAK,SAAS;QACrD,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC;QACjD,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAA;IAC9D,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAA;IACtB,MAAM,YAAY,GAAuB,EAAE,CAAA;IAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAA;QAC9B,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,SAAS,KAAK,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAClG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;gBACZ,IAAI,EAAE,iBAAiB;gBACvB,OAAO,EAAE,QAAQ,KAAK,oBAAoB;oBACxC,CAAC,CAAC,qCACA,IAAI,CAAC,KAAK,CAAE,CAAC,QACf,oBAAoB,MAAM,CAAC,aAAa,iBAAiB,MAAM,CAAC,SAAS,eAAe,KAAK,sBAC3F,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,GAAG;oBAChC,CAAC,CAAC,6CAA6C,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,aAAa,eAAe,KAAK,eACnH,WAAW,CAAC,OAAO,CAAC,OAAO,CAC7B,cAAc,OAAO,CAAC,OAAO,8CAA8C;gBAC7E,IAAI,EAAE,GAAG,QAAQ,IAAI,KAAK,GAAG;aAC9B,CAAC,CACH,CAAA;QACH,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAA;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAA;IACpD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAA;;YAChE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAChC,GAAG,UAAU;QACb,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE;KAChD,CAAC,CAAC,CAAA;AACL,CAAC,CAAC,CAAA;AAEJ;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,CACjB,KAAY,EACZ,OAAmB,EACgC,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,YAAY,CAAA;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAA;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,8CAA8C;YACvD,IAAI,EAAE,eAAe;SACtB,CAAC,CACH,CAAA;IACH,CAAC;IACD,IACE,CAAC,gDAAgD,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAClE,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAAE,EAC/C,CAAC;QACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EACL,+FAA+F,OAAO,CAAC,EAAE,GAAG;YAC9G,IAAI,EAAE,YAAY;SACnB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;IAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAA;IACxB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAChG,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,IAAI,SAAS,CAAC;YACZ,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,oFAAoF;YAC7F,IAAI,EAAE,gBAAgB;SACvB,CAAC,CACH,CAAA;IACH,CAAC;IACD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAA;IACrB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAA;IAC9C,2EAA2E;IAC3E,mFAAmF;IACnF,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAC9F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,EAAE,CAC7D,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE;YAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,OAAO,EAAE,KAAK;SACf,CAAC,CAAA;QACF,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE;YACxC,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,EAAE;YACF,KAAK;YACL,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;SACjE,CAAC,CAAA;IACJ,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAmC,EAAE,CAAA;IACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QACjD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACrC,KAAK,CAAC,KAAK,CAAE,CACd,CAAA;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACtC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAC9D,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,OAAO,IAAI,WAAW,CAAC,KAAK,KAAK,CAAC,CAAC,CACrG,CAAA;QACD,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAA;QACjC,OAAO;YACL,GAAG,QAAQ,CAAC,CAAC,CAAE;YACf,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,KAAK;YACrE,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;YACjE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;SACpC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,OAAO;QACL,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,KAAK;QACL,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;QACpE,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrC,CAAC;KACF,CAAA;AACH,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAA;AAEnG;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,SAAS,GAAwB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE;IACpE,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAA;IACjH,OAAO;QACL,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE,CAC3B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC;SACzD,CAAC,CAAC,CAAC;KACP,CAAA;AACH,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Deterministic fixed-suite execution and bound scorer evaluation.\n *\n * A run executes every case through the injected {@link CaseExecutor}, then\n * grades each execution with the scorers bound to the flow it executed. The\n * timestamp and the run identity come from the caller, so two runs of the same\n * suite over the same inputs produce byte-identical observations.\n *\n * @since 0.1.0\n */\n\nimport * as ScorerRunner from \"@smthrs/scorers/Runner\"\nimport * as Sampling from \"@smthrs/scorers/Sampling\"\nimport * as Scorer from \"@smthrs/scorers/Scorer\"\nimport type * as ScoreStore from \"@smthrs/scorers/ScoreStore\"\nimport * as Cause from \"effect/Cause\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { CaseExecutor, type Execution, type Service as CaseExecutorService } from \"./CaseExecutor.ts\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Binding, Case, Suite } from \"./Suite.ts\"\nimport * as Trials from \"./Trials.ts\"\n\n/**\n * One score observation emitted by a suite run.\n *\n * `scorer` is the scorer key, a digest of the scorer's own declaration, which\n * is what a baseline matches on. `scorerName` is the human name from the same\n * declaration, carried alongside so a report can be read without grepping for\n * the digest. `at` is the run's timestamp, not the scorer's: a run's\n * observations all carry one instant so a baseline stays reproducible.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Observation =\n  & {\n    readonly case: string\n    readonly scorer: string\n    readonly scorerName?: string | undefined\n    readonly stepKey: string\n    readonly at: string\n  }\n  & (\n    | {\n      readonly kind: \"score\"\n      readonly score: number\n      readonly reason?: string | undefined\n      readonly meta?: unknown\n    }\n    | { readonly kind: \"inconclusive\"; readonly reason: string; readonly meta?: unknown }\n  )\n\n/**\n * A request sent to the scorers batch runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface ScoreRequest {\n  readonly case: string\n  readonly stepKey: string\n  readonly binding: Binding\n  readonly input: {\n    readonly input: unknown\n    readonly output: unknown\n    readonly groundTruth?: unknown\n    readonly context?: unknown\n    readonly latencyMs?: number\n  }\n}\n\n/**\n * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreJob = ScorerRunner.Job\n\n/**\n * Structural adapter for `@smthrs/scorers`' blocking batch runner.\n *\n * A runner that implements `runBatchCorrelated` tags every observation with\n * its job identity and may return results in any order. A runner that implements\n * only `runBatch` is correlated positionally: it must return exactly one\n * observation per job, in order, and each observation must echo its job's\n * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before\n * scoring when two jobs share that pair, because their results could not be\n * attributed to a case.\n *\n * A run fails with `scorer_protocol` when either protocol is broken.\n * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its\n * service value can be used directly and `ambiguous_score_job` never applies\n * to it.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface ScoreBatchRunner {\n  readonly runBatch: (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>\n  readonly runBatchCorrelated?:\n    | ((\n      jobs: ReadonlyArray<ScoreJob>,\n      options?: { readonly concurrency?: number | undefined }\n    ) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>)\n    | undefined\n}\n\n/**\n * A score result aligned with a {@link ScoreRequest}.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ScoreObservation = ScoreStore.Observation\n\n/**\n * A batch result tagged with the identity of the job that produced it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BatchResult {\n  readonly identity: string\n  readonly observation: ScoreObservation\n}\n\n/**\n * Per-case result retained by the deterministic runner.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface CaseResult {\n  readonly case: string\n  readonly execution?: Execution | undefined\n  readonly error?: EvalError | undefined\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Summary | undefined\n}\n\n/**\n * Stable result of a suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunResult {\n  readonly runId: string\n  readonly suite: string\n  readonly cases: ReadonlyArray<CaseResult>\n  readonly observations: ReadonlyArray<Observation>\n  readonly trials?: Trials.Aggregate | undefined\n  readonly k?: number | undefined\n}\n\n/**\n * Options for a deterministic suite run.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface RunOptions {\n  readonly scorer?: ScoreBatchRunner | undefined\n  readonly runId: string\n  readonly sampleId?: string | undefined\n  readonly trials?: number | undefined\n  readonly k?: number | undefined\n  readonly at: string\n}\n\n/**\n * Injectable batch-runner service used when a caller wants a reusable adapter.\n *\n * {@link run} does not require it: a run scores with `options.scorer` when one\n * is passed, otherwise with this service when one is provided, otherwise\n * in process through {@link makeInline}.\n *\n * @category services\n * @since 0.1.0\n */\nexport class Runner extends Context.Service<Runner, ScoreBatchRunner>()(\"flows/evals/Runner\") {}\n\n/** The longest cause summary a public reason carries. */\nconst maxReasonLength = 2048\n\nconst boundedReason = (reason: string): string =>\n  reason.length <= maxReasonLength ? reason : `${reason.slice(0, maxReasonLength)}[truncated]`\n\nconst scorerKeyOf = (binding: Binding): string => binding.scorer.scorerKey\n\n// A scorer is a flow and every flow carries a name, so a run always has one to\n// report. A stored observation may still lack it, which is why `scorerLabel`\n// and the `Observation` field both keep the absent case.\nconst scorerNameOf = (binding: Binding): string => binding.scorer.name\n\nconst label = (binding: Binding): string =>\n  scorerLabel({ scorer: scorerKeyOf(binding), scorerName: scorerNameOf(binding) })\n\nconst inconclusive = (request: ScoreRequest, reason: string, at: string): Observation => {\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"inconclusive\",\n    reason,\n    at\n  }\n}\n\n// A case failure always locates itself. An executor that named the offending\n// value keeps its own path; one that named nothing still gets the case, because\n// a failure with no path leaves a caller nothing to branch on but prose.\nconst casePath = (name: string): string => `cases['${name}']`\n\n// The executor boundary handles every non-interruption cause, not only typed\n// failures: an executor that wraps a parser in `Effect.sync` dies with a\n// defect, and one case's defect must not abort its siblings or the report.\n// The wrapped message is flattened because a gate prints it to a CI log, where\n// a newline from the target would let it forge a workflow command; the\n// original message survives untouched on `cause`.\nconst caseError = (suiteCase: Case, cause: Cause.Cause<unknown>): EvalError => {\n  const reason = Cause.squash(cause)\n  return reason instanceof EvalError\n    ? new EvalError({\n      code: reason.code,\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${reason.message}`),\n      path: reason.path ?? casePath(suiteCase.name),\n      cause: reason\n    })\n    : new EvalError({\n      code: \"executor\",\n      message: flattenControlCharacters(`Target failed for case '${suiteCase.name}': ${String(reason)}`),\n      path: casePath(suiteCase.name),\n      cause: reason\n    })\n}\n\n// A step key is a runtime value the target returns, so unlike a suite name it\n// is flattened rather than rejected: it reaches baselines, reports and the\n// gate summary, and a control character in any of them corrupts a CI log line.\nconst runCase = (executor: CaseExecutorService, suiteCase: Case): Effect.Effect<CaseResult> =>\n  Effect.suspend(() => executor.run(structuredClone(suiteCase))).pipe(\n    Effect.map((execution): CaseResult => ({\n      case: suiteCase.name,\n      execution: { ...execution, stepKey: flattenControlCharacters(execution.stepKey) },\n      observations: []\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed<CaseResult>({ case: suiteCase.name, error: caseError(suiteCase, cause), observations: [] })\n    ),\n    Effect.withSpan(\"Eval.case\", { attributes: { case: suiteCase.name } })\n  )\n\n// A declared case value is ground truth even when it is `null`, `false`, `0`\n// or the empty string; only an absent `expected` defers to the binding.\nconst groundTruthFor = (suiteCase: Case, binding: Binding): { readonly groundTruth?: unknown } => {\n  const groundTruth = suiteCase.expected !== undefined ? suiteCase.expected : binding.groundTruth\n  return groundTruth === undefined ? {} : { groundTruth: structuredClone(groundTruth) }\n}\n\nconst requestFor = (suiteCase: Case, execution: Execution, binding: Binding): ScoreRequest => ({\n  case: suiteCase.name,\n  stepKey: execution.stepKey,\n  binding: {\n    ...structuredClone({\n      sampling: binding.sampling,\n      ...(binding.groundTruth === undefined ? {} : { groundTruth: binding.groundTruth }),\n      ...(binding.context === undefined ? {} : { context: binding.context })\n    }),\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo\n  },\n  input: {\n    input: structuredClone(suiteCase.input),\n    output: execution.output,\n    ...(groundTruthFor(suiteCase, binding)),\n    ...(binding.context === undefined ? {} : { context: structuredClone(binding.context) }),\n    latencyMs: execution.latencyMs\n  }\n})\n\nconst observationFor = (request: ScoreRequest, result: ScoreObservation, at: string): Observation => {\n  const source = result as {\n    readonly kind: unknown\n    readonly score?: unknown\n    readonly reason?: unknown\n    readonly meta?: unknown\n  }\n  const kind = source.kind\n  if (kind !== \"score\" && kind !== \"inconclusive\") {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned an unusable observation kind '${String(kind)}'`),\n      at\n    )\n  }\n  const reason = source.reason\n  if (kind === \"inconclusive\") {\n    return inconclusive(\n      request,\n      typeof reason === \"string\" && reason.length > 0\n        ? boundedReason(reason)\n        : boundedReason(`Scorer ${label(request.binding)} returned an inconclusive observation with no reason`),\n      at\n    )\n  }\n  const score = source.score\n  if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n    return inconclusive(\n      request,\n      boundedReason(`Scorer ${label(request.binding)} returned a score outside [0, 1]: ${String(score)}`),\n      at\n    )\n  }\n  const meta = source.meta\n  return {\n    case: request.case,\n    scorer: scorerKeyOf(request.binding),\n    scorerName: scorerNameOf(request.binding),\n    stepKey: request.stepKey,\n    kind: \"score\",\n    score,\n    ...(typeof reason === \"string\" ? { reason: boundedReason(reason) } : {}),\n    ...(meta === undefined ? {} : { meta }),\n    at\n  }\n}\n\nconst executeInline = (job: ScoreJob): Effect.Effect<ScoreObservation> =>\n  job.score.pipe(\n    Effect.flatMap(Scorer.validate),\n    Effect.map((result): ScoreStore.ScoreObservation => ({\n      ...job.observation,\n      kind: \"score\",\n      score: result.score,\n      ...(result.reason === undefined ? {} : { reason: result.reason }),\n      ...(result.meta === undefined ? {} : { meta: result.meta }),\n      at: job.at\n    })),\n    Effect.catchCause((cause) =>\n      Cause.hasInterrupts(cause)\n        ? Effect.interrupt\n        : Effect.succeed(ScorerRunner.inconclusive(job, cause))\n    )\n  )\n\n/**\n * Builds the in-process batch runner a run scores with by default.\n *\n * Each job's scorer runs in the current process, its result is checked by\n * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an\n * inconclusive observation naming its cause rather than failing the run. It\n * honours the {@link ScoreBatchRunner} protocol, so it is also the reference\n * a custom adapter can be compared against.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeInline = (): ScoreBatchRunner => {\n  const runBatchCorrelated = (\n    jobs: ReadonlyArray<ScoreJob>,\n    options?: { readonly concurrency?: number | undefined }\n  ): Effect.Effect<ReadonlyArray<BatchResult>> =>\n    Effect.forEach(\n      jobs,\n      (job) => executeInline(job).pipe(Effect.map((observation) => ({ identity: job.identity, observation }))),\n      { concurrency: options?.concurrency ?? 1 }\n    )\n  return {\n    runBatchCorrelated,\n    runBatch: (jobs, options) =>\n      runBatchCorrelated(jobs, options).pipe(Effect.map((results) => results.map((result) => result.observation)))\n  }\n}\n\n/**\n * Provides the in-process batch runner built by {@link makeInline}.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerInline: Layer.Layer<Runner> = Layer.succeed(Runner)(makeInline())\n\ninterface ScoreContext {\n  readonly at: string\n  readonly runId: string\n  readonly sampleId: string\n  readonly concurrency: number\n}\n\nconst buildJobs = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  { at, runId, sampleId, concurrency }: ScoreContext\n): Effect.Effect<\n  { readonly requests: ReadonlyArray<ScoreRequest>; readonly jobs: ReadonlyArray<ScoreJob> },\n  EvalError\n> =>\n  Effect.gen(function*() {\n    // `cases` is the result of one `Effect.forEach` over `suite.cases`, so the\n    // two arrays are index-aligned by construction; nothing here has to look a\n    // case back up by name.\n    const candidates = cases.flatMap((caseResult, index) => {\n      const execution = caseResult.execution\n      if (execution === undefined) return []\n      const suiteCase = suite.cases[index]!\n      return suite.bindings.flatMap((binding, bindingIndex) =>\n        binding.appliesTo === execution.target\n          ? [{ request: requestFor(suiteCase, execution, binding), bindingIndex }]\n          : []\n      )\n    })\n    const sampled = yield* Effect.forEach(\n      candidates,\n      ({ request, bindingIndex }) =>\n        Sampling.decide(\n          request.binding.sampling,\n          request.stepKey,\n          scorerKeyOf(request.binding)\n        ).pipe(\n          Effect.map((selected) => selected ? request : undefined),\n          Effect.mapError((cause) =>\n            new EvalError({\n              code: \"invalid_suite\",\n              message: `Invalid sampling policy for scorer ${label(request.binding)}`,\n              path: `bindings[${bindingIndex}].sampling`,\n              cause\n            })\n          )\n        ),\n      { concurrency }\n    )\n    const requests = sampled.filter((request): request is ScoreRequest => request !== undefined)\n    const jobs: Array<ScoreJob> = requests.map((request, index) => ({\n      identity: tupleKey(\n        suite.name,\n        runId,\n        sampleId,\n        request.case,\n        request.stepKey,\n        scorerKeyOf(request.binding),\n        `${index}`\n      ),\n      observation: { targetStepKey: request.stepKey, scorerKey: scorerKeyOf(request.binding) },\n      score: Effect.suspend(() => request.binding.scorer.score(request.input)),\n      at: Date.parse(at)\n    }))\n    return { requests, jobs }\n  })\n\nconst batchFallback = (\n  jobs: ReadonlyArray<ScoreJob>,\n  cause: Cause.Cause<unknown>\n): Effect.Effect<ReadonlyArray<ScoreObservation>> =>\n  Cause.hasInterrupts(cause)\n    ? Effect.interrupt\n    : Effect.succeed(jobs.map((job) => ScorerRunner.inconclusive(job, cause)))\n\nconst scoreCorrelated = (\n  jobs: ReadonlyArray<ScoreJob>,\n  scorer: ScoreBatchRunner,\n  runBatchCorrelated: NonNullable<ScoreBatchRunner[\"runBatchCorrelated\"]>,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const batchResults = yield* Effect.suspend(() => runBatchCorrelated.call(scorer, jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) =>\n        batchFallback(jobs, cause).pipe(\n          Effect.map((observations) =>\n            observations.map((observation, index) => ({\n              identity: jobs[index]!.identity,\n              observation\n            }))\n          )\n        )\n      )\n    )\n    if (batchResults.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned ${batchResults.length} results for ${jobs.length} jobs; a batch runner must return exactly one result per job identity`,\n          path: \"runBatchCorrelated\"\n        })\n      )\n    }\n    const jobIndexByIdentity = new Map(jobs.map((job, index) => [job.identity, index] as const))\n    const byIdentity = new Map<string, ScoreObservation>()\n    let unknown: { readonly identity: string; readonly index: number } | undefined\n    for (const [index, result] of batchResults.entries()) {\n      const jobIndex = jobIndexByIdentity.get(result.identity)\n      if (jobIndex !== undefined && byIdentity.has(result.identity)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message:\n              `Correlated scorer batch returned duplicate identity '${result.identity}' for job ${jobIndex} at result index ${index}`,\n            path: `runBatchCorrelated[${index}]`\n          })\n        )\n      }\n      if (jobIndex === undefined) unknown = { identity: result.identity, index }\n      byIdentity.set(result.identity, result.observation)\n    }\n    if (unknown !== undefined) {\n      const missingJobIndex = jobs.findIndex((job) => !byIdentity.has(job.identity))\n      const missingIdentity = jobs[missingJobIndex]!.identity\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Correlated scorer batch returned unknown identity '${unknown.identity}' at result index ${unknown.index}; job ${missingJobIndex} identity '${missingIdentity}' is absent`,\n          path: `runBatchCorrelated[${unknown.index}]`\n        })\n      )\n    }\n    return jobs.map((job) => byIdentity.get(job.identity)!)\n  })\n\nconst scoreOrderOnly = (\n  jobs: ReadonlyArray<ScoreJob>,\n  requests: ReadonlyArray<ScoreRequest>,\n  scorer: ScoreBatchRunner,\n  { concurrency }: ScoreContext\n): Effect.Effect<ReadonlyArray<ScoreObservation>, EvalError> =>\n  Effect.gen(function*() {\n    const pairIndexes = new Map<string, number>()\n    for (const [index, job] of jobs.entries()) {\n      const pair = tupleKey(job.observation.targetStepKey, job.observation.scorerKey)\n      const previous = pairIndexes.get(pair)\n      if (previous !== undefined) {\n        const first = requests[previous]!\n        const second = requests[index]!\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"ambiguous_score_job\",\n            message:\n              `Cannot attribute score jobs for cases '${first.case}' and '${second.case}': both use step key '${job.observation.targetStepKey}' and scorer ${\n                label(second.binding)\n              }. Give each case its own step key, or provide a batch runner that implements runBatchCorrelated`,\n            path: \"runBatch\"\n          })\n        )\n      }\n      pairIndexes.set(pair, index)\n    }\n    const results = yield* Effect.suspend(() => scorer.runBatch(jobs, { concurrency })).pipe(\n      Effect.catchCause((cause) => batchFallback(jobs, cause))\n    )\n    if (results.length !== jobs.length) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"scorer_protocol\",\n          message:\n            `Scorer batch returned ${results.length} observations for ${jobs.length} jobs; a batch runner must return exactly one observation per job, in order`,\n          path: \"runBatch\"\n        })\n      )\n    }\n    return results\n  })\n\nconst score = (\n  cases: ReadonlyArray<CaseResult>,\n  suite: Suite,\n  scorer: ScoreBatchRunner,\n  context: ScoreContext\n): Effect.Effect<ReadonlyArray<CaseResult>, EvalError> =>\n  Effect.gen(function*() {\n    const { requests, jobs } = yield* buildJobs(cases, suite, context)\n    if (requests.length === 0) return cases\n    const runBatchCorrelated = scorer.runBatchCorrelated\n    const protocol = runBatchCorrelated === undefined ? \"runBatch\" : \"runBatchCorrelated\"\n    const results = yield* runBatchCorrelated === undefined\n      ? scoreOrderOnly(jobs, requests, scorer, context)\n      : scoreCorrelated(jobs, scorer, runBatchCorrelated, context)\n    const { at } = context\n    const observations: Array<Observation> = []\n    for (const [index, request] of requests.entries()) {\n      const result = results[index]!\n      if (result.targetStepKey !== request.stepKey || result.scorerKey !== scorerKeyOf(request.binding)) {\n        return yield* Effect.fail(\n          new EvalError({\n            code: \"scorer_protocol\",\n            message: protocol === \"runBatchCorrelated\"\n              ? `Correlated scorer batch identity '${\n                jobs[index]!.identity\n              }' returned step '${result.targetStepKey}' and scorer '${result.scorerKey}' where job ${index} asked for scorer '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'`\n              : `Scorer batch returned an observation for '${result.scorerKey}' at step '${result.targetStepKey}' where job ${index} asked for '${\n                scorerKeyOf(request.binding)\n              }' at step '${request.stepKey}'; results must stay aligned with their jobs`,\n            path: `${protocol}[${index}]`\n          })\n        )\n      }\n      observations.push(observationFor(request, result, at))\n    }\n    const byCase = new Map<string, Array<Observation>>()\n    for (const observation of observations) {\n      const bucket = byCase.get(observation.case)\n      if (bucket === undefined) byCase.set(observation.case, [observation])\n      else bucket.push(observation)\n    }\n    return cases.map((caseResult) => ({\n      ...caseResult,\n      observations: byCase.get(caseResult.case) ?? []\n    }))\n  })\n\n/**\n * Runs a fixed suite with bounded execution and declaration-order results.\n *\n * Cases run through the provided {@link CaseExecutor} at the suite's\n * concurrency and are returned in declaration order. Each executor receives\n * its own mutable case copy. Each scorer receives independent copies of the\n * original case input, expected value, and binding data. Every execution is then\n * graded by the bindings whose `appliesTo` is the flow the execution reports as\n * its target, matched by reference identity, so a binding attached to another\n * flow contributes no observations.\n *\n * Scoring goes through `options.scorer` when the caller passes one, the\n * {@link Runner} service when one is provided, and {@link makeInline}\n * otherwise. A case whose target failed keeps its typed error, locates itself\n * with the executor's own `path` or with `cases['<name>']`, and produces no\n * observations; a scorer that failed produces an inconclusive observation. The\n * run itself fails only with `invalid_run_options` for a non-canonical run\n * identity or timestamp, `invalid_suite` for an unusable sampling policy,\n * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,\n * or `scorer_protocol` for a batch runner that broke the\n * {@link ScoreBatchRunner} contract.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const run = (\n  suite: Suite,\n  options: RunOptions\n): Effect.Effect<RunResult, EvalError, CaseExecutor> =>\n  Effect.gen(function*() {\n    const executor = yield* CaseExecutor\n    const injected = yield* Effect.serviceOption(Runner)\n    const scorer = options.scorer ?? (injected._tag === \"Some\" ? injected.value : makeInline())\n    const atMillis = Date.parse(options.at)\n    if (options.runId.trim().length === 0) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message: \"Deterministic runs require a non-empty runId\",\n          path: \"options.runId\"\n        })\n      )\n    }\n    if (\n      !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/u.test(options.at) ||\n      !Number.isFinite(atMillis) ||\n      new Date(atMillis).toISOString() !== options.at\n    ) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_run_options\",\n          message:\n            `Deterministic runs require a canonical UTC timestamp such as 2026-01-01T00:00:00.000Z, got '${options.at}'`,\n          path: \"options.at\"\n        })\n      )\n    }\n    const n = options.trials ?? suite.trials ?? 1\n    const k = options.k ?? 1\n    if (!Number.isSafeInteger(n) || n < 1 || n > 1000 || !Number.isSafeInteger(k) || k < 1 || k > n) {\n      return yield* Effect.fail(\n        new EvalError({\n          code: \"invalid_trials\",\n          message: \"trials must be an integer from 1 to 1000 and k must be an integer from 1 to trials\",\n          path: \"options.trials\"\n        })\n      )\n    }\n    const at = options.at\n    const runId = options.runId\n    const sampleId = options.sampleId ?? \"default\"\n    // Await each batch before starting the next: concurrency within a trial is\n    // bounded by the suite, and neither scheduling nor scorer latency reorders trials.\n    const batches = yield* Effect.forEach(Array.from({ length: n }, (_, index) => index), (index) =>\n      Effect.gen(function*() {\n        const cases = yield* Effect.forEach(suite.cases, (suiteCase) =>\n          runCase(executor, suiteCase), {\n          concurrency: suite.concurrency,\n          discard: false\n        })\n        return yield* score(cases, suite, scorer, {\n          concurrency: suite.concurrency,\n          at,\n          runId,\n          sampleId: n === 1 ? sampleId : tupleKey(sampleId, String(index))\n        })\n      }), { concurrency: 1 })\n    const outcomes: Record<string, Array<boolean>> = {}\n    const cases = suite.cases.map((suiteCase, index) => {\n      const attempts = batches.map((batch) =>\n        batch[index]!\n      )\n      const passed = attempts.map((attempt) =>\n        attempt.error === undefined && attempt.observations.length > 0 &&\n        attempt.observations.every((observation) => observation.kind === \"score\" && observation.score === 1)\n      )\n      outcomes[suiteCase.name] = passed\n      return {\n        ...attempts[0]!,\n        error: attempts.find((attempt) => attempt.error !== undefined)?.error,\n        observations: attempts.flatMap((attempt) => attempt.observations),\n        trials: Trials.summarize(passed, k)\n      }\n    })\n    return {\n      runId,\n      suite: suite.name,\n      cases,\n      observations: cases.flatMap((caseResult) => caseResult.observations),\n      trials: Trials.aggregate(outcomes, k),\n      k\n    }\n  }).pipe(Effect.withSpan(\"Eval.run\", { attributes: { suite: suite.name, runId: options.runId } }))\n\n/**\n * Provides a batch runner that is never available.\n *\n * Every bound score under this layer becomes an inconclusive observation, which\n * a gate grades as an undecidable run rather than a red. Provide it to state\n * that a suite must not score, and {@link layerInline} to score in process.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<Runner> = Layer.sync(Runner)(() => {\n  const unavailable = new EvalError({ code: \"scorer_unavailable\", message: \"No scorer batch runner is available\" })\n  return {\n    runBatch: () => Effect.fail(unavailable),\n    runBatchCorrelated: (jobs) =>\n      Effect.succeed(jobs.map((job) => ({\n        identity: job.identity,\n        observation: ScorerRunner.inconclusive(job, unavailable)\n      })))\n  }\n})\n"]}