@smthrs/evals 0.0.0-stage → 1.0.0-rc.12

package/dist/esm/Suite.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Fixed evaluation suite declarations.
+ *
+ * A suite is the fixed input half of an evaluation: named cases, the scorer
+ * bindings that grade them, and the concurrency the runner is allowed. It is
+ * validated once and then never changes, because the run that produced a
+ * committed baseline has to be reproducible from the suite value that was
+ * validated.
+ *
+ * @since 0.1.0
+ */
+import type * as ScorerBinding from "@smthrs/scorers/Binding";
+import * as Effect from "effect/Effect";
+import { EvalError } from "./EvalError.ts";
+/**
+ * A scorer binding accepted from `@smthrs/scorers`.
+ *
+ * The binding's `appliesTo` flow is matched against an execution's `target` by
+ * reference identity, so a binding only ever grades the exact flow value it was
+ * declared against.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Binding = ScorerBinding.Binding;
+/**
+ * One immutable fixed-suite case.
+ *
+ * `input` is handed to the executor and `expected` is offered to a bound scorer
+ * as ground truth. A declared `expected`, including `null`, wins over the
+ * binding's `groundTruth`; only an absent one defers to the binding. Both are
+ * snapshots taken by {@link make}: mutating the object a caller passed in never
+ * changes the suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Case {
+    readonly name: string;
+    readonly input: unknown;
+    readonly expected?: unknown | undefined;
+}
+/**
+ * Options for constructing a suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MakeOptions {
+    readonly name: string;
+    readonly cases: ReadonlyArray<Case>;
+    readonly bindings?: ReadonlyArray<Binding> | undefined;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * A validated, named collection of fixed cases and scorer bindings.
+ *
+ * The suite, cases, bindings, and their inert data are deeply frozen snapshots.
+ * Executable scorer and target identities remain unchanged.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Suite {
+    readonly name: string;
+    readonly cases: ReadonlyArray<Case>;
+    readonly bindings: ReadonlyArray<Binding>;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * The declared ceilings a suite is validated against.
+ *
+ * They exist so a mistake fails at construction with a sentence instead of
+ * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold
+ * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines
+ * fixture.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const limits: {
+    readonly concurrency: 1024;
+    readonly cases: 10000;
+    /** Longest JSON Lines fixture, in UTF-16 code units. */
+    readonly fixtureLength: number;
+};
+/**
+ * Builds and validates a fixed suite.
+ *
+ * When the effect runs, it reads every option, case field, and binding field
+ * exactly once, then validates and copies only what it read. Validating one
+ * value and copying another let a getter hand the suite something validation
+ * never saw. Every case and binding is then copied, including a binding's
+ * sampling policy, so the suite is a snapshot the caller can no longer reach:
+ * mutating an array, case input, or ratio policy afterwards leaves the
+ * validated suite unchanged. Data is checked before `structuredClone` and
+ * deeply frozen afterwards. Only plain objects, arrays, and cloneable
+ * primitives are admitted, with enumerable string-keyed data properties.
+ * Functions, class instances, mutable built-ins, and accessors fail with
+ * `invalid_suite` naming the offending path. Cycles and shared references are
+ * supported; null-prototype records become ordinary objects in the clone.
+ *
+ * Fails with `invalid_suite` for an empty or control-character name, no cases,
+ * more than `limits.cases` cases, a duplicate case name, or a concurrency that
+ * is not a safe integer in `[1, limits.concurrency]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const make: (options: MakeOptions) => Effect.Effect<Suite, EvalError>;
+/**
+ * Options used when decoding JSON Lines.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface JsonLinesOptions {
+    readonly name: string;
+    readonly bindings?: ReadonlyArray<Binding> | undefined;
+    readonly concurrency: number;
+    readonly trials?: number | undefined;
+}
+/**
+ * Loads the `{ name, input, expected? }` JSON Lines fixture format.
+ *
+ * Blank lines are skipped, a leading byte-order mark is stripped, and both LF
+ * and CRLF terminate a line. A malformed line fails with `invalid_suite`
+ * carrying the 1-based line number in both the message and the path; a fixture
+ * larger than `limits.fixtureLength` is rejected before any of it is parsed.
+ * The first non-blank line after `limits.cases` cases fails with
+ * `invalid_suite` at `cases` before it or any later line is parsed or decoded.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const fromJsonLines: (text: string, options: JsonLinesOptions) => Effect.Effect<Suite, EvalError>;
+//# sourceMappingURL=Suite.d.ts.map

package/dist/esm/Suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Suite.d.ts","sourceRoot":"","sources":["../../src/Suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,KAAK,aAAa,MAAM,yBAAyB,CAAA;AAC7D,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAG1C;;;;;;;;;GASG;AACH,MAAM,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAA;AAE3C;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;CACxC;AAED;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,SAAS,CAAA;IACtD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACzC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,MAAM;;;IAGjB,wDAAwD;;CAEhD,CAAA;AAkJV;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,IAAI,GAAI,SAAS,WAAW,KAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAqBtE,CAAA;AAEJ;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,SAAS,CAAA;IACtD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACrC;AAQD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,SAAS,gBAAgB,KAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAsClG,CAAA"}

package/dist/esm/Suite.js

@@ -0,0 +1,255 @@
+/**
+ * Fixed evaluation suite declarations.
+ *
+ * A suite is the fixed input half of an evaluation: named cases, the scorer
+ * bindings that grade them, and the concurrency the runner is allowed. It is
+ * validated once and then never changes, because the run that produced a
+ * committed baseline has to be reproducible from the suite value that was
+ * validated.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { EvalError } from "./EvalError.js";
+import { controlCharacter } from "./internal/controlCharacters.js";
+/**
+ * The declared ceilings a suite is validated against.
+ *
+ * They exist so a mistake fails at construction with a sentence instead of
+ * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold
+ * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines
+ * fixture.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const limits = {
+    concurrency: 1024,
+    cases: 10_000,
+    /** Longest JSON Lines fixture, in UTF-16 code units. */
+    fixtureLength: 8 * 1024 * 1024
+};
+const invalid = (message, path, cause) => new EvalError({ code: "invalid_suite", message, path, ...(cause === undefined ? {} : { cause }) });
+const cloneMessage = "Suite data must be structured-cloneable so the suite cannot change after it is validated";
+// Validate before structuredClone can erase a prototype or invoke a getter.
+// Only records and arrays can be protected completely with Object.freeze.
+const validateData = (value, path, seen = new WeakSet()) => {
+    if (typeof value === "function" || typeof value === "symbol")
+        throw invalid(cloneMessage, path);
+    if (value === null || typeof value !== "object")
+        return;
+    if (seen.has(value))
+        return;
+    seen.add(value);
+    const array = Array.isArray(value);
+    const prototype = Object.getPrototypeOf(value);
+    if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) {
+        throw invalid("Suite data must contain only plain objects, arrays, and structured-cloneable primitives", path);
+    }
+    for (const key of Reflect.ownKeys(value)) {
+        if (array && key === "length")
+            continue;
+        const childPath = typeof key === "string" && array && /^(0|[1-9]\d*)$/.test(key)
+            ? `${path}[${key}]`
+            : `${path}.${String(key)}`;
+        const descriptor = Object.getOwnPropertyDescriptor(value, key);
+        if (typeof key === "symbol" || !descriptor.enumerable || !("value" in descriptor)) {
+            throw invalid("Suite data properties must be enumerable string-keyed data fields", childPath);
+        }
+        validateData(descriptor.value, childPath, seen);
+    }
+};
+const freezeData = (value, seen = new WeakSet()) => {
+    if (value === null || typeof value !== "object" || seen.has(value))
+        return value;
+    seen.add(value);
+    for (const child of Object.values(value))
+        freezeData(child, seen);
+    return Object.freeze(value);
+};
+const clone = (value, path) => Effect.try({
+    try: () => {
+        validateData(value, path);
+        return freezeData(structuredClone(value));
+    },
+    catch: (cause) => cause instanceof EvalError ? cause : invalid(cloneMessage, path, cause)
+});
+const validate = (options) => {
+    if (options.name.trim().length === 0)
+        return invalid("Suite name must not be empty", "name");
+    const nameControl = controlCharacter(options.name);
+    if (nameControl !== undefined) {
+        return invalid(`Suite name must not contain the control character ${nameControl}`, "name");
+    }
+    if (!Number.isSafeInteger(options.trials) || options.trials < 1 || options.trials > 1000) {
+        return invalid("Suite trials must be an integer from 1 to 1000", "trials");
+    }
+    if (options.cases.length === 0)
+        return invalid("Suite must contain at least one case", "cases");
+    if (options.cases.length > limits.cases) {
+        return invalid(`Suite must contain at most ${limits.cases} cases, got ${options.cases.length}`, "cases");
+    }
+    if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {
+        return invalid(`Suite concurrency must be a positive safe integer, got ${String(options.concurrency)}`, "concurrency");
+    }
+    if (options.concurrency > limits.concurrency) {
+        return invalid(`Suite concurrency must be at most ${limits.concurrency}, got ${options.concurrency}`, "concurrency");
+    }
+    const names = new Set();
+    for (const [index, suiteCase] of options.cases.entries()) {
+        if (suiteCase.name.trim().length === 0) {
+            return invalid("Suite case name must not be empty", `cases[${index}].name`);
+        }
+        const caseControl = controlCharacter(suiteCase.name);
+        if (caseControl !== undefined) {
+            return invalid(`Suite case name must not contain the control character ${caseControl}`, `cases[${index}].name`);
+        }
+        if (names.has(suiteCase.name)) {
+            return invalid(`Duplicate suite case: ${suiteCase.name}`, `cases[${index}].name`);
+        }
+        names.add(suiteCase.name);
+    }
+    return undefined;
+};
+// Every field is read exactly once, into these two shapes, before anything is
+// validated or copied. Reading a field twice let a getter return the name that
+// passed validation and then a different one into the suite, which is the same
+// hole a baseline record had.
+const readCase = (suiteCase) => ({
+    name: suiteCase.name,
+    input: suiteCase.input,
+    expected: suiteCase.expected
+});
+const readBinding = (binding) => {
+    const read = {
+        scorer: binding.scorer,
+        appliesTo: binding.appliesTo,
+        sampling: binding.sampling
+    };
+    const groundTruth = binding.groundTruth;
+    if (groundTruth !== undefined)
+        read.groundTruth = groundTruth;
+    const context = binding.context;
+    if (context !== undefined)
+        read.context = context;
+    return read;
+};
+const copyCase = (suiteCase, index) => Effect.gen(function* () {
+    const input = yield* clone(suiteCase.input, `cases[${index}].input`);
+    if (suiteCase.expected === undefined)
+        return Object.freeze({ name: suiteCase.name, input });
+    const expected = yield* clone(suiteCase.expected, `cases[${index}].expected`);
+    return Object.freeze({ name: suiteCase.name, input, expected });
+});
+const copyBinding = (binding, index) => Effect.gen(function* () {
+    // `scorer` and `appliesTo` are executable identities matched by reference,
+    // so they are carried over unchanged; sampling, ground truth, and context
+    // are inert data that must not remain caller-owned.
+    const sampling = binding.sampling === undefined
+        ? undefined
+        : yield* clone(binding.sampling, `bindings[${index}].sampling`);
+    const copied = {
+        scorer: binding.scorer,
+        appliesTo: binding.appliesTo,
+        sampling: sampling
+    };
+    if (binding.groundTruth !== undefined) {
+        copied.groundTruth = yield* clone(binding.groundTruth, `bindings[${index}].groundTruth`);
+    }
+    if (binding.context !== undefined) {
+        copied.context = yield* clone(binding.context, `bindings[${index}].context`);
+    }
+    return Object.freeze(copied);
+});
+/**
+ * Builds and validates a fixed suite.
+ *
+ * When the effect runs, it reads every option, case field, and binding field
+ * exactly once, then validates and copies only what it read. Validating one
+ * value and copying another let a getter hand the suite something validation
+ * never saw. Every case and binding is then copied, including a binding's
+ * sampling policy, so the suite is a snapshot the caller can no longer reach:
+ * mutating an array, case input, or ratio policy afterwards leaves the
+ * validated suite unchanged. Data is checked before `structuredClone` and
+ * deeply frozen afterwards. Only plain objects, arrays, and cloneable
+ * primitives are admitted, with enumerable string-keyed data properties.
+ * Functions, class instances, mutable built-ins, and accessors fail with
+ * `invalid_suite` naming the offending path. Cycles and shared references are
+ * supported; null-prototype records become ordinary objects in the clone.
+ *
+ * Fails with `invalid_suite` for an empty or control-character name, no cases,
+ * more than `limits.cases` cases, a duplicate case name, or a concurrency that
+ * is not a safe integer in `[1, limits.concurrency]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (options) => Effect.suspend(() => {
+    const name = options.name;
+    const concurrency = options.concurrency;
+    const cases = [...options.cases].map(readCase);
+    const bindings = [...(options.bindings ?? [])].map(readBinding);
+    const trials = options.trials ?? 1;
+    const snapshot = { name, concurrency, cases, bindings, trials };
+    const error = validate(snapshot);
+    if (error !== undefined)
+        return Effect.fail(error);
+    return Effect.gen(function* () {
+        const copiedCases = yield* Effect.forEach(cases, copyCase);
+        const copiedBindings = yield* Effect.forEach(bindings, copyBinding);
+        return Object.freeze({
+            name,
+            cases: Object.freeze(copiedCases),
+            bindings: Object.freeze(copiedBindings),
+            concurrency,
+            trials
+        });
+    });
+});
+const jsonCase = Schema.Struct({
+    name: Schema.String,
+    input: Schema.Unknown,
+    expected: Schema.optional(Schema.Unknown)
+});
+/**
+ * Loads the `{ name, input, expected? }` JSON Lines fixture format.
+ *
+ * Blank lines are skipped, a leading byte-order mark is stripped, and both LF
+ * and CRLF terminate a line. A malformed line fails with `invalid_suite`
+ * carrying the 1-based line number in both the message and the path; a fixture
+ * larger than `limits.fixtureLength` is rejected before any of it is parsed.
+ * The first non-blank line after `limits.cases` cases fails with
+ * `invalid_suite` at `cases` before it or any later line is parsed or decoded.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const fromJsonLines = (text, options) => Effect.gen(function* () {
+    if (text.length > limits.fixtureLength) {
+        return yield* Effect.fail(invalid(`JSON Lines fixture must be at most ${limits.fixtureLength} characters, got ${text.length}`, "text"));
+    }
+    const cases = [];
+    const body = text.startsWith("\uFEFF") ? text.slice(1) : text;
+    for (const [index, line] of body.split(/\r?\n/).entries()) {
+        const trimmed = line.trim();
+        if (trimmed.length === 0)
+            continue;
+        // Stop at the ceiling before parsing or decoding the excess line: a
+        // fixture under the length limit can still hold hundreds of thousands
+        // of tiny lines, and reporting the count only in make() would parse and
+        // materialize every one of them first.
+        if (cases.length === limits.cases) {
+            return yield* Effect.fail(invalid(`Suite must contain at most ${limits.cases} cases, got ${cases.length + 1}`, "cases"));
+        }
+        const decoded = yield* Effect.try({
+            try: () => JSON.parse(trimmed),
+            catch: (cause) => invalid(`Invalid JSON on line ${index + 1}`, `line[${index + 1}]`, cause)
+        }).pipe(Effect.flatMap(Schema.decodeUnknownEffect(jsonCase)), Effect.mapError((cause) => cause instanceof EvalError
+            ? cause
+            : invalid(`Invalid suite case on line ${index + 1}`, `line[${index + 1}]`, cause)));
+        cases.push(decoded);
+    }
+    return yield* make({ ...options, cases });
+});
+//# sourceMappingURL=Suite.js.map

package/dist/esm/Suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Suite.js","sourceRoot":"","sources":["../../src/Suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AA+DlE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,WAAW,EAAE,IAAI;IACjB,KAAK,EAAE,MAAM;IACb,wDAAwD;IACxD,aAAa,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;CACtB,CAAA;AAEV,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAAY,EAAE,KAAe,EAAa,EAAE,CAC5E,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;AAEpG,MAAM,YAAY,GAAG,0FAA0F,CAAA;AAE/G,4EAA4E;AAC5E,0EAA0E;AAC1E,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,IAAY,EAAE,OAAO,IAAI,OAAO,EAAU,EAAQ,EAAE;IACxF,IAAI,OAAO,KAAK,KAAK,UAAU,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,MAAM,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAA;IAC/F,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAM;IACvD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAM;IAC3B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACf,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACjG,MAAM,OAAO,CAAC,yFAAyF,EAAE,IAAI,CAAC,CAAA;IAChH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ;YAAE,SAAQ;QACvC,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC;YAC9E,CAAC,CAAC,GAAG,IAAI,IAAI,GAAG,GAAG;YACnB,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;QAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,KAAK,EAAE,GAAG,CAAE,CAAA;QAC/D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC;YAClF,MAAM,OAAO,CAAC,mEAAmE,EAAE,SAAS,CAAC,CAAA;QAC/F,CAAC;QACD,YAAY,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IACjD,CAAC;AACH,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,KAAc,EAAE,OAAO,IAAI,OAAO,EAAU,EAAW,EAAE;IAC3E,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAChF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACjE,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC7B,CAAC,CAAA;AAED,MAAM,KAAK,GAAG,CAAC,KAAc,EAAE,IAAY,EAAqC,EAAE,CAChF,MAAM,CAAC,GAAG,CAAC;IACT,GAAG,EAAE,GAAG,EAAE;QACR,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACzB,OAAO,UAAU,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAA;IAC3C,CAAC;IACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,YAAY,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC;CAC1F,CAAC,CAAA;AAEJ,MAAM,QAAQ,GAAG,CAAC,OAAkD,EAAyB,EAAE;IAC7F,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAA;IAC5F,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAClD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,OAAO,CAAC,qDAAqD,WAAW,EAAE,EAAE,MAAM,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QACzF,OAAO,OAAO,CAAC,gDAAgD,EAAE,QAAQ,CAAC,CAAA;IAC5E,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,sCAAsC,EAAE,OAAO,CAAC,CAAA;IAC/F,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,8BAA8B,MAAM,CAAC,KAAK,eAAe,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAA;IAC1G,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1E,OAAO,OAAO,CACZ,0DAA0D,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EACvF,aAAa,CACd,CAAA;IACH,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO,OAAO,CACZ,qCAAqC,MAAM,CAAC,WAAW,SAAS,OAAO,CAAC,WAAW,EAAE,EACrF,aAAa,CACd,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,OAAO,CAAC,mCAAmC,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAA;QAC7E,CAAC;QACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACpD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,OAAO,CACZ,0DAA0D,WAAW,EAAE,EACvE,SAAS,KAAK,QAAQ,CACvB,CAAA;QACH,CAAC;QACD,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,OAAO,OAAO,CAAC,yBAAyB,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAA;QACnF,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC3B,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC,CAAA;AAED,8EAA8E;AAC9E,+EAA+E;AAC/E,+EAA+E;AAC/E,8BAA8B;AAC9B,MAAM,QAAQ,GAAG,CAAC,SAAe,EAAQ,EAAE,CAAC,CAAC;IAC3C,IAAI,EAAE,SAAS,CAAC,IAAI;IACpB,KAAK,EAAE,SAAS,CAAC,KAAK;IACtB,QAAQ,EAAE,SAAS,CAAC,QAAQ;CAC7B,CAAC,CAAA;AAEF,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAW,EAAE;IAChD,MAAM,IAAI,GAAmD;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAA;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAA;IACvC,IAAI,WAAW,KAAK,SAAS;QAAE,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;IAC7D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAA;IAC/B,IAAI,OAAO,KAAK,SAAS;QAAE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACjD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED,MAAM,QAAQ,GAAG,CAAC,SAAe,EAAE,KAAa,EAAkC,EAAE,CAClF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,KAAK,SAAS,CAAC,CAAA;IACpE,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,KAAK,YAAY,CAAC,CAAA;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;AACjE,CAAC,CAAC,CAAA;AAEJ,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,KAAa,EAAqC,EAAE,CACzF,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,2EAA2E;IAC3E,0EAA0E;IAC1E,oDAAoD;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS;QAC7C,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,KAAK,YAAY,CAAC,CAAA;IACjE,MAAM,MAAM,GAAmD;QAC7D,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,QAA+B;KAC1C,CAAA;IACD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,KAAK,eAAe,CAAC,CAAA;IAC1F,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,KAAK,WAAW,CAAC,CAAA;IAC9E,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAC9B,CAAC,CAAC,CAAA;AAEJ;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,OAAoB,EAAmC,EAAE,CAC5E,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE;IAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAA;IACzB,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAA;IACvC,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC9C,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAA;IAClC,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;IAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAChC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAClD,OAAO,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACnE,OAAO,MAAM,CAAC,MAAM,CAAC;YACnB,IAAI;YACJ,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;YACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;YACvC,WAAW;YACX,MAAM;SACP,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAeJ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,KAAK,EAAE,MAAM,CAAC,OAAO;IACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;CAC1C,CAAC,CAAA;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,OAAyB,EAAmC,EAAE,CACxG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,OAAO,CACL,sCAAsC,MAAM,CAAC,aAAa,oBAAoB,IAAI,CAAC,MAAM,EAAE,EAC3F,MAAM,CACP,CACF,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC7D,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,oEAAoE;QACpE,sEAAsE;QACtE,wEAAwE;QACxE,uCAAuC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CACvB,OAAO,CAAC,8BAA8B,MAAM,CAAC,KAAK,eAAe,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAC9F,CAAA;QACH,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;YAChC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY;YACzC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,wBAAwB,KAAK,GAAG,CAAC,EAAE,EAAE,QAAQ,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC;SAC5F,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,EACpD,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CACxB,KAAK,YAAY,SAAS;YACxB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,OAAO,CAAC,8BAA8B,KAAK,GAAG,CAAC,EAAE,EAAE,QAAQ,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CACpF,CACF,CAAA;QACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;AAC3C,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Fixed evaluation suite declarations.\n *\n * A suite is the fixed input half of an evaluation: named cases, the scorer\n * bindings that grade them, and the concurrency the runner is allowed. It is\n * validated once and then never changes, because the run that produced a\n * committed baseline has to be reproducible from the suite value that was\n * validated.\n *\n * @since 0.1.0\n */\n\nimport type * as ScorerBinding from \"@smthrs/scorers/Binding\"\nimport * as Effect from \"effect/Effect\"\nimport * as Schema from \"effect/Schema\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\n\n/**\n * A scorer binding accepted from `@smthrs/scorers`.\n *\n * The binding's `appliesTo` flow is matched against an execution's `target` by\n * reference identity, so a binding only ever grades the exact flow value it was\n * declared against.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Binding = ScorerBinding.Binding\n\n/**\n * One immutable fixed-suite case.\n *\n * `input` is handed to the executor and `expected` is offered to a bound scorer\n * as ground truth. A declared `expected`, including `null`, wins over the\n * binding's `groundTruth`; only an absent one defers to the binding. Both are\n * snapshots taken by {@link make}: mutating the object a caller passed in never\n * changes the suite.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Case {\n  readonly name: string\n  readonly input: unknown\n  readonly expected?: unknown | undefined\n}\n\n/**\n * Options for constructing a suite.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MakeOptions {\n  readonly name: string\n  readonly cases: ReadonlyArray<Case>\n  readonly bindings?: ReadonlyArray<Binding> | undefined\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\n/**\n * A validated, named collection of fixed cases and scorer bindings.\n *\n * The suite, cases, bindings, and their inert data are deeply frozen snapshots.\n * Executable scorer and target identities remain unchanged.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Suite {\n  readonly name: string\n  readonly cases: ReadonlyArray<Case>\n  readonly bindings: ReadonlyArray<Binding>\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\n/**\n * The declared ceilings a suite is validated against.\n *\n * They exist so a mistake fails at construction with a sentence instead of\n * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold\n * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines\n * fixture.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  concurrency: 1024,\n  cases: 10_000,\n  /** Longest JSON Lines fixture, in UTF-16 code units. */\n  fixtureLength: 8 * 1024 * 1024\n} as const\n\nconst invalid = (message: string, path: string, cause?: unknown): EvalError =>\n  new EvalError({ code: \"invalid_suite\", message, path, ...(cause === undefined ? {} : { cause }) })\n\nconst cloneMessage = \"Suite data must be structured-cloneable so the suite cannot change after it is validated\"\n\n// Validate before structuredClone can erase a prototype or invoke a getter.\n// Only records and arrays can be protected completely with Object.freeze.\nconst validateData = (value: unknown, path: string, seen = new WeakSet<object>()): void => {\n  if (typeof value === \"function\" || typeof value === \"symbol\") throw invalid(cloneMessage, path)\n  if (value === null || typeof value !== \"object\") return\n  if (seen.has(value)) return\n  seen.add(value)\n  const array = Array.isArray(value)\n  const prototype = Object.getPrototypeOf(value)\n  if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) {\n    throw invalid(\"Suite data must contain only plain objects, arrays, and structured-cloneable primitives\", path)\n  }\n  for (const key of Reflect.ownKeys(value)) {\n    if (array && key === \"length\") continue\n    const childPath = typeof key === \"string\" && array && /^(0|[1-9]\\d*)$/.test(key)\n      ? `${path}[${key}]`\n      : `${path}.${String(key)}`\n    const descriptor = Object.getOwnPropertyDescriptor(value, key)!\n    if (typeof key === \"symbol\" || !descriptor.enumerable || !(\"value\" in descriptor)) {\n      throw invalid(\"Suite data properties must be enumerable string-keyed data fields\", childPath)\n    }\n    validateData(descriptor.value, childPath, seen)\n  }\n}\n\nconst freezeData = (value: unknown, seen = new WeakSet<object>()): unknown => {\n  if (value === null || typeof value !== \"object\" || seen.has(value)) return value\n  seen.add(value)\n  for (const child of Object.values(value)) freezeData(child, seen)\n  return Object.freeze(value)\n}\n\nconst clone = (value: unknown, path: string): Effect.Effect<unknown, EvalError> =>\n  Effect.try({\n    try: () => {\n      validateData(value, path)\n      return freezeData(structuredClone(value))\n    },\n    catch: (cause) => cause instanceof EvalError ? cause : invalid(cloneMessage, path, cause)\n  })\n\nconst validate = (options: MakeOptions & { readonly trials: number }): EvalError | undefined => {\n  if (options.name.trim().length === 0) return invalid(\"Suite name must not be empty\", \"name\")\n  const nameControl = controlCharacter(options.name)\n  if (nameControl !== undefined) {\n    return invalid(`Suite name must not contain the control character ${nameControl}`, \"name\")\n  }\n  if (!Number.isSafeInteger(options.trials) || options.trials < 1 || options.trials > 1000) {\n    return invalid(\"Suite trials must be an integer from 1 to 1000\", \"trials\")\n  }\n  if (options.cases.length === 0) return invalid(\"Suite must contain at least one case\", \"cases\")\n  if (options.cases.length > limits.cases) {\n    return invalid(`Suite must contain at most ${limits.cases} cases, got ${options.cases.length}`, \"cases\")\n  }\n  if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {\n    return invalid(\n      `Suite concurrency must be a positive safe integer, got ${String(options.concurrency)}`,\n      \"concurrency\"\n    )\n  }\n  if (options.concurrency > limits.concurrency) {\n    return invalid(\n      `Suite concurrency must be at most ${limits.concurrency}, got ${options.concurrency}`,\n      \"concurrency\"\n    )\n  }\n  const names = new Set<string>()\n  for (const [index, suiteCase] of options.cases.entries()) {\n    if (suiteCase.name.trim().length === 0) {\n      return invalid(\"Suite case name must not be empty\", `cases[${index}].name`)\n    }\n    const caseControl = controlCharacter(suiteCase.name)\n    if (caseControl !== undefined) {\n      return invalid(\n        `Suite case name must not contain the control character ${caseControl}`,\n        `cases[${index}].name`\n      )\n    }\n    if (names.has(suiteCase.name)) {\n      return invalid(`Duplicate suite case: ${suiteCase.name}`, `cases[${index}].name`)\n    }\n    names.add(suiteCase.name)\n  }\n  return undefined\n}\n\n// Every field is read exactly once, into these two shapes, before anything is\n// validated or copied. Reading a field twice let a getter return the name that\n// passed validation and then a different one into the suite, which is the same\n// hole a baseline record had.\nconst readCase = (suiteCase: Case): Case => ({\n  name: suiteCase.name,\n  input: suiteCase.input,\n  expected: suiteCase.expected\n})\n\nconst readBinding = (binding: Binding): Binding => {\n  const read: { -readonly [K in keyof Binding]: Binding[K] } = {\n    scorer: binding.scorer,\n    appliesTo: binding.appliesTo,\n    sampling: binding.sampling\n  }\n  const groundTruth = binding.groundTruth\n  if (groundTruth !== undefined) read.groundTruth = groundTruth\n  const context = binding.context\n  if (context !== undefined) read.context = context\n  return read\n}\n\nconst copyCase = (suiteCase: Case, index: number): Effect.Effect<Case, EvalError> =>\n  Effect.gen(function*() {\n    const input = yield* clone(suiteCase.input, `cases[${index}].input`)\n    if (suiteCase.expected === undefined) return Object.freeze({ name: suiteCase.name, input })\n    const expected = yield* clone(suiteCase.expected, `cases[${index}].expected`)\n    return Object.freeze({ name: suiteCase.name, input, expected })\n  })\n\nconst copyBinding = (binding: Binding, index: number): Effect.Effect<Binding, EvalError> =>\n  Effect.gen(function*() {\n    // `scorer` and `appliesTo` are executable identities matched by reference,\n    // so they are carried over unchanged; sampling, ground truth, and context\n    // are inert data that must not remain caller-owned.\n    const sampling = binding.sampling === undefined\n      ? undefined\n      : yield* clone(binding.sampling, `bindings[${index}].sampling`)\n    const copied: { -readonly [K in keyof Binding]: Binding[K] } = {\n      scorer: binding.scorer,\n      appliesTo: binding.appliesTo,\n      sampling: sampling as Binding[\"sampling\"]\n    }\n    if (binding.groundTruth !== undefined) {\n      copied.groundTruth = yield* clone(binding.groundTruth, `bindings[${index}].groundTruth`)\n    }\n    if (binding.context !== undefined) {\n      copied.context = yield* clone(binding.context, `bindings[${index}].context`)\n    }\n    return Object.freeze(copied)\n  })\n\n/**\n * Builds and validates a fixed suite.\n *\n * When the effect runs, it reads every option, case field, and binding field\n * exactly once, then validates and copies only what it read. Validating one\n * value and copying another let a getter hand the suite something validation\n * never saw. Every case and binding is then copied, including a binding's\n * sampling policy, so the suite is a snapshot the caller can no longer reach:\n * mutating an array, case input, or ratio policy afterwards leaves the\n * validated suite unchanged. Data is checked before `structuredClone` and\n * deeply frozen afterwards. Only plain objects, arrays, and cloneable\n * primitives are admitted, with enumerable string-keyed data properties.\n * Functions, class instances, mutable built-ins, and accessors fail with\n * `invalid_suite` naming the offending path. Cycles and shared references are\n * supported; null-prototype records become ordinary objects in the clone.\n *\n * Fails with `invalid_suite` for an empty or control-character name, no cases,\n * more than `limits.cases` cases, a duplicate case name, or a concurrency that\n * is not a safe integer in `[1, limits.concurrency]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (options: MakeOptions): Effect.Effect<Suite, EvalError> =>\n  Effect.suspend(() => {\n    const name = options.name\n    const concurrency = options.concurrency\n    const cases = [...options.cases].map(readCase)\n    const bindings = [...(options.bindings ?? [])].map(readBinding)\n    const trials = options.trials ?? 1\n    const snapshot = { name, concurrency, cases, bindings, trials }\n    const error = validate(snapshot)\n    if (error !== undefined) return Effect.fail(error)\n    return Effect.gen(function*() {\n      const copiedCases = yield* Effect.forEach(cases, copyCase)\n      const copiedBindings = yield* Effect.forEach(bindings, copyBinding)\n      return Object.freeze({\n        name,\n        cases: Object.freeze(copiedCases),\n        bindings: Object.freeze(copiedBindings),\n        concurrency,\n        trials\n      })\n    })\n  })\n\n/**\n * Options used when decoding JSON Lines.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface JsonLinesOptions {\n  readonly name: string\n  readonly bindings?: ReadonlyArray<Binding> | undefined\n  readonly concurrency: number\n  readonly trials?: number | undefined\n}\n\nconst jsonCase = Schema.Struct({\n  name: Schema.String,\n  input: Schema.Unknown,\n  expected: Schema.optional(Schema.Unknown)\n})\n\n/**\n * Loads the `{ name, input, expected? }` JSON Lines fixture format.\n *\n * Blank lines are skipped, a leading byte-order mark is stripped, and both LF\n * and CRLF terminate a line. A malformed line fails with `invalid_suite`\n * carrying the 1-based line number in both the message and the path; a fixture\n * larger than `limits.fixtureLength` is rejected before any of it is parsed.\n * The first non-blank line after `limits.cases` cases fails with\n * `invalid_suite` at `cases` before it or any later line is parsed or decoded.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromJsonLines = (text: string, options: JsonLinesOptions): Effect.Effect<Suite, EvalError> =>\n  Effect.gen(function*() {\n    if (text.length > limits.fixtureLength) {\n      return yield* Effect.fail(\n        invalid(\n          `JSON Lines fixture must be at most ${limits.fixtureLength} characters, got ${text.length}`,\n          \"text\"\n        )\n      )\n    }\n    const cases: Array<Case> = []\n    const body = text.startsWith(\"\\uFEFF\") ? text.slice(1) : text\n    for (const [index, line] of body.split(/\\r?\\n/).entries()) {\n      const trimmed = line.trim()\n      if (trimmed.length === 0) continue\n      // Stop at the ceiling before parsing or decoding the excess line: a\n      // fixture under the length limit can still hold hundreds of thousands\n      // of tiny lines, and reporting the count only in make() would parse and\n      // materialize every one of them first.\n      if (cases.length === limits.cases) {\n        return yield* Effect.fail(\n          invalid(`Suite must contain at most ${limits.cases} cases, got ${cases.length + 1}`, \"cases\")\n        )\n      }\n      const decoded = yield* Effect.try({\n        try: () => JSON.parse(trimmed) as unknown,\n        catch: (cause) => invalid(`Invalid JSON on line ${index + 1}`, `line[${index + 1}]`, cause)\n      }).pipe(\n        Effect.flatMap(Schema.decodeUnknownEffect(jsonCase)),\n        Effect.mapError((cause) =>\n          cause instanceof EvalError\n            ? cause\n            : invalid(`Invalid suite case on line ${index + 1}`, `line[${index + 1}]`, cause)\n        )\n      )\n      cases.push(decoded)\n    }\n    return yield* make({ ...options, cases })\n  })\n"]}

package/dist/esm/Trials.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Repeated-run statistics for live model evaluations.
+ *
+ * A live case runs `n` times and passes `c` of them. {@link passAtK} is the
+ * chance that at least one of `k` runs passes, the capability a best-of-k
+ * caller sees; {@link passHatK} is the chance that all `k` pass, the
+ * consistency a user sees on every attempt. Both are exact for sampling `k` of
+ * the `n` recorded runs without replacement.
+ *
+ * Every function is pure and throws an `EvalError` with code `invalid_trials`
+ * for counts outside the documented bounds.
+ *
+ * @since 0.1.0
+ */
+/**
+ * Repeated-run statistics for one case.
+ *
+ * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,
+ * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Summary {
+    readonly n: number;
+    readonly passes: number;
+    readonly rate: number;
+    readonly passAt1: number;
+    readonly passAtK: number;
+    readonly passHatK: number;
+    readonly stderr: number;
+}
+/**
+ * Repeated-run statistics across cases.
+ *
+ * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is
+ * the fraction of cases whose every run passed.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Aggregate {
+    readonly cases: number;
+    readonly passAt1: number;
+    readonly passAtK: number;
+    readonly passHatK: number;
+    readonly allPass: number;
+    readonly perCase: Readonly<Record<string, Summary>>;
+}
+/**
+ * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,
+ * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is
+ * computed as a product, so large `n` neither overflows nor loses precision.
+ *
+ * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with
+ * `0 <= c <= n` and `1 <= k <= n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const passAtK: (n: number, c: number, k: number) => number;
+/**
+ * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`
+ * without replacement pass, the consistency measure of tau-bench.
+ *
+ * Throws `invalid_trials` under the same bounds as {@link passAtK}.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const passHatK: (n: number, c: number, k: number) => number;
+/**
+ * Summarizes one case's run outcomes at `k`.
+ *
+ * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const summarize: (outcomes: ReadonlyArray<boolean>, k: number) => Summary;
+/**
+ * Summarizes every case at `k` and averages over cases.
+ *
+ * A case with fewer than `k` runs is summarized at `k = n`, its own run
+ * count, so a short case still contributes rather than failing the whole
+ * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case
+ * with no runs or a `k` that is not a positive integer.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export declare const aggregate: (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number) => Aggregate;
+//# sourceMappingURL=Trials.d.ts.map

package/dist/esm/Trials.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Trials.d.ts","sourceRoot":"","sources":["../../src/Trials.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;;;;GAQG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CACpD;AAYD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,OAAO,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAMzD,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,KAAG,MAK1D,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,GAAI,UAAU,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,KAAG,OAcvE,CAAA;AAKD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,KAAG,SAiB9F,CAAA"}

package/dist/esm/Trials.js

@@ -0,0 +1,117 @@
+/**
+ * Repeated-run statistics for live model evaluations.
+ *
+ * A live case runs `n` times and passes `c` of them. {@link passAtK} is the
+ * chance that at least one of `k` runs passes, the capability a best-of-k
+ * caller sees; {@link passHatK} is the chance that all `k` pass, the
+ * consistency a user sees on every attempt. Both are exact for sampling `k` of
+ * the `n` recorded runs without replacement.
+ *
+ * Every function is pure and throws an `EvalError` with code `invalid_trials`
+ * for counts outside the documented bounds.
+ *
+ * @since 0.1.0
+ */
+import { EvalError } from "./EvalError.js";
+const invalid = (message, path) => new EvalError({ code: "invalid_trials", message, path });
+const validate = (n, c, k) => {
+    if (!Number.isInteger(n) || n < 1)
+        throw invalid(`Run count must be a positive integer, received ${n}`, "n");
+    if (!Number.isInteger(c) || c < 0 || c > n) {
+        throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, "c");
+    }
+    if (!Number.isInteger(k) || k < 1 || k > n)
+        throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, "k");
+};
+/**
+ * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,
+ * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is
+ * computed as a product, so large `n` neither overflows nor loses precision.
+ *
+ * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with
+ * `0 <= c <= n` and `1 <= k <= n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const passAtK = (n, c, k) => {
+    validate(n, c, k);
+    if (n - c < k)
+        return 1;
+    let fail = 1;
+    for (let i = n - c + 1; i <= n; i++)
+        fail *= 1 - k / i;
+    return 1 - fail;
+};
+/**
+ * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`
+ * without replacement pass, the consistency measure of tau-bench.
+ *
+ * Throws `invalid_trials` under the same bounds as {@link passAtK}.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const passHatK = (n, c, k) => {
+    validate(n, c, k);
+    let all = 1;
+    for (let i = 0; i < k; i++)
+        all *= (c - i) / (n - i);
+    return Math.max(0, all);
+};
+/**
+ * Summarizes one case's run outcomes at `k`.
+ *
+ * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const summarize = (outcomes, k) => {
+    const n = outcomes.length;
+    const passes = outcomes.filter(Boolean).length;
+    validate(n, passes, k);
+    const rate = passes / n;
+    return {
+        n,
+        passes,
+        rate,
+        passAt1: rate,
+        passAtK: passAtK(n, passes, k),
+        passHatK: passHatK(n, passes, k),
+        stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)
+    };
+};
+const mean = (values) => values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length;
+/**
+ * Summarizes every case at `k` and averages over cases.
+ *
+ * A case with fewer than `k` runs is summarized at `k = n`, its own run
+ * count, so a short case still contributes rather than failing the whole
+ * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case
+ * with no runs or a `k` that is not a positive integer.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const aggregate = (cases, k) => {
+    if (!Number.isInteger(k) || k < 1)
+        throw invalid(`k must be a positive integer, received ${k}`, "k");
+    const entries = Object.entries(cases).map(([name, outcomes]) => {
+        if (outcomes.length === 0)
+            throw invalid(`Case '${name}' has no runs`, `cases['${name}']`);
+        return [name, summarize(outcomes, Math.min(k, outcomes.length))];
+    });
+    // fromEntries defines own properties, so a case named `__proto__` stays a case.
+    const perCase = Object.fromEntries(entries);
+    const summaries = Object.values(perCase);
+    return {
+        cases: summaries.length,
+        passAt1: mean(summaries.map((summary) => summary.passAt1)),
+        passAtK: mean(summaries.map((summary) => summary.passAtK)),
+        passHatK: mean(summaries.map((summary) => summary.passHatK)),
+        allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),
+        perCase
+    };
+};
+//# sourceMappingURL=Trials.js.map