@smthrs/evals 0.0.0-stage → 1.0.0-rc.13

package/dist/esm/Report.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Canonical JSON and Markdown regression reports.
+ *
+ * The JSON report is the machine-readable artifact; the Markdown report is what
+ * an operator reads in a CI log, so it names every case the run could not
+ * decide rather than only counting them.
+ *
+ * @since 0.1.0
+ */
+import type { Baseline } from "./Baseline.ts";
+import type { MissingObservation, Nondeterminism, Regression, Report as RegressionReport } from "./Regression.ts";
+import type { CaseResult, RunResult } from "./Runner.ts";
+/**
+ * Current serialized report format version.
+ *
+ * It versions {@link Data}, the wire shape, independently of the nested
+ * `Baseline.version`: the comparison can change shape without the committed
+ * baseline artifact changing at all.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const version: 1;
+/** An index into {@link Data.run}'s `observations` table. */
+type ObservationRef = number;
+/**
+ * The serialized shape of a regression report.
+ *
+ * `run.observations` is the observation table: every observation the report
+ * mentions appears in it exactly once, and everything else refers to one by
+ * index. `run.cases[i].observations` is a list of indexes, and a regression or
+ * nondeterminism entry's `actual` is a single index. An in-memory
+ * `Regression.Report` carries the same observation object in up to four
+ * places, so encoding it directly repeated every reason and every `meta` that
+ * many times.
+ *
+ * `samples` and `inconclusive` are absent by construction: both are filters of
+ * `run.observations` by `kind`, so a reader recomputes them rather than
+ * downloading a third copy.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export interface Data {
+    readonly reportVersion: typeof version;
+    readonly suite: string;
+    readonly baseline: Baseline;
+    readonly run: Omit<RunResult, "cases"> & {
+        readonly cases: ReadonlyArray<Omit<CaseResult, "observations"> & {
+            readonly observations: ReadonlyArray<ObservationRef>;
+        }>;
+    };
+    readonly regressions: ReadonlyArray<Omit<Regression, "actual"> & {
+        readonly actual: ObservationRef;
+    }>;
+    readonly nondeterminism: ReadonlyArray<Omit<Nondeterminism, "actual"> & {
+        readonly actual: ObservationRef;
+    }>;
+    readonly missing: ReadonlyArray<MissingObservation>;
+}
+/**
+ * Projects a comparison into its serialized form.
+ *
+ * The observation table starts as `run.observations` and grows to hold any
+ * observation reached only through a case or a comparison entry, so a
+ * hand-built report whose case lists are not flattened still serializes every
+ * reference. Observations are pooled by identity, not by value: two distinct
+ * objects that happen to be equal stay two rows, because collapsing them would
+ * claim one grading where there were two.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const data: (report: RegressionReport) => Data;
+/**
+ * Serializes a regression report as stable, sorted-key JSON.
+ *
+ * The wire shape is {@link Data}, stamped with {@link version}: each
+ * observation is encoded once and referred to by index everywhere else. The
+ * report embeds each case's raw `execution.output`, which comes from an
+ * arbitrary target flow, so the encoding is total rather than trusting: keys
+ * are sorted by code unit, embedded strings are capped, total traversal is
+ * bounded, and anything JSON cannot express becomes a named marker
+ * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a
+ * `RangeError` or a silent `null`. Two identical runs therefore produce
+ * byte-identical JSON.
+ *
+ * Nothing redacts the embedded output. A suite whose cases carry secrets must
+ * not print this report where the log is readable.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const json: (report: RegressionReport) => string;
+/**
+ * Renders a concise stable Markdown regression report.
+ *
+ * Every count in the summary that is not zero has a section naming its rows:
+ * the regressions and the nondeterminism a gate reads as red, and the case
+ * failures, missing observations, and inconclusive observations that leave a
+ * gate undecided. A report that only counted them named nothing an operator
+ * could act on, which is exactly the run that needs debugging.
+ *
+ * Every cell and the suite heading value replace C0 controls and DEL with
+ * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,
+ * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,
+ * plus an ellipsis when truncated, without splitting escapes or code points.
+ *
+ * @category rendering
+ * @since 0.1.0
+ */
+export declare const markdown: (report: RegressionReport) => string;
+export {};
+//# sourceMappingURL=Report.d.ts.map

package/dist/esm/Report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Report.d.ts","sourceRoot":"","sources":["../../src/Report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAI7C,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,IAAI,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACjH,OAAO,KAAK,EAAE,UAAU,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAErE;;;;;;;;;GASG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC,6DAA6D;AAC7D,KAAK,cAAc,GAAG,MAAM,CAAA;AAE5B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,aAAa,EAAE,OAAO,OAAO,CAAA;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EACR,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,GACxB;QACA,QAAQ,CAAC,KAAK,EAAE,aAAa,CACzB,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,GAChC;YAAE,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;SAAE,CAC3D,CAAA;KACF,CAAA;IACH,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAA;IACrG,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAA;IAC5G,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;CACpD;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,IAAI,GAAI,QAAQ,gBAAgB,KAAG,IAsB/C,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,IAAI,GAAI,QAAQ,gBAAgB,KAAG,MAC2C,CAAA;AAkD3F;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,QAAQ,GAAI,QAAQ,gBAAgB,KAAG,MA4EnD,CAAA"}

package/dist/esm/Report.js

@@ -0,0 +1,188 @@
+/**
+ * Canonical JSON and Markdown regression reports.
+ *
+ * The JSON report is the machine-readable artifact; the Markdown report is what
+ * an operator reads in a CI log, so it names every case the run could not
+ * decide rather than only counting them.
+ *
+ * @since 0.1.0
+ */
+import { CanonicalJson } from "./internal/CanonicalJson.js";
+import { isControlCodePoint } from "./internal/controlCharacters.js";
+import { scorerLabel } from "./internal/scorerLabel.js";
+/**
+ * Current serialized report format version.
+ *
+ * It versions {@link Data}, the wire shape, independently of the nested
+ * `Baseline.version`: the comparison can change shape without the committed
+ * baseline artifact changing at all.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const version = 1;
+/**
+ * Projects a comparison into its serialized form.
+ *
+ * The observation table starts as `run.observations` and grows to hold any
+ * observation reached only through a case or a comparison entry, so a
+ * hand-built report whose case lists are not flattened still serializes every
+ * reference. Observations are pooled by identity, not by value: two distinct
+ * objects that happen to be equal stay two rows, because collapsing them would
+ * claim one grading where there were two.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const data = (report) => {
+    const positions = new Map();
+    const observations = [];
+    const refer = (observation) => {
+        const known = positions.get(observation);
+        if (known !== undefined)
+            return known;
+        const position = observations.length;
+        observations.push(observation);
+        positions.set(observation, position);
+        return position;
+    };
+    for (const observation of report.run.observations)
+        refer(observation);
+    const cases = report.run.cases.map((result) => ({ ...result, observations: result.observations.map(refer) }));
+    return {
+        reportVersion: version,
+        suite: report.suite,
+        baseline: report.baseline,
+        run: { ...report.run, cases, observations },
+        regressions: report.regressions.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+        nondeterminism: report.nondeterminism.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+        missing: report.missing
+    };
+};
+/**
+ * Serializes a regression report as stable, sorted-key JSON.
+ *
+ * The wire shape is {@link Data}, stamped with {@link version}: each
+ * observation is encoded once and referred to by index everywhere else. The
+ * report embeds each case's raw `execution.output`, which comes from an
+ * arbitrary target flow, so the encoding is total rather than trusting: keys
+ * are sorted by code unit, embedded strings are capped, total traversal is
+ * bounded, and anything JSON cannot express becomes a named marker
+ * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a
+ * `RangeError` or a silent `null`. Two identical runs therefore produce
+ * byte-identical JSON.
+ *
+ * Nothing redacts the embedded output. A suite whose cases carry secrets must
+ * not print this report where the log is readable.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const json = (report) => CanonicalJson.stringify(data(report), { maxStringLength: CanonicalJson.maxStringLength });
+/** Maximum escaped UTF-16 code units per value, before a truncation ellipsis. */
+const maxCellLength = 240;
+const cell = (value) => {
+    let text = "";
+    if (value !== undefined && value !== null) {
+        if (typeof value === "string")
+            text = value;
+        else {
+            try {
+                text = String(value);
+            }
+            catch {
+                text = "[unreadable]";
+            }
+        }
+    }
+    let escaped = "";
+    for (const character of text) {
+        const code = character.codePointAt(0);
+        // Escape each input character once, including backslashes before pipes.
+        // Ampersands prevent entities; dots, colons, and @ prevent GFM autolinks.
+        const token = isControlCodePoint(code)
+            ? " "
+            : /[\\|`*_[\]<>!#~&.:@]/u.test(character)
+                ? `\\${character}`
+                : character;
+        if (escaped.length + token.length > maxCellLength)
+            return `${escaped}…`;
+        escaped += token;
+    }
+    return escaped;
+};
+const scorerCell = (scorer, scorerName) => cell(scorerLabel({ scorer, scorerName }));
+const section = (heading, header, alignment, rows) => rows.length === 0 ? [] : [
+    `## ${heading}`,
+    "",
+    `| ${header.join(" | ")} |`,
+    `| ${alignment.join(" | ")} |`,
+    ...rows.map((row) => `| ${row.join(" | ")} |`),
+    ""
+];
+/**
+ * Renders a concise stable Markdown regression report.
+ *
+ * Every count in the summary that is not zero has a section naming its rows:
+ * the regressions and the nondeterminism a gate reads as red, and the case
+ * failures, missing observations, and inconclusive observations that leave a
+ * gate undecided. A report that only counted them named nothing an operator
+ * could act on, which is exactly the run that needs debugging.
+ *
+ * Every cell and the suite heading value replace C0 controls and DEL with
+ * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,
+ * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,
+ * plus an ellipsis when truncated, without splitting escapes or code points.
+ *
+ * @category rendering
+ * @since 0.1.0
+ */
+export const markdown = (report) => {
+    const lines = [
+        `# Evaluation report: ${cell(report.suite)}`,
+        "",
+        `- Regressions: ${report.regressions.length}`,
+        `- Nondeterminism: ${report.nondeterminism.length}`,
+        `- Missing observations: ${report.missing.length}`,
+        `- Inconclusive observations: ${report.inconclusive.length}`,
+        `- Failed cases: ${report.run.cases.filter((result) => result.error !== undefined).length}`,
+        ...(report.run.trials === undefined ? [] : [
+            `- pass@1: ${report.run.trials.passAt1.toFixed(6)}`,
+            `- pass@${report.run.k ?? 1}: ${report.run.trials.passAtK.toFixed(6)}`,
+            `- pass^${report.run.k ?? 1}: ${report.run.trials.passHatK.toFixed(6)}`,
+            `- stderr: ${(Object.values(report.run.trials.perCase).reduce((sum, item) => sum + item.stderr, 0) / report.run.trials.cases)
+                .toFixed(6)}`
+        ]),
+        "",
+        ...section("Regressions", ["Case", "Scorer", "Baseline", "Actual", "Drop"], ["---", "---", "---:", "---:", "---:"], report.regressions.map((item) => [
+            cell(item.case),
+            scorerCell(item.scorer, item.actual.scorerName),
+            item.baseline.score.toFixed(6),
+            item.actual.score.toFixed(6),
+            item.drop.toFixed(6)
+        ])),
+        ...section("Nondeterminism", ["Case", "Scorer", "Step key", "Baseline", "Actual", "Delta"], ["---", "---", "---", "---:", "---:", "---:"], report.nondeterminism.map((item) => [
+            cell(item.case),
+            scorerCell(item.scorer, item.actual.scorerName),
+            cell(item.actual.stepKey),
+            item.baseline.score.toFixed(6),
+            item.actual.score.toFixed(6),
+            item.delta.toFixed(6)
+        ])),
+        ...section("Case failures", ["Case", "Code", "Message"], ["---", "---", "---"], report.run.cases.flatMap((result) => result.error === undefined ? [] : [[cell(result.case), cell(result.error.code), cell(result.error.message)]])),
+        ...section("Missing observations", ["Side", "Case", "Scorer", "Step key"], ["---", "---", "---", "---"], report.missing.map((item) => [
+            cell(item.side),
+            cell(item.case),
+            scorerCell(item.scorer, item.scorerName),
+            cell(item.stepKey)
+        ])),
+        ...section("Inconclusive", ["Case", "Scorer", "Step key", "Reason"], ["---", "---", "---", "---"], report.inconclusive.map((item) => [
+            cell(item.case),
+            scorerCell(item.scorer, item.scorerName),
+            cell(item.stepKey),
+            cell(item.reason)
+        ]))
+    ];
+    return `${lines.join("\n").trim()}\n`;
+};
+//# sourceMappingURL=Report.js.map

package/dist/esm/Report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Report.js","sourceRoot":"","sources":["../../src/Report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAIvD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAU,CAAA;AAwCjC;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAwB,EAAQ,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,YAAY,GAAuB,EAAE,CAAA;IAC3C,MAAM,KAAK,GAAG,CAAC,WAAwB,EAAkB,EAAE;QACzD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAA;QACrC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAA;QACpC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QAC9B,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QACpC,OAAO,QAAQ,CAAA;IACjB,CAAC,CAAA;IACD,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,GAAG,CAAC,YAAY;QAAE,KAAK,CAAC,WAAW,CAAC,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7G,OAAO;QACL,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE;QAC3C,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3F,cAAc,EAAE,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjG,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAA;AACH,CAAC,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAwB,EAAU,EAAE,CACvD,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,aAAa,CAAC,eAAe,EAAE,CAAC,CAAA;AAE3F,iFAAiF;AACjF,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,IAAI,GAAG,CAAC,KAAc,EAAU,EAAE;IACtC,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,IAAI,GAAG,KAAK,CAAA;aACtC,CAAC;YACJ,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,GAAG,cAAc,CAAA;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAE,CAAA;QACtC,wEAAwE;QACxE,0EAA0E;QAC1E,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC;YACpC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC;gBACzC,CAAC,CAAC,KAAK,SAAS,EAAE;gBAClB,CAAC,CAAC,SAAS,CAAA;QACb,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,aAAa;YAAE,OAAO,GAAG,OAAO,GAAG,CAAA;QACvE,OAAO,IAAI,KAAK,CAAA;IAClB,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,UAA8B,EAAU,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAA;AAExH,MAAM,OAAO,GAAG,CACd,OAAe,EACf,MAA6B,EAC7B,SAAgC,EAChC,IAA0C,EACnB,EAAE,CACzB,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvB,MAAM,OAAO,EAAE;IACf,EAAE;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;IAC3B,KAAK,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI;IAC9B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IAC9C,EAAE;CACH,CAAA;AAEH;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAwB,EAAU,EAAE;IAC3D,MAAM,KAAK,GAAG;QACZ,wBAAwB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QAC5C,EAAE;QACF,kBAAkB,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE;QAC7C,qBAAqB,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE;QACnD,2BAA2B,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE;QAClD,gCAAgC,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;QAC5D,mBAAmB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE;QAC3F,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACzC,aAAa,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACnD,UAAU,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACtE,UAAU,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACvE,aACE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC7G,OAAO,CAAC,CAAC,CACd,EAAE;SACH,CAAC;QACF,EAAE;QACF,GAAG,OAAO,CACR,aAAa,EACb,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,EAChD,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EACtC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;SACrB,CAAC,CACH;QACD,GAAG,OAAO,CACR,gBAAgB,EAChB,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAC7D,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAC7C,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YAC/C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;YACzB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;SACtB,CAAC,CACH;QACD,GAAG,OAAO,CACR,eAAe,EACf,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,EAC3B,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EACrB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAClC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAC7G,CACF;QACD,GAAG,OAAO,CACR,sBAAsB,EACtB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC,EACtC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;SACnB,CAAC,CACH;QACD,GAAG,OAAO,CACR,cAAc,EACd,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,EACxC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EAC5B,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;SAClB,CAAC,CACH;KACF,CAAA;IACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAA;AACvC,CAAC,CAAA","sourcesContent":["/**\n * Canonical JSON and Markdown regression reports.\n *\n * The JSON report is the machine-readable artifact; the Markdown report is what\n * an operator reads in a CI log, so it names every case the run could not\n * decide rather than only counting them.\n *\n * @since 0.1.0\n */\n\nimport type { Baseline } from \"./Baseline.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { isControlCodePoint } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport type { MissingObservation, Nondeterminism, Regression, Report as RegressionReport } from \"./Regression.ts\"\nimport type { CaseResult, Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current serialized report format version.\n *\n * It versions {@link Data}, the wire shape, independently of the nested\n * `Baseline.version`: the comparison can change shape without the committed\n * baseline artifact changing at all.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/** An index into {@link Data.run}'s `observations` table. */\ntype ObservationRef = number\n\n/**\n * The serialized shape of a regression report.\n *\n * `run.observations` is the observation table: every observation the report\n * mentions appears in it exactly once, and everything else refers to one by\n * index. `run.cases[i].observations` is a list of indexes, and a regression or\n * nondeterminism entry's `actual` is a single index. An in-memory\n * `Regression.Report` carries the same observation object in up to four\n * places, so encoding it directly repeated every reason and every `meta` that\n * many times.\n *\n * `samples` and `inconclusive` are absent by construction: both are filters of\n * `run.observations` by `kind`, so a reader recomputes them rather than\n * downloading a third copy.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport interface Data {\n  readonly reportVersion: typeof version\n  readonly suite: string\n  readonly baseline: Baseline\n  readonly run:\n    & Omit<RunResult, \"cases\">\n    & {\n      readonly cases: ReadonlyArray<\n        & Omit<CaseResult, \"observations\">\n        & { readonly observations: ReadonlyArray<ObservationRef> }\n      >\n    }\n  readonly regressions: ReadonlyArray<Omit<Regression, \"actual\"> & { readonly actual: ObservationRef }>\n  readonly nondeterminism: ReadonlyArray<Omit<Nondeterminism, \"actual\"> & { readonly actual: ObservationRef }>\n  readonly missing: ReadonlyArray<MissingObservation>\n}\n\n/**\n * Projects a comparison into its serialized form.\n *\n * The observation table starts as `run.observations` and grows to hold any\n * observation reached only through a case or a comparison entry, so a\n * hand-built report whose case lists are not flattened still serializes every\n * reference. Observations are pooled by identity, not by value: two distinct\n * objects that happen to be equal stay two rows, because collapsing them would\n * claim one grading where there were two.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const data = (report: RegressionReport): Data => {\n  const positions = new Map<Observation, ObservationRef>()\n  const observations: Array<Observation> = []\n  const refer = (observation: Observation): ObservationRef => {\n    const known = positions.get(observation)\n    if (known !== undefined) return known\n    const position = observations.length\n    observations.push(observation)\n    positions.set(observation, position)\n    return position\n  }\n  for (const observation of report.run.observations) refer(observation)\n  const cases = report.run.cases.map((result) => ({ ...result, observations: result.observations.map(refer) }))\n  return {\n    reportVersion: version,\n    suite: report.suite,\n    baseline: report.baseline,\n    run: { ...report.run, cases, observations },\n    regressions: report.regressions.map((entry) => ({ ...entry, actual: refer(entry.actual) })),\n    nondeterminism: report.nondeterminism.map((entry) => ({ ...entry, actual: refer(entry.actual) })),\n    missing: report.missing\n  }\n}\n\n/**\n * Serializes a regression report as stable, sorted-key JSON.\n *\n * The wire shape is {@link Data}, stamped with {@link version}: each\n * observation is encoded once and referred to by index everywhere else. The\n * report embeds each case's raw `execution.output`, which comes from an\n * arbitrary target flow, so the encoding is total rather than trusting: keys\n * are sorted by code unit, embedded strings are capped, total traversal is\n * bounded, and anything JSON cannot express becomes a named marker\n * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a\n * `RangeError` or a silent `null`. Two identical runs therefore produce\n * byte-identical JSON.\n *\n * Nothing redacts the embedded output. A suite whose cases carry secrets must\n * not print this report where the log is readable.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const json = (report: RegressionReport): string =>\n  CanonicalJson.stringify(data(report), { maxStringLength: CanonicalJson.maxStringLength })\n\n/** Maximum escaped UTF-16 code units per value, before a truncation ellipsis. */\nconst maxCellLength = 240\n\nconst cell = (value: unknown): string => {\n  let text = \"\"\n  if (value !== undefined && value !== null) {\n    if (typeof value === \"string\") text = value\n    else {\n      try {\n        text = String(value)\n      } catch {\n        text = \"[unreadable]\"\n      }\n    }\n  }\n  let escaped = \"\"\n  for (const character of text) {\n    const code = character.codePointAt(0)!\n    // Escape each input character once, including backslashes before pipes.\n    // Ampersands prevent entities; dots, colons, and @ prevent GFM autolinks.\n    const token = isControlCodePoint(code)\n      ? \" \"\n      : /[\\\\|`*_[\\]<>!#~&.:@]/u.test(character)\n      ? `\\\\${character}`\n      : character\n    if (escaped.length + token.length > maxCellLength) return `${escaped}…`\n    escaped += token\n  }\n  return escaped\n}\n\nconst scorerCell = (scorer: string, scorerName: string | undefined): string => cell(scorerLabel({ scorer, scorerName }))\n\nconst section = (\n  heading: string,\n  header: ReadonlyArray<string>,\n  alignment: ReadonlyArray<string>,\n  rows: ReadonlyArray<ReadonlyArray<string>>\n): ReadonlyArray<string> =>\n  rows.length === 0 ? [] : [\n    `## ${heading}`,\n    \"\",\n    `| ${header.join(\" | \")} |`,\n    `| ${alignment.join(\" | \")} |`,\n    ...rows.map((row) => `| ${row.join(\" | \")} |`),\n    \"\"\n  ]\n\n/**\n * Renders a concise stable Markdown regression report.\n *\n * Every count in the summary that is not zero has a section naming its rows:\n * the regressions and the nondeterminism a gate reads as red, and the case\n * failures, missing observations, and inconclusive observations that leave a\n * gate undecided. A report that only counted them named nothing an operator\n * could act on, which is exactly the run that needs debugging.\n *\n * Every cell and the suite heading value replace C0 controls and DEL with\n * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,\n * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,\n * plus an ellipsis when truncated, without splitting escapes or code points.\n *\n * @category rendering\n * @since 0.1.0\n */\nexport const markdown = (report: RegressionReport): string => {\n  const lines = [\n    `# Evaluation report: ${cell(report.suite)}`,\n    \"\",\n    `- Regressions: ${report.regressions.length}`,\n    `- Nondeterminism: ${report.nondeterminism.length}`,\n    `- Missing observations: ${report.missing.length}`,\n    `- Inconclusive observations: ${report.inconclusive.length}`,\n    `- Failed cases: ${report.run.cases.filter((result) => result.error !== undefined).length}`,\n    ...(report.run.trials === undefined ? [] : [\n      `- pass@1: ${report.run.trials.passAt1.toFixed(6)}`,\n      `- pass@${report.run.k ?? 1}: ${report.run.trials.passAtK.toFixed(6)}`,\n      `- pass^${report.run.k ?? 1}: ${report.run.trials.passHatK.toFixed(6)}`,\n      `- stderr: ${\n        (Object.values(report.run.trials.perCase).reduce((sum, item) => sum + item.stderr, 0) / report.run.trials.cases)\n          .toFixed(6)\n      }`\n    ]),\n    \"\",\n    ...section(\n      \"Regressions\",\n      [\"Case\", \"Scorer\", \"Baseline\", \"Actual\", \"Drop\"],\n      [\"---\", \"---\", \"---:\", \"---:\", \"---:\"],\n      report.regressions.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.actual.scorerName),\n        item.baseline.score.toFixed(6),\n        item.actual.score.toFixed(6),\n        item.drop.toFixed(6)\n      ])\n    ),\n    ...section(\n      \"Nondeterminism\",\n      [\"Case\", \"Scorer\", \"Step key\", \"Baseline\", \"Actual\", \"Delta\"],\n      [\"---\", \"---\", \"---\", \"---:\", \"---:\", \"---:\"],\n      report.nondeterminism.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.actual.scorerName),\n        cell(item.actual.stepKey),\n        item.baseline.score.toFixed(6),\n        item.actual.score.toFixed(6),\n        item.delta.toFixed(6)\n      ])\n    ),\n    ...section(\n      \"Case failures\",\n      [\"Case\", \"Code\", \"Message\"],\n      [\"---\", \"---\", \"---\"],\n      report.run.cases.flatMap((result) =>\n        result.error === undefined ? [] : [[cell(result.case), cell(result.error.code), cell(result.error.message)]]\n      )\n    ),\n    ...section(\n      \"Missing observations\",\n      [\"Side\", \"Case\", \"Scorer\", \"Step key\"],\n      [\"---\", \"---\", \"---\", \"---\"],\n      report.missing.map((item) => [\n        cell(item.side),\n        cell(item.case),\n        scorerCell(item.scorer, item.scorerName),\n        cell(item.stepKey)\n      ])\n    ),\n    ...section(\n      \"Inconclusive\",\n      [\"Case\", \"Scorer\", \"Step key\", \"Reason\"],\n      [\"---\", \"---\", \"---\", \"---\"],\n      report.inconclusive.map((item) => [\n        cell(item.case),\n        scorerCell(item.scorer, item.scorerName),\n        cell(item.stepKey),\n        cell(item.reason)\n      ])\n    )\n  ]\n  return `${lines.join(\"\\n\").trim()}\\n`\n}\n"]}

package/dist/esm/Runner.d.ts

@@ -0,0 +1,229 @@
+/**
+ * Deterministic fixed-suite execution and bound scorer evaluation.
+ *
+ * A run executes every case through the injected {@link CaseExecutor}, then
+ * grades each execution with the scorers bound to the flow it executed. The
+ * timestamp and the run identity come from the caller, so two runs of the same
+ * suite over the same inputs produce byte-identical observations.
+ *
+ * @since 0.1.0
+ */
+import * as ScorerRunner from "@smthrs/scorers/Runner";
+import type * as ScoreStore from "@smthrs/scorers/ScoreStore";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { CaseExecutor, type Execution } from "./CaseExecutor.ts";
+import { EvalError } from "./EvalError.ts";
+import type { Binding, Suite } from "./Suite.ts";
+import * as Trials from "./Trials.ts";
+/**
+ * One score observation emitted by a suite run.
+ *
+ * `scorer` is the scorer key, a digest of the scorer's own declaration, which
+ * is what a baseline matches on. `scorerName` is the human name from the same
+ * declaration, carried alongside so a report can be read without grepping for
+ * the digest. `at` is the run's timestamp, not the scorer's: a run's
+ * observations all carry one instant so a baseline stays reproducible.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Observation = {
+    readonly case: string;
+    readonly scorer: string;
+    readonly scorerName?: string | undefined;
+    readonly stepKey: string;
+    readonly at: string;
+} & ({
+    readonly kind: "score";
+    readonly score: number;
+    readonly reason?: string | undefined;
+    readonly meta?: unknown;
+} | {
+    readonly kind: "inconclusive";
+    readonly reason: string;
+    readonly meta?: unknown;
+});
+/**
+ * A request sent to the scorers batch runner.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface ScoreRequest {
+    readonly case: string;
+    readonly stepKey: string;
+    readonly binding: Binding;
+    readonly input: {
+        readonly input: unknown;
+        readonly output: unknown;
+        readonly groundTruth?: unknown;
+        readonly context?: unknown;
+        readonly latencyMs?: number;
+    };
+}
+/**
+ * A blocking scorer job, matching the `Runner` module of `@smthrs/scorers`.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ScoreJob = ScorerRunner.Job;
+/**
+ * Structural adapter for `@smthrs/scorers`' blocking batch runner.
+ *
+ * A runner that implements `runBatchCorrelated` tags every observation with
+ * its job identity and may return results in any order. A runner that implements
+ * only `runBatch` is correlated positionally: it must return exactly one
+ * observation per job, in order, and each observation must echo its job's
+ * `targetStepKey` and `scorerKey`. A run refuses the order-only protocol before
+ * scoring when two jobs share that pair, because their results could not be
+ * attributed to a case.
+ *
+ * A run fails with `scorer_protocol` when either protocol is broken.
+ * `@smthrs/scorers`' `Runner.Service` implements `runBatchCorrelated`, so its
+ * service value can be used directly and `ambiguous_score_job` never applies
+ * to it.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export interface ScoreBatchRunner {
+    readonly runBatch: (jobs: ReadonlyArray<ScoreJob>, options?: {
+        readonly concurrency?: number | undefined;
+    }) => Effect.Effect<ReadonlyArray<ScoreObservation>, unknown>;
+    readonly runBatchCorrelated?: ((jobs: ReadonlyArray<ScoreJob>, options?: {
+        readonly concurrency?: number | undefined;
+    }) => Effect.Effect<ReadonlyArray<BatchResult>, unknown>) | undefined;
+}
+/**
+ * A score result aligned with a {@link ScoreRequest}.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ScoreObservation = ScoreStore.Observation;
+/**
+ * A batch result tagged with the identity of the job that produced it.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface BatchResult {
+    readonly identity: string;
+    readonly observation: ScoreObservation;
+}
+/**
+ * Per-case result retained by the deterministic runner.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface CaseResult {
+    readonly case: string;
+    readonly execution?: Execution | undefined;
+    readonly error?: EvalError | undefined;
+    readonly observations: ReadonlyArray<Observation>;
+    readonly trials?: Trials.Summary | undefined;
+}
+/**
+ * Stable result of a suite run.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface RunResult {
+    readonly runId: string;
+    readonly suite: string;
+    readonly cases: ReadonlyArray<CaseResult>;
+    readonly observations: ReadonlyArray<Observation>;
+    readonly trials?: Trials.Aggregate | undefined;
+    readonly k?: number | undefined;
+}
+/**
+ * Options for a deterministic suite run.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface RunOptions {
+    readonly scorer?: ScoreBatchRunner | undefined;
+    readonly runId: string;
+    readonly sampleId?: string | undefined;
+    readonly trials?: number | undefined;
+    readonly k?: number | undefined;
+    readonly at: string;
+}
+declare const Runner_base: Context.ServiceClass<Runner, "flows/evals/Runner", ScoreBatchRunner>;
+/**
+ * Injectable batch-runner service used when a caller wants a reusable adapter.
+ *
+ * {@link run} does not require it: a run scores with `options.scorer` when one
+ * is passed, otherwise with this service when one is provided, otherwise
+ * in process through {@link makeInline}.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export declare class Runner extends Runner_base {
+}
+/**
+ * Builds the in-process batch runner a run scores with by default.
+ *
+ * Each job's scorer runs in the current process, its result is checked by
+ * `@smthrs/scorers`' own `Scorer.validate`, and a scorer that fails becomes an
+ * inconclusive observation naming its cause rather than failing the run. It
+ * honours the {@link ScoreBatchRunner} protocol, so it is also the reference
+ * a custom adapter can be compared against.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const makeInline: () => ScoreBatchRunner;
+/**
+ * Provides the in-process batch runner built by {@link makeInline}.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layerInline: Layer.Layer<Runner>;
+/**
+ * Runs a fixed suite with bounded execution and declaration-order results.
+ *
+ * Cases run through the provided {@link CaseExecutor} at the suite's
+ * concurrency and are returned in declaration order. Each executor receives
+ * its own mutable case copy. Each scorer receives independent copies of the
+ * original case input, expected value, and binding data. Every execution is then
+ * graded by the bindings whose `appliesTo` is the flow the execution reports as
+ * its target, matched by reference identity, so a binding attached to another
+ * flow contributes no observations.
+ *
+ * Scoring goes through `options.scorer` when the caller passes one, the
+ * {@link Runner} service when one is provided, and {@link makeInline}
+ * otherwise. A case whose target failed keeps its typed error, locates itself
+ * with the executor's own `path` or with `cases['<name>']`, and produces no
+ * observations; a scorer that failed produces an inconclusive observation. The
+ * run itself fails only with `invalid_run_options` for a non-canonical run
+ * identity or timestamp, `invalid_suite` for an unusable sampling policy,
+ * `ambiguous_score_job` when an order-only runner cannot distinguish two jobs,
+ * or `scorer_protocol` for a batch runner that broke the
+ * {@link ScoreBatchRunner} contract.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const run: (suite: Suite, options: RunOptions) => Effect.Effect<RunResult, EvalError, CaseExecutor>;
+/**
+ * Provides a batch runner that is never available.
+ *
+ * Every bound score under this layer becomes an inconclusive observation, which
+ * a gate grades as an undecidable run rather than a red. Provide it to state
+ * that a suite must not score, and {@link layerInline} to score in process.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layerNoop: Layer.Layer<Runner>;
+export {};
+//# sourceMappingURL=Runner.d.ts.map

package/dist/esm/Runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Runner.d.ts","sourceRoot":"","sources":["../../src/Runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,YAAY,MAAM,wBAAwB,CAAA;AAGtD,OAAO,KAAK,KAAK,UAAU,MAAM,4BAA4B,CAAA;AAE7D,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAuC,MAAM,mBAAmB,CAAA;AACrG,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,OAAO,EAAQ,KAAK,EAAE,MAAM,YAAY,CAAA;AACtD,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAErC;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,WAAW,GACnB;IACA,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB,GACC,CACE;IACA,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB,GACC;IAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CAAE,CACtF,CAAA;AAEH;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;QAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;QAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAA;AAEvC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAA;IAC5D,QAAQ,CAAC,kBAAkB,CAAC,EACxB,CAAC,CACD,IAAI,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC7B,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,KACpD,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,GACtD,SAAS,CAAA;CACd;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;AAErD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAA;CAC7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IACzC,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IACjD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,SAAS,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACpB;;AAED;;;;;;;;;GASG;AACH,qBAAa,MAAO,SAAQ,WAAiE;CAAG;AAsMhG;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,UAAU,QAAO,gBAe7B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAuC,CAAA;AAuOnF;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,GAAG,GACd,OAAO,KAAK,EACZ,SAAS,UAAU,KAClB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAqFgD,CAAA;AAEnG;;;;;;;;;GASG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAUxC,CAAA"}