@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/internal/tupleKey.js

@@ -0,0 +1,25 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tupleKey_exports = {};
+__export(tupleKey_exports, {
+  tupleKey: () => tupleKey
+});
+module.exports = __toCommonJS(tupleKey_exports);
+const tupleKey = (...parts) => JSON.stringify(parts);
+//# sourceMappingURL=tupleKey.js.map

package/dist/cjs/internal/tupleKey.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/internal/tupleKey.ts"],
+  "sourcesContent": ["/**\n * The one injective tuple encoder every key in this package is built with.\n *\n * Joining caller-supplied strings on a delimiter is not injective: under a NUL\n * separator the tuples `[\"a\", \"b\" + NUL + \"c\"]` and `[\"a\" + NUL + \"b\", \"c\"]`\n * produce one key, so two distinct score jobs could share an identity and two\n * distinct baseline records could sort as one. Encoding the tuple as a JSON\n * array keeps the components separated by the encoding itself, whatever they\n * contain.\n *\n * @since 0.1.0\n */\n\n/**\n * Encodes a tuple of strings as one key.\n *\n * @since 0.1.0\n * @private\n */\nexport const tupleKey = (...parts: ReadonlyArray<string>): string => JSON.stringify(parts)\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBO,MAAM,WAAW,IAAI,UAAyC,KAAK,UAAU,KAAK;",
+  "names": []
+}

package/dist/cjs/package.json

@@ -0,0 +1 @@
+{"type":"commonjs"}

package/dist/esm/Baseline.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Canonical committed evaluation baselines.
+ *
+ * A baseline is the record of what a suite used to score, committed beside the
+ * suite it belongs to. It holds only the fields a comparison reads, and
+ * validation rebuilds every record from those fields, so nothing a caller
+ * happened to attach to an object travels into a committed artifact.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import { EvalError } from "./EvalError.ts";
+import type { RunResult } from "./Runner.ts";
+/**
+ * Current committed baseline artifact version.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const version: 1;
+/**
+ * One successful score retained by a baseline.
+ *
+ * `scorer` is the scorer key a comparison matches on. `scorerName` is the
+ * optional readable name of the same scorer.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface BaselineRecord {
+    readonly suite: string;
+    readonly case: string;
+    readonly scorer: string;
+    readonly scorerName?: string | undefined;
+    readonly stepKey: string;
+    readonly score: number;
+}
+/**
+ * Canonical committed evaluation baseline.
+ *
+ * `suite` records artifact ownership even when `records` is empty.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Baseline {
+    readonly version: typeof version;
+    readonly suite: string;
+    readonly records: ReadonlyArray<BaselineRecord>;
+}
+/**
+ * The declared ceilings a committed baseline is validated against.
+ *
+ * `length` bounds the JSON text `load` will parse, so an oversized committed
+ * file fails with a sentence instead of exhausting a CI runner's memory.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const limits: {
+    /** Longest baseline JSON text, in UTF-16 code units. */
+    readonly length: number;
+};
+/**
+ * Builds and validates a baseline from a run's successful observations.
+ *
+ * Inconclusive observations are dropped: a baseline records what was measured,
+ * and an inconclusive observation measured nothing.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const fromRun: (run: RunResult) => Effect.Effect<Baseline, EvalError>;
+/**
+ * Validates an in-memory baseline.
+ *
+ * The result is a snapshot: every known field is read once, then records are
+ * rebuilt from those validated values and the array is frozen. Mutating the
+ * array or the objects that were passed in cannot change the validated
+ * baseline, and a stateful getter cannot substitute a value after validation.
+ *
+ * Fails with `invalid_baseline` carrying the record index and field name in
+ * `path` for a wrong version, a non-array `records`, a record that is not an
+ * object, an identity field that is not a string or holds a control
+ * character, or a score that is not finite in `[0, 1]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const make: (baseline: Omit<Baseline, "version"> & {
+    readonly version?: typeof version;
+}) => Effect.Effect<Baseline, EvalError>;
+/**
+ * Serializes a baseline with recursively sorted keys and stable numbers.
+ *
+ * Records are ordered by an injective encoding of `(suite, case, scorer,
+ * stepKey)`, so two records whose names differ only in where a separator falls
+ * still sort apart. The result ends with a newline and never throws: a
+ * validated baseline holds only strings and finite numbers.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const write: (baseline: Baseline) => string;
+/**
+ * Loads and validates baseline JSON.
+ *
+ * A version-1 artifact without a top-level `suite` is accepted only when its
+ * nonempty records all name the same suite. That suite becomes the artifact's
+ * owner, so writing the result includes it. Empty or ambiguous legacy artifacts
+ * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`
+ * fails with `invalid_baseline` at `text` before any of it is parsed.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export declare const load: (text: string) => Effect.Effect<Baseline, EvalError>;
+//# sourceMappingURL=Baseline.d.ts.map

package/dist/esm/Baseline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Baseline.d.ts","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAe,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAG,CAAU,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,OAAO,OAAO,CAAA;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAChD;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,MAAM;IACjB,wDAAwD;;CAEhD,CAAA;AAiGV;;;;;;;;GAQG;AACH,eAAO,MAAM,OAAO,GAAI,KAAK,SAAS,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAczE,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,IAAI,GACf,UAAU,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,GAAG;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,OAAO,CAAA;CAAE,KAC1E,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAInC,CAAA;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAAI,UAAU,QAAQ,KAAG,MAW1C,CAAA;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,IAAI,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAMhC,CAAA"}

package/dist/esm/Baseline.js

@@ -0,0 +1,203 @@
+/**
+ * Canonical committed evaluation baselines.
+ *
+ * A baseline is the record of what a suite used to score, committed beside the
+ * suite it belongs to. It holds only the fields a comparison reads, and
+ * validation rebuilds every record from those fields, so nothing a caller
+ * happened to attach to an object travels into a committed artifact.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import { EvalError } from "./EvalError.js";
+import { CanonicalJson } from "./internal/CanonicalJson.js";
+import { controlCharacter } from "./internal/controlCharacters.js";
+import { tupleKey } from "./internal/tupleKey.js";
+/**
+ * Current committed baseline artifact version.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const version = 1;
+/**
+ * The declared ceilings a committed baseline is validated against.
+ *
+ * `length` bounds the JSON text `load` will parse, so an oversized committed
+ * file fails with a sentence instead of exhausting a CI runner's memory.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const limits = {
+    /** Longest baseline JSON text, in UTF-16 code units. */
+    length: 16 * 1024 * 1024
+};
+const fail = (message, path) => Effect.fail(new EvalError({ code: "invalid_baseline", message, ...(path === undefined ? {} : { path }) }));
+// A committed record is read by a gate whose summary goes to a CI log, so it
+// follows the same control-character rule as a suite name: rejected where it
+// enters the system, not flattened later.
+const stringField = (value, field, path) => {
+    if (typeof value !== "string") {
+        return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path);
+    }
+    const control = controlCharacter(value);
+    return control === undefined
+        ? Effect.succeed(value)
+        : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path);
+};
+const decodeRecord = (value, index) => Effect.gen(function* () {
+    const at = `records[${index}]`;
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        return yield* fail(`Baseline record must be an object, got ${value === null ? "null" : typeof value}`, at);
+    }
+    const source = value;
+    const rawSuite = source.suite;
+    const rawCase = source.case;
+    const rawScorer = source.scorer;
+    const rawScorerName = source.scorerName;
+    const rawStepKey = source.stepKey;
+    const rawScore = source.score;
+    const suite = yield* stringField(rawSuite, "suite", `${at}.suite`);
+    const caseName = yield* stringField(rawCase, "case", `${at}.case`);
+    const scorer = yield* stringField(rawScorer, "scorer", `${at}.scorer`);
+    const stepKey = yield* stringField(rawStepKey, "stepKey", `${at}.stepKey`);
+    const score = rawScore;
+    if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) {
+        return yield* fail(`Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`, `${at}.score`);
+    }
+    const scorerName = rawScorerName === undefined
+        ? undefined
+        : yield* stringField(rawScorerName, "scorerName", `${at}.scorerName`);
+    // Every record is rebuilt field by field. Keeping the caller's object would
+    // carry unknown keys, and any getter among them, straight into a committed
+    // artifact.
+    return {
+        suite,
+        case: caseName,
+        scorer,
+        ...(scorerName === undefined ? {} : { scorerName }),
+        stepKey,
+        score: Object.is(score, -0) ? 0 : score
+    };
+});
+const validate = (value) => Effect.gen(function* () {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+        return yield* fail("Baseline must be an object");
+    }
+    const artifact = value;
+    const rawVersion = artifact.version;
+    const rawSuite = artifact.suite;
+    const rawRecords = artifact.records;
+    if (rawVersion !== version) {
+        return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, "version");
+    }
+    if (rawSuite !== undefined && typeof rawSuite !== "string") {
+        return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, "suite");
+    }
+    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite);
+    if (suiteControl !== undefined) {
+        return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, "suite");
+    }
+    if (!Array.isArray(rawRecords)) {
+        return yield* fail("Baseline records must be an array", "records");
+    }
+    const records = yield* Effect.forEach(rawRecords, decodeRecord);
+    let suite = rawSuite;
+    if (suite === undefined) {
+        const first = records[0];
+        if (first === undefined) {
+            return yield* fail("Cannot infer baseline suite: legacy artifact has no records", "suite");
+        }
+        if (records.some((record) => record.suite !== first.suite)) {
+            return yield* fail("Cannot infer baseline suite: legacy records name multiple suites", "suite");
+        }
+        suite = first.suite;
+    }
+    return { version, suite, records: Object.freeze(records) };
+});
+/**
+ * Builds and validates a baseline from a run's successful observations.
+ *
+ * Inconclusive observations are dropped: a baseline records what was measured,
+ * and an inconclusive observation measured nothing.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const fromRun = (run) => {
+    const records = run.observations.flatMap((observation) => observation.kind === "score"
+        ? [{
+                suite: run.suite,
+                case: observation.case,
+                scorer: observation.scorer,
+                ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),
+                stepKey: observation.stepKey,
+                score: observation.score
+            }]
+        : []);
+    return validate({ version, suite: run.suite, records });
+};
+/**
+ * Validates an in-memory baseline.
+ *
+ * The result is a snapshot: every known field is read once, then records are
+ * rebuilt from those validated values and the array is frozen. Mutating the
+ * array or the objects that were passed in cannot change the validated
+ * baseline, and a stateful getter cannot substitute a value after validation.
+ *
+ * Fails with `invalid_baseline` carrying the record index and field name in
+ * `path` for a wrong version, a non-array `records`, a record that is not an
+ * object, an identity field that is not a string or holds a control
+ * character, or a score that is not finite in `[0, 1]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (baseline) => {
+    const artifactVersion = baseline.version ?? version;
+    const records = baseline.records;
+    return validate({ version: artifactVersion, suite: baseline.suite, records });
+};
+/**
+ * Serializes a baseline with recursively sorted keys and stable numbers.
+ *
+ * Records are ordered by an injective encoding of `(suite, case, scorer,
+ * stepKey)`, so two records whose names differ only in where a separator falls
+ * still sort apart. The result ends with a newline and never throws: a
+ * validated baseline holds only strings and finite numbers.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const write = (baseline) => {
+    const sortKey = (record) => tupleKey(record.suite, record.case, record.scorer, record.stepKey);
+    const records = baseline.records
+        .map((record) => [sortKey(record), record])
+        .sort(([left], [right]) => CanonicalJson.compareText(left, right))
+        .map(([, record]) => record);
+    return CanonicalJson.stringify({
+        version: baseline.version,
+        suite: baseline.suite,
+        records
+    });
+};
+/**
+ * Loads and validates baseline JSON.
+ *
+ * A version-1 artifact without a top-level `suite` is accepted only when its
+ * nonempty records all name the same suite. That suite becomes the artifact's
+ * owner, so writing the result includes it. Empty or ambiguous legacy artifacts
+ * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`
+ * fails with `invalid_baseline` at `text` before any of it is parsed.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const load = (text) => text.length > limits.length
+    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, "text")
+    : Effect.try({
+        try: () => JSON.parse(text),
+        catch: (cause) => new EvalError({ code: "invalid_baseline", message: "Baseline is not valid JSON", cause })
+    }).pipe(Effect.flatMap(validate));
+//# sourceMappingURL=Baseline.js.map

package/dist/esm/Baseline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Baseline.js","sourceRoot":"","sources":["../../src/Baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AAGjD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAU,CAAA;AAkCjC;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,wDAAwD;IACxD,MAAM,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;CAChB,CAAA;AAEV,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,IAAa,EAAmC,EAAE,CAC/E,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAC9F,CAAA;AAEH,6EAA6E;AAC7E,6EAA6E;AAC7E,0CAA0C;AAC1C,MAAM,WAAW,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,IAAY,EAAoC,EAAE;IACpG,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,0BAA0B,KAAK,2BAA2B,OAAO,KAAK,EAAE,EAAE,IAAI,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;IACvC,OAAO,OAAO,KAAK,SAAS;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;QACvB,CAAC,CAAC,IAAI,CAAC,0BAA0B,KAAK,4CAA4C,OAAO,EAAE,EAAE,IAAI,CAAC,CAAA;AACtG,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,KAAc,EAAE,KAAa,EAA4C,EAAE,CAC/F,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,EAAE,GAAG,WAAW,KAAK,GAAG,CAAA;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,0CAA0C,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,CAAC,CAAA;IAC5G,CAAC;IACD,MAAM,MAAM,GAAG,KAA4C,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAA;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAA;IAC/B,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAA;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAA;IACjC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAA;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,UAAU,CAAC,CAAA;IAC1E,MAAM,KAAK,GAAG,QAAQ,CAAA;IACtB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC,CAAC,IAAI,CAChB,wEAAwE,MAAM,CAAC,KAAK,CAAC,EAAE,EACvF,GAAG,EAAE,QAAQ,CACd,CAAA;IACH,CAAC;IACD,MAAM,UAAU,GAAG,aAAa,KAAK,SAAS;QAC5C,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,CAAC,CAAA;IACvE,4EAA4E;IAC5E,2EAA2E;IAC3E,YAAY;IACZ,OAAO;QACL,KAAK;QACL,IAAI,EAAE,QAAQ;QACd,MAAM;QACN,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,OAAO;QACP,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;KACxC,CAAA;AACH,CAAC,CAAC,CAAA;AAEJ,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsC,EAAE,CACtE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,QAAQ,GAAG,KAA6F,CAAA;IAC9G,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAA;IAC/B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAA;IACnC,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,4BAA4B,OAAO,SAAS,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;IACjG,CAAC;IACD,IAAI,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,gDAAgD,OAAO,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;IAChG,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACpF,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,iEAAiE,YAAY,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9G,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,mCAAmC,EAAE,SAAS,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAC/D,IAAI,KAAK,GAAG,QAAQ,CAAA;IACpB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACxB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,6DAA6D,EAAE,OAAO,CAAC,CAAA;QAC5F,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,kEAAkE,EAAE,OAAO,CAAC,CAAA;QACjG,CAAC;QACD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;IACrB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;AAC5D,CAAC,CAAC,CAAA;AAEJ;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAc,EAAsC,EAAE;IAC5E,MAAM,OAAO,GAA0B,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,WAAwB,EAAE,EAAE,CAC3F,WAAW,CAAC,IAAI,KAAK,OAAO;QAC1B,CAAC,CAAC,CAAC;gBACD,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,GAAG,CAAC,WAAW,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,CAAC,UAAU,EAAE,CAAC;gBACvF,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,KAAK,EAAE,WAAW,CAAC,KAAK;aACzB,CAAC;QACF,CAAC,CAAC,EAAE,CACP,CAAA;IACD,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AACzD,CAAC,CAAA;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAClB,QAA2E,EACvC,EAAE;IACtC,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAA;IACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;IAChC,OAAO,QAAQ,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,QAAkB,EAAU,EAAE;IAClD,MAAM,OAAO,GAAG,CAAC,MAAsB,EAAU,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO;SAC7B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC;SACnD,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SACjE,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAA;IAC9B,OAAO,aAAa,CAAC,SAAS,CAAC;QAC7B,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,OAAO;KACR,CAAC,CAAA;AACJ,CAAC,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAsC,EAAE,CACvE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;IACzB,CAAC,CAAC,IAAI,CAAC,4BAA4B,MAAM,CAAC,MAAM,oBAAoB,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC;IAC1F,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QACX,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY;QACtC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,4BAA4B,EAAE,KAAK,EAAE,CAAC;KAC5G,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA","sourcesContent":["/**\n * Canonical committed evaluation baselines.\n *\n * A baseline is the record of what a suite used to score, committed beside the\n * suite it belongs to. It holds only the fields a comparison reads, and\n * validation rebuilds every record from those fields, so nothing a caller\n * happened to attach to an object travels into a committed artifact.\n *\n * @since 0.1.0\n */\n\nimport * as Effect from \"effect/Effect\"\nimport { EvalError } from \"./EvalError.ts\"\nimport { CanonicalJson } from \"./internal/CanonicalJson.ts\"\nimport { controlCharacter } from \"./internal/controlCharacters.ts\"\nimport { tupleKey } from \"./internal/tupleKey.ts\"\nimport type { Observation, RunResult } from \"./Runner.ts\"\n\n/**\n * Current committed baseline artifact version.\n *\n * @category models\n * @since 0.1.0\n */\nexport const version = 1 as const\n\n/**\n * One successful score retained by a baseline.\n *\n * `scorer` is the scorer key a comparison matches on. `scorerName` is the\n * optional readable name of the same scorer.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface BaselineRecord {\n  readonly suite: string\n  readonly case: string\n  readonly scorer: string\n  readonly scorerName?: string | undefined\n  readonly stepKey: string\n  readonly score: number\n}\n\n/**\n * Canonical committed evaluation baseline.\n *\n * `suite` records artifact ownership even when `records` is empty.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Baseline {\n  readonly version: typeof version\n  readonly suite: string\n  readonly records: ReadonlyArray<BaselineRecord>\n}\n\n/**\n * The declared ceilings a committed baseline is validated against.\n *\n * `length` bounds the JSON text `load` will parse, so an oversized committed\n * file fails with a sentence instead of exhausting a CI runner's memory.\n *\n * @category models\n * @since 0.1.0\n */\nexport const limits = {\n  /** Longest baseline JSON text, in UTF-16 code units. */\n  length: 16 * 1024 * 1024\n} as const\n\nconst fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>\n  Effect.fail(\n    new EvalError({ code: \"invalid_baseline\", message, ...(path === undefined ? {} : { path }) })\n  )\n\n// A committed record is read by a gate whose summary goes to a CI log, so it\n// follows the same control-character rule as a suite name: rejected where it\n// enters the system, not flattened later.\nconst stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {\n  if (typeof value !== \"string\") {\n    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)\n  }\n  const control = controlCharacter(value)\n  return control === undefined\n    ? Effect.succeed(value)\n    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)\n}\n\nconst decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>\n  Effect.gen(function*() {\n    const at = `records[${index}]`\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(`Baseline record must be an object, got ${value === null ? \"null\" : typeof value}`, at)\n    }\n    const source = value as { readonly [key: string]: unknown }\n    const rawSuite = source.suite\n    const rawCase = source.case\n    const rawScorer = source.scorer\n    const rawScorerName = source.scorerName\n    const rawStepKey = source.stepKey\n    const rawScore = source.score\n    const suite = yield* stringField(rawSuite, \"suite\", `${at}.suite`)\n    const caseName = yield* stringField(rawCase, \"case\", `${at}.case`)\n    const scorer = yield* stringField(rawScorer, \"scorer\", `${at}.scorer`)\n    const stepKey = yield* stringField(rawStepKey, \"stepKey\", `${at}.stepKey`)\n    const score = rawScore\n    if (typeof score !== \"number\" || !Number.isFinite(score) || score < 0 || score > 1) {\n      return yield* fail(\n        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,\n        `${at}.score`\n      )\n    }\n    const scorerName = rawScorerName === undefined\n      ? undefined\n      : yield* stringField(rawScorerName, \"scorerName\", `${at}.scorerName`)\n    // Every record is rebuilt field by field. Keeping the caller's object would\n    // carry unknown keys, and any getter among them, straight into a committed\n    // artifact.\n    return {\n      suite,\n      case: caseName,\n      scorer,\n      ...(scorerName === undefined ? {} : { scorerName }),\n      stepKey,\n      score: Object.is(score, -0) ? 0 : score\n    }\n  })\n\nconst validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>\n  Effect.gen(function*() {\n    if (typeof value !== \"object\" || value === null || Array.isArray(value)) {\n      return yield* fail(\"Baseline must be an object\")\n    }\n    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }\n    const rawVersion = artifact.version\n    const rawSuite = artifact.suite\n    const rawRecords = artifact.records\n    if (rawVersion !== version) {\n      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, \"version\")\n    }\n    if (rawSuite !== undefined && typeof rawSuite !== \"string\") {\n      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, \"suite\")\n    }\n    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)\n    if (suiteControl !== undefined) {\n      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, \"suite\")\n    }\n    if (!Array.isArray(rawRecords)) {\n      return yield* fail(\"Baseline records must be an array\", \"records\")\n    }\n    const records = yield* Effect.forEach(rawRecords, decodeRecord)\n    let suite = rawSuite\n    if (suite === undefined) {\n      const first = records[0]\n      if (first === undefined) {\n        return yield* fail(\"Cannot infer baseline suite: legacy artifact has no records\", \"suite\")\n      }\n      if (records.some((record) => record.suite !== first.suite)) {\n        return yield* fail(\"Cannot infer baseline suite: legacy records name multiple suites\", \"suite\")\n      }\n      suite = first.suite\n    }\n    return { version, suite, records: Object.freeze(records) }\n  })\n\n/**\n * Builds and validates a baseline from a run's successful observations.\n *\n * Inconclusive observations are dropped: a baseline records what was measured,\n * and an inconclusive observation measured nothing.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {\n  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>\n    observation.kind === \"score\"\n      ? [{\n        suite: run.suite,\n        case: observation.case,\n        scorer: observation.scorer,\n        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),\n        stepKey: observation.stepKey,\n        score: observation.score\n      }]\n      : []\n  )\n  return validate({ version, suite: run.suite, records })\n}\n\n/**\n * Validates an in-memory baseline.\n *\n * The result is a snapshot: every known field is read once, then records are\n * rebuilt from those validated values and the array is frozen. Mutating the\n * array or the objects that were passed in cannot change the validated\n * baseline, and a stateful getter cannot substitute a value after validation.\n *\n * Fails with `invalid_baseline` carrying the record index and field name in\n * `path` for a wrong version, a non-array `records`, a record that is not an\n * object, an identity field that is not a string or holds a control\n * character, or a score that is not finite in `[0, 1]`.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  baseline: Omit<Baseline, \"version\"> & { readonly version?: typeof version }\n): Effect.Effect<Baseline, EvalError> => {\n  const artifactVersion = baseline.version ?? version\n  const records = baseline.records\n  return validate({ version: artifactVersion, suite: baseline.suite, records })\n}\n\n/**\n * Serializes a baseline with recursively sorted keys and stable numbers.\n *\n * Records are ordered by an injective encoding of `(suite, case, scorer,\n * stepKey)`, so two records whose names differ only in where a separator falls\n * still sort apart. The result ends with a newline and never throws: a\n * validated baseline holds only strings and finite numbers.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const write = (baseline: Baseline): string => {\n  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)\n  const records = baseline.records\n    .map((record) => [sortKey(record), record] as const)\n    .sort(([left], [right]) => CanonicalJson.compareText(left, right))\n    .map(([, record]) => record)\n  return CanonicalJson.stringify({\n    version: baseline.version,\n    suite: baseline.suite,\n    records\n  })\n}\n\n/**\n * Loads and validates baseline JSON.\n *\n * A version-1 artifact without a top-level `suite` is accepted only when its\n * nonempty records all name the same suite. That suite becomes the artifact's\n * owner, so writing the result includes it. Empty or ambiguous legacy artifacts\n * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`\n * fails with `invalid_baseline` at `text` before any of it is parsed.\n *\n * @category serialization\n * @since 0.1.0\n */\nexport const load = (text: string): Effect.Effect<Baseline, EvalError> =>\n  text.length > limits.length\n    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, \"text\")\n    : Effect.try({\n      try: () => JSON.parse(text) as unknown,\n      catch: (cause) => new EvalError({ code: \"invalid_baseline\", message: \"Baseline is not valid JSON\", cause })\n    }).pipe(Effect.flatMap(validate))\n"]}

package/dist/esm/CaseExecutor.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Injectable target-flow execution boundary.
+ *
+ * A suite says what to run; this service says how. Keeping it injectable is
+ * what lets the same suite run against a real flow, a scripted host, or nothing
+ * at all, and it is the only place a target's failure is converted into a typed
+ * evaluation failure.
+ *
+ * @since 0.1.0
+ */
+import type { Flow } from "@smthrs/core";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { EvalError } from "./EvalError.ts";
+import type { Case } from "./Suite.ts";
+/**
+ * The result of executing one target-flow case.
+ *
+ * `target` is the flow value the case actually executed. A run matches it
+ * against each binding's `appliesTo` by reference identity, so it has to be the
+ * declared flow itself rather than a copy of it.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Execution {
+    readonly output: unknown;
+    readonly stepKey: string;
+    readonly latencyMs: number;
+    readonly target: Pick<Flow.Any, "name">;
+}
+/**
+ * The one callback a case executor is.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Run = (suiteCase: Case) => Effect.Effect<Execution, EvalError>;
+/**
+ * Runtime shape for an injectable target-flow executor.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export interface Service {
+    readonly run: Run;
+}
+declare const CaseExecutor_base: Context.ServiceClass<CaseExecutor, "flows/evals/CaseExecutor", Service>;
+/**
+ * Injectable execution boundary for a target flow.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export declare class CaseExecutor extends CaseExecutor_base {
+}
+/**
+ * Builds an executor from its one callback.
+ *
+ * Throws a `TypeError` when there is no callback. An executor that silently
+ * degraded to {@link makeNoop} turned one wiring mistake into a whole suite of
+ * cases failing with `executor`, which reads as a broken target rather than a
+ * missing one.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const make: (run: Run) => Service;
+/**
+ * Builds an executor that fails every case with a typed executor error.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const makeNoop: () => Service;
+/**
+ * Provides the unavailable executor.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export declare const layerNoop: Layer.Layer<CaseExecutor>;
+export {};
+//# sourceMappingURL=CaseExecutor.d.ts.map

package/dist/esm/CaseExecutor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CaseExecutor.d.ts","sourceRoot":"","sources":["../../src/CaseExecutor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAA;AAEtC;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;CACxC;AAED;;;;;GAKG;AACH,MAAM,MAAM,GAAG,GAAG,CAAC,SAAS,EAAE,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;AAE1E;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAA;CAClB;;AAED;;;;;GAKG;AACH,qBAAa,YAAa,SAAQ,iBAAoE;CAAG;AAEzG;;;;;;;;;;GAUG;AACH,eAAO,MAAM,IAAI,GAAI,KAAK,GAAG,KAAG,OAK/B,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,QAAO,OAUxB,CAAA;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY,CAA2C,CAAA"}

package/dist/esm/CaseExecutor.js

@@ -0,0 +1,60 @@
+/**
+ * Injectable target-flow execution boundary.
+ *
+ * A suite says what to run; this service says how. Keeping it injectable is
+ * what lets the same suite run against a real flow, a scripted host, or nothing
+ * at all, and it is the only place a target's failure is converted into a typed
+ * evaluation failure.
+ *
+ * @since 0.1.0
+ */
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { EvalError } from "./EvalError.js";
+/**
+ * Injectable execution boundary for a target flow.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class CaseExecutor extends Context.Service()("flows/evals/CaseExecutor") {
+}
+/**
+ * Builds an executor from its one callback.
+ *
+ * Throws a `TypeError` when there is no callback. An executor that silently
+ * degraded to {@link makeNoop} turned one wiring mistake into a whole suite of
+ * cases failing with `executor`, which reads as a broken target rather than a
+ * missing one.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (run) => {
+    if (typeof run !== "function") {
+        throw new TypeError("CaseExecutor.make needs a callback");
+    }
+    return CaseExecutor.of({ run });
+};
+/**
+ * Builds an executor that fails every case with a typed executor error.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeNoop = () => CaseExecutor.of({
+    run: (suiteCase) => Effect.fail(new EvalError({
+        code: "executor",
+        message: `No executor is available for case '${suiteCase.name}'`,
+        path: `cases['${suiteCase.name}']`
+    }))
+});
+/**
+ * Provides the unavailable executor.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop = Layer.succeed(CaseExecutor)(makeNoop());
+//# sourceMappingURL=CaseExecutor.js.map

package/dist/esm/CaseExecutor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CaseExecutor.js","sourceRoot":"","sources":["../../src/CaseExecutor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAsC1C;;;;;GAKG;AACH,MAAM,OAAO,YAAa,SAAQ,OAAO,CAAC,OAAO,EAAyB,CAAC,0BAA0B,CAAC;CAAG;AAEzG;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,GAAQ,EAAW,EAAE;IACxC,IAAI,OAAO,GAAG,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAA;IAC3D,CAAC;IACD,OAAO,YAAY,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;AACjC,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAY,EAAE,CACpC,YAAY,CAAC,EAAE,CAAC;IACd,GAAG,EAAE,CAAC,SAAS,EAAE,EAAE,CACjB,MAAM,CAAC,IAAI,CACT,IAAI,SAAS,CAAC;QACZ,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,sCAAsC,SAAS,CAAC,IAAI,GAAG;QAChE,IAAI,EAAE,UAAU,SAAS,CAAC,IAAI,IAAI;KACnC,CAAC,CACH;CACJ,CAAC,CAAA;AAEJ;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAA8B,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA","sourcesContent":["/**\n * Injectable target-flow execution boundary.\n *\n * A suite says what to run; this service says how. Keeping it injectable is\n * what lets the same suite run against a real flow, a scripted host, or nothing\n * at all, and it is the only place a target's failure is converted into a typed\n * evaluation failure.\n *\n * @since 0.1.0\n */\n\nimport type { Flow } from \"@smthrs/core\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { EvalError } from \"./EvalError.ts\"\nimport type { Case } from \"./Suite.ts\"\n\n/**\n * The result of executing one target-flow case.\n *\n * `target` is the flow value the case actually executed. A run matches it\n * against each binding's `appliesTo` by reference identity, so it has to be the\n * declared flow itself rather than a copy of it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Execution {\n  readonly output: unknown\n  readonly stepKey: string\n  readonly latencyMs: number\n  readonly target: Pick<Flow.Any, \"name\">\n}\n\n/**\n * The one callback a case executor is.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Run = (suiteCase: Case) => Effect.Effect<Execution, EvalError>\n\n/**\n * Runtime shape for an injectable target-flow executor.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface Service {\n  readonly run: Run\n}\n\n/**\n * Injectable execution boundary for a target flow.\n *\n * @category services\n * @since 0.1.0\n */\nexport class CaseExecutor extends Context.Service<CaseExecutor, Service>()(\"flows/evals/CaseExecutor\") {}\n\n/**\n * Builds an executor from its one callback.\n *\n * Throws a `TypeError` when there is no callback. An executor that silently\n * degraded to {@link makeNoop} turned one wiring mistake into a whole suite of\n * cases failing with `executor`, which reads as a broken target rather than a\n * missing one.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (run: Run): Service => {\n  if (typeof run !== \"function\") {\n    throw new TypeError(\"CaseExecutor.make needs a callback\")\n  }\n  return CaseExecutor.of({ run })\n}\n\n/**\n * Builds an executor that fails every case with a typed executor error.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeNoop = (): Service =>\n  CaseExecutor.of({\n    run: (suiteCase) =>\n      Effect.fail(\n        new EvalError({\n          code: \"executor\",\n          message: `No executor is available for case '${suiteCase.name}'`,\n          path: `cases['${suiteCase.name}']`\n        })\n      )\n  })\n\n/**\n * Provides the unavailable executor.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<CaseExecutor> = Layer.succeed(CaseExecutor)(makeNoop())\n"]}

package/dist/esm/EvalError.d.ts

@@ -0,0 +1,64 @@
+/**
+ * The typed failures evaluation raises, with the stable codes a gate or a
+ * report branches on.
+ *
+ * @since 0.1.0
+ */
+import * as Schema from "effect/Schema";
+/**
+ * Stable evaluation failure codes.
+ *
+ * Each code names one thing a caller can do about it, so a gate that branches
+ * on the code routes the failure to the right owner:
+ *
+ * - `invalid_suite`: the suite declaration is wrong. Fix the suite.
+ * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.
+ * - `invalid_baseline`: the committed baseline is unreadable, belongs to
+ *   another suite, or holds a record the schema rejects. Regenerate it.
+ * - `invalid_tolerance`: the caller passed a tolerance that is not a finite
+ *   non-negative number. Fix the call, not the baseline.
+ * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`
+ *   outside their bounds. Fix the call.
+ * - `executor`: the target flow failed for a case.
+ * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so
+ *   an order-only batch runner's results cannot be attributed to a case. Give
+ *   each case its own step key, or use a runner that implements
+ *   `runBatchCorrelated`.
+ * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning
+ *   the wrong number of observations, or observations that identify jobs other
+ *   than the ones it was given. Nothing it returned can be trusted.
+ * - `scorer_unavailable`: no batch runner was available to score with.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const EvalErrorCode: Schema.Literals<readonly ["invalid_suite", "invalid_run_options", "invalid_baseline", "invalid_tolerance", "invalid_trials", "executor", "ambiguous_score_job", "scorer_protocol", "scorer_unavailable"]>;
+/**
+ * Stable evaluation failure code.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type EvalErrorCode = typeof EvalErrorCode.Type;
+declare const EvalError_base: Schema.Class<EvalError, Schema.TaggedStruct<"flows/evals/EvalError", {
+    readonly code: Schema.Literals<readonly ["invalid_suite", "invalid_run_options", "invalid_baseline", "invalid_tolerance", "invalid_trials", "executor", "ambiguous_score_job", "scorer_protocol", "scorer_unavailable"]>;
+    readonly message: Schema.String;
+    readonly path: Schema.optional<Schema.String>;
+    readonly cause: Schema.optional<Schema.Defect>;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * A typed failure raised while loading or executing an evaluation.
+ *
+ * `code` is the stable branch point, `message` is the sentence a CI log shows,
+ * and `path` locates the offending value inside the input the caller supplied
+ * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the
+ * original failure; it is never rendered on its own, so a message that matters
+ * to an operator has to say so itself.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export declare class EvalError extends EvalError_base {
+}
+export {};
+//# sourceMappingURL=EvalError.d.ts.map

package/dist/esm/EvalError.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalError.d.ts","sourceRoot":"","sources":["../../src/EvalError.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,eAAO,MAAM,aAAa,2MAUxB,CAAA;AAEF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,aAAa,CAAC,IAAI,CAAA;;;;;;;AAErD;;;;;;;;;;;GAWG;AACH,qBAAa,SAAU,SAAQ,cAK7B;CAAG"}