@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/CaseExecutor.js

@@ -0,0 +1,59 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var CaseExecutor_exports = {};
+__export(CaseExecutor_exports, {
+  CaseExecutor: () => CaseExecutor,
+  layerNoop: () => layerNoop,
+  make: () => make,
+  makeNoop: () => makeNoop
+});
+module.exports = __toCommonJS(CaseExecutor_exports);
+var Context = __toESM(require("effect/Context"), 1);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Layer = __toESM(require("effect/Layer"), 1);
+var import_EvalError = require("./EvalError.js");
+class CaseExecutor extends Context.Service()("flows/evals/CaseExecutor") {
+}
+const make = (run) => {
+  if (typeof run !== "function") {
+    throw new TypeError("CaseExecutor.make needs a callback");
+  }
+  return CaseExecutor.of({ run });
+};
+const makeNoop = () => CaseExecutor.of({
+  run: (suiteCase) => Effect.fail(
+    new import_EvalError.EvalError({
+      code: "executor",
+      message: `No executor is available for case '${suiteCase.name}'`,
+      path: `cases['${suiteCase.name}']`
+    })
+  )
+});
+const layerNoop = Layer.succeed(CaseExecutor)(makeNoop());
+//# sourceMappingURL=CaseExecutor.js.map

package/dist/cjs/CaseExecutor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/CaseExecutor.ts"],
+  "sourcesContent": ["/**\n * Injectable target-flow execution boundary.\n *\n * A suite says what to run; this service says how. Keeping it injectable is\n * what lets the same suite run against a real flow, a scripted host, or nothing\n * at all, and it is the only place a target's failure is converted into a typed\n * evaluation failure.\n *\n * @since 0.1.0\n */\n\nimport type { Flow } from \"@smthrs/core\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport { EvalError } from \"./EvalError.ts\"\nimport type { Case } from \"./Suite.ts\"\n\n/**\n * The result of executing one target-flow case.\n *\n * `target` is the flow value the case actually executed. A run matches it\n * against each binding's `appliesTo` by reference identity, so it has to be the\n * declared flow itself rather than a copy of it.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Execution {\n  readonly output: unknown\n  readonly stepKey: string\n  readonly latencyMs: number\n  readonly target: Pick<Flow.Any, \"name\">\n}\n\n/**\n * The one callback a case executor is.\n *\n * @category models\n * @since 0.1.0\n */\nexport type Run = (suiteCase: Case) => Effect.Effect<Execution, EvalError>\n\n/**\n * Runtime shape for an injectable target-flow executor.\n *\n * @category services\n * @since 0.1.0\n */\nexport interface Service {\n  readonly run: Run\n}\n\n/**\n * Injectable execution boundary for a target flow.\n *\n * @category services\n * @since 0.1.0\n */\nexport class CaseExecutor extends Context.Service<CaseExecutor, Service>()(\"flows/evals/CaseExecutor\") {}\n\n/**\n * Builds an executor from its one callback.\n *\n * Throws a `TypeError` when there is no callback. An executor that silently\n * degraded to {@link makeNoop} turned one wiring mistake into a whole suite of\n * cases failing with `executor`, which reads as a broken target rather than a\n * missing one.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (run: Run): Service => {\n  if (typeof run !== \"function\") {\n    throw new TypeError(\"CaseExecutor.make needs a callback\")\n  }\n  return CaseExecutor.of({ run })\n}\n\n/**\n * Builds an executor that fails every case with a typed executor error.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const makeNoop = (): Service =>\n  CaseExecutor.of({\n    run: (suiteCase) =>\n      Effect.fail(\n        new EvalError({\n          code: \"executor\",\n          message: `No executor is available for case '${suiteCase.name}'`,\n          path: `cases['${suiteCase.name}']`\n        })\n      )\n  })\n\n/**\n * Provides the unavailable executor.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerNoop: Layer.Layer<CaseExecutor> = Layer.succeed(CaseExecutor)(makeNoop())\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYA,cAAyB;AACzB,aAAwB;AACxB,YAAuB;AACvB,uBAA0B;AA4CnB,MAAM,qBAAqB,QAAQ,QAA+B,EAAE,0BAA0B,EAAE;AAAC;AAajG,MAAM,OAAO,CAAC,QAAsB;AACzC,MAAI,OAAO,QAAQ,YAAY;AAC7B,UAAM,IAAI,UAAU,oCAAoC;AAAA,EAC1D;AACA,SAAO,aAAa,GAAG,EAAE,IAAI,CAAC;AAChC;AAQO,MAAM,WAAW,MACtB,aAAa,GAAG;AAAA,EACd,KAAK,CAAC,cACJ,OAAO;AAAA,IACL,IAAI,2BAAU;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,sCAAsC,UAAU,IAAI;AAAA,MAC7D,MAAM,UAAU,UAAU,IAAI;AAAA,IAChC,CAAC;AAAA,EACH;AACJ,CAAC;AAQI,MAAM,YAAuC,MAAM,QAAQ,YAAY,EAAE,SAAS,CAAC;",
+  "names": []
+}

package/dist/cjs/EvalError.d.ts

@@ -0,0 +1,64 @@
+/**
+ * The typed failures evaluation raises, with the stable codes a gate or a
+ * report branches on.
+ *
+ * @since 0.1.0
+ */
+import * as Schema from "effect/Schema";
+/**
+ * Stable evaluation failure codes.
+ *
+ * Each code names one thing a caller can do about it, so a gate that branches
+ * on the code routes the failure to the right owner:
+ *
+ * - `invalid_suite`: the suite declaration is wrong. Fix the suite.
+ * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.
+ * - `invalid_baseline`: the committed baseline is unreadable, belongs to
+ *   another suite, or holds a record the schema rejects. Regenerate it.
+ * - `invalid_tolerance`: the caller passed a tolerance that is not a finite
+ *   non-negative number. Fix the call, not the baseline.
+ * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`
+ *   outside their bounds. Fix the call.
+ * - `executor`: the target flow failed for a case.
+ * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so
+ *   an order-only batch runner's results cannot be attributed to a case. Give
+ *   each case its own step key, or use a runner that implements
+ *   `runBatchCorrelated`.
+ * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning
+ *   the wrong number of observations, or observations that identify jobs other
+ *   than the ones it was given. Nothing it returned can be trusted.
+ * - `scorer_unavailable`: no batch runner was available to score with.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export declare const EvalErrorCode: Schema.Literals<readonly ["invalid_suite", "invalid_run_options", "invalid_baseline", "invalid_tolerance", "invalid_trials", "executor", "ambiguous_score_job", "scorer_protocol", "scorer_unavailable"]>;
+/**
+ * Stable evaluation failure code.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type EvalErrorCode = typeof EvalErrorCode.Type;
+declare const EvalError_base: Schema.Class<EvalError, Schema.TaggedStruct<"flows/evals/EvalError", {
+    readonly code: Schema.Literals<readonly ["invalid_suite", "invalid_run_options", "invalid_baseline", "invalid_tolerance", "invalid_trials", "executor", "ambiguous_score_job", "scorer_protocol", "scorer_unavailable"]>;
+    readonly message: Schema.String;
+    readonly path: Schema.optional<Schema.String>;
+    readonly cause: Schema.optional<Schema.Defect>;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * A typed failure raised while loading or executing an evaluation.
+ *
+ * `code` is the stable branch point, `message` is the sentence a CI log shows,
+ * and `path` locates the offending value inside the input the caller supplied
+ * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the
+ * original failure; it is never rendered on its own, so a message that matters
+ * to an operator has to say so itself.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export declare class EvalError extends EvalError_base {
+}
+export {};
+//# sourceMappingURL=EvalError.d.ts.map

package/dist/cjs/EvalError.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalError.d.ts","sourceRoot":"","sources":["../../src/EvalError.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,eAAO,MAAM,aAAa,2MAUxB,CAAA;AAEF;;;;;GAKG;AACH,MAAM,MAAM,aAAa,GAAG,OAAO,aAAa,CAAC,IAAI,CAAA;;;;;;;AAErD;;;;;;;;;;;GAWG;AACH,qBAAa,SAAU,SAAQ,cAK7B;CAAG"}

package/dist/cjs/EvalError.js

@@ -0,0 +1,54 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var EvalError_exports = {};
+__export(EvalError_exports, {
+  EvalError: () => EvalError,
+  EvalErrorCode: () => EvalErrorCode
+});
+module.exports = __toCommonJS(EvalError_exports);
+var Schema = __toESM(require("effect/Schema"), 1);
+const EvalErrorCode = Schema.Literals([
+  "invalid_suite",
+  "invalid_run_options",
+  "invalid_baseline",
+  "invalid_tolerance",
+  "invalid_trials",
+  "executor",
+  "ambiguous_score_job",
+  "scorer_protocol",
+  "scorer_unavailable"
+]);
+class EvalError extends Schema.TaggedError()("flows/evals/EvalError", {
+  code: EvalErrorCode,
+  message: Schema.String,
+  path: Schema.optional(Schema.String),
+  cause: Schema.optional(Schema.Defect())
+}) {
+}
+//# sourceMappingURL=EvalError.js.map

package/dist/cjs/EvalError.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/EvalError.ts"],
+  "sourcesContent": ["/**\n * The typed failures evaluation raises, with the stable codes a gate or a\n * report branches on.\n *\n * @since 0.1.0\n */\n\nimport * as Schema from \"effect/Schema\"\n\n/**\n * Stable evaluation failure codes.\n *\n * Each code names one thing a caller can do about it, so a gate that branches\n * on the code routes the failure to the right owner:\n *\n * - `invalid_suite`: the suite declaration is wrong. Fix the suite.\n * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.\n * - `invalid_baseline`: the committed baseline is unreadable, belongs to\n *   another suite, or holds a record the schema rejects. Regenerate it.\n * - `invalid_tolerance`: the caller passed a tolerance that is not a finite\n *   non-negative number. Fix the call, not the baseline.\n * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`\n *   outside their bounds. Fix the call.\n * - `executor`: the target flow failed for a case.\n * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so\n *   an order-only batch runner's results cannot be attributed to a case. Give\n *   each case its own step key, or use a runner that implements\n *   `runBatchCorrelated`.\n * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning\n *   the wrong number of observations, or observations that identify jobs other\n *   than the ones it was given. Nothing it returned can be trusted.\n * - `scorer_unavailable`: no batch runner was available to score with.\n *\n * @category models\n * @since 0.1.0\n */\nexport const EvalErrorCode = Schema.Literals([\n  \"invalid_suite\",\n  \"invalid_run_options\",\n  \"invalid_baseline\",\n  \"invalid_tolerance\",\n  \"invalid_trials\",\n  \"executor\",\n  \"ambiguous_score_job\",\n  \"scorer_protocol\",\n  \"scorer_unavailable\"\n])\n\n/**\n * Stable evaluation failure code.\n *\n * @category models\n * @since 0.1.0\n */\nexport type EvalErrorCode = typeof EvalErrorCode.Type\n\n/**\n * A typed failure raised while loading or executing an evaluation.\n *\n * `code` is the stable branch point, `message` is the sentence a CI log shows,\n * and `path` locates the offending value inside the input the caller supplied\n * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the\n * original failure; it is never rendered on its own, so a message that matters\n * to an operator has to say so itself.\n *\n * @category errors\n * @since 0.1.0\n */\nexport class EvalError extends Schema.TaggedError<EvalError>()(\"flows/evals/EvalError\", {\n  code: EvalErrorCode,\n  message: Schema.String,\n  path: Schema.optional(Schema.String),\n  cause: Schema.optional(Schema.Defect())\n}) {}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,aAAwB;AA6BjB,MAAM,gBAAgB,OAAO,SAAS;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAsBM,MAAM,kBAAkB,OAAO,YAAuB,EAAE,yBAAyB;AAAA,EACtF,MAAM;AAAA,EACN,SAAS,OAAO;AAAA,EAChB,MAAM,OAAO,SAAS,OAAO,MAAM;AAAA,EACnC,OAAO,OAAO,SAAS,OAAO,OAAO,CAAC;AACxC,CAAC,EAAE;AAAC;",
+  "names": []
+}

package/dist/cjs/Gate.d.ts

@@ -0,0 +1,55 @@
+/**
+ * The pass/fail decision over a finished evaluation run: how a threshold
+ * is applied and what a failing gate reports.
+ *
+ * @since 0.1.0
+ */
+import { type ScoreGateError, type Verdict } from "@smthrs/scorers/ScoreGate";
+import * as Effect from "effect/Effect";
+import type { Report } from "./Regression.ts";
+/**
+ * Thresholds accepted by a CI score gate.
+ *
+ * `mean` gates the arithmetic mean of every score observation, `min` gates the
+ * lowest one, and `perCase` gates each named case's lowest score. With none of
+ * them set the gate still runs, as `mean(0)`, so a run with no score at all is
+ * undecidable rather than a pass.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Options {
+    readonly mean?: number | undefined;
+    readonly min?: number | undefined;
+    readonly perCase?: Readonly<Record<string, number>> | undefined;
+}
+/**
+ * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.
+ *
+ * The threshold gates always run: an unobserved case cannot excuse the cases
+ * that were observed. Faults and findings are kept apart, and the verdict
+ * carries both, so a regression is a red while an unusable harness stays
+ * undecided.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const check: (report: Report, options?: Options) => Effect.Effect<Verdict, ScoreGateError>;
+/**
+ * Maps a gate verdict to the shared CI convention: a finding is exit code 1,
+ * an undecidable run is exit code 5.
+ *
+ * The summary is one log line: every C0 control and DEL in it becomes a
+ * space, the way a report cell is written, so a value that reached a verdict
+ * reason from a baseline, a step key or a target's failure cannot emit its own
+ * line, which on GitHub Actions would be a workflow command, or a terminal
+ * escape.
+ *
+ * @category grading
+ * @since 0.1.0
+ */
+export declare const ciGrade: (verdict: Verdict) => {
+    readonly exitCode: 0 | 1 | 5;
+    readonly summary: string;
+};
+//# sourceMappingURL=Gate.d.ts.map

package/dist/cjs/Gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Gate.d.ts","sourceRoot":"","sources":["../../src/Gate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAIL,KAAK,cAAc,EAEnB,KAAK,OAAO,EACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;AAE7C;;;;;;;;;;GAUG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAClC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACjC,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAA;CAChE;AAkDD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,KAAK,GAAI,QAAQ,MAAM,EAAE,UAAS,OAAY,KAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAa/F,CAAA;AAEJ;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,OAAO,GAAI,SAAS,OAAO,KAAG;IAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAGlG,CAAA"}

package/dist/cjs/Gate.js

@@ -0,0 +1,88 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var Gate_exports = {};
+__export(Gate_exports, {
+  check: () => check,
+  ciGrade: () => ciGrade
+});
+module.exports = __toCommonJS(Gate_exports);
+var import_ScoreGate = require("@smthrs/scorers/ScoreGate");
+var Effect = __toESM(require("effect/Effect"), 1);
+var import_controlCharacters = require("./internal/controlCharacters.js");
+var import_scorerLabel = require("./internal/scorerLabel.js");
+const samples = (report) => report.run.observations.map(
+  (observation) => observation.kind === "score" ? {
+    case: observation.case,
+    scorer: observation.scorer,
+    stepKey: observation.stepKey,
+    kind: "score",
+    value: observation.score,
+    reason: observation.reason
+  } : {
+    case: observation.case,
+    scorer: observation.scorer,
+    stepKey: observation.stepKey,
+    kind: "inconclusive",
+    reason: observation.reason
+  }
+);
+const environmentFaults = (report) => [
+  ...report.run.cases.flatMap(
+    (result) => result.error === void 0 ? [] : [`case '${result.case}' failed: ${result.error.code}: ${result.error.message}`]
+  ),
+  ...report.missing.map(
+    (item) => `missing ${item.side} observation for ${item.case}/${(0, import_scorerLabel.scorerLabel)(item)}/${item.stepKey}`
+  )
+];
+const findings = (report) => [
+  ...report.regressions.map(
+    (item) => `regression for ${item.case}/${(0, import_scorerLabel.scorerLabel)({ scorer: item.scorer, scorerName: item.actual.scorerName })}`
+  ),
+  ...report.nondeterminism.map(
+    (item) => `nondeterminism for ${item.case}/${(0, import_scorerLabel.scorerLabel)({ scorer: item.scorer, scorerName: item.actual.scorerName })}`
+  )
+];
+const check = (report, options = {}) => Effect.gen(function* () {
+  const expectation = (0, import_ScoreGate.expectScores)(samples(report));
+  const verdicts = [];
+  if (options.mean !== void 0) verdicts.push(yield* expectation.mean(options.mean));
+  if (options.min !== void 0) verdicts.push(yield* expectation.min(options.min));
+  if (options.perCase !== void 0) verdicts.push(yield* expectation.perCase(options.perCase));
+  if (options.mean === void 0 && options.min === void 0 && options.perCase === void 0) {
+    verdicts.push(yield* expectation.mean(0));
+  }
+  const reasons = findings(report);
+  if (reasons.length > 0) verdicts.push({ _tag: "Failed", reasons, inconclusive: [] });
+  return (0, import_ScoreGate.combine)(verdicts, environmentFaults(report));
+});
+const ciGrade = (verdict) => {
+  const graded = (0, import_ScoreGate.grade)(verdict);
+  return { exitCode: graded.exitCode, summary: (0, import_controlCharacters.flattenControlCharacters)(graded.summary) };
+};
+//# sourceMappingURL=Gate.js.map

package/dist/cjs/Gate.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/Gate.ts"],
+  "sourcesContent": ["/**\n * The pass/fail decision over a finished evaluation run: how a threshold\n * is applied and what a failing gate reports.\n *\n * @since 0.1.0\n */\n\nimport {\n  combine,\n  expectScores,\n  grade,\n  type ScoreGateError,\n  type ScoreSample,\n  type Verdict\n} from \"@smthrs/scorers/ScoreGate\"\nimport * as Effect from \"effect/Effect\"\nimport { flattenControlCharacters } from \"./internal/controlCharacters.ts\"\nimport { scorerLabel } from \"./internal/scorerLabel.ts\"\nimport type { Report } from \"./Regression.ts\"\n\n/**\n * Thresholds accepted by a CI score gate.\n *\n * `mean` gates the arithmetic mean of every score observation, `min` gates the\n * lowest one, and `perCase` gates each named case's lowest score. With none of\n * them set the gate still runs, as `mean(0)`, so a run with no score at all is\n * undecidable rather than a pass.\n *\n * @since 0.1.0\n * @category models\n */\nexport interface Options {\n  readonly mean?: number | undefined\n  readonly min?: number | undefined\n  readonly perCase?: Readonly<Record<string, number>> | undefined\n}\n\nconst samples = (report: Report): ReadonlyArray<ScoreSample> =>\n  report.run.observations.map((observation) =>\n    observation.kind === \"score\"\n      ? {\n        case: observation.case,\n        scorer: observation.scorer,\n        stepKey: observation.stepKey,\n        kind: \"score\",\n        value: observation.score,\n        reason: observation.reason\n      }\n      : {\n        case: observation.case,\n        scorer: observation.scorer,\n        stepKey: observation.stepKey,\n        kind: \"inconclusive\",\n        reason: observation.reason\n      }\n  )\n\n/**\n * An environment fault: something the comparison needed was never observed, so\n * the harness owes an answer it cannot give. A fault withholds a decision;\n * it does not make one.\n */\nconst environmentFaults = (report: Report): ReadonlyArray<string> => [\n  ...report.run.cases.flatMap((result) =>\n    result.error === undefined ? [] : [`case '${result.case}' failed: ${result.error.code}: ${result.error.message}`]\n  ),\n  ...report.missing.map((item) =>\n    `missing ${item.side} observation for ${item.case}/${scorerLabel(item)}/${item.stepKey}`\n  )\n]\n\n/**\n * A finding: the run measured something the baseline says it should not have.\n * A regression scored lower at a changed step key, and nondeterminism moved a\n * score at an unchanged one. Both are results, so both are red.\n */\nconst findings = (report: Report): ReadonlyArray<string> => [\n  ...report.regressions.map((item) =>\n    `regression for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`\n  ),\n  ...report.nondeterminism.map((item) =>\n    `nondeterminism for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`\n  )\n]\n\n/**\n * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.\n *\n * The threshold gates always run: an unobserved case cannot excuse the cases\n * that were observed. Faults and findings are kept apart, and the verdict\n * carries both, so a regression is a red while an unusable harness stays\n * undecided.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const check = (report: Report, options: Options = {}): Effect.Effect<Verdict, ScoreGateError> =>\n  Effect.gen(function*() {\n    const expectation = expectScores(samples(report))\n    const verdicts: Array<Verdict> = []\n    if (options.mean !== undefined) verdicts.push(yield* expectation.mean(options.mean))\n    if (options.min !== undefined) verdicts.push(yield* expectation.min(options.min))\n    if (options.perCase !== undefined) verdicts.push(yield* expectation.perCase(options.perCase))\n    if (options.mean === undefined && options.min === undefined && options.perCase === undefined) {\n      verdicts.push(yield* expectation.mean(0))\n    }\n    const reasons = findings(report)\n    if (reasons.length > 0) verdicts.push({ _tag: \"Failed\", reasons, inconclusive: [] })\n    return combine(verdicts, environmentFaults(report))\n  })\n\n/**\n * Maps a gate verdict to the shared CI convention: a finding is exit code 1,\n * an undecidable run is exit code 5.\n *\n * The summary is one log line: every C0 control and DEL in it becomes a\n * space, the way a report cell is written, so a value that reached a verdict\n * reason from a baseline, a step key or a target's failure cannot emit its own\n * line, which on GitHub Actions would be a workflow command, or a terminal\n * escape.\n *\n * @category grading\n * @since 0.1.0\n */\nexport const ciGrade = (verdict: Verdict): { readonly exitCode: 0 | 1 | 5; readonly summary: string } => {\n  const graded = grade(verdict)\n  return { exitCode: graded.exitCode, summary: flattenControlCharacters(graded.summary) }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,uBAOO;AACP,aAAwB;AACxB,+BAAyC;AACzC,yBAA4B;AAoB5B,MAAM,UAAU,CAAC,WACf,OAAO,IAAI,aAAa;AAAA,EAAI,CAAC,gBAC3B,YAAY,SAAS,UACjB;AAAA,IACA,MAAM,YAAY;AAAA,IAClB,QAAQ,YAAY;AAAA,IACpB,SAAS,YAAY;AAAA,IACrB,MAAM;AAAA,IACN,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,EACtB,IACE;AAAA,IACA,MAAM,YAAY;AAAA,IAClB,QAAQ,YAAY;AAAA,IACpB,SAAS,YAAY;AAAA,IACrB,MAAM;AAAA,IACN,QAAQ,YAAY;AAAA,EACtB;AACJ;AAOF,MAAM,oBAAoB,CAAC,WAA0C;AAAA,EACnE,GAAG,OAAO,IAAI,MAAM;AAAA,IAAQ,CAAC,WAC3B,OAAO,UAAU,SAAY,CAAC,IAAI,CAAC,SAAS,OAAO,IAAI,aAAa,OAAO,MAAM,IAAI,KAAK,OAAO,MAAM,OAAO,EAAE;AAAA,EAClH;AAAA,EACA,GAAG,OAAO,QAAQ;AAAA,IAAI,CAAC,SACrB,WAAW,KAAK,IAAI,oBAAoB,KAAK,IAAI,QAAI,gCAAY,IAAI,CAAC,IAAI,KAAK,OAAO;AAAA,EACxF;AACF;AAOA,MAAM,WAAW,CAAC,WAA0C;AAAA,EAC1D,GAAG,OAAO,YAAY;AAAA,IAAI,CAAC,SACzB,kBAAkB,KAAK,IAAI,QAAI,gCAAY,EAAE,QAAQ,KAAK,QAAQ,YAAY,KAAK,OAAO,WAAW,CAAC,CAAC;AAAA,EACzG;AAAA,EACA,GAAG,OAAO,eAAe;AAAA,IAAI,CAAC,SAC5B,sBAAsB,KAAK,IAAI,QAAI,gCAAY,EAAE,QAAQ,KAAK,QAAQ,YAAY,KAAK,OAAO,WAAW,CAAC,CAAC;AAAA,EAC7G;AACF;AAaO,MAAM,QAAQ,CAAC,QAAgB,UAAmB,CAAC,MACxD,OAAO,IAAI,aAAY;AACrB,QAAM,kBAAc,+BAAa,QAAQ,MAAM,CAAC;AAChD,QAAM,WAA2B,CAAC;AAClC,MAAI,QAAQ,SAAS,OAAW,UAAS,KAAK,OAAO,YAAY,KAAK,QAAQ,IAAI,CAAC;AACnF,MAAI,QAAQ,QAAQ,OAAW,UAAS,KAAK,OAAO,YAAY,IAAI,QAAQ,GAAG,CAAC;AAChF,MAAI,QAAQ,YAAY,OAAW,UAAS,KAAK,OAAO,YAAY,QAAQ,QAAQ,OAAO,CAAC;AAC5F,MAAI,QAAQ,SAAS,UAAa,QAAQ,QAAQ,UAAa,QAAQ,YAAY,QAAW;AAC5F,aAAS,KAAK,OAAO,YAAY,KAAK,CAAC,CAAC;AAAA,EAC1C;AACA,QAAM,UAAU,SAAS,MAAM;AAC/B,MAAI,QAAQ,SAAS,EAAG,UAAS,KAAK,EAAE,MAAM,UAAU,SAAS,cAAc,CAAC,EAAE,CAAC;AACnF,aAAO,0BAAQ,UAAU,kBAAkB,MAAM,CAAC;AACpD,CAAC;AAeI,MAAM,UAAU,CAAC,YAAiF;AACvG,QAAM,aAAS,wBAAM,OAAO;AAC5B,SAAO,EAAE,UAAU,OAAO,UAAU,aAAS,mDAAyB,OAAO,OAAO,EAAE;AACxF;",
+  "names": []
+}

package/dist/cjs/Regression.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Step-key-aware evaluation regression comparison.
+ *
+ * The step key is what separates the two findings this module reports. A score
+ * that moved at a *changed* step key is a regression: the target produced
+ * different work and it graded worse. A score that moved at an *unchanged* step
+ * key is nondeterminism: the same work graded differently twice. Both are
+ * results, and a gate reads both as red.
+ *
+ * @since 0.1.0
+ */
+import * as Effect from "effect/Effect";
+import type { Baseline, BaselineRecord } from "./Baseline.ts";
+import { EvalError } from "./EvalError.ts";
+import type { Observation, RunResult } from "./Runner.ts";
+/**
+ * Tolerances used for score comparisons.
+ *
+ * A move is reported only when it exceeds both tolerances, so either one alone
+ * is enough to silence it. Both default to `0`, which reports every move.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Tolerances {
+    readonly absolute?: number | undefined;
+    readonly relative?: number | undefined;
+}
+/**
+ * A score drop at a changed step key.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Regression {
+    readonly case: string;
+    readonly scorer: string;
+    readonly baseline: BaselineRecord;
+    readonly actual: Extract<Observation, {
+        readonly kind: "score";
+    }>;
+    readonly drop: number;
+}
+/**
+ * A changed score at the same step key, indicating nondeterminism.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Nondeterminism {
+    readonly case: string;
+    readonly scorer: string;
+    readonly baseline: BaselineRecord;
+    readonly actual: Extract<Observation, {
+        readonly kind: "score";
+    }>;
+    readonly delta: number;
+}
+/**
+ * An observation present on only one side of a comparison.
+ *
+ * `side` names the side it is missing from: `"run"` for a baseline record the
+ * run never reproduced, `"baseline"` for a score the run produced that no
+ * baseline record accounts for. `scorerName` comes from the observation that
+ * is present, when that artifact carried one.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MissingObservation {
+    readonly side: "baseline" | "run";
+    readonly case: string;
+    readonly scorer: string;
+    readonly scorerName?: string | undefined;
+    readonly stepKey: string;
+}
+/**
+ * Complete regression comparison.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Report {
+    readonly suite: string;
+    readonly baseline: Baseline;
+    readonly run: RunResult;
+    readonly regressions: ReadonlyArray<Regression>;
+    readonly nondeterminism: ReadonlyArray<Nondeterminism>;
+    readonly missing: ReadonlyArray<MissingObservation>;
+    readonly samples: ReadonlyArray<Extract<Observation, {
+        readonly kind: "score";
+    }>>;
+    readonly inconclusive: ReadonlyArray<Extract<Observation, {
+        readonly kind: "inconclusive";
+    }>>;
+}
+/**
+ * Compares a run to a baseline, preserving missing and inconclusive observations.
+ *
+ * Records and observations are grouped by `(case, scorer)` and then paired by
+ * step key, so a scorer that ran several times against one case is compared
+ * pairwise instead of by array position. A pair whose step key is unchanged and
+ * whose score moved is nondeterminism; a pair whose step key changed and whose
+ * score dropped is a regression; an unpaired record on either side is a missing
+ * observation. Inconclusive observations are carried through untouched, because
+ * they decide nothing and a gate has to see them.
+ *
+ * Fails with `invalid_tolerance` when a tolerance is not finite and
+ * non-negative, and with `invalid_baseline` when the artifact or any of its
+ * records belongs to a suite other than the one the run reports.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export declare const compare: (baseline: Baseline, run: RunResult, tolerances?: Tolerances) => Effect.Effect<Report, EvalError>;
+//# sourceMappingURL=Regression.d.ts.map

package/dist/cjs/Regression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Regression.d.ts","sourceRoot":"","sources":["../../src/Regression.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAI1C,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAEzD;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACvC;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;IACjC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,KAAK,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED;;;;;GAKG;AACH,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IACvB,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;IAC/C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACnD,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC,CAAA;IACjF,QAAQ,CAAC,YAAY,EAAE,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC,CAAA;CAC9F;AA2ED;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,OAAO,GAClB,UAAU,QAAQ,EAClB,KAAK,SAAS,EACd,aAAY,UAAe,KAC1B,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAwHjC,CAAA"}