@smthrs/evals 0.0.0-stage → 1.0.0-rc.3

package/src/Baseline.ts

@@ -0,0 +1,259 @@
+/**
+ * Canonical committed evaluation baselines.
+ *
+ * A baseline is the record of what a suite used to score, committed beside the
+ * suite it belongs to. It holds only the fields a comparison reads, and
+ * validation rebuilds every record from those fields, so nothing a caller
+ * happened to attach to an object travels into a committed artifact.
+ *
+ * @since 0.1.0
+ */
+
+import * as Effect from "effect/Effect"
+import { EvalError } from "./EvalError.ts"
+import { CanonicalJson } from "./internal/CanonicalJson.ts"
+import { controlCharacter } from "./internal/controlCharacters.ts"
+import { tupleKey } from "./internal/tupleKey.ts"
+import type { Observation, RunResult } from "./Runner.ts"
+
+/**
+ * Current committed baseline artifact version.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const version = 1 as const
+
+/**
+ * One successful score retained by a baseline.
+ *
+ * `scorer` is the scorer key a comparison matches on. `scorerName` is the
+ * optional readable name of the same scorer.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface BaselineRecord {
+  readonly suite: string
+  readonly case: string
+  readonly scorer: string
+  readonly scorerName?: string | undefined
+  readonly stepKey: string
+  readonly score: number
+}
+
+/**
+ * Canonical committed evaluation baseline.
+ *
+ * `suite` records artifact ownership even when `records` is empty.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Baseline {
+  readonly version: typeof version
+  readonly suite: string
+  readonly records: ReadonlyArray<BaselineRecord>
+}
+
+/**
+ * The declared ceilings a committed baseline is validated against.
+ *
+ * `length` bounds the JSON text `load` will parse, so an oversized committed
+ * file fails with a sentence instead of exhausting a CI runner's memory.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const limits = {
+  /** Longest baseline JSON text, in UTF-16 code units. */
+  length: 16 * 1024 * 1024
+} as const
+
+const fail = (message: string, path?: string): Effect.Effect<never, EvalError> =>
+  Effect.fail(
+    new EvalError({ code: "invalid_baseline", message, ...(path === undefined ? {} : { path }) })
+  )
+
+// A committed record is read by a gate whose summary goes to a CI log, so it
+// follows the same control-character rule as a suite name: rejected where it
+// enters the system, not flattened later.
+const stringField = (value: unknown, field: string, path: string): Effect.Effect<string, EvalError> => {
+  if (typeof value !== "string") {
+    return fail(`Baseline record field '${field}' must be a string, got ${typeof value}`, path)
+  }
+  const control = controlCharacter(value)
+  return control === undefined
+    ? Effect.succeed(value)
+    : fail(`Baseline record field '${field}' must not contain the control character ${control}`, path)
+}
+
+const decodeRecord = (value: unknown, index: number): Effect.Effect<BaselineRecord, EvalError> =>
+  Effect.gen(function*() {
+    const at = `records[${index}]`
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+      return yield* fail(`Baseline record must be an object, got ${value === null ? "null" : typeof value}`, at)
+    }
+    const source = value as { readonly [key: string]: unknown }
+    const rawSuite = source.suite
+    const rawCase = source.case
+    const rawScorer = source.scorer
+    const rawScorerName = source.scorerName
+    const rawStepKey = source.stepKey
+    const rawScore = source.score
+    const suite = yield* stringField(rawSuite, "suite", `${at}.suite`)
+    const caseName = yield* stringField(rawCase, "case", `${at}.case`)
+    const scorer = yield* stringField(rawScorer, "scorer", `${at}.scorer`)
+    const stepKey = yield* stringField(rawStepKey, "stepKey", `${at}.stepKey`)
+    const score = rawScore
+    if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) {
+      return yield* fail(
+        `Baseline record field 'score' must be a finite number in [0, 1], got ${String(score)}`,
+        `${at}.score`
+      )
+    }
+    const scorerName = rawScorerName === undefined
+      ? undefined
+      : yield* stringField(rawScorerName, "scorerName", `${at}.scorerName`)
+    // Every record is rebuilt field by field. Keeping the caller's object would
+    // carry unknown keys, and any getter among them, straight into a committed
+    // artifact.
+    return {
+      suite,
+      case: caseName,
+      scorer,
+      ...(scorerName === undefined ? {} : { scorerName }),
+      stepKey,
+      score: Object.is(score, -0) ? 0 : score
+    }
+  })
+
+const validate = (value: unknown): Effect.Effect<Baseline, EvalError> =>
+  Effect.gen(function*() {
+    if (typeof value !== "object" || value === null || Array.isArray(value)) {
+      return yield* fail("Baseline must be an object")
+    }
+    const artifact = value as { readonly version?: unknown; readonly suite?: unknown; readonly records?: unknown }
+    const rawVersion = artifact.version
+    const rawSuite = artifact.suite
+    const rawRecords = artifact.records
+    if (rawVersion !== version) {
+      return yield* fail(`Baseline version must be ${version}, got ${String(rawVersion)}`, "version")
+    }
+    if (rawSuite !== undefined && typeof rawSuite !== "string") {
+      return yield* fail(`Baseline field 'suite' must be a string, got ${typeof rawSuite}`, "suite")
+    }
+    const suiteControl = rawSuite === undefined ? undefined : controlCharacter(rawSuite)
+    if (suiteControl !== undefined) {
+      return yield* fail(`Baseline field 'suite' must not contain the control character ${suiteControl}`, "suite")
+    }
+    if (!Array.isArray(rawRecords)) {
+      return yield* fail("Baseline records must be an array", "records")
+    }
+    const records = yield* Effect.forEach(rawRecords, decodeRecord)
+    let suite = rawSuite
+    if (suite === undefined) {
+      const first = records[0]
+      if (first === undefined) {
+        return yield* fail("Cannot infer baseline suite: legacy artifact has no records", "suite")
+      }
+      if (records.some((record) => record.suite !== first.suite)) {
+        return yield* fail("Cannot infer baseline suite: legacy records name multiple suites", "suite")
+      }
+      suite = first.suite
+    }
+    return { version, suite, records: Object.freeze(records) }
+  })
+
+/**
+ * Builds and validates a baseline from a run's successful observations.
+ *
+ * Inconclusive observations are dropped: a baseline records what was measured,
+ * and an inconclusive observation measured nothing.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const fromRun = (run: RunResult): Effect.Effect<Baseline, EvalError> => {
+  const records: Array<BaselineRecord> = run.observations.flatMap((observation: Observation) =>
+    observation.kind === "score"
+      ? [{
+        suite: run.suite,
+        case: observation.case,
+        scorer: observation.scorer,
+        ...(observation.scorerName === undefined ? {} : { scorerName: observation.scorerName }),
+        stepKey: observation.stepKey,
+        score: observation.score
+      }]
+      : []
+  )
+  return validate({ version, suite: run.suite, records })
+}
+
+/**
+ * Validates an in-memory baseline.
+ *
+ * The result is a snapshot: every known field is read once, then records are
+ * rebuilt from those validated values and the array is frozen. Mutating the
+ * array or the objects that were passed in cannot change the validated
+ * baseline, and a stateful getter cannot substitute a value after validation.
+ *
+ * Fails with `invalid_baseline` carrying the record index and field name in
+ * `path` for a wrong version, a non-array `records`, a record that is not an
+ * object, an identity field that is not a string or holds a control
+ * character, or a score that is not finite in `[0, 1]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (
+  baseline: Omit<Baseline, "version"> & { readonly version?: typeof version }
+): Effect.Effect<Baseline, EvalError> => {
+  const artifactVersion = baseline.version ?? version
+  const records = baseline.records
+  return validate({ version: artifactVersion, suite: baseline.suite, records })
+}
+
+/**
+ * Serializes a baseline with recursively sorted keys and stable numbers.
+ *
+ * Records are ordered by an injective encoding of `(suite, case, scorer,
+ * stepKey)`, so two records whose names differ only in where a separator falls
+ * still sort apart. The result ends with a newline and never throws: a
+ * validated baseline holds only strings and finite numbers.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const write = (baseline: Baseline): string => {
+  const sortKey = (record: BaselineRecord): string => tupleKey(record.suite, record.case, record.scorer, record.stepKey)
+  const records = baseline.records
+    .map((record) => [sortKey(record), record] as const)
+    .sort(([left], [right]) => CanonicalJson.compareText(left, right))
+    .map(([, record]) => record)
+  return CanonicalJson.stringify({
+    version: baseline.version,
+    suite: baseline.suite,
+    records
+  })
+}
+
+/**
+ * Loads and validates baseline JSON.
+ *
+ * A version-1 artifact without a top-level `suite` is accepted only when its
+ * nonempty records all name the same suite. That suite becomes the artifact's
+ * owner, so writing the result includes it. Empty or ambiguous legacy artifacts
+ * fail with `invalid_baseline` at `suite`. Text longer than `limits.length`
+ * fails with `invalid_baseline` at `text` before any of it is parsed.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const load = (text: string): Effect.Effect<Baseline, EvalError> =>
+  text.length > limits.length
+    ? fail(`Baseline must be at most ${limits.length} characters, got ${text.length}`, "text")
+    : Effect.try({
+      try: () => JSON.parse(text) as unknown,
+      catch: (cause) => new EvalError({ code: "invalid_baseline", message: "Baseline is not valid JSON", cause })
+    }).pipe(Effect.flatMap(validate))

package/src/CaseExecutor.ts

@@ -0,0 +1,104 @@
+/**
+ * Injectable target-flow execution boundary.
+ *
+ * A suite says what to run; this service says how. Keeping it injectable is
+ * what lets the same suite run against a real flow, a scripted host, or nothing
+ * at all, and it is the only place a target's failure is converted into a typed
+ * evaluation failure.
+ *
+ * @since 0.1.0
+ */
+
+import type { Flow } from "@smthrs/core"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import { EvalError } from "./EvalError.ts"
+import type { Case } from "./Suite.ts"
+
+/**
+ * The result of executing one target-flow case.
+ *
+ * `target` is the flow value the case actually executed. A run matches it
+ * against each binding's `appliesTo` by reference identity, so it has to be the
+ * declared flow itself rather than a copy of it.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Execution {
+  readonly output: unknown
+  readonly stepKey: string
+  readonly latencyMs: number
+  readonly target: Pick<Flow.Any, "name">
+}
+
+/**
+ * The one callback a case executor is.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Run = (suiteCase: Case) => Effect.Effect<Execution, EvalError>
+
+/**
+ * Runtime shape for an injectable target-flow executor.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export interface Service {
+  readonly run: Run
+}
+
+/**
+ * Injectable execution boundary for a target flow.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class CaseExecutor extends Context.Service<CaseExecutor, Service>()("flows/evals/CaseExecutor") {}
+
+/**
+ * Builds an executor from its one callback.
+ *
+ * Throws a `TypeError` when there is no callback. An executor that silently
+ * degraded to {@link makeNoop} turned one wiring mistake into a whole suite of
+ * cases failing with `executor`, which reads as a broken target rather than a
+ * missing one.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (run: Run): Service => {
+  if (typeof run !== "function") {
+    throw new TypeError("CaseExecutor.make needs a callback")
+  }
+  return CaseExecutor.of({ run })
+}
+
+/**
+ * Builds an executor that fails every case with a typed executor error.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const makeNoop = (): Service =>
+  CaseExecutor.of({
+    run: (suiteCase) =>
+      Effect.fail(
+        new EvalError({
+          code: "executor",
+          message: `No executor is available for case '${suiteCase.name}'`,
+          path: `cases['${suiteCase.name}']`
+        })
+      )
+  })
+
+/**
+ * Provides the unavailable executor.
+ *
+ * @category layers
+ * @since 0.1.0
+ */
+export const layerNoop: Layer.Layer<CaseExecutor> = Layer.succeed(CaseExecutor)(makeNoop())

package/src/EvalError.ts

@@ -0,0 +1,74 @@
+/**
+ * The typed failures evaluation raises, with the stable codes a gate or a
+ * report branches on.
+ *
+ * @since 0.1.0
+ */
+
+import * as Schema from "effect/Schema"
+
+/**
+ * Stable evaluation failure codes.
+ *
+ * Each code names one thing a caller can do about it, so a gate that branches
+ * on the code routes the failure to the right owner:
+ *
+ * - `invalid_suite`: the suite declaration is wrong. Fix the suite.
+ * - `invalid_run_options`: `Runner.run`'s own options are wrong, not the suite.
+ * - `invalid_baseline`: the committed baseline is unreadable, belongs to
+ *   another suite, or holds a record the schema rejects. Regenerate it.
+ * - `invalid_tolerance`: the caller passed a tolerance that is not a finite
+ *   non-negative number. Fix the call, not the baseline.
+ * - `invalid_trials`: the caller passed run, pass, or `k` counts to `Trials`
+ *   outside their bounds. Fix the call.
+ * - `executor`: the target flow failed for a case.
+ * - `ambiguous_score_job`: two score jobs shared a step key and a scorer, so
+ *   an order-only batch runner's results cannot be attributed to a case. Give
+ *   each case its own step key, or use a runner that implements
+ *   `runBatchCorrelated`.
+ * - `scorer_protocol`: a batch runner broke the `runBatch` contract by returning
+ *   the wrong number of observations, or observations that identify jobs other
+ *   than the ones it was given. Nothing it returned can be trusted.
+ * - `scorer_unavailable`: no batch runner was available to score with.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const EvalErrorCode = Schema.Literals([
+  "invalid_suite",
+  "invalid_run_options",
+  "invalid_baseline",
+  "invalid_tolerance",
+  "invalid_trials",
+  "executor",
+  "ambiguous_score_job",
+  "scorer_protocol",
+  "scorer_unavailable"
+])
+
+/**
+ * Stable evaluation failure code.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type EvalErrorCode = typeof EvalErrorCode.Type
+
+/**
+ * A typed failure raised while loading or executing an evaluation.
+ *
+ * `code` is the stable branch point, `message` is the sentence a CI log shows,
+ * and `path` locates the offending value inside the input the caller supplied
+ * (`records[3].score`, `cases[1].input`, `options.at`). `cause` retains the
+ * original failure; it is never rendered on its own, so a message that matters
+ * to an operator has to say so itself.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export class EvalError extends Schema.TaggedError<EvalError>()("flows/evals/EvalError", {
+  code: EvalErrorCode,
+  message: Schema.String,
+  path: Schema.optional(Schema.String),
+  cause: Schema.optional(Schema.Defect())
+}) {}

package/src/Gate.ts

@@ -0,0 +1,128 @@
+/**
+ * The pass/fail decision over a finished evaluation run: how a threshold
+ * is applied and what a failing gate reports.
+ *
+ * @since 0.1.0
+ */
+
+import {
+  combine,
+  expectScores,
+  grade,
+  type ScoreGateError,
+  type ScoreSample,
+  type Verdict
+} from "@smthrs/scorers/ScoreGate"
+import * as Effect from "effect/Effect"
+import { flattenControlCharacters } from "./internal/controlCharacters.ts"
+import { scorerLabel } from "./internal/scorerLabel.ts"
+import type { Report } from "./Regression.ts"
+
+/**
+ * Thresholds accepted by a CI score gate.
+ *
+ * `mean` gates the arithmetic mean of every score observation, `min` gates the
+ * lowest one, and `perCase` gates each named case's lowest score. With none of
+ * them set the gate still runs, as `mean(0)`, so a run with no score at all is
+ * undecidable rather than a pass.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Options {
+  readonly mean?: number | undefined
+  readonly min?: number | undefined
+  readonly perCase?: Readonly<Record<string, number>> | undefined
+}
+
+const samples = (report: Report): ReadonlyArray<ScoreSample> =>
+  report.run.observations.map((observation) =>
+    observation.kind === "score"
+      ? {
+        case: observation.case,
+        scorer: observation.scorer,
+        stepKey: observation.stepKey,
+        kind: "score",
+        value: observation.score,
+        reason: observation.reason
+      }
+      : {
+        case: observation.case,
+        scorer: observation.scorer,
+        stepKey: observation.stepKey,
+        kind: "inconclusive",
+        reason: observation.reason
+      }
+  )
+
+/**
+ * An environment fault: something the comparison needed was never observed, so
+ * the harness owes an answer it cannot give. A fault withholds a decision;
+ * it does not make one.
+ */
+const environmentFaults = (report: Report): ReadonlyArray<string> => [
+  ...report.run.cases.flatMap((result) =>
+    result.error === undefined ? [] : [`case '${result.case}' failed: ${result.error.code}: ${result.error.message}`]
+  ),
+  ...report.missing.map((item) =>
+    `missing ${item.side} observation for ${item.case}/${scorerLabel(item)}/${item.stepKey}`
+  )
+]
+
+/**
+ * A finding: the run measured something the baseline says it should not have.
+ * A regression scored lower at a changed step key, and nondeterminism moved a
+ * score at an unchanged one. Both are results, so both are red.
+ */
+const findings = (report: Report): ReadonlyArray<string> => [
+  ...report.regressions.map((item) =>
+    `regression for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`
+  ),
+  ...report.nondeterminism.map((item) =>
+    `nondeterminism for ${item.case}/${scorerLabel({ scorer: item.scorer, scorerName: item.actual.scorerName })}`
+  )
+]
+
+/**
+ * Checks thresholds through `@smthrs/scorers`' shared ScoreGate arithmetic.
+ *
+ * The threshold gates always run: an unobserved case cannot excuse the cases
+ * that were observed. Faults and findings are kept apart, and the verdict
+ * carries both, so a regression is a red while an unusable harness stays
+ * undecided.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const check = (report: Report, options: Options = {}): Effect.Effect<Verdict, ScoreGateError> =>
+  Effect.gen(function*() {
+    const expectation = expectScores(samples(report))
+    const verdicts: Array<Verdict> = []
+    if (options.mean !== undefined) verdicts.push(yield* expectation.mean(options.mean))
+    if (options.min !== undefined) verdicts.push(yield* expectation.min(options.min))
+    if (options.perCase !== undefined) verdicts.push(yield* expectation.perCase(options.perCase))
+    if (options.mean === undefined && options.min === undefined && options.perCase === undefined) {
+      verdicts.push(yield* expectation.mean(0))
+    }
+    const reasons = findings(report)
+    if (reasons.length > 0) verdicts.push({ _tag: "Failed", reasons, inconclusive: [] })
+    return combine(verdicts, environmentFaults(report))
+  })
+
+/**
+ * Maps a gate verdict to the shared CI convention: a finding is exit code 1,
+ * an undecidable run is exit code 5.
+ *
+ * The summary is one log line: every C0 control and DEL in it becomes a
+ * space, the way a report cell is written, so a value that reached a verdict
+ * reason from a baseline, a step key or a target's failure cannot emit its own
+ * line, which on GitHub Actions would be a workflow command, or a terminal
+ * escape.
+ *
+ * @category grading
+ * @since 0.1.0
+ */
+export const ciGrade = (verdict: Verdict): { readonly exitCode: 0 | 1 | 5; readonly summary: string } => {
+  const graded = grade(verdict)
+  return { exitCode: graded.exitCode, summary: flattenControlCharacters(graded.summary) }
+}