@smthrs/evals 0.0.0-stage → 1.0.0-rc.12

package/src/Suite.ts

@@ -0,0 +1,358 @@
+/**
+ * Fixed evaluation suite declarations.
+ *
+ * A suite is the fixed input half of an evaluation: named cases, the scorer
+ * bindings that grade them, and the concurrency the runner is allowed. It is
+ * validated once and then never changes, because the run that produced a
+ * committed baseline has to be reproducible from the suite value that was
+ * validated.
+ *
+ * @since 0.1.0
+ */
+
+import type * as ScorerBinding from "@smthrs/scorers/Binding"
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { EvalError } from "./EvalError.ts"
+import { controlCharacter } from "./internal/controlCharacters.ts"
+
+/**
+ * A scorer binding accepted from `@smthrs/scorers`.
+ *
+ * The binding's `appliesTo` flow is matched against an execution's `target` by
+ * reference identity, so a binding only ever grades the exact flow value it was
+ * declared against.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Binding = ScorerBinding.Binding
+
+/**
+ * One immutable fixed-suite case.
+ *
+ * `input` is handed to the executor and `expected` is offered to a bound scorer
+ * as ground truth. A declared `expected`, including `null`, wins over the
+ * binding's `groundTruth`; only an absent one defers to the binding. Both are
+ * snapshots taken by {@link make}: mutating the object a caller passed in never
+ * changes the suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Case {
+  readonly name: string
+  readonly input: unknown
+  readonly expected?: unknown | undefined
+}
+
+/**
+ * Options for constructing a suite.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MakeOptions {
+  readonly name: string
+  readonly cases: ReadonlyArray<Case>
+  readonly bindings?: ReadonlyArray<Binding> | undefined
+  readonly concurrency: number
+  readonly trials?: number | undefined
+}
+
+/**
+ * A validated, named collection of fixed cases and scorer bindings.
+ *
+ * The suite, cases, bindings, and their inert data are deeply frozen snapshots.
+ * Executable scorer and target identities remain unchanged.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Suite {
+  readonly name: string
+  readonly cases: ReadonlyArray<Case>
+  readonly bindings: ReadonlyArray<Binding>
+  readonly concurrency: number
+  readonly trials?: number | undefined
+}
+
+/**
+ * The declared ceilings a suite is validated against.
+ *
+ * They exist so a mistake fails at construction with a sentence instead of
+ * exhausting memory in a runner: `concurrency` bounds the fibers a run may hold
+ * open, `cases` bounds one suite, and `fixtureLength` bounds one JSON Lines
+ * fixture.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const limits = {
+  concurrency: 1024,
+  cases: 10_000,
+  /** Longest JSON Lines fixture, in UTF-16 code units. */
+  fixtureLength: 8 * 1024 * 1024
+} as const
+
+const invalid = (message: string, path: string, cause?: unknown): EvalError =>
+  new EvalError({ code: "invalid_suite", message, path, ...(cause === undefined ? {} : { cause }) })
+
+const cloneMessage = "Suite data must be structured-cloneable so the suite cannot change after it is validated"
+
+// Validate before structuredClone can erase a prototype or invoke a getter.
+// Only records and arrays can be protected completely with Object.freeze.
+const validateData = (value: unknown, path: string, seen = new WeakSet<object>()): void => {
+  if (typeof value === "function" || typeof value === "symbol") throw invalid(cloneMessage, path)
+  if (value === null || typeof value !== "object") return
+  if (seen.has(value)) return
+  seen.add(value)
+  const array = Array.isArray(value)
+  const prototype = Object.getPrototypeOf(value)
+  if (array ? prototype !== Array.prototype : prototype !== Object.prototype && prototype !== null) {
+    throw invalid("Suite data must contain only plain objects, arrays, and structured-cloneable primitives", path)
+  }
+  for (const key of Reflect.ownKeys(value)) {
+    if (array && key === "length") continue
+    const childPath = typeof key === "string" && array && /^(0|[1-9]\d*)$/.test(key)
+      ? `${path}[${key}]`
+      : `${path}.${String(key)}`
+    const descriptor = Object.getOwnPropertyDescriptor(value, key)!
+    if (typeof key === "symbol" || !descriptor.enumerable || !("value" in descriptor)) {
+      throw invalid("Suite data properties must be enumerable string-keyed data fields", childPath)
+    }
+    validateData(descriptor.value, childPath, seen)
+  }
+}
+
+const freezeData = (value: unknown, seen = new WeakSet<object>()): unknown => {
+  if (value === null || typeof value !== "object" || seen.has(value)) return value
+  seen.add(value)
+  for (const child of Object.values(value)) freezeData(child, seen)
+  return Object.freeze(value)
+}
+
+const clone = (value: unknown, path: string): Effect.Effect<unknown, EvalError> =>
+  Effect.try({
+    try: () => {
+      validateData(value, path)
+      return freezeData(structuredClone(value))
+    },
+    catch: (cause) => cause instanceof EvalError ? cause : invalid(cloneMessage, path, cause)
+  })
+
+const validate = (options: MakeOptions & { readonly trials: number }): EvalError | undefined => {
+  if (options.name.trim().length === 0) return invalid("Suite name must not be empty", "name")
+  const nameControl = controlCharacter(options.name)
+  if (nameControl !== undefined) {
+    return invalid(`Suite name must not contain the control character ${nameControl}`, "name")
+  }
+  if (!Number.isSafeInteger(options.trials) || options.trials < 1 || options.trials > 1000) {
+    return invalid("Suite trials must be an integer from 1 to 1000", "trials")
+  }
+  if (options.cases.length === 0) return invalid("Suite must contain at least one case", "cases")
+  if (options.cases.length > limits.cases) {
+    return invalid(`Suite must contain at most ${limits.cases} cases, got ${options.cases.length}`, "cases")
+  }
+  if (!Number.isSafeInteger(options.concurrency) || options.concurrency < 1) {
+    return invalid(
+      `Suite concurrency must be a positive safe integer, got ${String(options.concurrency)}`,
+      "concurrency"
+    )
+  }
+  if (options.concurrency > limits.concurrency) {
+    return invalid(
+      `Suite concurrency must be at most ${limits.concurrency}, got ${options.concurrency}`,
+      "concurrency"
+    )
+  }
+  const names = new Set<string>()
+  for (const [index, suiteCase] of options.cases.entries()) {
+    if (suiteCase.name.trim().length === 0) {
+      return invalid("Suite case name must not be empty", `cases[${index}].name`)
+    }
+    const caseControl = controlCharacter(suiteCase.name)
+    if (caseControl !== undefined) {
+      return invalid(
+        `Suite case name must not contain the control character ${caseControl}`,
+        `cases[${index}].name`
+      )
+    }
+    if (names.has(suiteCase.name)) {
+      return invalid(`Duplicate suite case: ${suiteCase.name}`, `cases[${index}].name`)
+    }
+    names.add(suiteCase.name)
+  }
+  return undefined
+}
+
+// Every field is read exactly once, into these two shapes, before anything is
+// validated or copied. Reading a field twice let a getter return the name that
+// passed validation and then a different one into the suite, which is the same
+// hole a baseline record had.
+const readCase = (suiteCase: Case): Case => ({
+  name: suiteCase.name,
+  input: suiteCase.input,
+  expected: suiteCase.expected
+})
+
+const readBinding = (binding: Binding): Binding => {
+  const read: { -readonly [K in keyof Binding]: Binding[K] } = {
+    scorer: binding.scorer,
+    appliesTo: binding.appliesTo,
+    sampling: binding.sampling
+  }
+  const groundTruth = binding.groundTruth
+  if (groundTruth !== undefined) read.groundTruth = groundTruth
+  const context = binding.context
+  if (context !== undefined) read.context = context
+  return read
+}
+
+const copyCase = (suiteCase: Case, index: number): Effect.Effect<Case, EvalError> =>
+  Effect.gen(function*() {
+    const input = yield* clone(suiteCase.input, `cases[${index}].input`)
+    if (suiteCase.expected === undefined) return Object.freeze({ name: suiteCase.name, input })
+    const expected = yield* clone(suiteCase.expected, `cases[${index}].expected`)
+    return Object.freeze({ name: suiteCase.name, input, expected })
+  })
+
+const copyBinding = (binding: Binding, index: number): Effect.Effect<Binding, EvalError> =>
+  Effect.gen(function*() {
+    // `scorer` and `appliesTo` are executable identities matched by reference,
+    // so they are carried over unchanged; sampling, ground truth, and context
+    // are inert data that must not remain caller-owned.
+    const sampling = binding.sampling === undefined
+      ? undefined
+      : yield* clone(binding.sampling, `bindings[${index}].sampling`)
+    const copied: { -readonly [K in keyof Binding]: Binding[K] } = {
+      scorer: binding.scorer,
+      appliesTo: binding.appliesTo,
+      sampling: sampling as Binding["sampling"]
+    }
+    if (binding.groundTruth !== undefined) {
+      copied.groundTruth = yield* clone(binding.groundTruth, `bindings[${index}].groundTruth`)
+    }
+    if (binding.context !== undefined) {
+      copied.context = yield* clone(binding.context, `bindings[${index}].context`)
+    }
+    return Object.freeze(copied)
+  })
+
+/**
+ * Builds and validates a fixed suite.
+ *
+ * When the effect runs, it reads every option, case field, and binding field
+ * exactly once, then validates and copies only what it read. Validating one
+ * value and copying another let a getter hand the suite something validation
+ * never saw. Every case and binding is then copied, including a binding's
+ * sampling policy, so the suite is a snapshot the caller can no longer reach:
+ * mutating an array, case input, or ratio policy afterwards leaves the
+ * validated suite unchanged. Data is checked before `structuredClone` and
+ * deeply frozen afterwards. Only plain objects, arrays, and cloneable
+ * primitives are admitted, with enumerable string-keyed data properties.
+ * Functions, class instances, mutable built-ins, and accessors fail with
+ * `invalid_suite` naming the offending path. Cycles and shared references are
+ * supported; null-prototype records become ordinary objects in the clone.
+ *
+ * Fails with `invalid_suite` for an empty or control-character name, no cases,
+ * more than `limits.cases` cases, a duplicate case name, or a concurrency that
+ * is not a safe integer in `[1, limits.concurrency]`.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const make = (options: MakeOptions): Effect.Effect<Suite, EvalError> =>
+  Effect.suspend(() => {
+    const name = options.name
+    const concurrency = options.concurrency
+    const cases = [...options.cases].map(readCase)
+    const bindings = [...(options.bindings ?? [])].map(readBinding)
+    const trials = options.trials ?? 1
+    const snapshot = { name, concurrency, cases, bindings, trials }
+    const error = validate(snapshot)
+    if (error !== undefined) return Effect.fail(error)
+    return Effect.gen(function*() {
+      const copiedCases = yield* Effect.forEach(cases, copyCase)
+      const copiedBindings = yield* Effect.forEach(bindings, copyBinding)
+      return Object.freeze({
+        name,
+        cases: Object.freeze(copiedCases),
+        bindings: Object.freeze(copiedBindings),
+        concurrency,
+        trials
+      })
+    })
+  })
+
+/**
+ * Options used when decoding JSON Lines.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface JsonLinesOptions {
+  readonly name: string
+  readonly bindings?: ReadonlyArray<Binding> | undefined
+  readonly concurrency: number
+  readonly trials?: number | undefined
+}
+
+const jsonCase = Schema.Struct({
+  name: Schema.String,
+  input: Schema.Unknown,
+  expected: Schema.optional(Schema.Unknown)
+})
+
+/**
+ * Loads the `{ name, input, expected? }` JSON Lines fixture format.
+ *
+ * Blank lines are skipped, a leading byte-order mark is stripped, and both LF
+ * and CRLF terminate a line. A malformed line fails with `invalid_suite`
+ * carrying the 1-based line number in both the message and the path; a fixture
+ * larger than `limits.fixtureLength` is rejected before any of it is parsed.
+ * The first non-blank line after `limits.cases` cases fails with
+ * `invalid_suite` at `cases` before it or any later line is parsed or decoded.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const fromJsonLines = (text: string, options: JsonLinesOptions): Effect.Effect<Suite, EvalError> =>
+  Effect.gen(function*() {
+    if (text.length > limits.fixtureLength) {
+      return yield* Effect.fail(
+        invalid(
+          `JSON Lines fixture must be at most ${limits.fixtureLength} characters, got ${text.length}`,
+          "text"
+        )
+      )
+    }
+    const cases: Array<Case> = []
+    const body = text.startsWith("\uFEFF") ? text.slice(1) : text
+    for (const [index, line] of body.split(/\r?\n/).entries()) {
+      const trimmed = line.trim()
+      if (trimmed.length === 0) continue
+      // Stop at the ceiling before parsing or decoding the excess line: a
+      // fixture under the length limit can still hold hundreds of thousands
+      // of tiny lines, and reporting the count only in make() would parse and
+      // materialize every one of them first.
+      if (cases.length === limits.cases) {
+        return yield* Effect.fail(
+          invalid(`Suite must contain at most ${limits.cases} cases, got ${cases.length + 1}`, "cases")
+        )
+      }
+      const decoded = yield* Effect.try({
+        try: () => JSON.parse(trimmed) as unknown,
+        catch: (cause) => invalid(`Invalid JSON on line ${index + 1}`, `line[${index + 1}]`, cause)
+      }).pipe(
+        Effect.flatMap(Schema.decodeUnknownEffect(jsonCase)),
+        Effect.mapError((cause) =>
+          cause instanceof EvalError
+            ? cause
+            : invalid(`Invalid suite case on line ${index + 1}`, `line[${index + 1}]`, cause)
+        )
+      )
+      cases.push(decoded)
+    }
+    return yield* make({ ...options, cases })
+  })

package/src/Trials.ts

@@ -0,0 +1,155 @@
+/**
+ * Repeated-run statistics for live model evaluations.
+ *
+ * A live case runs `n` times and passes `c` of them. {@link passAtK} is the
+ * chance that at least one of `k` runs passes, the capability a best-of-k
+ * caller sees; {@link passHatK} is the chance that all `k` pass, the
+ * consistency a user sees on every attempt. Both are exact for sampling `k` of
+ * the `n` recorded runs without replacement.
+ *
+ * Every function is pure and throws an `EvalError` with code `invalid_trials`
+ * for counts outside the documented bounds.
+ *
+ * @since 0.1.0
+ */
+
+import { EvalError } from "./EvalError.ts"
+
+/**
+ * Repeated-run statistics for one case.
+ *
+ * `rate` equals `passAt1`; `stderr` is the standard error of the pass rate,
+ * `sqrt(rate * (1 - rate) / n)`, and 0 for a single run.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Summary {
+  readonly n: number
+  readonly passes: number
+  readonly rate: number
+  readonly passAt1: number
+  readonly passAtK: number
+  readonly passHatK: number
+  readonly stderr: number
+}
+
+/**
+ * Repeated-run statistics across cases.
+ *
+ * `passAt1`, `passAtK`, and `passHatK` are means over cases, and `allPass` is
+ * the fraction of cases whose every run passed.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Aggregate {
+  readonly cases: number
+  readonly passAt1: number
+  readonly passAtK: number
+  readonly passHatK: number
+  readonly allPass: number
+  readonly perCase: Readonly<Record<string, Summary>>
+}
+
+const invalid = (message: string, path: string): EvalError => new EvalError({ code: "invalid_trials", message, path })
+
+const validate = (n: number, c: number, k: number): void => {
+  if (!Number.isInteger(n) || n < 1) throw invalid(`Run count must be a positive integer, received ${n}`, "n")
+  if (!Number.isInteger(c) || c < 0 || c > n) {
+    throw invalid(`Pass count must be an integer from 0 to ${n}, received ${c}`, "c")
+  }
+  if (!Number.isInteger(k) || k < 1 || k > n) throw invalid(`k must be an integer from 1 to ${n}, received ${k}`, "k")
+}
+
+/**
+ * The unbiased pass@k estimator, `1 - C(n - c, k) / C(n, k)` (Chen et al.,
+ * 2021): the chance that at least one of `k` runs drawn from `n` passes. It is
+ * computed as a product, so large `n` neither overflows nor loses precision.
+ *
+ * Throws `invalid_trials` unless `n`, `c`, and `k` are integers with
+ * `0 <= c <= n` and `1 <= k <= n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const passAtK = (n: number, c: number, k: number): number => {
+  validate(n, c, k)
+  if (n - c < k) return 1
+  let fail = 1
+  for (let i = n - c + 1; i <= n; i++) fail *= 1 - k / i
+  return 1 - fail
+}
+
+/**
+ * pass^k, `C(c, k) / C(n, k)`: the chance that all `k` runs drawn from `n`
+ * without replacement pass, the consistency measure of tau-bench.
+ *
+ * Throws `invalid_trials` under the same bounds as {@link passAtK}.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const passHatK = (n: number, c: number, k: number): number => {
+  validate(n, c, k)
+  let all = 1
+  for (let i = 0; i < k; i++) all *= (c - i) / (n - i)
+  return Math.max(0, all)
+}
+
+/**
+ * Summarizes one case's run outcomes at `k`.
+ *
+ * Throws `invalid_trials` for no outcomes or a `k` outside `1..n`.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const summarize = (outcomes: ReadonlyArray<boolean>, k: number): Summary => {
+  const n = outcomes.length
+  const passes = outcomes.filter(Boolean).length
+  validate(n, passes, k)
+  const rate = passes / n
+  return {
+    n,
+    passes,
+    rate,
+    passAt1: rate,
+    passAtK: passAtK(n, passes, k),
+    passHatK: passHatK(n, passes, k),
+    stderr: n <= 1 ? 0 : Math.sqrt(rate * (1 - rate) / n)
+  }
+}
+
+const mean = (values: ReadonlyArray<number>): number =>
+  values.length === 0 ? 0 : values.reduce((total, value) => total + value, 0) / values.length
+
+/**
+ * Summarizes every case at `k` and averages over cases.
+ *
+ * A case with fewer than `k` runs is summarized at `k = n`, its own run
+ * count, so a short case still contributes rather than failing the whole
+ * aggregate. No cases aggregate to zeros. Throws `invalid_trials` for a case
+ * with no runs or a `k` that is not a positive integer.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export const aggregate = (cases: Readonly<Record<string, ReadonlyArray<boolean>>>, k: number): Aggregate => {
+  if (!Number.isInteger(k) || k < 1) throw invalid(`k must be a positive integer, received ${k}`, "k")
+  const entries = Object.entries(cases).map(([name, outcomes]): [string, Summary] => {
+    if (outcomes.length === 0) throw invalid(`Case '${name}' has no runs`, `cases['${name}']`)
+    return [name, summarize(outcomes, Math.min(k, outcomes.length))]
+  })
+  // fromEntries defines own properties, so a case named `__proto__` stays a case.
+  const perCase: Record<string, Summary> = Object.fromEntries(entries)
+  const summaries = Object.values(perCase)
+  return {
+    cases: summaries.length,
+    passAt1: mean(summaries.map((summary) => summary.passAt1)),
+    passAtK: mean(summaries.map((summary) => summary.passAtK)),
+    passHatK: mean(summaries.map((summary) => summary.passHatK)),
+    allPass: mean(summaries.map((summary) => summary.passes === summary.n ? 1 : 0)),
+    perCase
+  }
+}

package/src/index.ts

@@ -0,0 +1,76 @@
+/**
+ * Fixed-suite evaluation tools for flows.
+ *
+ * The pipeline runs in one direction: a {@link Suite} declares fixed cases and
+ * the scorers bound to them, a {@link Runner} executes them through an injected
+ * {@link CaseExecutor} and grades the executions, {@link Baseline} records what
+ * a run scored, {@link Regression} compares the next run with that record,
+ * {@link Report} renders the comparison, and {@link Gate} turns it into a CI
+ * exit code.
+ *
+ * @since 0.1.0
+ */
+
+/**
+ * Typed evaluation failures and the stable codes a caller branches on.
+ *
+ * @since 0.1.0
+ * @category errors
+ */
+export * as EvalError from "./EvalError.ts"
+/**
+ * Fixed suite declarations and their JSON Lines fixture format.
+ *
+ * @since 0.1.0
+ * @category suites
+ */
+export * as Suite from "./Suite.ts"
+/**
+ * The injectable boundary that executes one case against a target flow.
+ *
+ * @since 0.1.0
+ * @category services
+ */
+export * as CaseExecutor from "./CaseExecutor.ts"
+/**
+ * Deterministic suite execution and bound scorer evaluation.
+ *
+ * @since 0.1.0
+ * @category runners
+ */
+export * as Runner from "./Runner.ts"
+/**
+ * Committed baselines: what a suite used to score.
+ *
+ * @since 0.1.0
+ * @category baselines
+ */
+export * as Baseline from "./Baseline.ts"
+/**
+ * Step-key-aware comparison of a run against a baseline.
+ *
+ * @since 0.1.0
+ * @category regression
+ */
+export * as Regression from "./Regression.ts"
+/**
+ * Canonical JSON and Markdown renderings of a comparison.
+ *
+ * @since 0.1.0
+ * @category reports
+ */
+export * as Report from "./Report.ts"
+/**
+ * Score thresholds and the CI exit grade a comparison earns.
+ *
+ * @since 0.1.0
+ * @category gates
+ */
+export * as Gate from "./Gate.ts"
+/**
+ * Repeated-run statistics for live evaluations: pass@k and pass^k.
+ *
+ * @since 0.1.0
+ * @category statistics
+ */
+export * as Trials from "./Trials.ts"