@smthrs/evals 0.0.0-stage → 1.0.0-rc.4

package/src/Regression.ts

@@ -0,0 +1,314 @@
+/**
+ * Step-key-aware evaluation regression comparison.
+ *
+ * The step key is what separates the two findings this module reports. A score
+ * that moved at a *changed* step key is a regression: the target produced
+ * different work and it graded worse. A score that moved at an *unchanged* step
+ * key is nondeterminism: the same work graded differently twice. Both are
+ * results, and a gate reads both as red.
+ *
+ * @since 0.1.0
+ */
+
+import * as Effect from "effect/Effect"
+import type { Baseline, BaselineRecord } from "./Baseline.ts"
+import { EvalError } from "./EvalError.ts"
+import { CanonicalJson } from "./internal/CanonicalJson.ts"
+import { flattenControlCharacters } from "./internal/controlCharacters.ts"
+import { tupleKey } from "./internal/tupleKey.ts"
+import type { Observation, RunResult } from "./Runner.ts"
+
+/**
+ * Tolerances used for score comparisons.
+ *
+ * A move is reported only when it exceeds both tolerances, so either one alone
+ * is enough to silence it. Both default to `0`, which reports every move.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Tolerances {
+  readonly absolute?: number | undefined
+  readonly relative?: number | undefined
+}
+
+/**
+ * A score drop at a changed step key.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Regression {
+  readonly case: string
+  readonly scorer: string
+  readonly baseline: BaselineRecord
+  readonly actual: Extract<Observation, { readonly kind: "score" }>
+  readonly drop: number
+}
+
+/**
+ * A changed score at the same step key, indicating nondeterminism.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Nondeterminism {
+  readonly case: string
+  readonly scorer: string
+  readonly baseline: BaselineRecord
+  readonly actual: Extract<Observation, { readonly kind: "score" }>
+  readonly delta: number
+}
+
+/**
+ * An observation present on only one side of a comparison.
+ *
+ * `side` names the side it is missing from: `"run"` for a baseline record the
+ * run never reproduced, `"baseline"` for a score the run produced that no
+ * baseline record accounts for. `scorerName` comes from the observation that
+ * is present, when that artifact carried one.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface MissingObservation {
+  readonly side: "baseline" | "run"
+  readonly case: string
+  readonly scorer: string
+  readonly scorerName?: string | undefined
+  readonly stepKey: string
+}
+
+/**
+ * Complete regression comparison.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Report {
+  readonly suite: string
+  readonly baseline: Baseline
+  readonly run: RunResult
+  readonly regressions: ReadonlyArray<Regression>
+  readonly nondeterminism: ReadonlyArray<Nondeterminism>
+  readonly missing: ReadonlyArray<MissingObservation>
+  readonly samples: ReadonlyArray<Extract<Observation, { readonly kind: "score" }>>
+  readonly inconclusive: ReadonlyArray<Extract<Observation, { readonly kind: "inconclusive" }>>
+}
+
+type Score = Extract<Observation, { readonly kind: "score" }>
+
+// Injective: two distinct (case, scorer) pairs can never encode to one key,
+// which a delimiter join could not promise once a name may contain the
+// delimiter.
+const key = (caseName: string, scorer: string): string => tupleKey(caseName, scorer)
+const validTolerance = (value: number): boolean => Number.isFinite(value) && value >= 0
+
+const groupBy = <A>(items: ReadonlyArray<A>, of: (item: A) => string): Map<string, Array<A>> => {
+  const groups = new Map<string, Array<A>>()
+  for (const item of items) {
+    const bucket = groups.get(of(item))
+    if (bucket === undefined) groups.set(of(item), [item])
+    else bucket.push(item)
+  }
+  return groups
+}
+
+const byScore = <A extends { readonly score: number }>(items: ReadonlyArray<A>): Array<A> =>
+  [...items].sort((left, right) => left.score - right.score)
+
+const byStepThenScore = <A extends { readonly stepKey: string; readonly score: number }>(
+  items: ReadonlyArray<A>
+): Array<A> =>
+  [...items].sort((left, right) => CanonicalJson.compareText(left.stepKey, right.stepKey) || left.score - right.score)
+
+/** One matched pair, or one record that nothing on the other side accounts for. */
+type Pair =
+  | { readonly kind: "matched"; readonly expected: BaselineRecord; readonly observed: Score }
+  | { readonly kind: "missing-run"; readonly expected: BaselineRecord }
+  | { readonly kind: "missing-baseline"; readonly observed: Score }
+
+/**
+ * Pairs one case's baseline records with one case's observations.
+ *
+ * Same-step-key pairs are matched first, lowest score to lowest score, so a
+ * repeated scorer does not report a move that never happened. Whatever is left
+ * over is paired in stable order, and an unpaired record on either side is a
+ * missing observation rather than a silent drop.
+ */
+const pairsFor = (
+  expected: ReadonlyArray<BaselineRecord>,
+  observed: ReadonlyArray<Score>
+): ReadonlyArray<Pair> => {
+  const pairs: Array<Pair> = []
+  const expectedByStep = groupBy(expected, (record) => record.stepKey)
+  const observedByStep = groupBy(observed, (record) => record.stepKey)
+  const leftoverExpected: Array<BaselineRecord> = []
+  const leftoverObserved: Array<Score> = []
+  for (const stepKey of [...expectedByStep.keys()].sort(CanonicalJson.compareText)) {
+    const atStep = byScore(expectedByStep.get(stepKey)!)
+    const observedAtStep = byScore(observedByStep.get(stepKey) ?? [])
+    const count = Math.min(atStep.length, observedAtStep.length)
+    for (let index = 0; index < count; index++) {
+      pairs.push({ kind: "matched", expected: atStep[index]!, observed: observedAtStep[index]! })
+    }
+    leftoverExpected.push(...atStep.slice(count))
+    leftoverObserved.push(...observedAtStep.slice(count))
+  }
+  for (const [stepKey, atStep] of observedByStep) {
+    if (!expectedByStep.has(stepKey)) leftoverObserved.push(...atStep)
+  }
+  const remainingExpected = byStepThenScore(leftoverExpected)
+  const remainingObserved = byStepThenScore(leftoverObserved)
+  const matched = Math.min(remainingExpected.length, remainingObserved.length)
+  for (let index = 0; index < matched; index++) {
+    pairs.push({ kind: "matched", expected: remainingExpected[index]!, observed: remainingObserved[index]! })
+  }
+  for (const record of remainingExpected.slice(matched)) pairs.push({ kind: "missing-run", expected: record })
+  for (const record of remainingObserved.slice(matched)) pairs.push({ kind: "missing-baseline", observed: record })
+  return pairs
+}
+
+/**
+ * Compares a run to a baseline, preserving missing and inconclusive observations.
+ *
+ * Records and observations are grouped by `(case, scorer)` and then paired by
+ * step key, so a scorer that ran several times against one case is compared
+ * pairwise instead of by array position. A pair whose step key is unchanged and
+ * whose score moved is nondeterminism; a pair whose step key changed and whose
+ * score dropped is a regression; an unpaired record on either side is a missing
+ * observation. Inconclusive observations are carried through untouched, because
+ * they decide nothing and a gate has to see them.
+ *
+ * Fails with `invalid_tolerance` when a tolerance is not finite and
+ * non-negative, and with `invalid_baseline` when the artifact or any of its
+ * records belongs to a suite other than the one the run reports.
+ *
+ * @category constructors
+ * @since 0.1.0
+ */
+export const compare = (
+  baseline: Baseline,
+  run: RunResult,
+  tolerances: Tolerances = {}
+): Effect.Effect<Report, EvalError> => {
+  const absolute = tolerances.absolute ?? 0
+  const relative = tolerances.relative ?? 0
+  if (!validTolerance(absolute)) {
+    return Effect.fail(
+      new EvalError({
+        code: "invalid_tolerance",
+        message: `Regression tolerance 'absolute' must be finite and non-negative, got ${String(absolute)}`,
+        path: "tolerances.absolute"
+      })
+    )
+  }
+  if (!validTolerance(relative)) {
+    return Effect.fail(
+      new EvalError({
+        code: "invalid_tolerance",
+        message: `Regression tolerance 'relative' must be finite and non-negative, got ${String(relative)}`,
+        path: "tolerances.relative"
+      })
+    )
+  }
+  const baselineSuite = baseline.suite
+  if (baselineSuite !== run.suite) {
+    return Effect.fail(
+      new EvalError({
+        code: "invalid_baseline",
+        message: flattenControlCharacters(
+          `Baseline belongs to suite '${baselineSuite}', but the run is suite '${run.suite}'`
+        ),
+        path: "baseline.suite"
+      })
+    )
+  }
+  const foreign = [...new Set(baseline.records.filter((record) => record.suite !== run.suite).map((r) => r.suite))]
+  if (foreign.length > 0) {
+    return Effect.fail(
+      new EvalError({
+        code: "invalid_baseline",
+        message: flattenControlCharacters(
+          `Baseline holds records for suite ${
+            foreign.map((suite) => `'${suite}'`).join(", ")
+          }, but the run is suite '${run.suite}'`
+        ),
+        path: "baseline.records"
+      })
+    )
+  }
+  const actual = run.observations.filter((observation): observation is Score => observation.kind === "score")
+  const inconclusive = run.observations.filter((
+    observation
+  ): observation is Extract<Observation, { readonly kind: "inconclusive" }> => observation.kind === "inconclusive")
+  const baselineByKey = groupBy(baseline.records, (record) => key(record.case, record.scorer))
+  const actualByKey = groupBy(actual, (observation) => key(observation.case, observation.scorer))
+  const regressions: Array<Regression> = []
+  const nondeterminism: Array<Nondeterminism> = []
+  const missing: Array<MissingObservation> = []
+  for (const groupedKey of new Set([...baselineByKey.keys(), ...actualByKey.keys()])) {
+    const pairs = pairsFor(baselineByKey.get(groupedKey) ?? [], actualByKey.get(groupedKey) ?? [])
+    for (const pair of pairs) {
+      if (pair.kind === "missing-baseline") {
+        missing.push({
+          side: "baseline",
+          case: pair.observed.case,
+          scorer: pair.observed.scorer,
+          ...(pair.observed.scorerName === undefined ? {} : { scorerName: pair.observed.scorerName }),
+          stepKey: pair.observed.stepKey
+        })
+        continue
+      }
+      if (pair.kind === "missing-run") {
+        missing.push({
+          side: "run",
+          case: pair.expected.case,
+          scorer: pair.expected.scorer,
+          ...(pair.expected.scorerName === undefined ? {} : { scorerName: pair.expected.scorerName }),
+          stepKey: pair.expected.stepKey
+        })
+        continue
+      }
+      const baselineRecord = pair.expected
+      const observation = pair.observed
+      const delta = observation.score - baselineRecord.score
+      // Both branches divide by the same guarded magnitude. A baseline score of
+      // exactly 0 used to be special-cased in the regression branch, which was
+      // dead: a score is validated into [0, 1], so nothing can drop below 0.
+      if (observation.stepKey === baselineRecord.stepKey) {
+        if (
+          Math.abs(delta) > absolute && Math.abs(delta) / Math.max(Math.abs(baselineRecord.score), 1e-12) > relative
+        ) {
+          nondeterminism.push({
+            case: observation.case,
+            scorer: observation.scorer,
+            baseline: baselineRecord,
+            actual: observation,
+            delta
+          })
+        }
+      } else if (
+        delta < 0 && -delta > absolute && -delta / Math.max(baselineRecord.score, 1e-12) > relative
+      ) {
+        regressions.push({
+          case: observation.case,
+          scorer: observation.scorer,
+          baseline: baselineRecord,
+          actual: observation,
+          drop: -delta
+        })
+      }
+    }
+  }
+  return Effect.succeed({
+    suite: run.suite,
+    baseline,
+    run,
+    regressions,
+    nondeterminism,
+    missing,
+    samples: actual,
+    inconclusive
+  })
+}

package/src/Report.ts

@@ -0,0 +1,268 @@
+/**
+ * Canonical JSON and Markdown regression reports.
+ *
+ * The JSON report is the machine-readable artifact; the Markdown report is what
+ * an operator reads in a CI log, so it names every case the run could not
+ * decide rather than only counting them.
+ *
+ * @since 0.1.0
+ */
+
+import type { Baseline } from "./Baseline.ts"
+import { CanonicalJson } from "./internal/CanonicalJson.ts"
+import { isControlCodePoint } from "./internal/controlCharacters.ts"
+import { scorerLabel } from "./internal/scorerLabel.ts"
+import type { MissingObservation, Nondeterminism, Regression, Report as RegressionReport } from "./Regression.ts"
+import type { CaseResult, Observation, RunResult } from "./Runner.ts"
+
+/**
+ * Current serialized report format version.
+ *
+ * It versions {@link Data}, the wire shape, independently of the nested
+ * `Baseline.version`: the comparison can change shape without the committed
+ * baseline artifact changing at all.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const version = 1 as const
+
+/** An index into {@link Data.run}'s `observations` table. */
+type ObservationRef = number
+
+/**
+ * The serialized shape of a regression report.
+ *
+ * `run.observations` is the observation table: every observation the report
+ * mentions appears in it exactly once, and everything else refers to one by
+ * index. `run.cases[i].observations` is a list of indexes, and a regression or
+ * nondeterminism entry's `actual` is a single index. An in-memory
+ * `Regression.Report` carries the same observation object in up to four
+ * places, so encoding it directly repeated every reason and every `meta` that
+ * many times.
+ *
+ * `samples` and `inconclusive` are absent by construction: both are filters of
+ * `run.observations` by `kind`, so a reader recomputes them rather than
+ * downloading a third copy.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export interface Data {
+  readonly reportVersion: typeof version
+  readonly suite: string
+  readonly baseline: Baseline
+  readonly run:
+    & Omit<RunResult, "cases">
+    & {
+      readonly cases: ReadonlyArray<
+        & Omit<CaseResult, "observations">
+        & { readonly observations: ReadonlyArray<ObservationRef> }
+      >
+    }
+  readonly regressions: ReadonlyArray<Omit<Regression, "actual"> & { readonly actual: ObservationRef }>
+  readonly nondeterminism: ReadonlyArray<Omit<Nondeterminism, "actual"> & { readonly actual: ObservationRef }>
+  readonly missing: ReadonlyArray<MissingObservation>
+}
+
+/**
+ * Projects a comparison into its serialized form.
+ *
+ * The observation table starts as `run.observations` and grows to hold any
+ * observation reached only through a case or a comparison entry, so a
+ * hand-built report whose case lists are not flattened still serializes every
+ * reference. Observations are pooled by identity, not by value: two distinct
+ * objects that happen to be equal stay two rows, because collapsing them would
+ * claim one grading where there were two.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const data = (report: RegressionReport): Data => {
+  const positions = new Map<Observation, ObservationRef>()
+  const observations: Array<Observation> = []
+  const refer = (observation: Observation): ObservationRef => {
+    const known = positions.get(observation)
+    if (known !== undefined) return known
+    const position = observations.length
+    observations.push(observation)
+    positions.set(observation, position)
+    return position
+  }
+  for (const observation of report.run.observations) refer(observation)
+  const cases = report.run.cases.map((result) => ({ ...result, observations: result.observations.map(refer) }))
+  return {
+    reportVersion: version,
+    suite: report.suite,
+    baseline: report.baseline,
+    run: { ...report.run, cases, observations },
+    regressions: report.regressions.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+    nondeterminism: report.nondeterminism.map((entry) => ({ ...entry, actual: refer(entry.actual) })),
+    missing: report.missing
+  }
+}
+
+/**
+ * Serializes a regression report as stable, sorted-key JSON.
+ *
+ * The wire shape is {@link Data}, stamped with {@link version}: each
+ * observation is encoded once and referred to by index everywhere else. The
+ * report embeds each case's raw `execution.output`, which comes from an
+ * arbitrary target flow, so the encoding is total rather than trusting: keys
+ * are sorted by code unit, embedded strings are capped, total traversal is
+ * bounded, and anything JSON cannot express becomes a named marker
+ * (`[circular]`, `[NaN]`, `[function]`, `[budget exceeded]`) instead of a
+ * `RangeError` or a silent `null`. Two identical runs therefore produce
+ * byte-identical JSON.
+ *
+ * Nothing redacts the embedded output. A suite whose cases carry secrets must
+ * not print this report where the log is readable.
+ *
+ * @category serialization
+ * @since 0.1.0
+ */
+export const json = (report: RegressionReport): string =>
+  CanonicalJson.stringify(data(report), { maxStringLength: CanonicalJson.maxStringLength })
+
+/** Maximum escaped UTF-16 code units per value, before a truncation ellipsis. */
+const maxCellLength = 240
+
+const cell = (value: unknown): string => {
+  let text = ""
+  if (value !== undefined && value !== null) {
+    if (typeof value === "string") text = value
+    else {
+      try {
+        text = String(value)
+      } catch {
+        text = "[unreadable]"
+      }
+    }
+  }
+  let escaped = ""
+  for (const character of text) {
+    const code = character.codePointAt(0)!
+    // Escape each input character once, including backslashes before pipes.
+    // Ampersands prevent entities; dots, colons, and @ prevent GFM autolinks.
+    const token = isControlCodePoint(code)
+      ? " "
+      : /[\\|`*_[\]<>!#~&.:@]/u.test(character)
+      ? `\\${character}`
+      : character
+    if (escaped.length + token.length > maxCellLength) return `${escaped}…`
+    escaped += token
+  }
+  return escaped
+}
+
+const scorerCell = (scorer: string, scorerName: string | undefined): string => cell(scorerLabel({ scorer, scorerName }))
+
+const section = (
+  heading: string,
+  header: ReadonlyArray<string>,
+  alignment: ReadonlyArray<string>,
+  rows: ReadonlyArray<ReadonlyArray<string>>
+): ReadonlyArray<string> =>
+  rows.length === 0 ? [] : [
+    `## ${heading}`,
+    "",
+    `| ${header.join(" | ")} |`,
+    `| ${alignment.join(" | ")} |`,
+    ...rows.map((row) => `| ${row.join(" | ")} |`),
+    ""
+  ]
+
+/**
+ * Renders a concise stable Markdown regression report.
+ *
+ * Every count in the summary that is not zero has a section naming its rows:
+ * the regressions and the nondeterminism a gate reads as red, and the case
+ * failures, missing observations, and inconclusive observations that leave a
+ * gate undecided. A report that only counted them named nothing an operator
+ * could act on, which is exactly the run that needs debugging.
+ *
+ * Every cell and the suite heading value replace C0 controls and DEL with
+ * spaces and escape backslashes, pipes, inline GFM syntax, raw HTML, entities,
+ * and URL/email autolinks. Values are capped at 240 escaped UTF-16 code units,
+ * plus an ellipsis when truncated, without splitting escapes or code points.
+ *
+ * @category rendering
+ * @since 0.1.0
+ */
+export const markdown = (report: RegressionReport): string => {
+  const lines = [
+    `# Evaluation report: ${cell(report.suite)}`,
+    "",
+    `- Regressions: ${report.regressions.length}`,
+    `- Nondeterminism: ${report.nondeterminism.length}`,
+    `- Missing observations: ${report.missing.length}`,
+    `- Inconclusive observations: ${report.inconclusive.length}`,
+    `- Failed cases: ${report.run.cases.filter((result) => result.error !== undefined).length}`,
+    ...(report.run.trials === undefined ? [] : [
+      `- pass@1: ${report.run.trials.passAt1.toFixed(6)}`,
+      `- pass@${report.run.k ?? 1}: ${report.run.trials.passAtK.toFixed(6)}`,
+      `- pass^${report.run.k ?? 1}: ${report.run.trials.passHatK.toFixed(6)}`,
+      `- stderr: ${
+        (Object.values(report.run.trials.perCase).reduce((sum, item) => sum + item.stderr, 0) / report.run.trials.cases)
+          .toFixed(6)
+      }`
+    ]),
+    "",
+    ...section(
+      "Regressions",
+      ["Case", "Scorer", "Baseline", "Actual", "Drop"],
+      ["---", "---", "---:", "---:", "---:"],
+      report.regressions.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.actual.scorerName),
+        item.baseline.score.toFixed(6),
+        item.actual.score.toFixed(6),
+        item.drop.toFixed(6)
+      ])
+    ),
+    ...section(
+      "Nondeterminism",
+      ["Case", "Scorer", "Step key", "Baseline", "Actual", "Delta"],
+      ["---", "---", "---", "---:", "---:", "---:"],
+      report.nondeterminism.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.actual.scorerName),
+        cell(item.actual.stepKey),
+        item.baseline.score.toFixed(6),
+        item.actual.score.toFixed(6),
+        item.delta.toFixed(6)
+      ])
+    ),
+    ...section(
+      "Case failures",
+      ["Case", "Code", "Message"],
+      ["---", "---", "---"],
+      report.run.cases.flatMap((result) =>
+        result.error === undefined ? [] : [[cell(result.case), cell(result.error.code), cell(result.error.message)]]
+      )
+    ),
+    ...section(
+      "Missing observations",
+      ["Side", "Case", "Scorer", "Step key"],
+      ["---", "---", "---", "---"],
+      report.missing.map((item) => [
+        cell(item.side),
+        cell(item.case),
+        scorerCell(item.scorer, item.scorerName),
+        cell(item.stepKey)
+      ])
+    ),
+    ...section(
+      "Inconclusive",
+      ["Case", "Scorer", "Step key", "Reason"],
+      ["---", "---", "---", "---"],
+      report.inconclusive.map((item) => [
+        cell(item.case),
+        scorerCell(item.scorer, item.scorerName),
+        cell(item.stepKey),
+        cell(item.reason)
+      ])
+    )
+  ]
+  return `${lines.join("\n").trim()}\n`
+}