@namzu/sdk 62.11.0 → 62.13.0

package/src/eval/experiment.ts

@@ -1,3 +1,14 @@
+import { EvalUsageError, validateSeed } from './bootstrap.js'
+import { type SuiteFacts, buildDisclosure, describeDisclosure } from './disclosure.js'
+import {
+	describeIntervals,
+	describeSplits,
+	duplicateMessage,
+	duplicateName,
+	groupRuns,
+	statsOver,
+} from './report-stats.js'
+import { currentEvalRunOptions } from './run-options.js'
 import type {
 	CaseResult,
 	CaseStatus,
@@ -10,6 +21,8 @@ import type {
 import type { ScoreUncertainty } from './uncertainty.js'
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js'
 
+const MAX_REPEATS = 100
+
 const MAX_CASE_TIMEOUT_MS = 2_147_483_647
 
 export interface ExperimentConfig<TInput = unknown> {
@@ -42,7 +55,29 @@ export interface ExperimentConfig<TInput = unknown> {
 	passThreshold?: number
 	/** Cases to run at once. Default 1 — deterministic ordering by default. */
 	concurrency?: number
+	/** Called once per run of a case; with repeats, once per repeat. */
 	onCaseFinish?: (result: CaseResult) => void
+	/**
+	 * Times every case runs, from 1 to 100. Default 1.
+	 *
+	 * One run of a case says little about a model that does not answer the
+	 * same way twice. The report keeps every run and says per case how many
+	 * passed, and per suite how many cases passed every time. An
+	 * `EvalRunOptions` scope opened by `namzu eval --repeat` wins over this.
+	 */
+	repeats?: number
+	/**
+	 * Seed for the bootstrap intervals, an integer from 0 to 4294967295.
+	 * Default 1, so the same cases give the same interval on every machine.
+	 * Written into the report.
+	 */
+	seed?: number
+	/**
+	 * What the suite knows about how it runs: scripted or a real model, which
+	 * provider and model fill each role, routing, reasoning effort. Whatever is
+	 * left out is written `unknown` in the report, never guessed.
+	 */
+	disclosure?: SuiteFacts
 }
 
 /**
@@ -64,23 +99,33 @@ export async function runExperiment<TInput>(
 	const threshold = config.passThreshold ?? 1
 	const concurrency = Math.max(1, config.concurrency ?? 1)
 	const timeoutMs = validateCaseTimeoutMs(config.timeoutMs)
+	const options = currentEvalRunOptions()
+	const repeats = validateRepeats(options.repeats ?? config.repeats)
+	const seed = validateSeed(options.seed ?? config.seed)
+	for (const evalCase of config.cases) validateSplit(evalCase)
+	const alike = duplicateName(config.cases.map((c) => c.name))
+	if (alike !== undefined) throw new Error(duplicateMessage(config.name, alike))
 
-	const results: CaseResult[] = new Array(config.cases.length)
+	// Case-major, so a case's repeats sit together in the report.
+	const total = config.cases.length * repeats
+	const results: CaseResult[] = new Array(total)
 	let cursor = 0
 
 	const worker = async (): Promise<void> => {
 		for (;;) {
 			const index = cursor++
-			const evalCase = config.cases[index]
-			if (!evalCase) return
+			if (index >= total) return
+			const evalCase = config.cases[Math.floor(index / repeats)] as EvalCase<TInput>
 
 			const result = await evaluateCase(config, evalCase, threshold, timeoutMs)
+			result.repeat = (index % repeats) + 1
+			if (evalCase.split) result.split = evalCase.split
 			results[index] = result
 			config.onCaseFinish?.(result)
 		}
 	}
 
-	await Promise.all(Array.from({ length: Math.min(concurrency, config.cases.length) }, worker))
+	await Promise.all(Array.from({ length: Math.min(concurrency, total) }, worker))
 
 	const settled = results.filter(Boolean)
 	// An inconclusive case has no mean to contribute; averaging its zero in
@@ -88,6 +133,11 @@ export async function runExperiment<TInput>(
 	const scored = settled.filter((r) => r.status !== 'inconclusive')
 	const mean = scored.length === 0 ? 0 : scored.reduce((sum, r) => sum + r.mean, 0) / scored.length
 
+	const stats = statsOver(groupRuns(settled), seed)
+	// One number per CASE, so a case's repeats are not counted as independent
+	// evidence about the suite.
+	const caseMeans = stats.caseSummaries.flatMap((c) => (c.mean === null ? [] : [c.mean]))
+
 	return {
 		name: config.name,
 		cases: settled,
@@ -95,12 +145,43 @@ export async function runExperiment<TInput>(
 		// Over the same cases the mean is over. Computing spread across a
 		// different denominator than the average it qualifies would produce
 		// an interval that does not belong to the number beside it.
-		uncertainty: uncertaintyOf(scored.map((r) => r.mean)),
+		uncertainty: uncertaintyOf(caseMeans),
 		passed: settled.filter((r) => r.status === 'passed').length,
 		failed: settled.filter((r) => r.status === 'failed').length,
 		inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
 		byScorer: meanByScorer(settled),
 		durationMs: Date.now() - startedAt,
+		repeats,
+		seed,
+		...(stats.passRate === null ? {} : { passRate: stats.passRate }),
+		intervals: stats.intervals,
+		...(repeats > 1 ? { caseSummaries: stats.caseSummaries, reliability: stats.reliability } : {}),
+		splits: stats.splits,
+		disclosure: buildDisclosure(
+			config.disclosure,
+			{ cases: config.cases.length, repeats, seed },
+			options,
+			new Date().toISOString(),
+		),
+	}
+}
+
+function validateRepeats(value: number | undefined): number {
+	if (value === undefined) return 1
+	if (!Number.isInteger(value) || value < 1 || value > MAX_REPEATS) {
+		throw new EvalUsageError(
+			`repeats must be an integer from 1 to ${MAX_REPEATS}, or omitted; received ${String(value)}`,
+		)
+	}
+	return value
+}
+
+function validateSplit(evalCase: EvalCase<unknown>): void {
+	const split = evalCase.split as unknown
+	if (split !== undefined && split !== 'development' && split !== 'held-out') {
+		throw new RangeError(
+			`case "${evalCase.name}" declares split ${JSON.stringify(split)}; a split is "development" or "held-out", or left out`,
+		)
 	}
 }
 
@@ -351,17 +432,46 @@ function derivedUncertainty(report: ExperimentReport): ScoreUncertainty {
 }
 
 export function formatReport(report: ExperimentReport): string {
+	const repeats = report.repeats ?? 1
+	const caseCount = repeats > 1 ? report.cases.length / repeats : report.cases.length
 	const lines: string[] = [
-		`${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
-		// On its own line and always printed, including when the interval is
-		// undefined. A mean printed alone is the thing that has been
-		// over-read: two experiments three points apart look like a difference, and
-		// at the n a hand-built suite has they are usually the same experiment
-		// twice. Computing the interval and not showing it would leave the
-		// reader exactly where they started.
-		`  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
-		'',
+		repeats > 1
+			? `${report.name}: ${report.passed}/${report.cases.length} runs passed (${caseCount} cases x ${repeats} repeats, mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`
+			: `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
 	]
+	if (report.intervals) {
+		// The interval sits next to the number it belongs to. A mean printed
+		// alone is the thing that has been over-read: two experiments three
+		// points apart look like a difference, and at the n a hand-built
+		// suite has they are usually the same experiment twice.
+		for (const line of describeIntervals({
+			mean: report.cases.some((c) => c.status !== 'inconclusive') ? report.mean : null,
+			passRate: report.passRate ?? null,
+			intervals: report.intervals,
+		})) {
+			lines.push(`  ${line}`)
+		}
+	} else {
+		// A report from before bootstrap intervals, or one built by hand: the
+		// older interval, still never omitted.
+		lines.push(
+			`  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
+		)
+	}
+	if (report.reliability && repeats > 1) {
+		const r = report.reliability
+		lines.push(
+			`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`,
+		)
+		for (const c of report.caseSummaries ?? []) {
+			if (!c.passedEveryRepeat) lines.push(`    ${c.case}: ${c.passed} of ${c.repeats} passed`)
+		}
+	}
+	if (report.splits) {
+		for (const line of describeSplits(report.splits)) lines.push(`  ${line}`)
+	}
+	if (report.disclosure) lines.push(`  ${describeDisclosure(report.disclosure)}`)
+	lines.push('')
 
 	// Surfaced above the failures, because an inconclusive case means the
 	// harness is broken and every number below it is measured over less
@@ -381,7 +491,9 @@ export function formatReport(report: ExperimentReport): string {
 	if (failures.length > 0) {
 		lines.push('', 'Failures:')
 		for (const failure of failures) {
-			lines.push(`  ✗ ${failure.case} (${failure.mean.toFixed(2)})`)
+			lines.push(
+				`  ✗ ${failure.case}${repeats > 1 && failure.repeat ? ` [repeat ${failure.repeat} of ${repeats}]` : ''} (${failure.mean.toFixed(2)})`,
+			)
 			if (failure.failedGates && failure.failedGates.length > 0) {
 				// First line under the case, because it says WHY this failed.
 				// A mean of 0.75 next to four scores leaves the reader

package/src/eval/from-turn.ts

@@ -1,3 +1,4 @@
+import { compareAnswerToOutcome } from '../runtime/query/turn/answer-claims.js'
 import type { Turn } from '../types/session/turn.js'
 import type { EvalTurn } from './types.js'
 
@@ -12,6 +13,7 @@ import type { EvalTurn } from './types.js'
  */
 export function evalTurnFromTurn(turn: Turn, opts: { durationMs?: number } = {}): EvalTurn {
 	const steps = turn.steps ?? []
+	const claimMismatch = compareAnswerToOutcome(turn.result, turn.outcome)
 	return {
 		output: turn.result ?? null,
 		...(turn.structuredOutput !== undefined ? { structuredOutput: turn.structuredOutput } : {}),
@@ -23,6 +25,9 @@ export function evalTurnFromTurn(turn: Turn, opts: { durationMs?: number } = {})
 		totalCostUsd: turn.costInfo.totalCost,
 		durationMs: opts.durationMs ?? (turn.endedAt ?? Date.now()) - turn.startedAt,
 		...(turn.lastError ? { error: turn.lastError } : {}),
+		...(turn.outcome ? { outcome: turn.outcome } : {}),
+		...(claimMismatch ? { claimMismatch } : {}),
+		...(turn.handedBack ? { handedBack: turn.handedBack } : {}),
 	}
 }
 

package/src/eval/index.ts

@@ -15,16 +15,67 @@ export type { ExperimentConfig } from './experiment.js'
 export { judgeScorer } from './judge.js'
 export type { JudgeScorerConfig } from './judge.js'
 export {
+	claimMismatchScorer,
+	handedBackScorer,
 	completionScorer,
 	containsScorer,
 	customScorer,
+	outcomeScorer,
 	stepBudgetScorer,
 	trajectoryScorer,
 } from './scorers.js'
+export {
+	BOOTSTRAP_RESAMPLES,
+	DEFAULT_EVAL_SEED,
+	EvalUsageError,
+	INTERVAL_LEVEL,
+	MIN_CASES_FOR_INTERVAL,
+	bootstrapCases,
+	bootstrapPairedDifference,
+	createSeededRandom,
+	exactSignTestP,
+	isEvalUsageError,
+} from './bootstrap.js'
+export type {
+	BootstrapMethod,
+	CaseIntervals,
+	CaseTally,
+	DifferenceIntervals,
+	Interval,
+	PairedTally,
+} from './bootstrap.js'
+export type {
+	ReportDisclosure,
+	RoleDisclosure,
+	RoutingDisclosure,
+	RunMode,
+	SuiteFacts,
+} from './disclosure.js'
+export { sdkVersion } from './disclosure.js'
+export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js'
+export type { EvalRunOptions } from './run-options.js'
+export { formatRunSummary, summarizeRun } from './run-summary.js'
+export type { RunDisclosure, RunSummary } from './run-summary.js'
+export type { RepeatReliability, SplitSummary } from './report-stats.js'
+export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js'
+export type {
+	CaseRef,
+	DisclosureComparison,
+	DisclosureDifference,
+	DisclosureDimension,
+	DisclosureUnknown,
+	EvalComparison,
+	Flips,
+	LoadedReportFile,
+	LoadedSuite,
+	SideNumbers,
+} from './compare.js'
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js'
 export type { ScoreUncertainty } from './uncertainty.js'
 export type {
+	CaseRepeatSummary,
 	CaseResult,
+	EvalSplit,
 	CaseStatus,
 	EvalCase,
 	EvalTurn,

package/src/eval/report-stats.ts

@@ -0,0 +1,257 @@
+import { type CaseIntervals, type CaseTally, bootstrapCases } from './bootstrap.js'
+import type { CaseRepeatSummary, CaseResult, EvalSplit } from './types.js'
+
+/**
+ * The numbers a report carries about a set of runs, whether that set is one
+ * suite, one split of it, or every suite in a run.
+ *
+ * Cases are the unit. A case with several repeats is one case with several
+ * runs: the pass rate pools the runs, the intervals resample the cases, and
+ * reliability asks of each case whether it held up every time.
+ */
+
+export interface RepeatReliability {
+	/** Cases that ran. */
+	cases: number
+	/** Cases whose every repeat passed. */
+	passedEveryRepeat: number
+	/** `passedEveryRepeat / cases`; null when no case ran. */
+	fraction: number | null
+}
+
+export interface SplitSummary {
+	cases: number
+	runs: number
+	passed: number
+	failed: number
+	inconclusive: number
+	passRate: number | null
+	mean: number | null
+	intervals: CaseIntervals
+}
+
+/** The runs of one case, with the key that pairs it with the same case elsewhere. */
+export interface CaseRuns {
+	readonly key: string
+	readonly name: string
+	readonly split?: EvalSplit
+	readonly runs: readonly CaseResult[]
+}
+
+export interface RunStats {
+	cases: number
+	runs: number
+	passed: number
+	failed: number
+	inconclusive: number
+	passRate: number | null
+	mean: number | null
+	intervals: CaseIntervals
+	caseSummaries: CaseRepeatSummary[]
+	reliability: RepeatReliability
+	splits: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>
+}
+
+export const SPLIT_ORDER = ['development', 'held-out', 'unassigned'] as const
+
+/** Group a flat list of runs by case name, in the order the cases first appear. */
+export function groupRuns(
+	results: readonly CaseResult[],
+	keyOf: (name: string) => string = (name) => name,
+): CaseRuns[] {
+	const groups = new Map<string, { name: string; split?: EvalSplit; runs: CaseResult[] }>()
+	for (const run of results) {
+		const key = keyOf(run.case)
+		let group = groups.get(key)
+		if (!group) {
+			group = { name: run.case, ...(run.split ? { split: run.split } : {}), runs: [] }
+			groups.set(key, group)
+		}
+		group.runs.push(run)
+	}
+	return [...groups].map(([key, g]) => ({ key, ...g }))
+}
+
+/** The first name that appears more than once, or undefined. */
+export function duplicateName(names: readonly string[]): string | undefined {
+	const seen = new Set<string>()
+	for (const name of names) {
+		if (seen.has(name)) return name
+		seen.add(name)
+	}
+	return undefined
+}
+
+export function duplicateMessage(suite: string, name: string): string {
+	return `suite "${suite}" has the case "${name}" more than once; a case is named by its id, and two cases sharing one would be merged into one in every count and interval. Give each case its own name.`
+}
+
+/**
+ * The first case in a list of runs that appears twice for the same repeat: a
+ * second run of a case is a repeat only when it says it is a later one.
+ */
+export function duplicateRun(
+	runs: readonly Pick<CaseResult, 'case' | 'repeat'>[],
+): string | undefined {
+	const seen = new Set<string>()
+	for (const run of runs) {
+		const key = `${run.case}\u0000${run.repeat ?? 1}`
+		if (seen.has(key)) return run.case
+		seen.add(key)
+	}
+	return undefined
+}
+
+/** Status of a run, tolerating a report old enough to carry only `passed`. */
+export function statusOf(run: Pick<CaseResult, 'status' | 'passed'>): CaseResult['status'] {
+	if (run.status === 'passed' || run.status === 'failed' || run.status === 'inconclusive') {
+		return run.status
+	}
+	return run.passed === true ? 'passed' : 'failed'
+}
+
+export function tallyOf(runs: readonly CaseResult[]): CaseTally | null {
+	let scoreSum = 0
+	let passes = 0
+	let judged = 0
+	for (const run of runs) {
+		const status = statusOf(run)
+		if (status === 'inconclusive') continue
+		judged++
+		scoreSum += run.mean
+		if (status === 'passed') passes++
+	}
+	return judged === 0 ? null : { scoreSum, passes, judged }
+}
+
+/** True when the case has at least one judged run and every run passed. */
+export function passedEveryRun(runs: readonly CaseResult[]): boolean {
+	return runs.length > 0 && runs.every((run) => statusOf(run) === 'passed')
+}
+
+function summaryOf(runs: readonly CaseResult[]) {
+	let passed = 0
+	let failed = 0
+	let inconclusive = 0
+	for (const run of runs) {
+		const status = statusOf(run)
+		if (status === 'passed') passed++
+		else if (status === 'failed') failed++
+		else inconclusive++
+	}
+	return { passed, failed, inconclusive }
+}
+
+function numbersOver(cases: readonly CaseRuns[], seed: number) {
+	const tallies: CaseTally[] = []
+	let runs = 0
+	for (const c of cases) {
+		runs += c.runs.length
+		const tally = tallyOf(c.runs)
+		if (tally) tallies.push(tally)
+	}
+	const counts = summaryOf(cases.flatMap((c) => c.runs))
+	const judged = counts.passed + counts.failed
+	const scoreSum = tallies.reduce((sum, t) => sum + t.scoreSum, 0)
+	return {
+		runs,
+		...counts,
+		passRate: judged === 0 ? null : counts.passed / judged,
+		mean: judged === 0 ? null : scoreSum / judged,
+		intervals: bootstrapCases(tallies, seed),
+	}
+}
+
+export function statsOver(cases: readonly CaseRuns[], seed: number): RunStats {
+	const whole = numbersOver(cases, seed)
+	const caseSummaries: CaseRepeatSummary[] = cases.map((c) => {
+		const counts = summaryOf(c.runs)
+		const tally = tallyOf(c.runs)
+		return {
+			case: c.name,
+			...(c.split ? { split: c.split } : {}),
+			repeats: c.runs.length,
+			...counts,
+			mean: tally ? tally.scoreSum / tally.judged : null,
+			passedEveryRepeat: passedEveryRun(c.runs),
+		}
+	})
+	const everyRepeat = caseSummaries.filter((c) => c.passedEveryRepeat).length
+
+	const splits: RunStats['splits'] = {}
+	for (const name of SPLIT_ORDER) {
+		const members = cases.filter((c) => (c.split ?? 'unassigned') === name)
+		if (members.length === 0) continue
+		const part = numbersOver(members, seed)
+		splits[name] = { cases: members.length, ...part }
+	}
+
+	return {
+		cases: cases.length,
+		...whole,
+		caseSummaries,
+		reliability: {
+			cases: cases.length,
+			passedEveryRepeat: everyRepeat,
+			fraction: cases.length === 0 ? null : everyRepeat / cases.length,
+		},
+		splits,
+	}
+}
+
+const percent = (value: number): string => `${(value * 100).toFixed(1)}%`
+const score = (value: number): string => value.toFixed(3)
+
+/**
+ * The intervals, as lines a person reads next to the numbers. A number whose
+ * interval is not computed is printed with the reason, never alone.
+ */
+export function describeIntervals(args: {
+	mean: number | null
+	passRate: number | null
+	intervals: CaseIntervals
+}): string[] {
+	const { intervals } = args
+	const line = (
+		label: string,
+		estimate: number | null,
+		interval: CaseIntervals['mean'],
+		show: (n: number) => string,
+	): string => {
+		if (estimate === null) return `${label}: none, no run was judged`
+		if (!interval.computed) return `${label} ${show(estimate)}, no interval: ${interval.reason}`
+		return `${label} ${show(estimate)}, 95% interval ${show(interval.low)} to ${show(interval.high)}`
+	}
+	const lines = [
+		line('mean score', args.mean, intervals.mean, score),
+		line('pass rate', args.passRate, intervals.passRate, percent),
+	]
+	if (intervals.mean.computed || intervals.passRate.computed) {
+		lines.push(
+			`intervals: ${intervals.method.kind} (${intervals.method.cases} cases, ${intervals.method.resamples} resamples), seed ${intervals.method.seed}`,
+		)
+	}
+	return lines
+}
+
+/** The line that says which splits a suite has, and says so plainly when none is held out. */
+export function describeSplits(splits: RunStats['splits']): string[] {
+	const lines: string[] = []
+	// A suite that declares no split has one bucket and nothing to break down.
+	const declared = SPLIT_ORDER.some((name) => name !== 'unassigned' && splits[name])
+	for (const name of SPLIT_ORDER) {
+		const part = splits[name]
+		if (!part || !declared) continue
+		const rate = part.passRate === null ? 'no judged run' : `pass rate ${percent(part.passRate)}`
+		const interval = part.intervals.passRate.computed
+			? `, 95% interval ${percent(part.intervals.passRate.low)} to ${percent(part.intervals.passRate.high)}`
+			: ''
+		lines.push(`${name}: ${part.cases} case${part.cases === 1 ? '' : 's'}, ${rate}${interval}`)
+	}
+	if (!splits['held-out']) {
+		lines.push(
+			'no held-out cases: every number here is measured on cases the suite was developed against',
+		)
+	}
+	return lines
+}

package/src/eval/run-options.ts

@@ -0,0 +1,49 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+/**
+ * Settings an operator gives a whole eval run, handed to every
+ * `runExperiment` that executes inside it.
+ *
+ * A suite file calls `runExperiment` itself and returns the report, so
+ * `namzu eval --repeat 5` has no argument to pass the five through. Rather
+ * than ask every suite to forward it, the runner opens a scope with
+ * {@link withEvalRunOptions} and `runExperiment` reads the scope. A suite
+ * that was written before the option existed repeats like any other.
+ *
+ * An option given here wins over the same option in the suite's own config:
+ * the person running the command asked for it.
+ */
+export interface EvalRunOptions {
+	/** Times every case runs. */
+	readonly repeats?: number
+	/** Seed for the bootstrap. */
+	readonly seed?: number
+	/** Version of the program that is running the suites, for the disclosure. */
+	readonly harnessVersion?: string
+	/** Source revision the suites came from, when known. */
+	readonly commit?: string
+	/** Version of the suite being run, when known. */
+	readonly suiteVersion?: string
+}
+
+const STORE_KEY = Symbol.for('namzu.eval.runOptions')
+
+/**
+ * One store per process, found through a global symbol, so a suite that
+ * resolves its own copy of the SDK still sees the scope the runner opened.
+ */
+function store(): AsyncLocalStorage<EvalRunOptions> {
+	const holder = globalThis as { [STORE_KEY]?: AsyncLocalStorage<EvalRunOptions> }
+	holder[STORE_KEY] ??= new AsyncLocalStorage<EvalRunOptions>()
+	return holder[STORE_KEY]
+}
+
+/** Run `work` with `options` in force for every experiment it starts. */
+export function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T {
+	return store().run(options, work)
+}
+
+/** The options in force here, or an empty set outside any scope. */
+export function currentEvalRunOptions(): EvalRunOptions {
+	return store().getStore() ?? {}
+}