@namzu/sdk 2.0.0 → 3.0.0

package/src/eval/__tests__/scorer-gates.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, it } from 'vitest'
+
+import { formatReport, runExperiment } from '../experiment.js'
+import type { EvalRun, Score, Scorer } from '../types.js'
+
+/**
+ * A case's verdict was one unweighted mean over every scorer against one
+ * suite-wide threshold, and the two halves of that fought each other.
+ *
+ * At the default threshold of 1 the harness never reports a false pass —
+ * but a trajectory F1 and a graded judge can essentially never reach 1, so
+ * every real suite lowers it. And every step down buys tolerance for the
+ * fuzzy scorers by buying exactly the same tolerance for the deterministic
+ * ones: at 0.75, a trajectory score of 0 alongside three perfect scores
+ * averages to 0.75 and reports PASSED. The regression the harness exists
+ * to catch comes back green.
+ *
+ * The workarounds were lossy too — folding every hard check into one
+ * custom scorer destroys per-dimension reporting, and two experiments at
+ * two thresholds re-runs the dataset and splits one case's verdict across
+ * two reports.
+ */
+
+const run = (): EvalRun => ({
+	output: 'answer',
+	steps: [],
+	toolCalls: [],
+	totalTokens: 0,
+	totalCostUsd: 0,
+	durationMs: 0,
+})
+
+const scorer = (name: string, score: number, extra: Partial<Scorer> = {}): Scorer => ({
+	name,
+	score: (): Score => ({ score, reason: `${name} scored ${score}` }),
+	...extra,
+})
+
+const experiment = (scorers: Scorer[], passThreshold = 0.75) =>
+	runExperiment({
+		name: 'gates',
+		cases: [{ name: 'one', input: null }],
+		scorers,
+		passThreshold,
+		run: async () => run(),
+	})
+
+describe('a gate scorer', () => {
+	it('fails the case even when the mean clears the threshold', async () => {
+		const report = await experiment([
+			scorer('trajectory', 0, { severity: 'gate' }),
+			scorer('completion', 1),
+			scorer('contains', 1),
+			scorer('judge', 1),
+		])
+
+		// Mean is exactly 0.75 against a 0.75 threshold — reported passed
+		// before, with the one deterministic check at zero.
+		expect(report.cases[0]?.mean).toBeCloseTo(0.75)
+		expect(report.cases[0]?.status).toBe('failed')
+	})
+
+	it('names itself, so the reader does not have to guess', async () => {
+		const report = await experiment([
+			scorer('trajectory', 0, { severity: 'gate' }),
+			scorer('judge', 1),
+		])
+		expect(report.cases[0]?.failedGates).toEqual(['trajectory'])
+	})
+
+	it('honours its own threshold when it sets one', async () => {
+		// "Good enough" is not one number across dimensions: a trajectory
+		// match at 0.8 may be fine while a completion check at 0.8 is
+		// meaningless.
+		const lenient = await experiment([
+			scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.7 }),
+		])
+		expect(lenient.cases[0]?.status).toBe('passed')
+
+		const strict = await experiment([
+			scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.9 }),
+		])
+		expect(strict.cases[0]?.status).toBe('failed')
+	})
+
+	it('falls back to the suite threshold when it sets none', async () => {
+		const report = await experiment([scorer('trajectory', 0.5, { severity: 'gate' })], 0.75)
+		expect(report.cases[0]?.failedGates).toEqual(['trajectory'])
+	})
+
+	it('does not fail the case when it could not judge at all', async () => {
+		// Unavailable is not zero. A gate that threw did not judge the run
+		// badly, it failed to judge it — the inconclusive path, not a
+		// failure, and the two demand opposite responses.
+		const report = await runExperiment({
+			name: 'gates',
+			cases: [{ name: 'one', input: null }],
+			scorers: [
+				{
+					name: 'judge-gate',
+					severity: 'gate',
+					score: () => {
+						throw new Error('provider unreachable')
+					},
+				},
+			],
+			run: async () => run(),
+		})
+
+		expect(report.cases[0]?.status).toBe('inconclusive')
+		expect(report.cases[0]?.failedGates).toBeUndefined()
+	})
+
+	it('lets a passing gate through', async () => {
+		const report = await experiment([scorer('trajectory', 1, { severity: 'gate' })])
+		expect(report.cases[0]?.status).toBe('passed')
+	})
+})
+
+describe('a soft scorer', () => {
+	it('only moves the mean', async () => {
+		// The fuzzy dimensions, where a number below 1 is normal and a
+		// threshold is a judgement call.
+		const report = await experiment([scorer('judge', 0.5), scorer('other', 1)])
+		expect(report.cases[0]?.status).toBe('passed')
+		expect(report.cases[0]?.failedGates).toBeUndefined()
+	})
+
+	it('is the default, so a suite that sets nothing is unchanged', async () => {
+		const report = await experiment([
+			scorer('a', 0),
+			scorer('b', 1),
+			scorer('c', 1),
+			scorer('d', 1),
+		])
+		expect(report.cases[0]?.status).toBe('passed')
+	})
+
+	it('can still fail the case through the mean', async () => {
+		const report = await experiment([scorer('a', 0), scorer('b', 0.5)])
+		expect(report.cases[0]?.status).toBe('failed')
+		expect(report.cases[0]?.failedGates).toBeUndefined()
+	})
+})
+
+describe('the report', () => {
+	it('says which gate missed, above the individual scores', async () => {
+		const report = await experiment([
+			scorer('trajectory', 0, { severity: 'gate' }),
+			scorer('judge', 1),
+		])
+
+		const text = formatReport(report)
+		expect(text).toContain('gate missed: trajectory')
+	})
+})

package/src/eval/experiment.ts

@@ -0,0 +1,335 @@
+import type {
+	CaseResult,
+	CaseStatus,
+	EvalCase,
+	EvalRun,
+	ExperimentReport,
+	Score,
+	Scorer,
+} from './types.js'
+
+export interface ExperimentConfig<TInput = unknown> {
+	name: string
+	cases: readonly EvalCase<TInput>[]
+	/** Applied to every case unless the case overrides them. */
+	scorers: readonly Scorer[]
+	/**
+	 * Execute one case. Returning an `EvalRun` rather than driving `query()`
+	 * here keeps the harness independent of how you construct a run —
+	 * scripted mock, real provider, or a whole agent behind a facade.
+	 */
+	run: (input: TInput, evalCase: EvalCase<TInput>, signal: AbortSignal) => Promise<EvalRun>
+	/**
+	 * Deadline for a single case. Unset means no deadline.
+	 *
+	 * `executeCase` used to be a bare await, so a `run` closure that never
+	 * settled blocked its worker and `runExperiment` never returned — no
+	 * report, no partial results, nothing to read. The documented path
+	 * inherits deadlines from the runtime it drives, so the residual is a
+	 * closure that does not go through `query()` and a mid-iteration
+	 * provider stall the between-iterations guard cannot see. Both are
+	 * reachable, and neither is the suite's fault to absorb silently.
+	 *
+	 * A timed-out case is REPORTED and the suite continues, exactly like a
+	 * case that threw: forty cases should not be lost to one that hung.
+	 */
+	timeoutMs?: number
+	/** Mean score a case must reach to count as passed. Default 1. */
+	passThreshold?: number
+	/** Cases to run at once. Default 1 — deterministic ordering by default. */
+	concurrency?: number
+	onCaseFinish?: (result: CaseResult) => void
+}
+
+/**
+ * Run a dataset and score it.
+ *
+ * There was no evaluation harness of any kind: no dataset, no scorer, no
+ * judge, no trajectory assertion. So every behavior change in this SDK —
+ * a tool description, the `search_tools` top-k, the compaction threshold —
+ * shipped with no regression signal behind it, and the only way to notice
+ * a degradation was for a user to hit it.
+ *
+ * The scorer contract requires a `reason` on every score, so a report says
+ * what got worse rather than only that something did.
+ */
+export async function runExperiment<TInput>(
+	config: ExperimentConfig<TInput>,
+): Promise<ExperimentReport> {
+	const startedAt = Date.now()
+	const threshold = config.passThreshold ?? 1
+	const concurrency = Math.max(1, config.concurrency ?? 1)
+
+	const results: CaseResult[] = new Array(config.cases.length)
+	let cursor = 0
+
+	const worker = async (): Promise<void> => {
+		for (;;) {
+			const index = cursor++
+			const evalCase = config.cases[index]
+			if (!evalCase) return
+
+			const run = await executeCase(config, evalCase)
+			const scorers = evalCase.scorers ?? config.scorers
+			const scores: Record<string, Score> = {}
+
+			// Scores are keyed by name, so two scorers sharing one collapse:
+			// the mean's denominator becomes the count of distinct NAMES and
+			// the surviving score is whichever ran last. Two
+			// `containsScorer(...)` instances are both called 'contains', so
+			// this is easy to hit by accident and silently halves the
+			// evidence. Ambiguous results are worse than a loud failure.
+			const seen = new Set<string>()
+			for (const scorer of scorers) {
+				if (seen.has(scorer.name)) {
+					throw new Error(
+						`Duplicate scorer name "${scorer.name}" for case "${evalCase.name}". Scores are keyed by name, so the second would overwrite the first and the case mean would be computed over the wrong denominator. Give each scorer a distinct name.`,
+					)
+				}
+				seen.add(scorer.name)
+				scores[scorer.name] = await safeScore(scorer, run, evalCase)
+			}
+
+			// Only scores that were actually produced count — an unavailable
+			// scorer leaves the denominator alone rather than dragging the
+			// mean toward zero with a measurement that never happened.
+			const values = Object.values(scores)
+				.filter((s) => s.unavailable !== true)
+				.map((s) => s.score)
+			const mean = values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length
+
+			// A gate miss fails the case whatever the mean says. Averaging a
+			// hard check together with a fuzzy one lets three good scores
+			// carry a zero: trajectory 0 + completion 1 + contains 1 +
+			// judge 1 averages to 0.75 and reports passed at a threshold of
+			// 0.75 — the exact regression the harness exists to catch,
+			// reported green. An UNAVAILABLE gate does not fail the case;
+			// it did not judge the run at all, which is the inconclusive
+			// path, not a failure.
+			const failedGates = scorers
+				.filter((s) => s.severity === 'gate')
+				.filter((s) => {
+					const score = scores[s.name]
+					if (!score || score.unavailable === true) return false
+					return score.score < (s.threshold ?? threshold)
+				})
+				.map((s) => s.name)
+
+			const status: CaseStatus =
+				Object.keys(scores).length > 0 && values.length === 0
+					? 'inconclusive'
+					: failedGates.length > 0
+						? 'failed'
+						: mean >= threshold
+							? 'passed'
+							: 'failed'
+			const result: CaseResult = {
+				case: evalCase.name,
+				run,
+				scores,
+				mean,
+				status,
+				passed: status === 'passed',
+				// Named, not just counted: "failed" with a mean of 0.75 sends
+				// somebody to read four scores and guess which one mattered.
+				...(failedGates.length > 0 ? { failedGates } : {}),
+			}
+			results[index] = result
+			config.onCaseFinish?.(result)
+		}
+	}
+
+	await Promise.all(Array.from({ length: Math.min(concurrency, config.cases.length) }, worker))
+
+	const settled = results.filter(Boolean)
+	// An inconclusive case has no mean to contribute; averaging its zero in
+	// would report a suite-wide regression caused by the harness.
+	const scored = settled.filter((r) => r.status !== 'inconclusive')
+	const mean = scored.length === 0 ? 0 : scored.reduce((sum, r) => sum + r.mean, 0) / scored.length
+
+	return {
+		name: config.name,
+		cases: settled,
+		mean,
+		passed: settled.filter((r) => r.status === 'passed').length,
+		failed: settled.filter((r) => r.status === 'failed').length,
+		inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
+		byScorer: meanByScorer(settled),
+		durationMs: Date.now() - startedAt,
+	}
+}
+
+/**
+ * A case that throws is a RESULT, not a crash. An eval suite whose first
+ * broken case aborts the run tells you nothing about the other forty.
+ */
+async function executeCase<TInput>(
+	config: ExperimentConfig<TInput>,
+	evalCase: EvalCase<TInput>,
+): Promise<EvalRun> {
+	const startedAt = Date.now()
+	const controller = new AbortController()
+	const timeoutMs = config.timeoutMs
+	// The signal is handed to `run` so a closure that drives `query()` can
+	// pass it through and actually stop working. A closure that ignores it
+	// is merely detached rather than stopped — the same bargain every other
+	// deadline in the SDK makes — but the SUITE is unblocked either way,
+	// which is the part that was missing.
+	const timer =
+		timeoutMs !== undefined && timeoutMs > 0
+			? setTimeout(
+					() => controller.abort(new Error(`case timed out after ${timeoutMs}ms`)),
+					timeoutMs,
+				)
+			: undefined
+
+	const deadline =
+		timer === undefined
+			? undefined
+			: new Promise<never>((_resolve, reject) => {
+					controller.signal.addEventListener('abort', () => reject(controller.signal.reason), {
+						once: true,
+					})
+				})
+
+	try {
+		const work = config.run(evalCase.input, evalCase, controller.signal)
+		// A rejection of the loser must never surface as an unhandled
+		// rejection after the race has already been decided.
+		if (deadline) work.catch(() => {})
+		return await (deadline ? Promise.race([work, deadline]) : work)
+	} catch (err) {
+		return {
+			output: null,
+			steps: [],
+			toolCalls: [],
+			totalTokens: 0,
+			totalCostUsd: 0,
+			// Real elapsed time, not zero: a case that burned its whole
+			// deadline is the most interesting number in the report, and
+			// zero would hide it.
+			durationMs: Date.now() - startedAt,
+			error: err instanceof Error ? err.message : String(err),
+		}
+	} finally {
+		// Only the timer needs clearing. Aborting here would fire a spurious
+		// signal at a `run` that has already settled — the deadline branch
+		// has aborted already, and every other path is done.
+		clearTimeout(timer)
+	}
+}
+
+/** A throwing scorer scores zero with the throw as its reason. */
+async function safeScore(scorer: Scorer, run: EvalRun, evalCase: EvalCase): Promise<Score> {
+	// A run that THREW scores zero, whatever the scorer would have said.
+	// `executeCase` catches the failure and returns an empty run, and an
+	// empty run walks straight into every scorer's happy path:
+	// `stepBudgetScorer` sees 0 steps against its allowance and returns 1,
+	// `trajectoryScorer` sees no tools expected and none called and returns
+	// 1. So a suite whose runs were all dying reported green. The failure is
+	// recorded on `run.error` and nothing consulted it.
+	if (run.error !== undefined) {
+		return { score: 0, reason: `run failed: ${run.error}`, details: { error: run.error } }
+	}
+
+	try {
+		return await scorer.score(run, evalCase)
+	} catch (err) {
+		// UNAVAILABLE, not zero. A scorer that threw did not judge the run
+		// badly — it failed to judge it at all, and the two call for
+		// opposite responses. Scoring the throw zero was survivable while
+		// every scorer was a pure function; a scorer that reaches a provider
+		// makes it actively misleading, because one rate limit would look
+		// exactly like a behavioural regression.
+		return {
+			score: 0,
+			unavailable: true,
+			reason: `scorer "${scorer.name}" could not judge this run: ${
+				err instanceof Error ? err.message : String(err)
+			}`,
+		}
+	}
+}
+
+function meanByScorer(results: readonly CaseResult[]): Record<string, number> {
+	const sums = new Map<string, { total: number; count: number }>()
+	for (const result of results) {
+		for (const [name, score] of Object.entries(result.scores)) {
+			if (score.unavailable === true) continue
+			const entry = sums.get(name) ?? { total: 0, count: 0 }
+			entry.total += score.score
+			entry.count++
+			sums.set(name, entry)
+		}
+	}
+	const out: Record<string, number> = {}
+	// A scorer that was never available anywhere is omitted rather than
+	// reported as 0 — a dimension with no measurements is not a dimension
+	// that scored badly.
+	for (const [name, { total, count }] of sums) {
+		if (count > 0) out[name] = total / count
+	}
+	return out
+}
+
+/**
+ * Render a report as text.
+ *
+ * Failures print their scorer reasons, because a CI log that says
+ * "0.62" is a log that sends someone back to reproduce it by hand.
+ */
+export function formatReport(report: ExperimentReport): string {
+	const lines: string[] = [
+		`${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
+		'',
+	]
+
+	// Surfaced above the failures, because an inconclusive case means the
+	// harness is broken and every number below it is measured over less
+	// evidence than it looks like.
+	if (report.inconclusive > 0) {
+		lines.push(
+			`  ${report.inconclusive} case${report.inconclusive === 1 ? '' : 's'} could not be judged — the numbers below cover the rest`,
+			'',
+		)
+	}
+
+	for (const [name, mean] of Object.entries(report.byScorer)) {
+		lines.push(`  ${name}: ${mean.toFixed(2)}`)
+	}
+
+	const failures = report.cases.filter((c) => c.status === 'failed')
+	if (failures.length > 0) {
+		lines.push('', 'Failures:')
+		for (const failure of failures) {
+			lines.push(`  ✗ ${failure.case} (${failure.mean.toFixed(2)})`)
+			if (failure.failedGates && failure.failedGates.length > 0) {
+				// First line under the case, because it says WHY this failed.
+				// A mean of 0.75 next to four scores leaves the reader
+				// guessing which one mattered.
+				lines.push(`      gate missed: ${failure.failedGates.join(', ')}`)
+			}
+			for (const [name, score] of Object.entries(failure.scores)) {
+				if (score.unavailable === true) {
+					lines.push(`      ${name}: not judged — ${score.reason}`)
+				} else if (score.score < 1) {
+					lines.push(`      ${name}: ${score.reason}`)
+				}
+			}
+			if (failure.run.error) lines.push(`      error: ${failure.run.error}`)
+		}
+	}
+
+	const unjudged = report.cases.filter((c) => c.status === 'inconclusive')
+	if (unjudged.length > 0) {
+		lines.push('', 'Not judged:')
+		for (const item of unjudged) {
+			lines.push(`  ? ${item.case}`)
+			for (const [name, score] of Object.entries(item.scores)) {
+				lines.push(`      ${name}: ${score.reason}`)
+			}
+		}
+	}
+
+	return lines.join('\n')
+}

package/src/eval/from-run.ts

@@ -0,0 +1,48 @@
+import type { Run } from '../types/run/entity.js'
+import type { EvalRun } from './types.js'
+
+/**
+ * Project a finished `Run` into the shape scorers consume.
+ *
+ * This is the whole bridge between the harness and the runtime, and it is
+ * three lines of mapping because `Run.steps` now exists. Before that, a
+ * trajectory scorer would have had to correlate raw `RunEvent`s by
+ * iteration number and diff cumulative counters — which is why the step
+ * record had to land before the harness could.
+ */
+export function evalRunFromRun(run: Run, opts: { durationMs?: number } = {}): EvalRun {
+	const steps = run.steps ?? []
+	return {
+		output: run.result ?? null,
+		...(run.structuredOutput !== undefined ? { structuredOutput: run.structuredOutput } : {}),
+		steps,
+		// Flattened in call order across steps — the trajectory.
+		toolCalls: steps.flatMap((s) => s.toolCalls.map((c) => c.function.name)),
+		...(run.stopReason ? { stopReason: run.stopReason } : {}),
+		totalTokens: run.tokenUsage.totalTokens,
+		totalCostUsd: run.costInfo.totalCost,
+		durationMs: opts.durationMs ?? (run.endedAt ?? Date.now()) - run.startedAt,
+		...(run.lastError ? { error: run.lastError } : {}),
+	}
+}
+
+/**
+ * Drain a `query()` generator to its returned `Run`, then project it.
+ *
+ * The common case, so it does not need writing per suite:
+ *
+ * ```ts
+ * runExperiment({
+ *   name: 'file-editing',
+ *   cases,
+ *   scorers: [trajectoryScorer(), completionScorer()],
+ *   run: (input) => evalRunFromQuery(query({ provider, tools, messages: input, … })),
+ * })
+ * ```
+ */
+export async function evalRunFromQuery(generator: AsyncGenerator<unknown, Run>): Promise<EvalRun> {
+	const startedAt = Date.now()
+	let next = await generator.next()
+	while (!next.done) next = await generator.next()
+	return evalRunFromRun(next.value, { durationMs: Date.now() - startedAt })
+}

package/src/eval/index.ts

@@ -0,0 +1,21 @@
+export { evalRunFromQuery, evalRunFromRun } from './from-run.js'
+export { formatReport, runExperiment } from './experiment.js'
+export type { ExperimentConfig } from './experiment.js'
+export { judgeScorer } from './judge.js'
+export type { JudgeScorerConfig } from './judge.js'
+export {
+	completionScorer,
+	containsScorer,
+	customScorer,
+	stepBudgetScorer,
+	trajectoryScorer,
+} from './scorers.js'
+export type {
+	CaseResult,
+	CaseStatus,
+	EvalCase,
+	EvalRun,
+	ExperimentReport,
+	Score,
+	Scorer,
+} from './types.js'