@namzu/sdk 2.0.0 → 3.0.0

package/src/eval/__tests__/harness-honesty.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, it } from 'vitest'
+
+import { runExperiment } from '../experiment.js'
+import { containsScorer, stepBudgetScorer, trajectoryScorer } from '../scorers.js'
+import type { EvalRun } from '../types.js'
+
+/**
+ * An eval harness that reports green on a broken suite is worse than no
+ * harness: it converts "we do not know" into "we checked". Both defects
+ * here did exactly that.
+ */
+
+const emptyRun = (): EvalRun => ({
+	output: 'ok',
+	steps: [],
+	toolCalls: [],
+	totalTokens: 0,
+	totalCostUsd: 0,
+	durationMs: 0,
+})
+
+describe('a case whose run THREW cannot score above zero', () => {
+	it('scores 0 instead of a perfect step budget', async () => {
+		// `executeCase` catches the throw and returns an empty run, and an
+		// empty run walks into every scorer's happy path: 0 steps is within
+		// any allowance, and "no tools expected, none called" is a match. The
+		// suite reported 1.0 for a case that never ran.
+		const report = await runExperiment({
+			name: 'crash',
+			cases: [{ name: 'boom', input: 'x' }],
+			scorers: [stepBudgetScorer(5), trajectoryScorer()],
+			run: () => {
+				throw new Error('provider exploded')
+			},
+		})
+
+		expect(report.cases[0]?.scores['step-budget']?.score).toBe(0)
+		expect(report.cases[0]?.scores.trajectory?.score).toBe(0)
+		expect(report.mean).toBe(0)
+	})
+
+	it('says why, so the green-to-red change is diagnosable', async () => {
+		const report = await runExperiment({
+			name: 'crash',
+			cases: [{ name: 'boom', input: 'x' }],
+			scorers: [stepBudgetScorer(5)],
+			run: () => {
+				throw new Error('provider exploded')
+			},
+		})
+
+		expect(report.cases[0]?.scores['step-budget']?.reason).toContain('provider exploded')
+	})
+
+	it('still scores a healthy run normally', async () => {
+		const report = await runExperiment({
+			name: 'fine',
+			cases: [{ name: 'ok', input: 'x' }],
+			scorers: [stepBudgetScorer(5)],
+			run: () => Promise.resolve(emptyRun()),
+		})
+
+		expect(report.cases[0]?.scores['step-budget']?.score).toBe(1)
+	})
+})
+
+describe('two scorers cannot silently collapse into one', () => {
+	it('refuses a duplicate scorer name rather than overwriting', async () => {
+		// Scores are keyed by name, so two `containsScorer(...)` instances —
+		// both called 'contains' — meant the second overwrote the first and
+		// the case mean was computed over the wrong denominator. With 'a'
+		// scoring 0 and 'b' scoring 1 the suite reported 1.0 where the honest
+		// answer is 0.5.
+		await expect(
+			runExperiment({
+				name: 'dupe',
+				cases: [{ name: 'c', input: 'x' }],
+				scorers: [containsScorer('a'), containsScorer('b')],
+				run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
+			}),
+		).rejects.toThrow(/Duplicate scorer name "contains"/)
+	})
+
+	it('accepts distinct names', async () => {
+		const report = await runExperiment({
+			name: 'ok',
+			cases: [{ name: 'c', input: 'x' }],
+			scorers: [
+				{ ...containsScorer('a'), name: 'contains-a' },
+				{ ...containsScorer('b'), name: 'contains-b' },
+			],
+			run: () => Promise.resolve({ ...emptyRun(), output: 'b' }),
+		})
+
+		expect(report.cases[0]?.scores['contains-a']?.score).toBe(0)
+		expect(report.cases[0]?.scores['contains-b']?.score).toBe(1)
+		expect(report.cases[0]?.mean).toBe(0.5)
+	})
+})

package/src/eval/__tests__/harness.test.ts

@@ -0,0 +1,249 @@
+import { describe, expect, it } from 'vitest'
+
+import { formatReport, runExperiment } from '../experiment.js'
+import {
+	completionScorer,
+	containsScorer,
+	customScorer,
+	stepBudgetScorer,
+	trajectoryScorer,
+} from '../scorers.js'
+import type { EvalCase, EvalRun } from '../types.js'
+
+/**
+ * There was no evaluation harness of any kind — no dataset, no scorer, no
+ * judge, no trajectory assertion. So the constants that govern namzu's
+ * most load-bearing behavior (`search_tools` top-k, the compaction
+ * threshold, six state-list caps, every builtin tool description) were
+ * tunable but not measurable: change one and the only way to learn the
+ * agent now takes four tool calls where it took one was for a user to
+ * hit it.
+ */
+
+function run(overrides: Partial<EvalRun> = {}): EvalRun {
+	return {
+		output: 'done',
+		steps: [],
+		toolCalls: [],
+		stopReason: 'end_turn',
+		totalTokens: 0,
+		totalCostUsd: 0,
+		durationMs: 0,
+		...overrides,
+	}
+}
+
+const CASE: EvalCase = { name: 'c', input: null }
+
+describe('trajectoryScorer', () => {
+	// `Scorer.score` may be async (a model-graded judge is just an async
+	// predicate), so every call site awaits even when the built-in is sync.
+	const score = async (expectedTools: string[], toolCalls: string[]) =>
+		await trajectoryScorer().score(run({ toolCalls }), { ...CASE, expectedTools })
+
+	it('scores an exact trajectory 1', async () => {
+		expect((await score(['read', 'edit'], ['read', 'edit'])).score).toBe(1)
+	})
+
+	it('distinguishes ORDER — a set-based score could not', async () => {
+		// Reading before editing is not the same run as editing then reading.
+		const reversed = await score(['read', 'edit'], ['edit', 'read'])
+		expect(reversed.score).toBeLessThan(1)
+		expect(reversed.score).toBeGreaterThan(0)
+	})
+
+	it('penalises an extra call through precision', async () => {
+		const wasteful = await score(['read'], ['read', 'read', 'read'])
+		expect(wasteful.score).toBeLessThan(1)
+		expect(wasteful.details?.recall).toBe(1)
+		expect(wasteful.details?.precision).toBeLessThan(1)
+	})
+
+	it('penalises a missing call through recall', async () => {
+		const lazy = await score(['read', 'edit', 'bash'], ['read'])
+		expect(lazy.details?.precision).toBe(1)
+		expect(lazy.details?.recall).toBeLessThan(1)
+	})
+
+	it('separates "wasteful" from "skipped" — a final-answer score collapses them', async () => {
+		const wasteful = await score(['read', 'edit'], ['read', 'ls', 'edit'])
+		const skipped = await score(['read', 'edit'], ['read'])
+		expect(wasteful.score).not.toBe(skipped.score)
+	})
+
+	it('scores 1 when no tools were expected and none ran', async () => {
+		expect((await score([], [])).score).toBe(1)
+	})
+
+	it('scores 0 when tools were expected and none ran', async () => {
+		const s = await score(['read'], [])
+		expect(s.score).toBe(0)
+		expect(s.reason).toContain('no tools were called')
+	})
+
+	it('scores 0 when none were expected and some ran', async () => {
+		expect((await score([], ['bash'])).score).toBe(0)
+	})
+
+	it('always explains itself', async () => {
+		expect((await score(['read'], ['edit'])).reason.length).toBeGreaterThan(0)
+	})
+})
+
+describe('completionScorer', () => {
+	it('accepts a clean settle', async () => {
+		expect((await completionScorer().score(run({ stopReason: 'end_turn' }), CASE)).score).toBe(1)
+	})
+
+	it('rejects a budget-forced stop', async () => {
+		const s = await completionScorer().score(run({ stopReason: 'max_iterations' }), CASE)
+		expect(s.score).toBe(0)
+		expect(s.reason).toContain('max_iterations')
+	})
+
+	it('reports the error when the run threw', async () => {
+		const s = await completionScorer().score(run({ error: 'boom' }), CASE)
+		expect(s.score).toBe(0)
+		expect(s.reason).toContain('boom')
+	})
+})
+
+describe('stepBudgetScorer', () => {
+	const withSteps = (n: number) => run({ steps: Array.from({ length: n }, () => ({}) as never) })
+
+	it('passes within budget', async () => {
+		expect((await stepBudgetScorer(5).score(withSteps(3), CASE)).score).toBe(1)
+	})
+
+	it('degrades proportionally past it, rather than snapping to zero', async () => {
+		// A run that took 6 steps against a 5 allowance is worse than one
+		// that took 20; a binary score cannot say so.
+		const slight = (await stepBudgetScorer(5).score(withSteps(6), CASE)).score
+		const severe = (await stepBudgetScorer(5).score(withSteps(20), CASE)).score
+		expect(slight).toBeGreaterThan(severe)
+		expect(slight).toBeLessThan(1)
+	})
+})
+
+describe('containsScorer', () => {
+	it('scores the fraction found and names what is missing', async () => {
+		const s = await containsScorer('alpha', 'beta').score(run({ output: 'alpha only' }), CASE)
+		expect(s.score).toBe(0.5)
+		expect(s.reason).toContain('beta')
+	})
+})
+
+describe('runExperiment', () => {
+	const cases: EvalCase[] = [
+		{ name: 'good', input: 'a', expectedTools: ['read'] },
+		{ name: 'bad', input: 'b', expectedTools: ['read', 'edit'] },
+	]
+
+	it('scores every case and aggregates', async () => {
+		const report = await runExperiment({
+			name: 'demo',
+			cases,
+			scorers: [trajectoryScorer()],
+			run: async (input) => run({ toolCalls: input === 'a' ? ['read'] : ['bash'] }),
+		})
+
+		expect(report.cases).toHaveLength(2)
+		expect(report.passed).toBe(1)
+		expect(report.failed).toBe(1)
+		expect(report.byScorer.trajectory).toBeCloseTo(0.5, 5)
+	})
+
+	it('a throwing case is a RESULT, not a crash', async () => {
+		// A suite whose first broken case aborts tells you nothing about the
+		// other forty.
+		const report = await runExperiment({
+			name: 'demo',
+			cases,
+			scorers: [completionScorer()],
+			run: async (input) => {
+				if (input === 'a') throw new Error('exploded')
+				return run()
+			},
+		})
+
+		expect(report.cases).toHaveLength(2)
+		expect(report.cases[0]?.run.error).toBe('exploded')
+		expect(report.cases[1]?.passed).toBe(true)
+	})
+
+	it('a throwing scorer scores zero with the throw as its reason', async () => {
+		const report = await runExperiment({
+			name: 'demo',
+			cases: [cases[0] as EvalCase],
+			scorers: [
+				customScorer('boom', () => {
+					throw new Error('scorer broke')
+				}),
+			],
+			run: async () => run(),
+		})
+
+		expect(report.cases[0]?.scores.boom?.score).toBe(0)
+		expect(report.cases[0]?.scores.boom?.reason).toContain('scorer broke')
+	})
+
+	it('preserves case order under concurrency', async () => {
+		const many: EvalCase[] = Array.from({ length: 8 }, (_, i) => ({
+			name: `case-${i}`,
+			input: i,
+		}))
+		const report = await runExperiment({
+			name: 'demo',
+			cases: many,
+			concurrency: 4,
+			scorers: [completionScorer()],
+			run: async (input) => {
+				await new Promise((r) => setTimeout(r, (8 - (input as number)) * 2))
+				return run()
+			},
+		})
+
+		expect(report.cases.map((c) => c.case)).toEqual(many.map((c) => c.name))
+	})
+
+	it('honours a per-case scorer override', async () => {
+		const report = await runExperiment({
+			name: 'demo',
+			cases: [{ name: 'special', input: null, scorers: [containsScorer('xyz')] }],
+			scorers: [completionScorer()],
+			run: async () => run({ output: 'no match here' }),
+		})
+
+		expect(Object.keys(report.cases[0]?.scores ?? {})).toEqual(['contains'])
+	})
+
+	it('respects a partial pass threshold', async () => {
+		const report = await runExperiment({
+			name: 'demo',
+			cases: [{ name: 'partial', input: null, expectedTools: ['read', 'edit'] }],
+			scorers: [trajectoryScorer()],
+			passThreshold: 0.5,
+			run: async () => run({ toolCalls: ['read'] }),
+		})
+
+		expect(report.passed).toBe(1)
+	})
+})
+
+describe('formatReport', () => {
+	it('prints scorer reasons for failures, not just numbers', async () => {
+		// A CI log that says "0.62" sends someone back to reproduce it by hand.
+		const report = await runExperiment({
+			name: 'demo',
+			cases: [{ name: 'bad', input: null, expectedTools: ['read', 'edit'] }],
+			scorers: [trajectoryScorer()],
+			run: async () => run({ toolCalls: ['bash'] }),
+		})
+
+		const text = formatReport(report)
+		expect(text).toContain('0/1 passed')
+		expect(text).toContain('✗ bad')
+		expect(text).toContain('trajectory:')
+		expect(text).toContain('expected read → edit')
+	})
+})

package/src/eval/__tests__/judge.test.ts

@@ -0,0 +1,306 @@
+import { describe, expect, it } from 'vitest'
+
+import type { ChatCompletionParams, StreamChunk } from '../../types/provider/index.js'
+import type { LLMProvider } from '../../types/provider/interface.js'
+import { runExperiment } from '../experiment.js'
+import { judgeScorer } from '../judge.js'
+import type { EvalCase, EvalRun } from '../types.js'
+
+/**
+ * Every other scorer is a pure function over the run, which is what makes
+ * them reproducible and what makes them unable to say whether an answer is
+ * GOOD. A judge closes that, and brings a failure mode none of the others
+ * have: it can fail to answer at all. A failed measurement scored zero
+ * reads exactly like a regression, so the distinction is what most of this
+ * file is about.
+ */
+
+function fakeProvider(reply: string | (() => never), tokens = 42): LLMProvider {
+	const seen: ChatCompletionParams[] = []
+	const provider: LLMProvider & { seen: ChatCompletionParams[] } = {
+		id: 'fake',
+		name: 'fake',
+		capabilities: {
+			supportsTools: false,
+			supportsStreaming: true,
+			supportsFunctionCalling: false,
+		},
+		seen,
+		async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
+			seen.push(params)
+			if (typeof reply !== 'string') reply()
+			yield { id: 'j', delta: { content: reply as string } }
+			yield {
+				id: 'j',
+				delta: {},
+				finishReason: 'stop',
+				usage: {
+					promptTokens: tokens,
+					completionTokens: 0,
+					totalTokens: tokens,
+					cachedTokens: 0,
+					cacheWriteTokens: 0,
+				},
+			}
+		},
+		async listModels() {
+			return []
+		},
+		async healthCheck() {
+			return true
+		},
+	}
+	return provider
+}
+
+const RUN: EvalRun = {
+	output: 'The capital of France is Paris.',
+	steps: [],
+	toolCalls: ['search'],
+	stopReason: 'end_turn',
+	totalTokens: 100,
+	totalCostUsd: 0,
+	durationMs: 10,
+}
+
+const CASE: EvalCase = { name: 'capital', input: 'What is the capital of France?' }
+
+const RUBRIC = 'The answer names the correct city.'
+
+describe('grading', () => {
+	it('normalises the grade onto 0..1 against the scale it asked for', async () => {
+		const scorer = judgeScorer({
+			provider: fakeProvider('{"grade": 3, "reason": "names the right city"}'),
+			model: 'm',
+			rubric: RUBRIC,
+			scale: 4,
+		})
+		const score = await scorer.score(RUN, CASE)
+
+		expect(score.score).toBe(0.75)
+		expect(score.reason).toBe('names the right city')
+		expect(score.details?.grade).toBe(3)
+	})
+
+	it('reads a verdict wrapped in prose or fences', async () => {
+		const scorer = judgeScorer({
+			provider: fakeProvider('Here is my grade:\n```json\n{"grade": 4, "reason": "correct"}\n```'),
+			model: 'm',
+			rubric: RUBRIC,
+		})
+		expect((await scorer.score(RUN, CASE)).score).toBe(1)
+	})
+
+	it('handles a brace inside the reason string', async () => {
+		const scorer = judgeScorer({
+			provider: fakeProvider('{"grade": 2, "reason": "mentions {city} placeholder"}'),
+			model: 'm',
+			rubric: RUBRIC,
+			scale: 4,
+		})
+		// A naive scan to the first `}` would cut the object in half here.
+		expect((await scorer.score(RUN, CASE)).score).toBe(0.5)
+	})
+
+	it('carries what the judging itself cost', async () => {
+		const scorer = judgeScorer({
+			provider: fakeProvider('{"grade": 4, "reason": "ok"}', 1234),
+			model: 'm',
+			rubric: RUBRIC,
+		})
+		// A judge is the most expensive scorer there is; a bill nobody can
+		// attribute is a bill nobody controls.
+		expect((await scorer.score(RUN, CASE)).details?.judgeTokens).toBe(1234)
+	})
+
+	it('grades the same run the same way twice', async () => {
+		const provider = fakeProvider('{"grade": 4, "reason": "ok"}') as LLMProvider & {
+			seen: ChatCompletionParams[]
+		}
+		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
+		// Sampling noise is indistinguishable from a regression, so the
+		// judge does not sample.
+		expect(provider.seen[0]?.temperature).toBe(0)
+	})
+})
+
+describe('what the judge is shown', () => {
+	const promptOf = (provider: LLMProvider) =>
+		String((provider as unknown as { seen: ChatCompletionParams[] }).seen[0]?.messages[0]?.content)
+
+	it('shows the rubric, the task and the answer', async () => {
+		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
+		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
+
+		const prompt = promptOf(provider)
+		expect(prompt).toContain(RUBRIC)
+		expect(prompt).toContain('What is the capital of France?')
+		expect(prompt).toContain('The capital of France is Paris.')
+	})
+
+	it('withholds the trajectory unless asked', async () => {
+		const off = fakeProvider('{"grade": 4, "reason": "ok"}')
+		await judgeScorer({ provider: off, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
+		expect(promptOf(off)).not.toContain('TOOLS CALLED')
+
+		const on = fakeProvider('{"grade": 4, "reason": "ok"}')
+		await judgeScorer({
+			provider: on,
+			model: 'm',
+			rubric: RUBRIC,
+			includeTrajectory: true,
+		}).score(RUN, CASE)
+		expect(promptOf(on)).toContain('search')
+	})
+
+	it('says when it truncated the answer', async () => {
+		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
+		await judgeScorer({
+			provider,
+			model: 'm',
+			rubric: RUBRIC,
+			maxOutputChars: 10,
+		}).score({ ...RUN, output: 'x'.repeat(500) }, CASE)
+
+		// A judge shown a silently cut answer marks it down for stopping
+		// mid-sentence, which scores our truncation rather than the run.
+		expect(promptOf(provider)).toContain('cut at 10 characters by the harness')
+	})
+
+	it('offers the reference answer when the case has one', async () => {
+		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
+		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, {
+			...CASE,
+			expected: 'Paris',
+		})
+		expect(promptOf(provider)).toContain('Paris')
+	})
+})
+
+describe('a judge that cannot judge', () => {
+	const failing = (reply: string) =>
+		judgeScorer({ provider: fakeProvider(reply), model: 'm', rubric: RUBRIC, scale: 4 })
+
+	it('throws on a reply with no JSON at all', async () => {
+		await expect(failing('I think it was pretty good!').score(RUN, CASE)).rejects.toThrow(/no JSON/)
+	})
+
+	it('throws on a grade outside the scale it was given', async () => {
+		// Clamping would turn a judge that misread the scale into a
+		// confident score, and a judge that misread the scale did not apply
+		// the rubric either.
+		await expect(failing('{"grade": 9, "reason": "great"}').score(RUN, CASE)).rejects.toThrow(
+			/outside the 0\.\.4 scale/,
+		)
+	})
+
+	it('throws on a non-numeric grade', async () => {
+		await expect(failing('{"grade": "good", "reason": "x"}').score(RUN, CASE)).rejects.toThrow(
+			/no numeric grade/,
+		)
+	})
+
+	it('throws on an unterminated object', async () => {
+		await expect(failing('{"grade": 3, "reason": "cut off').score(RUN, CASE)).rejects.toThrow(
+			/unterminated/,
+		)
+	})
+
+	it('refuses to exist without a rubric', () => {
+		expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: '   ' })).toThrow(
+			/rubric is required/,
+		)
+	})
+
+	it('refuses a nonsense scale', () => {
+		expect(() =>
+			judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: RUBRIC, scale: 0 }),
+		).toThrow(/positive integer/)
+	})
+})
+
+describe('a broken judge is not a bad run', () => {
+	const dataset = {
+		name: 'suite',
+		cases: [CASE],
+		run: async (): Promise<EvalRun> => RUN,
+	}
+
+	it('reports the case as inconclusive rather than failed', async () => {
+		const report = await runExperiment({
+			...dataset,
+			scorers: [
+				judgeScorer({
+					provider: fakeProvider(() => {
+						throw new Error('429 rate limited')
+					}),
+					model: 'm',
+					rubric: RUBRIC,
+				}),
+			],
+		})
+
+		expect(report.inconclusive).toBe(1)
+		expect(report.failed).toBe(0)
+		expect(report.passed).toBe(0)
+		expect(report.cases[0]?.scores.judge?.unavailable).toBe(true)
+		expect(report.cases[0]?.scores.judge?.reason).toContain('429')
+	})
+
+	it('keeps an unavailable score out of the mean entirely', async () => {
+		const report = await runExperiment({
+			...dataset,
+			scorers: [
+				{ name: 'always-one', score: () => ({ score: 1, reason: 'fine' }) },
+				judgeScorer({
+					provider: fakeProvider(() => {
+						throw new Error('down')
+					}),
+					model: 'm',
+					rubric: RUBRIC,
+				}),
+			],
+		})
+
+		// Not 0.5. A measurement that never happened is not a zero, and
+		// averaging it in would report a regression the harness caused.
+		expect(report.cases[0]?.mean).toBe(1)
+		expect(report.cases[0]?.status).toBe('passed')
+		expect(report.mean).toBe(1)
+	})
+
+	it('omits a scorer that was never available from the per-dimension means', async () => {
+		const report = await runExperiment({
+			...dataset,
+			scorers: [
+				judgeScorer({
+					provider: fakeProvider(() => {
+						throw new Error('down')
+					}),
+					model: 'm',
+					rubric: RUBRIC,
+				}),
+			],
+		})
+
+		// A dimension with no measurements is not a dimension that scored 0.
+		expect(report.byScorer.judge).toBeUndefined()
+	})
+
+	it('still fails a case the judge actually judged badly', async () => {
+		const report = await runExperiment({
+			...dataset,
+			scorers: [
+				judgeScorer({
+					provider: fakeProvider('{"grade": 0, "reason": "names the wrong city"}'),
+					model: 'm',
+					rubric: RUBRIC,
+				}),
+			],
+		})
+
+		expect(report.failed).toBe(1)
+		expect(report.inconclusive).toBe(0)
+		expect(report.cases[0]?.scores.judge?.unavailable).toBeUndefined()
+	})
+})