@namzu/sdk 14.0.6 → 15.0.0

package/src/eval/__tests__/judge.test.ts

@@ -1,306 +0,0 @@
-import { describe, expect, it } from 'vitest'
-
-import type { ChatCompletionParams, StreamChunk } from '../../types/provider/index.js'
-import type { LLMProvider } from '../../types/provider/interface.js'
-import { runExperiment } from '../experiment.js'
-import { judgeScorer } from '../judge.js'
-import type { EvalCase, EvalRun } from '../types.js'
-
-/**
- * Every other scorer is a pure function over the run, which is what makes
- * them reproducible and what makes them unable to say whether an answer is
- * GOOD. A judge closes that, and brings a failure mode none of the others
- * have: it can fail to answer at all. A failed measurement scored zero
- * reads exactly like a regression, so the distinction is what most of this
- * file is about.
- */
-
-function fakeProvider(reply: string | (() => never), tokens = 42): LLMProvider {
-	const seen: ChatCompletionParams[] = []
-	const provider: LLMProvider & { seen: ChatCompletionParams[] } = {
-		id: 'fake',
-		name: 'fake',
-		capabilities: {
-			supportsTools: false,
-			supportsStreaming: true,
-			supportsFunctionCalling: false,
-		},
-		seen,
-		async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
-			seen.push(params)
-			if (typeof reply !== 'string') reply()
-			yield { id: 'j', delta: { content: reply as string } }
-			yield {
-				id: 'j',
-				delta: {},
-				finishReason: 'stop',
-				usage: {
-					promptTokens: tokens,
-					completionTokens: 0,
-					totalTokens: tokens,
-					cachedTokens: 0,
-					cacheWriteTokens: 0,
-				},
-			}
-		},
-		async listModels() {
-			return []
-		},
-		async healthCheck() {
-			return true
-		},
-	}
-	return provider
-}
-
-const RUN: EvalRun = {
-	output: 'The capital of France is Paris.',
-	steps: [],
-	toolCalls: ['search'],
-	stopReason: 'end_turn',
-	totalTokens: 100,
-	totalCostUsd: 0,
-	durationMs: 10,
-}
-
-const CASE: EvalCase = { name: 'capital', input: 'What is the capital of France?' }
-
-const RUBRIC = 'The answer names the correct city.'
-
-describe('grading', () => {
-	it('normalises the grade onto 0..1 against the scale it asked for', async () => {
-		const scorer = judgeScorer({
-			provider: fakeProvider('{"grade": 3, "reason": "names the right city"}'),
-			model: 'm',
-			rubric: RUBRIC,
-			scale: 4,
-		})
-		const score = await scorer.score(RUN, CASE)
-
-		expect(score.score).toBe(0.75)
-		expect(score.reason).toBe('names the right city')
-		expect(score.details?.grade).toBe(3)
-	})
-
-	it('reads a verdict wrapped in prose or fences', async () => {
-		const scorer = judgeScorer({
-			provider: fakeProvider('Here is my grade:\n```json\n{"grade": 4, "reason": "correct"}\n```'),
-			model: 'm',
-			rubric: RUBRIC,
-		})
-		expect((await scorer.score(RUN, CASE)).score).toBe(1)
-	})
-
-	it('handles a brace inside the reason string', async () => {
-		const scorer = judgeScorer({
-			provider: fakeProvider('{"grade": 2, "reason": "mentions {city} placeholder"}'),
-			model: 'm',
-			rubric: RUBRIC,
-			scale: 4,
-		})
-		// A naive scan to the first `}` would cut the object in half here.
-		expect((await scorer.score(RUN, CASE)).score).toBe(0.5)
-	})
-
-	it('carries what the judging itself cost', async () => {
-		const scorer = judgeScorer({
-			provider: fakeProvider('{"grade": 4, "reason": "ok"}', 1234),
-			model: 'm',
-			rubric: RUBRIC,
-		})
-		// A judge is the most expensive scorer there is; a bill nobody can
-		// attribute is a bill nobody controls.
-		expect((await scorer.score(RUN, CASE)).details?.judgeTokens).toBe(1234)
-	})
-
-	it('grades the same run the same way twice', async () => {
-		const provider = fakeProvider('{"grade": 4, "reason": "ok"}') as LLMProvider & {
-			seen: ChatCompletionParams[]
-		}
-		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
-		// Sampling noise is indistinguishable from a regression, so the
-		// judge does not sample.
-		expect(provider.seen[0]?.temperature).toBe(0)
-	})
-})
-
-describe('what the judge is shown', () => {
-	const promptOf = (provider: LLMProvider) =>
-		String((provider as unknown as { seen: ChatCompletionParams[] }).seen[0]?.messages[0]?.content)
-
-	it('shows the rubric, the task and the answer', async () => {
-		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
-		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
-
-		const prompt = promptOf(provider)
-		expect(prompt).toContain(RUBRIC)
-		expect(prompt).toContain('What is the capital of France?')
-		expect(prompt).toContain('The capital of France is Paris.')
-	})
-
-	it('withholds the trajectory unless asked', async () => {
-		const off = fakeProvider('{"grade": 4, "reason": "ok"}')
-		await judgeScorer({ provider: off, model: 'm', rubric: RUBRIC }).score(RUN, CASE)
-		expect(promptOf(off)).not.toContain('TOOLS CALLED')
-
-		const on = fakeProvider('{"grade": 4, "reason": "ok"}')
-		await judgeScorer({
-			provider: on,
-			model: 'm',
-			rubric: RUBRIC,
-			includeTrajectory: true,
-		}).score(RUN, CASE)
-		expect(promptOf(on)).toContain('search')
-	})
-
-	it('says when it truncated the answer', async () => {
-		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
-		await judgeScorer({
-			provider,
-			model: 'm',
-			rubric: RUBRIC,
-			maxOutputChars: 10,
-		}).score({ ...RUN, output: 'x'.repeat(500) }, CASE)
-
-		// A judge shown a silently cut answer marks it down for stopping
-		// mid-sentence, which scores our truncation rather than the run.
-		expect(promptOf(provider)).toContain('cut at 10 characters by the harness')
-	})
-
-	it('offers the reference answer when the case has one', async () => {
-		const provider = fakeProvider('{"grade": 4, "reason": "ok"}')
-		await judgeScorer({ provider, model: 'm', rubric: RUBRIC }).score(RUN, {
-			...CASE,
-			expected: 'Paris',
-		})
-		expect(promptOf(provider)).toContain('Paris')
-	})
-})
-
-describe('a judge that cannot judge', () => {
-	const failing = (reply: string) =>
-		judgeScorer({ provider: fakeProvider(reply), model: 'm', rubric: RUBRIC, scale: 4 })
-
-	it('throws on a reply with no JSON at all', async () => {
-		await expect(failing('I think it was pretty good!').score(RUN, CASE)).rejects.toThrow(/no JSON/)
-	})
-
-	it('throws on a grade outside the scale it was given', async () => {
-		// Clamping would turn a judge that misread the scale into a
-		// confident score, and a judge that misread the scale did not apply
-		// the rubric either.
-		await expect(failing('{"grade": 9, "reason": "great"}').score(RUN, CASE)).rejects.toThrow(
-			/outside the 0\.\.4 scale/,
-		)
-	})
-
-	it('throws on a non-numeric grade', async () => {
-		await expect(failing('{"grade": "good", "reason": "x"}').score(RUN, CASE)).rejects.toThrow(
-			/no numeric grade/,
-		)
-	})
-
-	it('throws on an unterminated object', async () => {
-		await expect(failing('{"grade": 3, "reason": "cut off').score(RUN, CASE)).rejects.toThrow(
-			/unterminated/,
-		)
-	})
-
-	it('refuses to exist without a rubric', () => {
-		expect(() => judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: '   ' })).toThrow(
-			/rubric is required/,
-		)
-	})
-
-	it('refuses a nonsense scale', () => {
-		expect(() =>
-			judgeScorer({ provider: fakeProvider('{}'), model: 'm', rubric: RUBRIC, scale: 0 }),
-		).toThrow(/positive integer/)
-	})
-})
-
-describe('a broken judge is not a bad run', () => {
-	const dataset = {
-		name: 'suite',
-		cases: [CASE],
-		run: async (): Promise<EvalRun> => RUN,
-	}
-
-	it('reports the case as inconclusive rather than failed', async () => {
-		const report = await runExperiment({
-			...dataset,
-			scorers: [
-				judgeScorer({
-					provider: fakeProvider(() => {
-						throw new Error('429 rate limited')
-					}),
-					model: 'm',
-					rubric: RUBRIC,
-				}),
-			],
-		})
-
-		expect(report.inconclusive).toBe(1)
-		expect(report.failed).toBe(0)
-		expect(report.passed).toBe(0)
-		expect(report.cases[0]?.scores.judge?.unavailable).toBe(true)
-		expect(report.cases[0]?.scores.judge?.reason).toContain('429')
-	})
-
-	it('keeps an unavailable score out of the mean entirely', async () => {
-		const report = await runExperiment({
-			...dataset,
-			scorers: [
-				{ name: 'always-one', score: () => ({ score: 1, reason: 'fine' }) },
-				judgeScorer({
-					provider: fakeProvider(() => {
-						throw new Error('down')
-					}),
-					model: 'm',
-					rubric: RUBRIC,
-				}),
-			],
-		})
-
-		// Not 0.5. A measurement that never happened is not a zero, and
-		// averaging it in would report a regression the harness caused.
-		expect(report.cases[0]?.mean).toBe(1)
-		expect(report.cases[0]?.status).toBe('passed')
-		expect(report.mean).toBe(1)
-	})
-
-	it('omits a scorer that was never available from the per-dimension means', async () => {
-		const report = await runExperiment({
-			...dataset,
-			scorers: [
-				judgeScorer({
-					provider: fakeProvider(() => {
-						throw new Error('down')
-					}),
-					model: 'm',
-					rubric: RUBRIC,
-				}),
-			],
-		})
-
-		// A dimension with no measurements is not a dimension that scored 0.
-		expect(report.byScorer.judge).toBeUndefined()
-	})
-
-	it('still fails a case the judge actually judged badly', async () => {
-		const report = await runExperiment({
-			...dataset,
-			scorers: [
-				judgeScorer({
-					provider: fakeProvider('{"grade": 0, "reason": "names the wrong city"}'),
-					model: 'm',
-					rubric: RUBRIC,
-				}),
-			],
-		})
-
-		expect(report.failed).toBe(1)
-		expect(report.inconclusive).toBe(0)
-		expect(report.cases[0]?.scores.judge?.unavailable).toBeUndefined()
-	})
-})

package/src/eval/__tests__/scorer-gates.test.ts

@@ -1,156 +0,0 @@
-import { describe, expect, it } from 'vitest'
-
-import { formatReport, runExperiment } from '../experiment.js'
-import type { EvalRun, Score, Scorer } from '../types.js'
-
-/**
- * A case's verdict was one unweighted mean over every scorer against one
- * suite-wide threshold, and the two halves of that fought each other.
- *
- * At the default threshold of 1 the harness never reports a false pass —
- * but a trajectory F1 and a graded judge can essentially never reach 1, so
- * every real suite lowers it. And every step down buys tolerance for the
- * fuzzy scorers by buying exactly the same tolerance for the deterministic
- * ones: at 0.75, a trajectory score of 0 alongside three perfect scores
- * averages to 0.75 and reports PASSED. The regression the harness exists
- * to catch comes back green.
- *
- * The workarounds were lossy too — folding every hard check into one
- * custom scorer destroys per-dimension reporting, and two experiments at
- * two thresholds re-runs the dataset and splits one case's verdict across
- * two reports.
- */
-
-const run = (): EvalRun => ({
-	output: 'answer',
-	steps: [],
-	toolCalls: [],
-	totalTokens: 0,
-	totalCostUsd: 0,
-	durationMs: 0,
-})
-
-const scorer = (name: string, score: number, extra: Partial<Scorer> = {}): Scorer => ({
-	name,
-	score: (): Score => ({ score, reason: `${name} scored ${score}` }),
-	...extra,
-})
-
-const experiment = (scorers: Scorer[], passThreshold = 0.75) =>
-	runExperiment({
-		name: 'gates',
-		cases: [{ name: 'one', input: null }],
-		scorers,
-		passThreshold,
-		run: async () => run(),
-	})
-
-describe('a gate scorer', () => {
-	it('fails the case even when the mean clears the threshold', async () => {
-		const report = await experiment([
-			scorer('trajectory', 0, { severity: 'gate' }),
-			scorer('completion', 1),
-			scorer('contains', 1),
-			scorer('judge', 1),
-		])
-
-		// Mean is exactly 0.75 against a 0.75 threshold — reported passed
-		// before, with the one deterministic check at zero.
-		expect(report.cases[0]?.mean).toBeCloseTo(0.75)
-		expect(report.cases[0]?.status).toBe('failed')
-	})
-
-	it('names itself, so the reader does not have to guess', async () => {
-		const report = await experiment([
-			scorer('trajectory', 0, { severity: 'gate' }),
-			scorer('judge', 1),
-		])
-		expect(report.cases[0]?.failedGates).toEqual(['trajectory'])
-	})
-
-	it('honours its own threshold when it sets one', async () => {
-		// "Good enough" is not one number across dimensions: a trajectory
-		// match at 0.8 may be fine while a completion check at 0.8 is
-		// meaningless.
-		const lenient = await experiment([
-			scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.7 }),
-		])
-		expect(lenient.cases[0]?.status).toBe('passed')
-
-		const strict = await experiment([
-			scorer('trajectory', 0.8, { severity: 'gate', threshold: 0.9 }),
-		])
-		expect(strict.cases[0]?.status).toBe('failed')
-	})
-
-	it('falls back to the suite threshold when it sets none', async () => {
-		const report = await experiment([scorer('trajectory', 0.5, { severity: 'gate' })], 0.75)
-		expect(report.cases[0]?.failedGates).toEqual(['trajectory'])
-	})
-
-	it('does not fail the case when it could not judge at all', async () => {
-		// Unavailable is not zero. A gate that threw did not judge the run
-		// badly, it failed to judge it — the inconclusive path, not a
-		// failure, and the two demand opposite responses.
-		const report = await runExperiment({
-			name: 'gates',
-			cases: [{ name: 'one', input: null }],
-			scorers: [
-				{
-					name: 'judge-gate',
-					severity: 'gate',
-					score: () => {
-						throw new Error('provider unreachable')
-					},
-				},
-			],
-			run: async () => run(),
-		})
-
-		expect(report.cases[0]?.status).toBe('inconclusive')
-		expect(report.cases[0]?.failedGates).toBeUndefined()
-	})
-
-	it('lets a passing gate through', async () => {
-		const report = await experiment([scorer('trajectory', 1, { severity: 'gate' })])
-		expect(report.cases[0]?.status).toBe('passed')
-	})
-})
-
-describe('a soft scorer', () => {
-	it('only moves the mean', async () => {
-		// The fuzzy dimensions, where a number below 1 is normal and a
-		// threshold is a judgement call.
-		const report = await experiment([scorer('judge', 0.5), scorer('other', 1)])
-		expect(report.cases[0]?.status).toBe('passed')
-		expect(report.cases[0]?.failedGates).toBeUndefined()
-	})
-
-	it('is the default, so a suite that sets nothing is unchanged', async () => {
-		const report = await experiment([
-			scorer('a', 0),
-			scorer('b', 1),
-			scorer('c', 1),
-			scorer('d', 1),
-		])
-		expect(report.cases[0]?.status).toBe('passed')
-	})
-
-	it('can still fail the case through the mean', async () => {
-		const report = await experiment([scorer('a', 0), scorer('b', 0.5)])
-		expect(report.cases[0]?.status).toBe('failed')
-		expect(report.cases[0]?.failedGates).toBeUndefined()
-	})
-})
-
-describe('the report', () => {
-	it('says which gate missed, above the individual scores', async () => {
-		const report = await experiment([
-			scorer('trajectory', 0, { severity: 'gate' }),
-			scorer('judge', 1),
-		])
-
-		const text = formatReport(report)
-		expect(text).toContain('gate missed: trajectory')
-	})
-})

package/src/execution/local.test.ts

@@ -1,46 +0,0 @@
-import { existsSync, mkdtempSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { LocalExecutionContext } from './local.js'
-
-// `command`/`args` on executeCommand() come from whatever tool call invokes it, so they
-// must be treated as untrusted. A single arg that smuggles a shell separator + a second
-// command proves whether the child process was actually handed a shell.
-function injectionArg(canaryPath: string): string {
-	return process.platform === 'win32'
-		? `& echo x> ${canaryPath} & rem`
-		: `; echo x > ${canaryPath} #`
-}
-
-describe('LocalExecutionContext.executeCommand', () => {
-	let dir: string
-	let canary: string
-
-	beforeEach(() => {
-		dir = mkdtempSync(join(tmpdir(), 'namzu-local-exec-'))
-		canary = join(dir, 'canary.txt')
-	})
-
-	afterEach(() => {
-		rmSync(dir, { recursive: true, force: true })
-	})
-
-	it('does not let a shell-metacharacter arg run a second command by default', async () => {
-		const ctx = new LocalExecutionContext({ id: 'test', cwd: dir })
-		await ctx.initialize()
-
-		await ctx.executeCommand('node', ['--version', injectionArg(canary)])
-
-		expect(existsSync(canary)).toBe(false)
-	})
-
-	it('still runs through a shell when explicitly requested', async () => {
-		const ctx = new LocalExecutionContext({ id: 'test', cwd: dir })
-		await ctx.initialize()
-
-		await ctx.executeCommand('node', ['--version', injectionArg(canary)], { shell: true })
-
-		expect(existsSync(canary)).toBe(true)
-	})
-})

package/src/gateway/__tests__/a-gateway-that-forgets-the-oldest.test.ts

@@ -1,130 +0,0 @@
-import { describe, expect, it } from 'vitest'
-
-import type { AgentManagerContract } from '../../types/agent/manager.js'
-import type { AgentTask, AgentTaskContext } from '../../types/agent/task.js'
-import type { AgentId, TaskId } from '../../types/ids/index.js'
-import { LocalTaskGateway } from '../local.js'
-
-/**
- * The gateway's two ledgers — `trackedTaskIds` and `settledHandles` — had `add`
- * and `set` and no removal anywhere. The doc called them "bounded by the number
- * the gateway itself launched", which is true and is not a bound: a gateway
- * built per run is bounded by that run, but `SupervisorAgentConfig.gateway`
- * lets a host supply its own, and a long-lived host reusing one accumulates an
- * id and a settled handle for every task it ever launched.
- *
- * Eviction is oldest-first and takes both together. Dropping a tracked id while
- * keeping its handle — or the reverse — would make a task that ran read as one
- * that never launched, which is precisely the defect the settled-handle map was
- * added to fix.
- */
-
-const CAP = 1_000
-
-class CountingManager {
-	private n = 0
-
-	async sendMessage(): Promise<AgentTask> {
-		this.n++
-		return { taskId: `tsk_${this.n}` as TaskId, state: 'running' } as AgentTask
-	}
-	async waitForCompletion(): Promise<void> {}
-	getInstance(): AgentTask | undefined {
-		return undefined
-	}
-	cancel(): void {}
-}
-
-function context(): AgentTaskContext {
-	return {
-		parentRunId: 'run_p' as never,
-		parentAgentId: 'sup',
-		parentAbortController: new AbortController(),
-		depth: 0,
-		budgetTracker: { total: 1_000_000, remaining: 1_000_000 },
-		tenantId: 'tnt_g' as never,
-		sessionId: 'ses_g' as never,
-		projectId: 'prj_g' as never,
-	} as unknown as AgentTaskContext
-}
-
-function tracked(gateway: LocalTaskGateway): Set<TaskId> {
-	return (gateway as unknown as { trackedTaskIds: Set<TaskId> }).trackedTaskIds
-}
-
-function settled(gateway: LocalTaskGateway): Map<TaskId, unknown> {
-	return (gateway as unknown as { settledHandles: Map<TaskId, unknown> }).settledHandles
-}
-
-async function launch(gateway: LocalTaskGateway, count: number): Promise<void> {
-	for (let i = 0; i < count; i++) {
-		await gateway.createTask({
-			agentId: 'w' as AgentId,
-			prompt: 'go',
-			workingDirectory: '/tmp',
-		} as never)
-	}
-}
-
-describe('a gateway that outlives its run forgets the oldest', () => {
-	it('keeps every task of a run that never reaches the cap', async () => {
-		// The control, and the case that must not change: a supervisor reading
-		// its listing at the end of a run sees everything it launched.
-		const gateway = new LocalTaskGateway(
-			new CountingManager() as unknown as AgentManagerContract,
-			context(),
-		)
-
-		await launch(gateway, 50)
-
-		expect(tracked(gateway).size).toBe(50)
-	})
-
-	it('stops growing once the cap is passed', async () => {
-		const gateway = new LocalTaskGateway(
-			new CountingManager() as unknown as AgentManagerContract,
-			context(),
-		)
-
-		await launch(gateway, CAP + 25)
-
-		expect(tracked(gateway).size).toBe(CAP)
-	})
-
-	it('drops the oldest, not the newest', async () => {
-		// A cap that evicted the most recent would keep the ledger small and
-		// answer every question wrongly.
-		const gateway = new LocalTaskGateway(
-			new CountingManager() as unknown as AgentManagerContract,
-			context(),
-		)
-
-		await launch(gateway, CAP + 3)
-
-		const ids = tracked(gateway)
-		expect(ids.has('tsk_1' as TaskId)).toBe(false)
-		expect(ids.has('tsk_3' as TaskId)).toBe(false)
-		expect(ids.has(`tsk_${CAP + 3}` as TaskId)).toBe(true)
-	})
-
-	it('never keeps a settled handle for an id it has forgotten', async () => {
-		// The two must be evicted together. A handle whose id is gone is
-		// unreachable through `listTasks`, which walks the ids — so it would be
-		// retained memory serving nothing, and the leak would survive its own
-		// fix.
-		const gateway = new LocalTaskGateway(
-			new CountingManager() as unknown as AgentManagerContract,
-			context(),
-		)
-		await launch(gateway, 10)
-		const handles = settled(gateway)
-		for (let i = 1; i <= 10; i++) handles.set(`tsk_${i}` as TaskId, { id: i })
-
-		await launch(gateway, CAP + 5)
-
-		const ids = tracked(gateway)
-		for (const id of handles.keys()) {
-			expect({ id, trackedToo: ids.has(id) }).toEqual({ id, trackedToo: true })
-		}
-	})
-})