@namzu/sdk 2.0.0 → 3.0.0

package/src/eval/judge.ts

@@ -0,0 +1,240 @@
+import { collect } from '../provider/collect.js'
+import type { LLMProvider } from '../types/provider/interface.js'
+import type { EvalCase, EvalRun, Score, Scorer } from './types.js'
+
+/**
+ * Grade an open-ended answer with a model.
+ *
+ * Every other scorer here is a pure function over the run, which is what
+ * makes them cheap and reproducible — and also what makes them unable to
+ * say anything about whether an answer is *good*. `containsScorer` can
+ * check that a required phrase appears; it cannot tell a correct
+ * explanation from a fluent wrong one. That was the hole: the dimension
+ * most worth guarding was the one with no scorer behind it.
+ *
+ * A judge is a network call, so this scorer can fail in ways a pure
+ * function cannot. It throws rather than returning a low score, and the
+ * harness files a throw as `unavailable` — a rate limit must not read as
+ * a regression. See `Score.unavailable`.
+ */
+export interface JudgeScorerConfig {
+	/** Defaults to `'judge'`. Give it a distinct name to run several. */
+	name?: string
+	provider: LLMProvider
+	model: string
+	/**
+	 * What "good" means, in the caller's words. REQUIRED.
+	 *
+	 * A judge asked to rate quality with no rubric rates fluency, which
+	 * correlates with almost nothing worth measuring and drifts whenever
+	 * the judge model changes. Making this optional would make the scorer
+	 * easy to use and its output meaningless, so it is not optional.
+	 */
+	rubric: string
+	/**
+	 * Highest grade on the scale. Default 4.
+	 *
+	 * An integer scale, not a 0..1 float: models place a continuous score
+	 * poorly and cluster on round numbers, while a short ordinal scale
+	 * against a written rubric is a judgement they can actually make. The
+	 * result is divided down to 0..1 for the report.
+	 *
+	 * The default is EVEN on purpose — an odd scale has a midpoint, and a
+	 * midpoint is where an uncertain judge parks. Forcing a side produces a
+	 * signal; a pile of 3-out-of-5s does not.
+	 */
+	scale?: number
+	/**
+	 * Show the judge which tools the run called. Default false.
+	 *
+	 * Useful when the rubric is about method rather than answer, and a
+	 * needless cost otherwise — the trajectory is usually longer than the
+	 * answer it produced.
+	 */
+	includeTrajectory?: boolean
+	/**
+	 * Cap on the answer text handed to the judge. Default 20000.
+	 *
+	 * Truncation is disclosed IN the prompt. A judge shown a silently cut
+	 * answer marks it down for stopping mid-sentence, which scores our
+	 * truncation rather than the run.
+	 */
+	maxOutputChars?: number
+}
+
+const DEFAULT_SCALE = 4
+const DEFAULT_MAX_OUTPUT_CHARS = 20_000
+
+interface Verdict {
+	grade: number
+	reason: string
+}
+
+/**
+ * Pull the verdict out of the reply.
+ *
+ * Models wrap JSON in prose or fences however firmly asked not to, so the
+ * first balanced object is extracted rather than the whole reply parsed.
+ * What is NOT tolerated is a missing or out-of-range grade: that is an
+ * unusable measurement, and the caller has to hear about it rather than
+ * receive a number invented from a fallback.
+ */
+function parseVerdict(reply: string, scale: number): Verdict {
+	const start = reply.indexOf('{')
+	if (start === -1) throw new Error(`judge returned no JSON object: ${reply.slice(0, 200)}`)
+
+	let depth = 0
+	let inString = false
+	let escaped = false
+	let end = -1
+	for (let i = start; i < reply.length; i++) {
+		const ch = reply[i]
+		if (escaped) {
+			escaped = false
+			continue
+		}
+		if (ch === '\\') {
+			escaped = true
+			continue
+		}
+		if (ch === '"') {
+			inString = !inString
+			continue
+		}
+		if (inString) continue
+		if (ch === '{') depth++
+		else if (ch === '}') {
+			depth--
+			if (depth === 0) {
+				end = i
+				break
+			}
+		}
+	}
+	if (end === -1)
+		throw new Error(`judge returned an unterminated JSON object: ${reply.slice(0, 200)}`)
+
+	let parsed: unknown
+	try {
+		parsed = JSON.parse(reply.slice(start, end + 1))
+	} catch (err) {
+		throw new Error(
+			`judge returned unparsable JSON: ${err instanceof Error ? err.message : String(err)}`,
+		)
+	}
+
+	const grade = (parsed as { grade?: unknown }).grade
+	if (typeof grade !== 'number' || !Number.isFinite(grade)) {
+		throw new Error(`judge returned no numeric grade: ${JSON.stringify(parsed).slice(0, 200)}`)
+	}
+	if (grade < 0 || grade > scale) {
+		// Clamping instead would quietly turn a judge that misread the scale
+		// into a confident score, and a judge that misread the scale did not
+		// apply the rubric either.
+		throw new Error(`judge returned ${grade}, outside the 0..${scale} scale it was given`)
+	}
+
+	const reason = (parsed as { reason?: unknown }).reason
+	return {
+		grade,
+		reason: typeof reason === 'string' && reason.length > 0 ? reason : '(judge gave no reason)',
+	}
+}
+
+function buildPrompt(
+	config: JudgeScorerConfig,
+	run: EvalRun,
+	evalCase: EvalCase,
+	scale: number,
+): string {
+	const limit = config.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS
+	const answer = run.output ?? ''
+	const truncated = answer.length > limit
+
+	const sections: string[] = [
+		'Grade the ANSWER against the RUBRIC.',
+		'',
+		`RUBRIC:\n${config.rubric}`,
+		'',
+		`TASK:\n${typeof evalCase.input === 'string' ? evalCase.input : JSON.stringify(evalCase.input)}`,
+	]
+
+	if (evalCase.expected !== undefined) {
+		sections.push(
+			'',
+			`REFERENCE (one good answer, not the only one):\n${
+				typeof evalCase.expected === 'string'
+					? evalCase.expected
+					: JSON.stringify(evalCase.expected)
+			}`,
+		)
+	}
+
+	if (config.includeTrajectory === true) {
+		sections.push(
+			'',
+			`TOOLS CALLED, in order:\n${run.toolCalls.length > 0 ? run.toolCalls.join(' -> ') : '(none)'}`,
+		)
+	}
+
+	sections.push('', `ANSWER:\n${truncated ? answer.slice(0, limit) : answer}`)
+	if (truncated) {
+		// Disclosed, so the judge does not mark the answer down for an
+		// ending we removed.
+		sections.push(
+			'',
+			`(The answer above was cut at ${limit} characters by the harness, not by the agent. Do not penalise it for ending abruptly.)`,
+		)
+	}
+
+	sections.push(
+		'',
+		`Reply with JSON and nothing else: {"grade": <integer 0..${scale}>, "reason": "<one or two sentences>"}.`,
+		`0 means the rubric is not met at all; ${scale} means it is fully met. Judge only against the rubric.`,
+	)
+
+	return sections.join('\n')
+}
+
+export function judgeScorer(config: JudgeScorerConfig): Scorer {
+	const scale = config.scale ?? DEFAULT_SCALE
+	if (!Number.isInteger(scale) || scale < 1) {
+		throw new Error(`judgeScorer: scale must be a positive integer, got ${scale}`)
+	}
+	if (config.rubric.trim().length === 0) {
+		throw new Error(
+			'judgeScorer: a rubric is required. A judge with no rubric rates fluency, which is not what anyone means to measure.',
+		)
+	}
+
+	return {
+		name: config.name ?? 'judge',
+		async score(run: EvalRun, evalCase: EvalCase): Promise<Score> {
+			const response = await collect(
+				config.provider.chatStream({
+					model: config.model,
+					messages: [{ role: 'user', content: buildPrompt(config, run, evalCase, scale) }],
+					// The same run must grade the same way twice, or a
+					// regression cannot be told from sampling noise.
+					temperature: 0,
+					maxTokens: 512,
+				}),
+			)
+
+			const verdict = parseVerdict(response.message.content ?? '', scale)
+			return {
+				score: verdict.grade / scale,
+				reason: verdict.reason,
+				details: {
+					grade: verdict.grade,
+					scale,
+					// Carried so a suite can account for what the judging
+					// itself cost. A judge is the most expensive scorer there
+					// is, and a bill nobody can attribute is a bill nobody
+					// controls.
+					judgeTokens: response.usage.totalTokens,
+				},
+			}
+		},
+	}
+}

package/src/eval/scorers.ts

@@ -0,0 +1,169 @@
+import type { EvalCase, EvalRun, Score, Scorer } from './types.js'
+
+/**
+ * Longest common subsequence length between two tool sequences.
+ *
+ * Subsequence, not set intersection: order carries meaning in a
+ * trajectory. Reading a file before editing it is not the same run as
+ * editing then reading, and a set-based score cannot tell them apart.
+ */
+function lcsLength(a: readonly string[], b: readonly string[]): number {
+	const table: number[][] = Array.from({ length: a.length + 1 }, () =>
+		new Array(b.length + 1).fill(0),
+	)
+	for (let i = 1; i <= a.length; i++) {
+		for (let j = 1; j <= b.length; j++) {
+			const row = table[i]
+			const prev = table[i - 1]
+			if (!row || !prev) continue
+			row[j] =
+				a[i - 1] === b[j - 1] ? (prev[j - 1] ?? 0) + 1 : Math.max(prev[j] ?? 0, row[j - 1] ?? 0)
+		}
+	}
+	return table[a.length]?.[b.length] ?? 0
+}
+
+/**
+ * How closely the run's tool sequence matched the expected one, as F1 over
+ * the longest common subsequence.
+ *
+ * Trajectory, not final answer. Namzu's most load-bearing behavior is
+ * tuned by constants nobody could measure — `search_tools` activates the
+ * top 5 deferred tools, compaction fires at 0.7, six state lists cap at
+ * 25 — and changing any of them, or a tool description, or the
+ * deferred-tools prompt block, could silently make the agent take four
+ * tool calls where it took one. Final-answer scoring cannot see that;
+ * this can.
+ *
+ * Extra calls cut precision, missing calls cut recall, so a run that does
+ * the right thing wastefully and a run that skips a step score
+ * differently — which is the distinction a final-answer score collapses.
+ */
+export function trajectoryScorer(): Scorer {
+	return {
+		name: 'trajectory',
+		score(run: EvalRun, evalCase: EvalCase): Score {
+			const expected = evalCase.expectedTools ?? []
+			const actual = run.toolCalls
+
+			if (expected.length === 0 && actual.length === 0) {
+				return { score: 1, reason: 'no tools expected and none called' }
+			}
+			if (expected.length === 0) {
+				return {
+					score: 0,
+					reason: `expected no tool calls, got ${actual.length}: ${actual.join(' → ')}`,
+					details: { actual },
+				}
+			}
+			if (actual.length === 0) {
+				return {
+					score: 0,
+					reason: `expected ${expected.join(' → ')}, but no tools were called`,
+					details: { expected },
+				}
+			}
+
+			const matched = lcsLength(expected, actual)
+			const precision = matched / actual.length
+			const recall = matched / expected.length
+			const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall)
+
+			return {
+				score: f1,
+				reason:
+					f1 === 1
+						? `exact trajectory: ${actual.join(' → ')}`
+						: `matched ${matched}/${expected.length} in order; expected ${expected.join(' → ')}, got ${actual.join(' → ')}`,
+				details: { precision, recall, matched, expected, actual },
+			}
+		},
+	}
+}
+
+/** The run settled cleanly rather than erroring or being cut off. */
+export function completionScorer(
+	acceptable: readonly string[] = ['end_turn', 'stop_condition'],
+): Scorer {
+	return {
+		name: 'completion',
+		// Binary by construction: the run either settled cleanly or it did
+		// not, so there is no score between 0 and 1 for a mean to soften.
+		// Averaged in, a hard 0 here is carried by three good fuzzy scores.
+		severity: 'gate',
+		score(run: EvalRun): Score {
+			if (run.error) {
+				return { score: 0, reason: `run failed: ${run.error}` }
+			}
+			const ok = run.stopReason !== undefined && acceptable.includes(run.stopReason)
+			return {
+				score: ok ? 1 : 0,
+				reason: ok
+					? `settled as ${run.stopReason}`
+					: `settled as ${run.stopReason ?? 'unknown'}, expected one of ${acceptable.join(', ')}`,
+				details: { stopReason: run.stopReason },
+			}
+		},
+	}
+}
+
+/**
+ * The run stayed within a step budget.
+ *
+ * A regression that makes the agent take four turns where it took one is
+ * invisible to correctness scoring and very visible on the bill.
+ */
+export function stepBudgetScorer(maxSteps: number): Scorer {
+	return {
+		name: 'step-budget',
+		score(run: EvalRun): Score {
+			const used = run.steps.length
+			return {
+				score: used <= maxSteps ? 1 : Math.max(0, maxSteps / used),
+				reason:
+					used <= maxSteps
+						? `${used}/${maxSteps} steps`
+						: `over budget: ${used} steps against a ${maxSteps} allowance`,
+				details: { used, maxSteps },
+			}
+		},
+	}
+}
+
+/** The final text contains every required substring. */
+export function containsScorer(...required: string[]): Scorer {
+	return {
+		name: 'contains',
+		// A required fragment is required. Partial credit here reads as
+		// "most of the answer was there", which is not what was asked.
+		severity: 'gate',
+		score(run: EvalRun): Score {
+			const text = run.output ?? ''
+			const missing = required.filter((r) => !text.includes(r))
+			return {
+				score: required.length === 0 ? 1 : (required.length - missing.length) / required.length,
+				reason:
+					missing.length === 0
+						? `found all ${required.length} required fragments`
+						: `missing: ${missing.join(', ')}`,
+				details: { missing },
+			}
+		},
+	}
+}
+
+/**
+ * Judge the run with a caller-supplied predicate.
+ *
+ * The escape hatch for anything the built-in scorers do not cover. For
+ * a model-graded judge, reach for `judgeScorer` first: it handles the
+ * parts that are easy to get wrong — a rubric it refuses to run without,
+ * an ordinal scale rather than a float, disclosed truncation, and a
+ * failed call that reports as unjudged rather than as a zero.
+ */
+export function customScorer(
+	name: string,
+	fn: (run: EvalRun, evalCase: EvalCase) => Score | Promise<Score>,
+): Scorer {
+	return { name, score: fn }
+}

package/src/eval/types.ts

@@ -0,0 +1,157 @@
+import type { StepResult } from '../types/run/step.js'
+
+/**
+ * One case in a dataset: an input, and what a good run looks like.
+ *
+ * `TInput` is the agent's own input type, so a dataset is checked against
+ * the thing it tests rather than against `unknown`.
+ */
+export interface EvalCase<TInput = unknown> {
+	name: string
+	input: TInput
+	/** Free-form expectation, consumed by whichever scorers you attach. */
+	expected?: unknown
+	/**
+	 * Tool names the run is expected to call, in order. Consumed by
+	 * `trajectoryScorer`.
+	 */
+	expectedTools?: readonly string[]
+	/** Per-case scorer overrides; falls back to the dataset's scorers. */
+	scorers?: readonly Scorer[]
+}
+
+/** What actually happened, handed to every scorer. */
+export interface EvalRun {
+	/** Final assistant text, when the run produced one. */
+	output: string | null
+	/** Schema-validated final output, when `structuredOutput` was requested. */
+	structuredOutput?: unknown
+	steps: readonly StepResult[]
+	/** Tool names in call order, flattened across steps. */
+	toolCalls: readonly string[]
+	stopReason?: string
+	totalTokens: number
+	totalCostUsd: number
+	durationMs: number
+	/** Set when the run threw rather than settling. */
+	error?: string
+}
+
+/**
+ * A judgement about one run.
+ *
+ * `reason` is REQUIRED, not optional. A bare number tells you a run got
+ * worse without telling you how, which is exactly when you need to know —
+ * so every scorer has to be able to explain itself, and one that cannot
+ * is a scorer that should not exist.
+ */
+export interface Score {
+	/** 0..1. Higher is better. Ignored when `unavailable` is set. */
+	score: number
+	reason: string
+	/** Anything the scorer wants to surface (counts, diffs, matched items). */
+	details?: Record<string, unknown>
+	/**
+	 * The scorer could not produce a judgement — it threw, its provider was
+	 * unreachable, its verdict did not parse.
+	 *
+	 * A failed measurement is not a measurement of zero. Scoring it zero
+	 * says "the run was bad" when the truth is "we do not know", and the
+	 * two demand opposite responses: one is a regression to chase, the
+	 * other is a broken harness to fix. This mattered little while every
+	 * scorer was a pure function — a throw there is a bug — and matters a
+	 * great deal now that a scorer can be a network call, where one 429
+	 * would otherwise turn a green suite red and send somebody hunting a
+	 * regression that never happened.
+	 *
+	 * An unavailable score is excluded from the case mean's numerator AND
+	 * its denominator, and the case it belongs to is reported as
+	 * inconclusive rather than passed or failed.
+	 */
+	unavailable?: boolean
+}
+
+/**
+ * How a scorer's verdict affects the case's.
+ *
+ * A single unweighted mean over every scorer, compared to one suite-wide
+ * threshold, made the two halves of the design fight each other. At the
+ * default threshold of 1 the harness never reports a false pass — but a
+ * trajectory F1 and a graded judge can essentially never reach 1, so every
+ * real suite lowers it. And every step down buys tolerance for the fuzzy
+ * scorers by buying the same tolerance for the deterministic ones: at 0.75,
+ * a trajectory score of 0 alongside three perfect scores averages to 0.75
+ * and reports **passed**. The exact regression the harness exists to catch
+ * comes back green.
+ *
+ *  - `gate` — a miss fails the case outright, whatever the mean says.
+ *    For the deterministic checks: it called the wrong tools, it did not
+ *    finish, the required phrase is absent.
+ *  - `soft` — contributes to the mean only. For the fuzzy ones, where a
+ *    number below 1 is normal and a threshold is a judgement call.
+ *
+ * Defaults to `soft`, so a suite that sets nothing behaves exactly as
+ * before.
+ */
+export type ScorerSeverity = 'gate' | 'soft'
+
+export interface Scorer {
+	name: string
+	score(run: EvalRun, evalCase: EvalCase): Score | Promise<Score>
+	/** See {@link ScorerSeverity}. Default `soft`. */
+	severity?: ScorerSeverity
+	/**
+	 * The score this scorer must reach. Defaults to the experiment's
+	 * `passThreshold`.
+	 *
+	 * Per-scorer because "good enough" is not one number across dimensions:
+	 * a trajectory match at 0.8 may be fine while a completion check at 0.8
+	 * is meaningless — it either finished or it did not.
+	 */
+	threshold?: number
+}
+
+/**
+ * `inconclusive` is a first-class outcome: every scorer that ran was
+ * unavailable, so there is no evidence either way. Folding it into
+ * `failed` invents a regression and folding it into `passed` invents
+ * evidence; both are worse than saying nothing was measured.
+ */
+export type CaseStatus = 'passed' | 'failed' | 'inconclusive'
+
+export interface CaseResult {
+	case: string
+	run: EvalRun
+	scores: Record<string, Score>
+	/** Mean of this case's AVAILABLE scores. Zero when none were. */
+	mean: number
+	status: CaseStatus
+	/** True only for `status: 'passed'`. */
+	passed: boolean
+	/**
+	 * Gate scorers that missed their threshold, by name.
+	 *
+	 * Named rather than counted: a case reported failed with a mean of 0.75
+	 * otherwise sends somebody to read four scores and guess which one
+	 * mattered. Absent when no gate missed.
+	 */
+	failedGates?: readonly string[]
+}
+
+export interface ExperimentReport {
+	name: string
+	cases: readonly CaseResult[]
+	/** Mean score across the cases that produced one. */
+	mean: number
+	passed: number
+	failed: number
+	/** Cases where no scorer could produce a judgement. */
+	inconclusive: number
+	/**
+	 * Per-scorer means over the cases each one actually judged, for
+	 * spotting which dimension regressed. A scorer that was unavailable on
+	 * some cases is averaged over the rest rather than dragged toward zero.
+	 */
+	byScorer: Record<string, number>
+	durationMs: number
+}

package/src/execution/local.test.ts

@@ -0,0 +1,46 @@
+import { existsSync, mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { LocalExecutionContext } from './local.js'
+
+// `command`/`args` on executeCommand() come from whatever tool call invokes it, so they
+// must be treated as untrusted. A single arg that smuggles a shell separator + a second
+// command proves whether the child process was actually handed a shell.
+function injectionArg(canaryPath: string): string {
+	return process.platform === 'win32'
+		? `& echo x> ${canaryPath} & rem`
+		: `; echo x > ${canaryPath} #`
+}
+
+describe('LocalExecutionContext.executeCommand', () => {
+	let dir: string
+	let canary: string
+
+	beforeEach(() => {
+		dir = mkdtempSync(join(tmpdir(), 'namzu-local-exec-'))
+		canary = join(dir, 'canary.txt')
+	})
+
+	afterEach(() => {
+		rmSync(dir, { recursive: true, force: true })
+	})
+
+	it('does not let a shell-metacharacter arg run a second command by default', async () => {
+		const ctx = new LocalExecutionContext({ id: 'test', cwd: dir })
+		await ctx.initialize()
+
+		await ctx.executeCommand('node', ['--version', injectionArg(canary)])
+
+		expect(existsSync(canary)).toBe(false)
+	})
+
+	it('still runs through a shell when explicitly requested', async () => {
+		const ctx = new LocalExecutionContext({ id: 'test', cwd: dir })
+		await ctx.initialize()
+
+		await ctx.executeCommand('node', ['--version', injectionArg(canary)], { shell: true })
+
+		expect(existsSync(canary)).toBe(true)
+	})
+})

package/src/execution/local.ts

@@ -102,7 +102,12 @@ export class LocalExecutionContext extends BaseExecutionContext implements Comma
 		const cwd = options?.cwd ? resolve(this.cwd, options.cwd) : this.cwd
 		const env = { ...process.env, ...this.envVars, ...options?.env }
 		const timeoutMs = options?.timeoutMs ?? 30_000
-		const shell = options?.shell ?? this.shell ?? true
+		// node's spawn() only keeps argv separation when it execs the binary directly;
+		// with shell enabled it re-joins command+args into one unescaped string and hands
+		// it to sh/cmd.exe, so any caller-supplied arg becomes shell syntax rather than a
+		// literal value. Since `command`/`args` here are whatever a tool call passed in,
+		// shell interpretation must be an explicit opt-in, never the silent default.
+		const shell = options?.shell ?? this.shell ?? false
 
 		return new Promise<CommandResult>((resolvePromise) => {
 			const start = performance.now()