@namzu/sdk 2.0.0 → 3.0.0

package/src/constants/continuation.ts

@@ -1,9 +1,9 @@
 /**
  * Synthetic user prompt injected by the iteration loop when a turn
- * ends with `stop_reason: max_tokens` AND no tool_use. Mirrors
- * Claude.ai's "Continue" affordance: the loop pushes this message
- * back into the conversation and fires another iteration, letting
- * the model pick up where it was cut off.
+ * ends with `stop_reason: max_tokens` AND no tool_use. A turn cut off
+ * at the output ceiling is not a finished answer: the loop pushes this
+ * message back into the conversation and fires another iteration,
+ * letting the model pick up where it was cut off.
  *
  * The exact string is the marker used by `resolveResult` (in
  * `manager/run/persistence.ts`) to detect auto-continuation

package/src/constants/mcp/index.ts

@@ -1 +1,35 @@
+/**
+ * The protocol version this client ADVERTISES in `initialize`.
+ *
+ * Deliberately the version namzu actually implements, not the newest one
+ * published. Advertising a version whose requirements are unimplemented is
+ * worse than advertising an older one honestly: the server tailors its
+ * behavior to what the client claims, and the mismatch surfaces later as
+ * a malformed exchange rather than a clean negotiation failure. Raising
+ * this is a conformance task, not a constant edit.
+ */
 export const MCP_PROTOCOL_VERSION = '2024-11-05'
+
+/**
+ * Versions this client can speak if a server negotiates down (or up) to
+ * one of them, newest first.
+ *
+ * A server is free to answer `initialize` with a version other than the one
+ * the client asked for — that is how the handshake is specified. namzu used
+ * to ignore the answer entirely and carry on regardless, so a server
+ * responding with a version we cannot speak looked exactly like a
+ * successful connection until something downstream broke oddly.
+ */
+export const MCP_SUPPORTED_PROTOCOL_VERSIONS: readonly string[] = ['2024-11-05']
+
+/**
+ * Default deadline for one MCP JSON-RPC round trip.
+ *
+ * Generous enough for a server that shells out or hits a network API,
+ * short enough that an unresponsive one surfaces as an error the model
+ * can react to rather than a hang.
+ */
+export const DEFAULT_MCP_REQUEST_TIMEOUT_MS = 30_000
+
+/** JSON-RPC 2.0 reserved code for an unimplemented method. */
+export const JSON_RPC_METHOD_NOT_FOUND = -32601

package/src/constants/plugin/index.ts

@@ -5,8 +5,21 @@ export const USER_PLUGIN_DIR = '.namzu/plugins'
 /** Manifest file name */
 export const PLUGIN_MANIFEST_FILENAME = 'plugin.json'
 
-/** Namespacing separator between plugin name and component name */
-export const PLUGIN_NAMESPACE_SEPARATOR = ':'
+/**
+ * Namespacing separator between plugin name and component name.
+ *
+ * `__`, not `:`. A tool name reaches the provider verbatim, and the major
+ * message APIs accept `[a-zA-Z0-9_-]` only — so a colon made EVERY
+ * plugin-contributed tool name illegal, unconditionally. Those tools are
+ * registered deferred, so the rejection fired the moment one was activated
+ * and took down the whole request rather than that one tool, with nothing
+ * naming the culprit.
+ *
+ * Double underscore rather than single so the boundary stays legible in a
+ * name that already contains underscores, and it matches the separator the
+ * remote-tool bridge next door already uses.
+ */
+export const PLUGIN_NAMESPACE_SEPARATOR = '__'
 
 /** Plugin name validation */
 export const PLUGIN_NAME_MAX_LENGTH = 64
@@ -22,4 +35,13 @@ export const MAX_PERSONAS_PER_PLUGIN = 5
 
 /** Hook execution */
 export const HOOK_TIMEOUT_MS = 5_000
+
+/**
+ * Priority a hook gets when it does not declare one.
+ *
+ * Mid-scale on purpose: a guard can sort itself decisively ahead of every
+ * existing hook without having to know what they chose, and an observer
+ * can sort itself after.
+ */
+export const DEFAULT_HOOK_PRIORITY = 100
 export const HOOK_MAX_CONCURRENT = 10

package/src/constants/sandbox/index.ts

@@ -34,8 +34,10 @@ export const SANDBOX_SAFE_ENV_KEYS = new Set([
 // ContainerSandboxLayout default container paths
 // ---------------------------------------------------------------------------
 //
-// Mirrors the taxonomy Anthropic's container architecture exposes to
-// the model (Claude container blueprint, Code Interpreter, "skills").
+// A container needs one place the user will see and several the user
+// will not, and the difference has to be legible to the model from the
+// path alone — otherwise everything lands in the output bind and the
+// user reads the agent's scratch work as its answer.
 // Exported so prompt-template consumers can write
 // `Outputs go to ${SANDBOX_DEFAULT_OUTPUTS_PATH}` instead of
 // hard-coding the string in two places that drift.
@@ -49,9 +51,8 @@ export const SANDBOX_DEFAULT_UPLOADS_PATH = '/mnt/user-data/uploads'
 /**
  * Default container path for the agent's working/scratch space (RW).
  * Sibling mount to {@link SANDBOX_DEFAULT_OUTPUTS_PATH} — anything
- * written here is invisible to the output collector by design,
- * mirroring the Anthropic Cowork pattern (`/home/claude` scratch vs.
- * `/mnt/user-data/outputs` user-visible).
+ * written here is invisible to the output collector by design: an
+ * agent needs somewhere to think out loud that is not the deliverable.
  */
 export const SANDBOX_DEFAULT_SCRATCH_PATH = '/mnt/user-data/scratch'
 

package/src/constants/telemetry/index.ts

@@ -13,6 +13,14 @@ export const GENAI = {
 	USAGE_INPUT_TOKENS: 'gen_ai.usage.input_tokens',
 	USAGE_OUTPUT_TOKENS: 'gen_ai.usage.output_tokens',
 
+	/**
+	 * Which kind of token a usage measurement counts. The convention keys
+	 * one metric by this rather than minting a metric name per kind, so a
+	 * dashboard that sums the metric gets the whole picture instead of one
+	 * slice of it.
+	 */
+	TOKEN_TYPE: 'gen_ai.token.type',
+
 	TOOL_NAME: 'gen_ai.tool.name',
 	TOOL_TYPE: 'gen_ai.tool.type',
 	TOOL_CALL_ID: 'gen_ai.tool.call_id',

package/src/constants/tools/index.ts

@@ -32,3 +32,23 @@ export const DANGEROUS_PATTERNS = [
 ]
 
 export const FILESYSTEM_TOOLS = new Set(['glob', 'read', 'write', 'bash'])
+
+/**
+ * Subdirectory of a run directory holding tool output that exceeded the
+ * model-visible budget. Kept beside the run so it is cleaned up with the
+ * run and reachable by the agent through its own `read`/`grep`.
+ */
+export const TOOL_OUTPUT_DIR_NAME = 'tool-output'
+
+/**
+ * Turns the loop will spend asking again for a valid structured output
+ * before giving up with `stop_reason: 'structured_output_failed'`.
+ *
+ * Bounded on purpose: a model that cannot satisfy the schema should fail
+ * loudly and cheaply, not iterate against `maxIterations`.
+ */
+export const DEFAULT_STRUCTURED_OUTPUT_RETRIES = 2
+
+/** What the loop sends when the model tries to finish in prose instead. */
+export const STRUCTURED_OUTPUT_REPROMPT =
+	'[SYSTEM] This task requires a structured result. Call the `structured_output` tool with your final answer, matching its schema exactly. Do not answer in prose.'

package/src/contracts/api.ts

@@ -58,6 +58,13 @@ export type WireRunStatus =
 	| 'failed'
 	| 'cancelled'
 	| 'cancelling'
+	/**
+	 * The run's approval window closed with nobody answering.
+	 *
+	 * Set by a host sweeping expired parks — there is no domain status that
+	 * collapses onto it, because the kernel is suspended mid-iteration while
+	 * parked and never gets to observe its own deadline passing.
+	 */
 	| 'expired'
 
 // Wire-side rename of types/run/events.StopReason. Kept distinct so the HTTP
@@ -142,7 +149,33 @@ export type StreamEventType =
 	| 'run.resuming'
 	| 'iteration.started'
 	| 'iteration.completed'
+	/**
+	 * A compaction pass replaced a span of history with a summary. Wire-
+	 * visible because the operation is destructive: a client rendering the
+	 * transcript needs to know its middle was dropped, not infer it.
+	 */
+	| 'compaction.completed'
+	/** A guardrail refused or corrected the run. */
+	| 'guardrail.triggered'
+	/**
+	 * Extended-thinking lifecycle. Wire-visible because without it a
+	 * client renders a multi-second stall with no events while the model
+	 * is demonstrably working.
+	 */
+	| 'reasoning.started'
+	| 'reasoning.delta'
+	| 'reasoning.completed'
+	/**
+	 * A model call failed transiently and is being retried after a backoff.
+	 * Wire-visible on the same grounds as `tool.progress` and the reasoning
+	 * events: without it a client sees no event and no keepalive for the
+	 * whole backoff, so a run that is about to succeed is indistinguishable
+	 * from one that has hung.
+	 */
+	| 'provider.retry'
 	| 'tool.executing'
+	/** Ephemeral progress from a long-running tool. Not in the transcript. */
+	| 'tool.progress'
 	| 'tool.completed'
 	| 'tool.error'
 	// v3 tool input lifecycle (ses_001-tool-stream-events). Additive; phase 4
@@ -156,6 +189,13 @@ export type StreamEventType =
 	| 'message.created'
 	| 'message.delta'
 	| 'message.completed'
+	/**
+	 * A tool asked the user a question and the run is parked on it. Wire-
+	 * visible for the same reason `review.requested` is: a client cannot
+	 * render an approval card for something it never hears about.
+	 */
+	| 'question.asked'
+	| 'question.answered'
 	| 'review.requested'
 	| 'review.completed'
 	| 'checkpoint.created'

package/src/contracts/run-status.ts

@@ -0,0 +1,43 @@
+import type { RunStatus } from '../types/run/status.js'
+import type { WireRunStatus } from './api.js'
+
+/**
+ * Collapse a domain run status onto the wire enum.
+ *
+ * The mapping was documented on `WireRunStatus` and implemented nowhere, so
+ * every surface that needed it open-coded the collapse or, more often,
+ * passed a domain status where a wire one was expected and relied on the
+ * two enums happening to share spellings. They do not share all of them:
+ * `succeeded` and `awaiting_*` have no wire counterpart at all.
+ *
+ * Total over the domain enum, and exhaustive by construction so a new
+ * domain status is a type error here rather than an `undefined` on the
+ * wire.
+ */
+export function toWireRunStatus(status: RunStatus): WireRunStatus {
+	switch (status) {
+		case 'queued':
+			return 'queued'
+		case 'running':
+			return 'running'
+		// A run waiting on a human is still a live run to a wire consumer:
+		// it has not settled, and nothing it can do will make it settle.
+		// `awaiting_hitl_resolution` is the persisted variant — the user is
+		// absent and the run survives until answered or cancelled — which is
+		// waiting, not finished.
+		case 'awaiting_hitl':
+		case 'awaiting_hitl_resolution':
+		case 'awaiting_subsession':
+			return 'running'
+		case 'succeeded':
+			return 'completed'
+		case 'failed':
+			return 'failed'
+		case 'cancelled':
+			return 'cancelled'
+		default: {
+			const _exhaustive: never = status
+			throw new Error(`Unmapped run status: ${_exhaustive as string}`)
+		}
+	}
+}

package/src/eval/__tests__/case-deadline.test.ts

@@ -0,0 +1,147 @@
+import { describe, expect, it } from 'vitest'
+
+import { runExperiment } from '../experiment.js'
+import type { EvalRun } from '../types.js'
+
+/**
+ * `executeCase` was a bare await, so a `run` closure that never settled
+ * blocked its worker and `runExperiment` never returned — no report, no
+ * partial results, nothing to read.
+ *
+ * Narrower than it sounds: the documented path inherits three deadlines
+ * from the runtime it drives. The residual is a closure that does not go
+ * through `query()`, and a mid-iteration provider stall the
+ * between-iterations guard cannot see. Both are reachable, and neither is
+ * the suite's to absorb silently.
+ */
+
+const emptyRun = (): EvalRun => ({
+	output: 'ok',
+	steps: [],
+	toolCalls: [],
+	totalTokens: 0,
+	totalCostUsd: 0,
+	durationMs: 0,
+})
+
+const never = () => new Promise<EvalRun>(() => {})
+
+const scorer = {
+	name: 'always-one',
+	score: async () => ({ score: 1, reason: 'fine' }),
+}
+
+describe('a case that never settles', () => {
+	it('is reported instead of hanging the suite', async () => {
+		const report = await runExperiment({
+			name: 'deadline',
+			cases: [{ name: 'hangs', input: null }],
+			scorers: [scorer],
+			timeoutMs: 20,
+			run: never,
+		})
+
+		expect(report.cases).toHaveLength(1)
+		expect(report.cases[0]?.run.error).toMatch(/timed out after 20ms/)
+	})
+
+	it('does not take the rest of the suite with it', async () => {
+		// Forty cases should not be lost to one that hung — the same rule
+		// the harness already applied to a case that threw.
+		const report = await runExperiment({
+			name: 'deadline',
+			cases: [
+				{ name: 'hangs', input: null },
+				{ name: 'fine', input: null },
+			],
+			scorers: [scorer],
+			timeoutMs: 20,
+			run: async (_input, evalCase) => (evalCase.name === 'hangs' ? await never() : emptyRun()),
+		})
+
+		expect(report.cases.map((c) => c.case).sort()).toEqual(['fine', 'hangs'])
+		expect(report.passed).toBe(1)
+		expect(report.failed).toBe(1)
+	})
+
+	it('reports the time it actually burned, not zero', async () => {
+		// A case that spent its whole deadline is the most interesting
+		// number in the report; zero would hide it.
+		const report = await runExperiment({
+			name: 'deadline',
+			cases: [{ name: 'hangs', input: null }],
+			scorers: [scorer],
+			timeoutMs: 30,
+			run: never,
+		})
+
+		expect(report.cases[0]?.run.durationMs).toBeGreaterThanOrEqual(20)
+	})
+})
+
+describe('the signal handed to run', () => {
+	it('fires when the deadline passes, so a cooperating closure can stop', async () => {
+		let aborted = false
+		await runExperiment({
+			name: 'deadline',
+			cases: [{ name: 'watches', input: null }],
+			scorers: [scorer],
+			timeoutMs: 20,
+			run: async (_input, _case, signal) => {
+				signal.addEventListener('abort', () => {
+					aborted = true
+				})
+				return await never()
+			},
+		})
+
+		expect(aborted).toBe(true)
+	})
+
+	it('stays quiet for a case that finished in time', async () => {
+		// A spurious abort after the work is done would train a closure to
+		// ignore the signal entirely.
+		let aborted = false
+		await runExperiment({
+			name: 'deadline',
+			cases: [{ name: 'fast', input: null }],
+			scorers: [scorer],
+			timeoutMs: 5_000,
+			run: async (_input, _case, signal) => {
+				signal.addEventListener('abort', () => {
+					aborted = true
+				})
+				return emptyRun()
+			},
+		})
+
+		expect(aborted).toBe(false)
+	})
+})
+
+describe('a suite with no deadline set', () => {
+	it('behaves exactly as before', async () => {
+		const report = await runExperiment({
+			name: 'no-deadline',
+			cases: [{ name: 'fine', input: null }],
+			scorers: [scorer],
+			run: async () => emptyRun(),
+		})
+
+		expect(report.passed).toBe(1)
+		expect(report.cases[0]?.run.error).toBeUndefined()
+	})
+
+	it('still turns a throw into a result rather than a crash', async () => {
+		const report = await runExperiment({
+			name: 'no-deadline',
+			cases: [{ name: 'throws', input: null }],
+			scorers: [scorer],
+			run: async () => {
+				throw new Error('broken case')
+			},
+		})
+
+		expect(report.cases[0]?.run.error).toBe('broken case')
+	})
+})

package/src/eval/__tests__/end-to-end.test.ts

@@ -0,0 +1,259 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { MockLLMProvider } from '../../provider/mock.js'
+import { ToolExecutor } from '../../runtime/query/executor.js'
+import { IterationOrchestrator } from '../../runtime/query/iteration/index.js'
+import { ActivityStore } from '../../store/activity/memory.js'
+import type { RunId } from '../../types/ids/index.js'
+import type { Message } from '../../types/message/index.js'
+import type { Run } from '../../types/run/entity.js'
+import type { RunEvent } from '../../types/run/index.js'
+import { hasToolCall } from '../../types/run/step.js'
+import type { ToolRegistryContract } from '../../types/tool/index.js'
+import type { Logger } from '../../utils/logger.js'
+import { runExperiment } from '../experiment.js'
+import { evalRunFromRun } from '../from-run.js'
+import { completionScorer, trajectoryScorer } from '../scorers.js'
+import type { EvalCase } from '../types.js'
+
+/**
+ * The harness against the REAL loop, driven by the scriptable mock.
+ *
+ * The unit tests above check the scorers in isolation; this checks the
+ * thing that actually matters — that a behavior change in the agent shows
+ * up as a score drop. Every piece this depends on had to land first:
+ * `Run.steps` (otherwise a trajectory scorer would correlate raw events by
+ * iteration number) and a mock that can emit tool calls (otherwise the
+ * loop cannot be driven at all).
+ */
+
+const RUN_ID = 'run_eval' as RunId
+
+function makeLogger(): Logger {
+	const stub = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }
+	return { ...stub, child: vi.fn(() => ({ ...stub, child: vi.fn() })) } as unknown as Logger
+}
+
+/** Drive the loop with a scripted model and return the finished `Run`. */
+async function driveAgent(turns: unknown[]): Promise<Run> {
+	const messages: Message[] = []
+	let iteration = 0
+	let stopReason: string | undefined
+	const log = makeLogger()
+
+	const tools = {
+		get: vi.fn((name: string) => ({
+			name,
+			isConcurrencySafe: () => true,
+			isReadOnly: () => true,
+			isDestructive: () => false,
+		})),
+		execute: vi.fn(async (name: string) => ({ success: true, output: `${name} ok` })),
+		has: vi.fn(() => true),
+		listNames: vi.fn(() => []),
+		getAvailability: vi.fn(() => 'active'),
+		toLLMTools: vi.fn(() => []),
+		register: vi.fn(),
+		unregister: vi.fn(),
+	} as unknown as ToolRegistryContract
+
+	const activityStore = new ActivityStore(RUN_ID, {
+		enabled: false,
+		trackToolCalls: false,
+		trackLlmTurns: false,
+	})
+
+	const runMgr = {
+		id: RUN_ID,
+		messages,
+		tokenUsage: {
+			promptTokens: 0,
+			completionTokens: 0,
+			totalTokens: 0,
+			cachedTokens: 0,
+			cacheWriteTokens: 0,
+		},
+		costInfo: { inputCostPer1M: 0, outputCostPer1M: 0, totalCost: 0, cacheDiscount: 0 },
+		get currentIteration() {
+			return iteration
+		},
+		incrementIteration: () => ++iteration,
+		pushMessage: (m: Message) => {
+			messages.push(m)
+		},
+		recordTurnUsage: vi.fn(),
+		accumulateUsage: vi.fn(),
+		clearLastPromptTokens: vi.fn(),
+		lastPromptTokens: undefined,
+		setStopReason: (r: string) => {
+			stopReason = r
+		},
+		setStructuredOutput: vi.fn(),
+		markCancelled: vi.fn(),
+	}
+
+	const orchestrator = new IterationOrchestrator({
+		provider: new MockLLMProvider({ turns: turns as never }),
+		runConfig: { model: 'mock', maxIterations: 10 },
+		tools,
+		runMgr,
+		toolExecutor: new ToolExecutor(
+			{
+				tools,
+				runId: RUN_ID,
+				workingDirectory: '/tmp',
+				permissionMode: 'auto',
+				env: {},
+				abortSignal: new AbortController().signal,
+			},
+			activityStore,
+			async () => {},
+			log,
+		),
+		activityStore,
+		abortController: new AbortController(),
+		log,
+		emitEvent: async () => {},
+		drainPending: function* (): Generator<RunEvent> {},
+		checkpointMgr: { create: async () => ({ id: 'cp_1' }) },
+		planManager: { active: undefined },
+		resumeHandler: async () => ({ action: 'approve_tools' }),
+		stopWhen: hasToolCall('finish'),
+		guard: {
+			beforeIteration: () => ({
+				shouldStop: iteration >= 10,
+				forceFinalize: false,
+				isCancelled: false,
+				stopReason: 'max_iterations',
+			}),
+		},
+	} as never)
+
+	const gen = orchestrator.runLoop()
+	let next = await gen.next()
+	while (!next.done) next = await gen.next()
+
+	return {
+		id: RUN_ID,
+		status: 'completed',
+		messages,
+		tokenUsage: runMgr.tokenUsage,
+		costInfo: runMgr.costInfo,
+		currentIteration: iteration,
+		startedAt: Date.now() - 10,
+		endedAt: Date.now(),
+		steps: orchestrator.getSteps(),
+		...(stopReason ? { stopReason } : {}),
+		result: 'done',
+	} as unknown as Run
+}
+
+/** The trajectory a healthy agent takes for this task. */
+const GOLDEN: EvalCase[] = [
+	{
+		name: 'reads before editing, then finishes',
+		input: 'edit the file',
+		expectedTools: ['read', 'edit', 'finish'],
+	},
+]
+
+const HEALTHY = [
+	{ toolCalls: [{ name: 'read' }] },
+	{ toolCalls: [{ name: 'edit' }] },
+	{ toolCalls: [{ name: 'finish' }] },
+]
+
+describe('the harness scores a real run', () => {
+	it('projects a finished Run into the shape scorers consume', async () => {
+		const run = await driveAgent(HEALTHY)
+		const projected = evalRunFromRun(run)
+
+		expect(projected.toolCalls).toEqual(['read', 'edit', 'finish'])
+		expect(projected.steps).toHaveLength(3)
+		expect(projected.stopReason).toBe('stop_condition')
+	})
+
+	it('scores the golden trajectory 1', async () => {
+		const report = await runExperiment({
+			name: 'file-editing',
+			cases: GOLDEN,
+			scorers: [trajectoryScorer(), completionScorer(['stop_condition'])],
+			run: async () => evalRunFromRun(await driveAgent(HEALTHY)),
+		})
+
+		expect(report.mean).toBe(1)
+		expect(report.failed).toBe(0)
+	})
+})
+
+describe('the harness catches a behavior regression', () => {
+	it('drops the score when the agent skips a step', async () => {
+		// The regression this is for: a tool-description change that makes the
+		// agent edit without reading first. Final-answer scoring cannot see
+		// it — the file still gets edited.
+		const regressed = [{ toolCalls: [{ name: 'edit' }] }, { toolCalls: [{ name: 'finish' }] }]
+
+		const report = await runExperiment({
+			name: 'file-editing',
+			cases: GOLDEN,
+			scorers: [trajectoryScorer()],
+			run: async () => evalRunFromRun(await driveAgent(regressed)),
+		})
+
+		expect(report.mean).toBeLessThan(1)
+		expect(report.failed).toBe(1)
+		expect(report.cases[0]?.scores.trajectory?.reason).toContain('read')
+	})
+
+	it('drops the score when the agent becomes wasteful', async () => {
+		// The regression this is for: a `search_tools` top-k change that makes
+		// the agent probe before acting. The answer is identical; the bill is
+		// not.
+		const wasteful = [
+			{ toolCalls: [{ name: 'ls' }] },
+			{ toolCalls: [{ name: 'glob' }] },
+			{ toolCalls: [{ name: 'read' }] },
+			{ toolCalls: [{ name: 'edit' }] },
+			{ toolCalls: [{ name: 'finish' }] },
+		]
+
+		const report = await runExperiment({
+			name: 'file-editing',
+			cases: GOLDEN,
+			scorers: [trajectoryScorer()],
+			run: async () => evalRunFromRun(await driveAgent(wasteful)),
+		})
+
+		expect(report.mean).toBeLessThan(1)
+		// Recall is intact — it did everything asked. Precision is what fell.
+		expect(report.cases[0]?.scores.trajectory?.details?.recall).toBe(1)
+		expect(report.cases[0]?.scores.trajectory?.details?.precision).toBeLessThan(1)
+	})
+
+	it('separates the two failures, which a pass/fail assertion would not', async () => {
+		const score = async (turns: unknown[]) => {
+			const report = await runExperiment({
+				name: 'x',
+				cases: GOLDEN,
+				scorers: [trajectoryScorer()],
+				run: async () => evalRunFromRun(await driveAgent(turns)),
+			})
+			return report.mean
+		}
+
+		const skipped = await score([
+			{ toolCalls: [{ name: 'edit' }] },
+			{ toolCalls: [{ name: 'finish' }] },
+		])
+		const wasteful = await score([
+			{ toolCalls: [{ name: 'ls' }] },
+			{ toolCalls: [{ name: 'read' }] },
+			{ toolCalls: [{ name: 'edit' }] },
+			{ toolCalls: [{ name: 'finish' }] },
+		])
+
+		expect(skipped).not.toBe(wasteful)
+		// Skipping required work is worse than doing it inefficiently.
+		expect(skipped).toBeLessThan(wasteful)
+	})
+})