@namzu/sdk 2.0.0 → 3.0.0

package/src/__tests__/default-path-conformance.test.ts

@@ -0,0 +1,306 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { findDanglingMessages } from '../compaction/dangling.js'
+import { MockLLMProvider } from '../provider/mock.js'
+import { ToolExecutor } from '../runtime/query/executor.js'
+import { IterationOrchestrator } from '../runtime/query/iteration/index.js'
+import { ActivityStore } from '../store/activity/memory.js'
+import type { HITLResumeDecision } from '../types/hitl/index.js'
+import type { RunId } from '../types/ids/index.js'
+import type { Message } from '../types/message/index.js'
+import type { RunEvent } from '../types/run/index.js'
+import type { ToolRegistryContract } from '../types/tool/index.js'
+import type { VerificationGateConfig } from '../types/verification/index.js'
+import type { Logger } from '../utils/logger.js'
+import { VerificationGate } from '../verification/gate.js'
+
+/**
+ * The loop driven the way the SHIPPED CLI drives it, with a human who says
+ * no.
+ *
+ * This is the test that was missing. Every P0 in the hardening audit was a
+ * defect on the default path, and every one of them was invisible because
+ * the existing tests configured their way around it — the SDK default
+ * `autoApproveHandler` means CI never once saw a rejection, so "every
+ * decline kills the run" shipped and stayed shipped. The CLI wires a real
+ * permission prompt straight onto `reject_tools`, so a user declining a
+ * tool was the first thing to hit it.
+ *
+ * The assertion is deliberately end-state rather than mechanism: after the
+ * turn, is the conversation still something a provider would accept?
+ */
+
+const RUN_ID = 'run_conformance' as RunId
+
+/** The CLI's gate, verbatim in shape: read-only allowed, dangerous denied. */
+const CLI_GATE: VerificationGateConfig = {
+	enabled: true,
+	rules: [],
+	allowReadOnlyTools: true,
+	denyDangerousPatterns: true,
+	logDecisions: false,
+}
+
+function makeLogger(): Logger {
+	const stub = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }
+	return { ...stub, child: vi.fn(() => ({ ...stub, child: vi.fn() })) } as unknown as Logger
+}
+
+function harness(opts: { decision: HITLResumeDecision; turns: unknown[] }) {
+	const messages: Message[] = []
+	const executed: string[] = []
+	let iteration = 0
+	let stopReason: string | undefined
+	const log = makeLogger()
+
+	const defs: Record<string, boolean> = { read: true, bash: false, ls: true }
+
+	const tools = {
+		get: vi.fn((name: string) => ({
+			name,
+			category: defs[name] ? 'filesystem' : 'shell',
+			isReadOnly: () => defs[name] === true,
+			isDestructive: () => defs[name] !== true,
+			isConcurrencySafe: () => defs[name] === true,
+		})),
+		execute: vi.fn(async (name: string) => {
+			executed.push(name)
+			return { success: true, output: `${name} ok` }
+		}),
+		has: vi.fn(() => true),
+		listNames: vi.fn(() => Object.keys(defs)),
+		getAvailability: vi.fn(() => 'active'),
+		toLLMTools: vi.fn(() => []),
+		register: vi.fn(),
+		unregister: vi.fn(),
+	} as unknown as ToolRegistryContract
+
+	const activityStore = new ActivityStore(RUN_ID, {
+		enabled: false,
+		trackToolCalls: false,
+		trackLlmTurns: false,
+	})
+
+	const runMgr = {
+		id: RUN_ID,
+		messages,
+		tokenUsage: {
+			promptTokens: 0,
+			completionTokens: 0,
+			totalTokens: 0,
+			cachedTokens: 0,
+			cacheWriteTokens: 0,
+		},
+		costInfo: { inputCostPer1M: 0, outputCostPer1M: 0, totalCost: 0, cacheDiscount: 0 },
+		get currentIteration() {
+			return iteration
+		},
+		incrementIteration: () => ++iteration,
+		pushMessage: (m: Message) => {
+			messages.push(m)
+		},
+		recordTurnUsage: vi.fn(),
+		accumulateUsage: vi.fn(),
+		clearLastPromptTokens: vi.fn(),
+		lastPromptTokens: undefined,
+		setStopReason: (r: string) => {
+			stopReason = r
+		},
+		setStructuredOutput: vi.fn(),
+		markCancelled: vi.fn(),
+	}
+
+	const orchestrator = new IterationOrchestrator({
+		provider: new MockLLMProvider({ turns: opts.turns as never }),
+		// The CLI's runConfig shape: a huge cumulative budget and a model id.
+		runConfig: { model: 'claude-opus-5', maxIterations: 50, tokenBudget: 1_000_000 },
+		tools,
+		runMgr,
+		toolExecutor: new ToolExecutor(
+			{
+				tools,
+				runId: RUN_ID,
+				workingDirectory: '/tmp',
+				permissionMode: 'auto',
+				env: {},
+				abortSignal: new AbortController().signal,
+			},
+			activityStore,
+			async () => {},
+			log,
+		),
+		activityStore,
+		abortController: new AbortController(),
+		log,
+		emitEvent: async () => {},
+		drainPending: function* (): Generator<RunEvent> {},
+		checkpointMgr: { create: async () => ({ id: 'cp_1' }) },
+		planManager: { active: undefined },
+		// The CLI's permission prompt resolves to exactly this.
+		resumeHandler: async () => opts.decision,
+		verificationGate: new VerificationGate(CLI_GATE, log),
+		guard: {
+			beforeIteration: () => ({
+				shouldStop: iteration >= 4,
+				forceFinalize: false,
+				isCancelled: false,
+				stopReason: 'max_iterations',
+			}),
+		},
+	} as never)
+
+	return { orchestrator, messages, executed, stopReason: () => stopReason }
+}
+
+async function drain(o: IterationOrchestrator) {
+	const gen = o.runLoop()
+	let next = await gen.next()
+	while (!next.done) next = await gen.next()
+}
+
+/**
+ * The only question that matters: could the next provider request be sent?
+ *
+ * An unanswered `tool_use` is a protocol violation — the wire replies
+ * `400 messages.N: Did not find 1 tool_result block(s)` — and with no
+ * provider retry that ends the run.
+ */
+function expectSendableHistory(messages: Message[]) {
+	const dangling = findDanglingMessages(messages)
+	expect(dangling.assistantsWithUnmatchedCalls).toEqual([])
+	expect(dangling.orphanedToolMessages).toEqual([])
+	expect(dangling.isValid).toBe(true)
+}
+
+const READ_THEN_ANSWER = [
+	{ toolCalls: [{ name: 'read', args: { path: 'a.txt' } }] },
+	{ text: 'Here is what I found.' },
+]
+
+describe('the invariant check itself has teeth', () => {
+	// Running the tests above against the pre-fix `tool-review.ts` makes them
+	// crash on a shape change rather than fail on the assertion, which proves
+	// incompatibility but not detection. So reproduce the exact history the
+	// old code produced and show the check rejects it. Without this, a future
+	// regression could weaken `expectSendableHistory` into something that
+	// passes everything.
+	it('rejects the history the old rejection path produced', () => {
+		const brokenHistory: Message[] = [
+			{ role: 'user', content: 'delete everything' },
+			{
+				role: 'assistant',
+				content: null,
+				toolCalls: [
+					{ id: 'call_0', type: 'function', function: { name: 'bash', arguments: '{}' } },
+				],
+			},
+			// What the four broken branches pushed instead of a tool_result.
+			{ role: 'user', content: '[SYSTEM] Tool calls rejected: no' },
+		]
+
+		const dangling = findDanglingMessages(brokenHistory)
+		expect(dangling.isValid).toBe(false)
+		expect(dangling.assistantsWithUnmatchedCalls).toEqual([1])
+		expect(() => expectSendableHistory(brokenHistory)).toThrow()
+	})
+
+	it('accepts the same turn once the call is answered', () => {
+		const repaired: Message[] = [
+			{ role: 'user', content: 'delete everything' },
+			{
+				role: 'assistant',
+				content: null,
+				toolCalls: [
+					{ id: 'call_0', type: 'function', function: { name: 'bash', arguments: '{}' } },
+				],
+			},
+			{ role: 'tool', toolCallId: 'call_0', content: 'Error: refused', isError: true },
+		]
+
+		expect(() => expectSendableHistory(repaired)).not.toThrow()
+	})
+})
+
+describe('the CLI default path survives a human saying no', () => {
+	it('a plain rejection leaves a conversation the provider would accept', async () => {
+		const h = harness({
+			decision: { action: 'reject_tools', feedback: 'Do not read that file.' },
+			turns: [
+				{ toolCalls: [{ name: 'bash', args: { command: 'ls -la' } }] },
+				{ text: 'Understood.' },
+			],
+		})
+
+		await drain(h.orchestrator)
+
+		expectSendableHistory(h.messages)
+		expect(h.executed).toEqual([])
+		// And the feedback actually reached the model, in the slot it reads.
+		expect(h.messages.some((m) => String(m.content).includes('Do not read that file'))).toBe(true)
+	})
+
+	it('a gate denial leaves a conversation the provider would accept', async () => {
+		const h = harness({
+			decision: { action: 'approve_tools' },
+			turns: [
+				// `rm -rf /` trips deny_dangerous_patterns in the CLI's gate.
+				{ toolCalls: [{ name: 'bash', args: { command: 'rm -rf /' } }] },
+				{ text: 'I will not do that.' },
+			],
+		})
+
+		await drain(h.orchestrator)
+
+		expectSendableHistory(h.messages)
+		expect(h.executed).toEqual([])
+	})
+
+	it('a human approval cannot release a call the gate denied', async () => {
+		const h = harness({
+			decision: { action: 'approve_tools' },
+			turns: [
+				{
+					toolCalls: [
+						{ name: 'read', args: { path: 'a.txt' } },
+						{ name: 'bash', args: { command: 'rm -rf /' } },
+					],
+				},
+				{ text: 'Done what I could.' },
+			],
+		})
+
+		await drain(h.orchestrator)
+
+		// The gate's decision is the floor; approval covers only what it left
+		// undecided.
+		expect(h.executed).toEqual(['read'])
+		expectSendableHistory(h.messages)
+	})
+
+	it('a partial deny answers the denied call too', async () => {
+		const h = harness({
+			decision: {
+				action: 'modify_tools',
+				modifications: [{ toolCallId: 'call_0_1', action: 'deny' }],
+			} as HITLResumeDecision,
+			turns: [
+				{ toolCalls: [{ name: 'ls' }, { name: 'bash', args: { command: 'echo hi' } }] },
+				{ text: 'ok' },
+			],
+		})
+
+		await drain(h.orchestrator)
+
+		expectSendableHistory(h.messages)
+	})
+
+	it('the approving path is unaffected', async () => {
+		const h = harness({ decision: { action: 'approve_tools' }, turns: READ_THEN_ANSWER })
+
+		await drain(h.orchestrator)
+
+		expect(h.executed).toEqual(['read'])
+		expect(h.stopReason()).toBe('end_turn')
+		expectSendableHistory(h.messages)
+	})
+})

package/src/__tests__/shipped-defaults.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+	DEFAULT_ASSUMED_CONTEXT_WINDOW,
+	resolveContextWindow,
+} from '../compaction/context-window.js'
+import { DEFAULT_MCP_REQUEST_TIMEOUT_MS } from '../constants/mcp/index.js'
+import { DEFAULT_STRUCTURED_OUTPUT_RETRIES } from '../constants/tools/index.js'
+import { MOCK_CAPABILITIES } from '../provider/mock-register.js'
+import { DEFAULT_PROVIDER_RETRY } from '../provider/retry.js'
+import { DEFAULT_TOOL_CONCURRENCY, DEFAULT_TOOL_TIMEOUT_MS } from '../runtime/query/executor.js'
+import { DEFAULT_MAX_TOOL_OUTPUT_CHARS } from '../runtime/query/tool-output-budget.js'
+
+/**
+ * A conformance test over the SHIPPED DEFAULTS.
+ *
+ * Every P0 in the hardening audit was a defect in what namzu does out of the
+ * box, masked by a non-default test path: `autoApproveHandler` hid the
+ * dangling-`tool_use` bug, the CLI's `tokenBudget: 1_000_000` hid the
+ * compaction trigger, and `MOCK_CAPABILITIES.supportsTools: false` meant no
+ * consumer could reach the tool loop at all. Each was individually tested
+ * and each was individually wrong, because the tests configured their way
+ * out of the default.
+ *
+ * So this file asserts the defaults themselves. It is deliberately boring:
+ * the value of a default is that it does not drift, and a number changing
+ * here should be a decision someone made, not a side effect of a refactor.
+ * Every expectation carries the reason the value is what it is.
+ */
+
+describe('a run cannot hang', () => {
+	it('tools have a deadline, and it is survivable rather than generous', () => {
+		// bash defaulted to ONE HOUR while ignoring Stop entirely.
+		expect(DEFAULT_TOOL_TIMEOUT_MS).toBe(120_000)
+		expect(DEFAULT_TOOL_TIMEOUT_MS).toBeLessThanOrEqual(5 * 60_000)
+	})
+
+	it('MCP round trips have a deadline', () => {
+		// stdio — the default transport for local servers — armed no timer at
+		// all, so a wedged server hung the run with no error and no run_failed.
+		expect(DEFAULT_MCP_REQUEST_TIMEOUT_MS).toBeGreaterThan(0)
+		expect(DEFAULT_MCP_REQUEST_TIMEOUT_MS).toBeLessThanOrEqual(60_000)
+	})
+
+	it('tool fan-out is bounded', () => {
+		// `Promise.all` over an unbounded batch: fifty parallel reads opened
+		// fifty handles and fifty activity records at once.
+		expect(DEFAULT_TOOL_CONCURRENCY).toBeGreaterThan(0)
+		expect(DEFAULT_TOOL_CONCURRENCY).toBeLessThanOrEqual(16)
+	})
+})
+
+describe('a run cannot be killed by one bad response', () => {
+	it('transient provider failures are retried by default', () => {
+		// A single 429 or 503 used to terminate a run outright.
+		expect(DEFAULT_PROVIDER_RETRY.maxRetries).toBeGreaterThan(0)
+	})
+
+	it('backoff is bounded on both ends', () => {
+		expect(DEFAULT_PROVIDER_RETRY.initialDelayMs).toBeGreaterThan(0)
+		expect(DEFAULT_PROVIDER_RETRY.maxDelayMs).toBeGreaterThan(DEFAULT_PROVIDER_RETRY.initialDelayMs)
+		// A server asking for 15 minutes must not silently park an
+		// interactive run for 15 minutes.
+		expect(DEFAULT_PROVIDER_RETRY.maxRetryAfterMs).toBeLessThanOrEqual(120_000)
+	})
+})
+
+describe('a run cannot blow its own context', () => {
+	it('tool output is capped, and the cap is a fraction of a small window', () => {
+		// `read` returned whole files, `bash` allowed a 100 MB buffer: a 2 MB
+		// lockfile became ~500k tokens in one tool_result.
+		expect(DEFAULT_MAX_TOOL_OUTPUT_CHARS).toBeGreaterThan(0)
+		// ~4 chars/token; one result must not be able to eat a fifth of a
+		// 200k window.
+		expect(DEFAULT_MAX_TOOL_OUTPUT_CHARS / 4).toBeLessThan(200_000 / 5)
+	})
+
+	it('compaction always resolves a real window, never a spend budget', () => {
+		// The trigger divided by `runConfig.tokenBudget` — a cumulative spend
+		// cap — whenever `contextWindowTokens` was absent, which was always.
+		for (const model of [undefined, '', 'totally-unknown-model', 'claude-opus-5', 'gpt-4']) {
+			const resolved = resolveContextWindow(undefined, model)
+			expect(resolved.tokens).toBeGreaterThan(0)
+			expect(resolved.source).not.toBe('config')
+		}
+	})
+
+	it('the unknown-model fallback is conservative, not optimistic', () => {
+		// Under-estimating costs a summarization pass. Over-estimating kills
+		// the run on a provider context-length error with nothing recoverable.
+		expect(DEFAULT_ASSUMED_CONTEXT_WINDOW).toBeLessThanOrEqual(200_000)
+	})
+})
+
+describe('a run cannot loop on a demand the model will not meet', () => {
+	it('structured-output re-prompts are bounded well below the iteration cap', () => {
+		expect(DEFAULT_STRUCTURED_OUTPUT_RETRIES).toBeGreaterThan(0)
+		// The shipped `maxIterations` default is 200; this must fail fast.
+		expect(DEFAULT_STRUCTURED_OUTPUT_RETRIES).toBeLessThan(10)
+	})
+})
+
+describe('the shipped test model can exercise the tool loop', () => {
+	it('declares tool support', () => {
+		// `supportsTools: false` made capability negotiation strip the tool
+		// surface before a request was built, so no consumer could test that
+		// the loop calls their tool — and namzu hand-rolled eight fakes.
+		expect(MOCK_CAPABILITIES.supportsTools).toBe(true)
+		expect(MOCK_CAPABILITIES.supportsFunctionCalling).toBe(true)
+	})
+})

package/src/advisory/__tests__/budget-enforcement.test.ts

@@ -0,0 +1,208 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import type { AdvisorDefinition, AdvisoryCallRecord } from '../../types/advisory/index.js'
+import type { StreamChunk } from '../../types/provider/index.js'
+import { assertBudgetEnforceable } from '../budget.js'
+import { AdvisoryContext } from '../context.js'
+import { TriggerEvaluator } from '../evaluator.js'
+import { AdvisoryExecutor } from '../executor.js'
+import { AdvisorRegistry } from '../registry.js'
+
+/**
+ * `AdvisoryBudget` declared six caps and enforced one. The other five were
+ * read by nothing: a host could set `maxCostPerRun` and watch an advisor
+ * spend without limit, and the only signal that the cap was inert was that
+ * nothing ever happened.
+ *
+ * Two of the six could not be honoured at all — the advisory stack is built
+ * per run, so a per-SESSION cap had no accumulator to count against — and
+ * those were removed rather than left as decoration. What remains is
+ * enforced here.
+ */
+
+function record(cost: number): AdvisoryCallRecord {
+	return {
+		advisorId: 'a',
+		request: { question: 'q' },
+		result: { advice: '' },
+		usage: {
+			promptTokens: 0,
+			completionTokens: 0,
+			totalTokens: 0,
+			cachedTokens: 0,
+			cacheWriteTokens: 0,
+		},
+		cost: { inputCostPer1M: 0, outputCostPer1M: 0, totalCost: cost, cacheDiscount: 0 },
+		durationMs: 1,
+		iteration: 1,
+		timestamp: 0,
+	}
+}
+
+function contextWith(budget: Parameters<typeof assertBudgetEnforceable>[0]['budget']) {
+	return new AdvisoryContext(
+		new AdvisorRegistry([]),
+		new AdvisoryExecutor(),
+		new TriggerEvaluator([]),
+		budget,
+	)
+}
+
+describe('a per-run cost cap actually stops the next call', () => {
+	it('allows a call while the accumulated cost is under the cap', () => {
+		const ctx = contextWith({ maxCostPerRun: 1 })
+		ctx.recordCall(record(0.4))
+		expect(ctx.checkBudget().allowed).toBe(true)
+	})
+
+	it('refuses once the accumulated cost reaches the cap', () => {
+		const ctx = contextWith({ maxCostPerRun: 1 })
+		ctx.recordCall(record(0.6))
+		ctx.recordCall(record(0.5))
+
+		const verdict = ctx.checkBudget()
+		expect(verdict.allowed).toBe(false)
+		// The operator gets the number they set and the number they hit;
+		// "budget exhausted" alone leaves them guessing which cap tripped.
+		expect(verdict.reason).toContain('1.1')
+		expect(verdict.reason).toContain('cost')
+	})
+
+	it('leaves the call cap working alongside it', () => {
+		const ctx = contextWith({ maxCallsPerRun: 1 })
+		ctx.recordCall(record(0))
+		expect(ctx.checkBudget().allowed).toBe(false)
+	})
+})
+
+describe('a cost cap without pricing is refused, not silently inert', () => {
+	const advisor = (pricing?: { inputCostPer1M: number; outputCostPer1M: number }) =>
+		({
+			id: 'a',
+			name: 'A',
+			model: 'm',
+			provider: {} as AdvisorDefinition['provider'],
+			...(pricing ? { pricing } : {}),
+		}) as AdvisorDefinition
+
+	it('throws when a cost cap is set and an advisor carries no pricing', () => {
+		expect(() =>
+			assertBudgetEnforceable({ advisors: [advisor()], budget: { maxCostPerRun: 5 } }),
+		).toThrow(/pricing/i)
+	})
+
+	it('accepts the same cap once pricing is supplied', () => {
+		expect(() =>
+			assertBudgetEnforceable({
+				advisors: [advisor({ inputCostPer1M: 1, outputCostPer1M: 2 })],
+				budget: { maxCostPerRun: 5 },
+			}),
+		).not.toThrow()
+	})
+
+	it('says nothing about pricing when no cost cap is set', () => {
+		expect(() =>
+			assertBudgetEnforceable({ advisors: [advisor()], budget: { maxCallsPerRun: 2 } }),
+		).not.toThrow()
+	})
+})
+
+describe('a per-call token cap reaches the provider', () => {
+	function providerCapturing(seen: { maxTokens?: number }) {
+		return {
+			chatStream: vi.fn(async function* (params: {
+				maxTokens?: number
+			}): AsyncGenerator<StreamChunk> {
+				seen.maxTokens = params.maxTokens
+				yield { id: 'r', delta: { content: 'ok' } } as StreamChunk
+				yield {
+					id: 'r',
+					delta: {},
+					finishReason: 'stop',
+					usage: {
+						promptTokens: 1,
+						completionTokens: 1,
+						totalTokens: 2,
+						cachedTokens: 0,
+						cacheWriteTokens: 0,
+					},
+				} as StreamChunk
+			}),
+		} as unknown as AdvisorDefinition['provider']
+	}
+
+	it('clamps an advisor that asks for more than the budget allows', async () => {
+		const seen: { maxTokens?: number } = {}
+		const exec = new AdvisoryExecutor(undefined, { maxTokensPerCall: 100 })
+		await exec.consult(
+			{
+				id: 'a',
+				name: 'A',
+				model: 'm',
+				provider: providerCapturing(seen),
+				maxResponseTokens: 5000,
+			},
+			{ question: 'q' },
+			{ messages: [], iteration: 1 },
+		)
+		expect(seen.maxTokens).toBe(100)
+	})
+
+	it('applies the cap to an advisor that names no ceiling of its own', async () => {
+		const seen: { maxTokens?: number } = {}
+		const exec = new AdvisoryExecutor(undefined, { maxTokensPerCall: 100 })
+		await exec.consult(
+			{ id: 'a', name: 'A', model: 'm', provider: providerCapturing(seen) },
+			{ question: 'q' },
+			{ messages: [], iteration: 1 },
+		)
+		expect(seen.maxTokens).toBe(100)
+	})
+
+	it('leaves a smaller advisor ceiling alone', async () => {
+		const seen: { maxTokens?: number } = {}
+		const exec = new AdvisoryExecutor(undefined, { maxTokensPerCall: 100 })
+		await exec.consult(
+			{ id: 'a', name: 'A', model: 'm', provider: providerCapturing(seen), maxResponseTokens: 40 },
+			{ question: 'q' },
+			{ messages: [], iteration: 1 },
+		)
+		expect(seen.maxTokens).toBe(40)
+	})
+})
+
+describe('cost is real when the advisor carries pricing', () => {
+	it('computes from the advisor pricing rather than reporting zero', async () => {
+		const provider = {
+			chatStream: vi.fn(async function* (): AsyncGenerator<StreamChunk> {
+				yield { id: 'r', delta: { content: 'ok' } } as StreamChunk
+				yield {
+					id: 'r',
+					delta: {},
+					finishReason: 'stop',
+					usage: {
+						promptTokens: 1_000_000,
+						completionTokens: 1_000_000,
+						totalTokens: 2_000_000,
+						cachedTokens: 0,
+						cacheWriteTokens: 0,
+					},
+				} as StreamChunk
+			}),
+		} as unknown as AdvisorDefinition['provider']
+
+		const exec = new AdvisoryExecutor()
+		const out = await exec.consult(
+			{
+				id: 'a',
+				name: 'A',
+				model: 'm',
+				provider,
+				pricing: { inputCostPer1M: 3, outputCostPer1M: 15 },
+			},
+			{ question: 'q' },
+			{ messages: [], iteration: 1 },
+		)
+		expect(out.cost.totalCost).toBe(18)
+	})
+})