@namzu/sdk 2.0.0 → 3.0.0

package/src/runtime/query/iteration/phases/retained-messages.test.ts

@@ -0,0 +1,226 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { findDanglingMessages } from '../../../../compaction/dangling.js'
+import { WorkingStateManager } from '../../../../compaction/manager.js'
+import { clearStaleToolResults } from '../../../../compaction/tool-result-editing.js'
+import { CompactionConfigSchema } from '../../../../config/runtime.js'
+import type { RunId } from '../../../../types/ids/index.js'
+import {
+	type Message,
+	createAssistantMessage,
+	createSystemMessage,
+	createToolMessage,
+	createUserMessage,
+} from '../../../../types/message/index.js'
+import type { Logger } from '../../../../utils/logger.js'
+import { runCompactionCheck } from './compaction.js'
+import type { IterationContext } from './context.js'
+
+/**
+ * Everything the run protected from compaction was protected by POSITION:
+ * the leading system run, the working-memory slot, the last N turns, the
+ * most recent tool results. So a standing constraint stated in the MIDDLE
+ * of a conversation aged out at the same rate as chatter, and no
+ * positional rule could express it.
+ */
+
+function makeLogger(): Logger {
+	const self = {
+		info: vi.fn(),
+		warn: vi.fn(),
+		error: vi.fn(),
+		debug: vi.fn(),
+	} as unknown as Logger
+	;(self as { child: (ctx: unknown) => Logger }).child = vi.fn(() => self)
+	return self
+}
+
+const FILLER = 'x'.repeat(200)
+const CONSTRAINT = 'the account id is 4471; never bill a different one'
+
+function makeCtx(messages: Message[]): IterationContext {
+	const config = CompactionConfigSchema.parse({
+		strategy: 'structured',
+		llmVerification: false,
+		contextWindowTokens: 100,
+		// Off, so these cases exercise the rebuild rather than being
+		// satisfied by the cheap in-place reclaim that runs first.
+		clearToolResults: false,
+	})
+	const manager = new WorkingStateManager(config)
+	manager.addDecision('built the report')
+
+	return {
+		runConfig: { tokenBudget: 0 },
+		compactionConfig: config,
+		workingStateManager: manager,
+		log: makeLogger(),
+		runMgr: {
+			id: 'run_1' as RunId,
+			currentIteration: 3,
+			messages,
+			lastPromptTokens: undefined,
+			clearLastPromptTokens: () => {},
+			accumulateUsage: () => {},
+		},
+		emitEvent: async () => {},
+	} as unknown as IterationContext
+}
+
+/** A long run whose middle holds one message worth keeping. */
+function longRun(pin: (m: Message) => Message = (m) => m): Message[] {
+	const msgs: Message[] = [createSystemMessage(`STATIC SYSTEM PROMPT ${FILLER}`, 'cache')]
+	for (let i = 0; i < 8; i++) {
+		msgs.push(
+			i === 1 ? pin(createUserMessage(CONSTRAINT)) : createUserMessage(`user turn ${i} ${FILLER}`),
+		)
+		msgs.push(createAssistantMessage(`assistant turn ${i} ${FILLER}`))
+	}
+	return msgs
+}
+
+const pinned = (m: Message): Message => ({ ...m, retain: true })
+const asText = (content: unknown): string => (typeof content === 'string' ? content : '')
+
+describe('a message the host marked as never-evictable', () => {
+	it('is evicted without the marker', async () => {
+		// The premise. Without this the next case proves nothing: the
+		// message might simply have been recent enough to survive.
+		const messages = longRun()
+		await runCompactionCheck(makeCtx(messages))
+
+		expect(messages.some((m) => asText(m.content) === CONSTRAINT)).toBe(false)
+	})
+
+	it('survives the pass verbatim', async () => {
+		const messages = longRun(pinned)
+		await runCompactionCheck(makeCtx(messages))
+
+		expect(messages.some((m) => asText(m.content).includes('[COMPACTED CONTEXT]'))).toBe(true)
+		expect(messages.some((m) => asText(m.content) === CONSTRAINT)).toBe(true)
+	})
+
+	it('sits after the summary and before the recent window', async () => {
+		const messages = longRun(pinned)
+		await runCompactionCheck(makeCtx(messages))
+
+		const summary = messages.findIndex((m) => asText(m.content).includes('[COMPACTED CONTEXT]'))
+		const kept = messages.findIndex((m) => asText(m.content) === CONSTRAINT)
+		const lastTurn = messages.findIndex((m) => asText(m.content).startsWith('assistant turn 7'))
+
+		expect(kept).toBeGreaterThan(summary)
+		expect(kept).toBeLessThan(lastTurn)
+	})
+
+	it('does not keep the run alive by pinning everything', async () => {
+		// The marker is a budget the setter spends, and the pass must still
+		// reclaim what was not pinned.
+		const messages = longRun(pinned)
+		const before = messages.length
+		await runCompactionCheck(makeCtx(messages))
+
+		expect(messages.length).toBeLessThan(before)
+	})
+})
+
+describe('a pinned half of a tool pair', () => {
+	function runWithPairs(pinIndex: 'call' | 'result'): Message[] {
+		const call = {
+			id: 'call_pinned',
+			type: 'function' as const,
+			function: { name: 'lookup_account', arguments: '{}' },
+		}
+		const msgs: Message[] = [createSystemMessage(`STATIC SYSTEM PROMPT ${FILLER}`, 'cache')]
+		msgs.push(createUserMessage(`user turn 0 ${FILLER}`))
+		const assistant = { ...createAssistantMessage(null, [call]) } as Message
+		const result = createToolMessage(CONSTRAINT, 'call_pinned') as Message
+		msgs.push(
+			pinIndex === 'call' ? { ...assistant, retain: true } : assistant,
+			pinIndex === 'result' ? { ...result, retain: true } : result,
+		)
+		for (let i = 1; i < 8; i++) {
+			msgs.push(createUserMessage(`user turn ${i} ${FILLER}`))
+			msgs.push(createAssistantMessage(`assistant turn ${i} ${FILLER}`))
+		}
+		return msgs
+	}
+
+	it('pulls in the result when the call is pinned', async () => {
+		const messages = runWithPairs('call')
+		await runCompactionCheck(makeCtx(messages))
+
+		// Half a pair is not a smaller history — a `tool_use` with no
+		// `tool_result` is rejected outright on the next turn.
+		expect(findDanglingMessages(messages).isValid).toBe(true)
+		expect(messages.some((m) => asText(m.content) === CONSTRAINT)).toBe(true)
+	})
+
+	it('pulls in the call when the result is pinned', async () => {
+		const messages = runWithPairs('result')
+		await runCompactionCheck(makeCtx(messages))
+
+		expect(findDanglingMessages(messages).isValid).toBe(true)
+		expect(messages.some((m) => m.role === 'assistant' && m.toolCalls?.length)).toBe(true)
+	})
+
+	it('pulls in the SIBLING results of a fan-out, not just the pinned one', async () => {
+		// The second hop. Pinning one result pulls in its assistant turn,
+		// and that turn issued two more calls — an assistant with three
+		// calls and one surviving result is the same dangling error in the
+		// other direction.
+		const calls = ['a', 'b', 'c'].map((suffix) => ({
+			id: `call_${suffix}`,
+			type: 'function' as const,
+			function: { name: `lookup_${suffix}`, arguments: '{}' },
+		}))
+		const messages: Message[] = [createSystemMessage(`STATIC SYSTEM PROMPT ${FILLER}`, 'cache')]
+		messages.push(createUserMessage(`user turn 0 ${FILLER}`))
+		messages.push(createAssistantMessage(null, calls) as Message)
+		messages.push(
+			{ ...(createToolMessage(CONSTRAINT, 'call_a') as Message), retain: true },
+			createToolMessage(`sibling b ${FILLER}`, 'call_b') as Message,
+			createToolMessage(`sibling c ${FILLER}`, 'call_c') as Message,
+		)
+		for (let i = 1; i < 8; i++) {
+			messages.push(createUserMessage(`user turn ${i} ${FILLER}`))
+			messages.push(createAssistantMessage(`assistant turn ${i} ${FILLER}`))
+		}
+
+		await runCompactionCheck(makeCtx(messages))
+
+		expect(findDanglingMessages(messages).isValid).toBe(true)
+		expect(messages.filter((m) => m.role === 'tool')).toHaveLength(3)
+	})
+})
+
+describe('a pinned tool result and the cheap in-place reclaim', () => {
+	function bigResults(pin: boolean): Message[] {
+		const calls = [0, 1, 2, 3].map((i) => ({
+			id: `call_${i}`,
+			type: 'function' as const,
+			function: { name: 'lookup_account', arguments: '{}' },
+		}))
+		const msgs: Message[] = [createUserMessage('go')]
+		for (const call of calls) {
+			msgs.push(createAssistantMessage(null, [call]) as Message)
+			const body = `${CONSTRAINT} ${'y'.repeat(5000)}`
+			const result = createToolMessage(body, call.id) as Message
+			msgs.push(pin && call.id === 'call_0' ? { ...result, retain: true } : result)
+		}
+		return msgs
+	}
+
+	it('is cleared without the marker', async () => {
+		const edit = clearStaleToolResults(bigResults(false), { keepRecentToolResults: 1 })
+		expect(edit.clearedCount).toBe(3)
+	})
+
+	it('keeps its content when marked', async () => {
+		// Clearing keeps the message and replaces its content, which is
+		// exactly the loss the marker was asked to prevent.
+		const edit = clearStaleToolResults(bigResults(true), { keepRecentToolResults: 1 })
+
+		expect(edit.clearedCount).toBe(2)
+		expect(asText(edit.messages[2]?.content)).toContain(CONSTRAINT)
+	})
+})

package/src/runtime/query/iteration/phases/tool-review-denials.test.ts

@@ -0,0 +1,270 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { findDanglingMessages } from '../../../../compaction/dangling.js'
+import { ActivityStore } from '../../../../store/activity/memory.js'
+import type { HITLResumeDecision } from '../../../../types/hitl/index.js'
+import type { RunId } from '../../../../types/ids/index.js'
+import type { Message } from '../../../../types/message/index.js'
+import type { ChatCompletionResponse } from '../../../../types/provider/index.js'
+import type { ToolRegistryContract } from '../../../../types/tool/index.js'
+import type { VerificationGateConfig } from '../../../../types/verification/index.js'
+import type { Logger } from '../../../../utils/logger.js'
+import { VerificationGate } from '../../../../verification/gate.js'
+import { ToolExecutor } from '../../executor.js'
+import type { IterationContext } from './context.js'
+import { runToolReview } from './tool-review.js'
+
+/**
+ * Regression suite for the tool-review invariant:
+ *
+ *   Every `tool_use` block the model emits is answered by exactly one
+ *   `tool_result`, on every path — gate denial, human rejection, partial
+ *   approval — because an unanswered `tool_use` makes the NEXT provider
+ *   request malformed and kills the run.
+ *
+ * Plus the policy invariant discovered alongside it: a human "approve" on
+ * the gate's mixed-decision path must not execute the calls the gate
+ * denied.
+ */
+
+const RUN_ID = 'run_denial_test' as RunId
+
+function makeLogger(): Logger {
+	const stub = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }
+	return { ...stub, child: vi.fn(() => ({ ...stub, child: vi.fn() })) } as unknown as Logger
+}
+
+/** `read` is read-only (gate-allowable); `bash rm -rf /` trips deny_dangerous_patterns. */
+function response(): ChatCompletionResponse {
+	return {
+		id: 'resp_1',
+		model: 'test',
+		message: {
+			role: 'assistant',
+			content: null,
+			toolCalls: [
+				{
+					id: 'call_safe',
+					type: 'function',
+					function: { name: 'read', arguments: JSON.stringify({ path: 'a.txt' }) },
+				},
+				{
+					id: 'call_danger',
+					type: 'function',
+					function: { name: 'bash', arguments: JSON.stringify({ command: 'rm -rf /' }) },
+				},
+			],
+		},
+		finishReason: 'tool_calls',
+		usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
+	} as ChatCompletionResponse
+}
+
+interface Harness {
+	ctx: IterationContext
+	messages: Message[]
+	executed: string[]
+}
+
+/** The schema fills defaults; tests only state the rules they care about. */
+function gateConfig(partial: Partial<VerificationGateConfig>): VerificationGateConfig {
+	return {
+		enabled: true,
+		rules: [],
+		allowReadOnlyTools: false,
+		denyDangerousPatterns: false,
+		logDecisions: false,
+		...partial,
+	}
+}
+
+function harness(opts: {
+	gate?: VerificationGateConfig
+	decision?: HITLResumeDecision
+}): Harness {
+	const executed: string[] = []
+	const messages: Message[] = []
+	const log = makeLogger()
+
+	const toolDefs: Record<string, { readOnly: boolean; category: string }> = {
+		read: { readOnly: true, category: 'filesystem' },
+		bash: { readOnly: false, category: 'shell' },
+	}
+
+	const tools = {
+		get: vi.fn((name: string) => {
+			const def = toolDefs[name]
+			if (!def) return undefined
+			return {
+				name,
+				category: def.category,
+				isReadOnly: () => def.readOnly,
+				isDestructive: () => !def.readOnly,
+				isConcurrencySafe: () => def.readOnly,
+			}
+		}),
+		execute: vi.fn(async (name: string) => {
+			executed.push(name)
+			return { success: true, output: `${name} ok` }
+		}),
+		has: vi.fn(() => true),
+		listNames: vi.fn(() => Object.keys(toolDefs)),
+		getAvailability: vi.fn(() => 'active'),
+		register: vi.fn(),
+		unregister: vi.fn(),
+	} as unknown as ToolRegistryContract
+
+	const toolExecutor = new ToolExecutor(
+		{
+			tools,
+			runId: RUN_ID,
+			workingDirectory: '/tmp',
+			permissionMode: 'auto',
+			env: {},
+			abortSignal: new AbortController().signal,
+		},
+		new ActivityStore(RUN_ID, { enabled: true, trackToolCalls: true, trackLlmTurns: true }),
+		async () => {},
+		log,
+	)
+
+	const ctx = {
+		tools,
+		toolExecutor,
+		log,
+		abortController: new AbortController(),
+		runMgr: {
+			id: RUN_ID,
+			messages,
+			pushMessage: (m: Message) => {
+				messages.push(m)
+			},
+			setStopReason: vi.fn(),
+			markCancelled: vi.fn(),
+		},
+		checkpointMgr: { create: async () => ({ id: 'cp_1' }) },
+		emitEvent: async () => {},
+		drainPending: async function* () {},
+		resumeHandler: async () => opts.decision ?? ({ action: 'approve_tools' } as HITLResumeDecision),
+		verificationGate: opts.gate ? new VerificationGate(opts.gate, log) : undefined,
+	} as unknown as IterationContext
+
+	return { ctx, messages, executed }
+}
+
+/** Drain the generator and return its decision. */
+async function run(ctx: IterationContext, resp: ChatCompletionResponse) {
+	const gen = runToolReview(ctx, resp, 1)
+	let next = await gen.next()
+	while (!next.done) next = await gen.next()
+	return next.value.decision
+}
+
+/** The whole point: history must be sendable to a provider afterwards. */
+function expectEveryToolCallAnswered(messages: Message[], resp: ChatCompletionResponse) {
+	const assistant: Message = {
+		role: 'assistant',
+		content: resp.message.content,
+		toolCalls: resp.message.toolCalls,
+	}
+	const history = [assistant, ...messages]
+	const dangling = findDanglingMessages(history)
+	expect(dangling.assistantsWithUnmatchedCalls).toEqual([])
+	expect(dangling.orphanedToolMessages).toEqual([])
+	expect(dangling.isValid).toBe(true)
+
+	const answered = messages
+		.filter((m): m is Message & { toolCallId: string } => m.role === 'tool')
+		.map((m) => m.toolCallId)
+	expect(answered.sort()).toEqual((resp.message.toolCalls ?? []).map((tc) => tc.id).sort())
+}
+
+describe('runToolReview — every tool_use is answered', () => {
+	it('gate all-deny still emits a tool_result per call', async () => {
+		const h = harness({
+			gate: gateConfig({ rules: [{ type: 'deny_by_name', toolNames: ['read', 'bash'] }] }),
+		})
+		const resp = response()
+		const outcome = await run(h.ctx, resp)
+
+		expect(outcome).toBe('rejected')
+		expect(h.executed).toEqual([])
+		expectEveryToolCallAnswered(h.messages, resp)
+		// The reason must travel inside the tool_result so it can steer.
+		expect(h.messages.some((m) => String(m.content).includes('verification gate'))).toBe(true)
+	})
+
+	it('human reject_tools still emits a tool_result per call, carrying the feedback', async () => {
+		const h = harness({
+			decision: { action: 'reject_tools', feedback: 'Use the API instead of shelling out.' },
+		})
+		const resp = response()
+		const outcome = await run(h.ctx, resp)
+
+		expect(outcome).toBe('rejected')
+		expect(h.executed).toEqual([])
+		expectEveryToolCallAnswered(h.messages, resp)
+		expect(h.messages.some((m) => String(m.content).includes('Use the API instead'))).toBe(true)
+	})
+
+	it('modify_tools with a PARTIAL deny answers the denied call too', async () => {
+		const h = harness({
+			decision: {
+				action: 'modify_tools',
+				modifications: [{ toolCallId: 'call_danger', action: 'deny' }],
+			} as HITLResumeDecision,
+		})
+		const resp = response()
+		const outcome = await run(h.ctx, resp)
+
+		expect(outcome).toBe('executed')
+		expect(h.executed).toEqual(['read'])
+		expectEveryToolCallAnswered(h.messages, resp)
+	})
+
+	it('plain approval executes everything and answers everything', async () => {
+		const h = harness({ decision: { action: 'approve_tools' } })
+		const resp = response()
+		const outcome = await run(h.ctx, resp)
+
+		expect(outcome).toBe('executed')
+		expect(h.executed.sort()).toEqual(['bash', 'read'])
+		expectEveryToolCallAnswered(h.messages, resp)
+	})
+})
+
+describe('runToolReview — a gate denial outranks a human approval', () => {
+	it('approve_tools on the gate MIXED path does not execute the gate-denied call', async () => {
+		// allow_read_only allows `read`; deny_dangerous_patterns denies the
+		// `rm -rf /` bash call → mixed → human review → human approves.
+		const h = harness({
+			gate: gateConfig({ allowReadOnlyTools: true, denyDangerousPatterns: true }),
+			decision: { action: 'approve_tools' },
+		})
+		const resp = response()
+		const outcome = await run(h.ctx, resp)
+
+		expect(outcome).toBe('executed')
+		// THE BUG: this used to be ['read', 'bash'] — approval replayed the
+		// full unfiltered response and ran the call the gate refused.
+		expect(h.executed).toEqual(['read'])
+		expectEveryToolCallAnswered(h.messages, resp)
+	})
+
+	it('modify_tools cannot rewrite a gate-denied call back into execution', async () => {
+		const h = harness({
+			gate: gateConfig({ allowReadOnlyTools: true, denyDangerousPatterns: true }),
+			decision: {
+				action: 'modify_tools',
+				modifications: [
+					{ toolCallId: 'call_danger', action: 'modify', modifiedInput: { command: 'ls' } },
+				],
+			} as HITLResumeDecision,
+		})
+		const resp = response()
+		await run(h.ctx, resp)
+
+		expect(h.executed).toEqual(['read'])
+		expectEveryToolCallAnswered(h.messages, resp)
+	})
+})