@namzu/sdk 2.0.0 → 3.0.0

package/src/runtime/query/__tests__/tool-pause.test.ts

@@ -0,0 +1,209 @@
+import { describe, expect, it, vi } from 'vitest'
+import { z } from 'zod'
+
+import { MockLLMProvider, registerMock } from '../../../provider/index.js'
+import { ToolRegistry } from '../../../registry/index.js'
+import { defineTool } from '../../../tools/defineTool.js'
+import type { HITLResumeDecision } from '../../../types/hitl/index.js'
+import type { MockTurn } from '../../../types/provider/index.js'
+import type { ToolPauseOutcome } from '../../../types/tool/index.js'
+import {
+	generateProjectId,
+	generateSessionId,
+	generateTenantId,
+	generateThreadId,
+} from '../../../utils/id.js'
+import { drainQuery } from '../index.js'
+import { PendingAnswers, QuestionParkBinding } from '../question-park.js'
+import { createToolPause, pauseId } from '../tool-pause.js'
+
+/**
+ * The pause machinery is durable and excellent, and it was reachable from
+ * exactly four kernel-owned points: the plan gate, the tool-review gate,
+ * the iteration cadence, and one built-in question tool. A host-authored
+ * tool — the spend, the outbound post, the destructive migration — had no
+ * seam to it, and nothing in the type a tool author is handed suggested
+ * one could be hand-wired.
+ */
+
+registerMock()
+
+const ANSWER = (id: string, option: string): HITLResumeDecision => ({
+	action: 'answer_question',
+	questionId: id,
+	selectedOptionIds: [option],
+})
+
+const request = {
+	name: 'target_environment',
+	prompt: 'which environment should this run against?',
+	options: [
+		{ id: 'staging', label: 'Staging' },
+		{ id: 'production', label: 'Production' },
+	],
+}
+
+describe('a pause raised from inside a tool', () => {
+	it('records a durable park and returns the human answer', async () => {
+		const record = vi.fn(async () => 'cp_1' as never)
+		const resolve = vi.fn(async () => {})
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async (r) =>
+				ANSWER(r.type === 'user_question' ? r.question.questionId : '', 'staging'),
+			recorder: { record, resolve },
+		})
+
+		expect(await pause(request)).toEqual({ status: 'answered', selectedOptionIds: ['staging'] })
+		expect(record).toHaveBeenCalledTimes(1)
+		// Cleared once answered, so an approval queue stops serving it.
+		expect(resolve).toHaveBeenCalledWith(
+			'cp_1',
+			expect.objectContaining({ action: 'answer_question' }),
+		)
+	})
+
+	it('names the pause so one call can raise more than one', () => {
+		// The tool-use id identifies the CALL, and a call may ask "which
+		// environment" and then "are you sure" — keying on the id alone would
+		// deliver the first answer to the second question.
+		expect(pauseId('call_1', 'target_environment')).not.toBe(pauseId('call_1', 'confirm'))
+	})
+
+	it('refuses an answer addressed to a different pause', async () => {
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async () => ANSWER('call_1:some_other_pause', 'production'),
+		})
+
+		// Host queues are keyed by run, so a stale client can answer pause N
+		// after pause N+1 opened. Answering the wrong one is worse than not
+		// answering.
+		expect(await pause(request)).toMatchObject({ status: 'unanswered' })
+	})
+
+	it('reports an unanswered pause as its own outcome, never as consent', async () => {
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async () => ({ action: 'continue' }),
+		})
+
+		const outcome: ToolPauseOutcome = await pause(request)
+		expect(outcome.status).toBe('unanswered')
+		expect(outcome).not.toHaveProperty('selectedOptionIds')
+	})
+
+	it('reports an abort separately from silence', async () => {
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async () => ({ action: 'abort', reason: 'stop' }),
+		})
+
+		expect(await pause(request)).toEqual({ status: 'aborted' })
+	})
+
+	it('drops a selection the tool never offered', async () => {
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async () => ANSWER('call_1:target_environment', 'delete_everything'),
+		})
+
+		expect(await pause(request)).toMatchObject({ status: 'unanswered' })
+	})
+
+	it('answers from a resumed run without parking again', async () => {
+		const record = vi.fn(async () => 'cp_1' as never)
+		const answers = new PendingAnswers()
+		answers.set(
+			pauseId('call_1', 'target_environment'),
+			ANSWER('call_1:target_environment', 'production'),
+		)
+
+		const parkHandler = vi.fn(async () => ({ action: 'continue' }) as HITLResumeDecision)
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler,
+			recorder: { record, resolve: async () => {} },
+			pendingAnswers: answers,
+		})
+
+		expect(await pause(request)).toEqual({ status: 'answered', selectedOptionIds: ['production'] })
+		// Re-entering the tool is HOW the answer is delivered, so it must not
+		// ask the human something they already answered.
+		expect(parkHandler).not.toHaveBeenCalled()
+		expect(record).not.toHaveBeenCalled()
+	})
+
+	it('still pauses in-process when nothing durable is attached', async () => {
+		// Unbound is the pre-run state, and the degradation is the same one
+		// the built-in question tool has: the await works, only the
+		// cross-process handoff is missing.
+		const pause = createToolPause({
+			runId: 'run_1' as never,
+			toolUseId: 'call_1',
+			parkHandler: async () => ANSWER('call_1:target_environment', 'staging'),
+			recorder: new QuestionParkBinding(),
+		})
+
+		expect(await pause(request)).toEqual({ status: 'answered', selectedOptionIds: ['staging'] })
+	})
+})
+
+describe('the seam a tool author is handed', () => {
+	async function runToolThatPauses(turns: MockTurn[], seen: { outcome?: ToolPauseOutcome }) {
+		const tools = new ToolRegistry()
+		tools.register(
+			defineTool({
+				name: 'deploy',
+				description: 'deploy tool',
+				inputSchema: z.object({}),
+				category: 'custom',
+				permissions: [],
+				readOnly: false,
+				destructive: true,
+				concurrencySafe: false,
+				execute: async (_input, context) => {
+					seen.outcome = await context.requestPause?.(request)
+					return { success: true, output: `pause: ${seen.outcome?.status ?? 'no seam'}` }
+				},
+			}),
+		)
+
+		return drainQuery({
+			provider: new MockLLMProvider({ turns }),
+			tools,
+			agentId: 'a',
+			agentName: 'A',
+			messages: [{ role: 'user', content: 'deploy it' }],
+			workingDirectory: process.cwd(),
+			runConfig: { model: 'mock', tokenBudget: 100_000, timeoutMs: 30_000, maxIterations: 3 },
+			projectId: generateProjectId(),
+			sessionId: generateSessionId(),
+			threadId: generateThreadId(),
+			tenantId: generateTenantId(),
+			resumeHandler: async () => ANSWER('call_1:target_environment', 'staging'),
+		} as never)
+	}
+
+	it('reaches a host-authored tool through the tool context', async () => {
+		const seen: { outcome?: ToolPauseOutcome } = {}
+		await runToolThatPauses(
+			[
+				{
+					toolCalls: [{ id: 'call_1', name: 'deploy', args: {} }],
+					finishReason: 'tool_calls',
+				},
+				{ text: 'deployed' },
+			],
+			seen,
+		)
+
+		expect(seen.outcome).toEqual({ status: 'answered', selectedOptionIds: ['staging'] })
+	})
+})

package/src/runtime/query/__tests__/tool-progress.test.ts

@@ -0,0 +1,154 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import { MockLLMProvider } from '../../../provider/mock.js'
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import type { SessionId, TenantId } from '../../../types/ids/index.js'
+import { createUserMessage } from '../../../types/message/index.js'
+import { isEphemeralEvent } from '../../../types/run/events.js'
+import type { RunEvent } from '../../../types/run/index.js'
+import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
+import type { ToolContext, ToolDefinition } from '../../../types/tool/index.js'
+import { drainQuery } from '../index.js'
+
+/**
+ * A tool may run for the full per-tool deadline — two minutes by default —
+ * and was silent for all of it. A host could show that a build had started
+ * and then nothing at all until it finished or timed out.
+ */
+
+const dirs: string[] = []
+
+afterEach(async () => {
+	await Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true })))
+	dirs.length = 0
+})
+
+function reportingTool(report: (ctx: ToolContext) => void): ToolDefinition {
+	return {
+		name: 'build',
+		description: 'Build the project',
+		inputSchema: z.object({}),
+		execute: (_input: unknown, ctx: ToolContext) => {
+			report(ctx)
+			return Promise.resolve({ success: true, output: 'built' })
+		},
+	} as unknown as ToolDefinition
+}
+
+async function run(tool: ToolDefinition) {
+	const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-progress-'))
+	dirs.push(workingDirectory)
+
+	const tools = new ToolRegistry()
+	tools.register(tool)
+	const events: RunEvent[] = []
+
+	const result = await drainQuery(
+		{
+			provider: new MockLLMProvider({
+				turns: [{ toolCalls: [{ name: 'build', args: {} }] }, { text: 'done' }],
+			}),
+			tools,
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 10_000,
+				tokenBudget: 100_000,
+				maxIterations: 4,
+				maxResponseTokens: 256,
+			},
+			agentId: 'agent_p',
+			agentName: 'Progress',
+			messages: [createUserMessage('build it')],
+			workingDirectory,
+			sessionId: 'ses_p' as SessionId,
+			threadId: 'thd_p' as ThreadId,
+			projectId: 'prj_p' as ProjectId,
+			tenantId: 'tnt_p' as TenantId,
+		},
+		(e) => {
+			events.push(e)
+		},
+	)
+
+	return { result, events }
+}
+
+describe('a long-running tool can say how far along it is', () => {
+	it('reaches the host as an event naming the call it belongs to', async () => {
+		const { events } = await run(
+			reportingTool((ctx) => {
+				ctx.report?.('compiled 40/120 files', 0.33)
+			}),
+		)
+
+		const progress = events.find(
+			(e): e is Extract<RunEvent, { type: 'tool_progress' }> => e.type === 'tool_progress',
+		)
+		expect(progress).toBeDefined()
+		expect(progress?.message).toBe('compiled 40/120 files')
+		expect(progress?.fraction).toBeCloseTo(0.33)
+		// A batch can run several tools at once, so a host rendering them
+		// side by side needs to know whose progress this is.
+		expect(progress?.toolName).toBe('build')
+		expect(progress?.toolUseId).toBeTruthy()
+	})
+
+	it('clamps a fraction a tool gets wrong rather than passing it on', async () => {
+		const { events } = await run(
+			reportingTool((ctx) => {
+				ctx.report?.('overshot', 4.2)
+				ctx.report?.('undershot', -1)
+			}),
+		)
+
+		const fractions = events
+			.filter((e): e is Extract<RunEvent, { type: 'tool_progress' }> => e.type === 'tool_progress')
+			.map((e) => e.fraction)
+		expect(fractions).toEqual([1, 0])
+	})
+
+	it('is ephemeral, so a chatty tool cannot bloat the durable record', () => {
+		expect(
+			isEphemeralEvent({
+				type: 'tool_progress',
+				runId: 'run_x' as never,
+				toolUseId: 'call_x' as never,
+				toolName: 'build',
+				message: 'x',
+			} as RunEvent),
+		).toBe(true)
+	})
+
+	it('reporting never throws back into the tool', async () => {
+		// The property a tool author relies on: `report()` is fire-and-forget
+		// and returns void, so a tool can call it without wrapping it.
+		//
+		// Note what this does NOT claim: a HOST listener that throws still
+		// kills the run, because `drainQuery` awaits the listener unguarded
+		// for every event type. That is a general contract, not something
+		// specific to progress, and it is not this feature's to change.
+		let threw = false
+		const { result } = await run(
+			reportingTool((ctx) => {
+				try {
+					ctx.report?.('tick', 0.5)
+					ctx.report?.('tock')
+				} catch {
+					threw = true
+				}
+			}),
+		)
+
+		expect(threw).toBe(false)
+		expect(result.result).toBe('done')
+	})
+
+	it('a tool that reports nothing produces no events', async () => {
+		const { events } = await run(reportingTool(() => {}))
+		expect(events.some((e) => e.type === 'tool_progress')).toBe(false)
+	})
+})

package/src/runtime/query/__tests__/tool-repair-retry.test.ts

@@ -0,0 +1,255 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { z } from 'zod'
+
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import { ActivityStore } from '../../../store/activity/memory.js'
+import type { RunId } from '../../../types/ids/index.js'
+import type { ChatCompletionResponse } from '../../../types/provider/index.js'
+import type { ToolDefinition, ToolResult } from '../../../types/tool/index.js'
+import type { RepairToolCall } from '../../../types/tool/repair.js'
+import type { Logger } from '../../../utils/logger.js'
+import { ToolExecutor, type ToolExecutorConfig } from '../executor.js'
+
+/**
+ * Two failure modes that both used to cost a full model round trip.
+ *
+ * A malformed tool call went back as a `tool_result` error; the model
+ * re-read the entire context and issued a second inference to add a
+ * missing brace. A transient tool failure did the same, except the model
+ * also had to decide on its own that retrying was worth it.
+ */
+
+const RUN_ID = 'run_repair' as RunId
+
+function makeLogger(): Logger {
+	const stub = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }
+	return { ...stub, child: vi.fn(() => ({ ...stub, child: vi.fn() })) } as unknown as Logger
+}
+
+function response(name: string, args: string): ChatCompletionResponse {
+	return {
+		id: 'resp_1',
+		model: 'mock',
+		message: {
+			role: 'assistant',
+			content: null,
+			toolCalls: [{ id: 'call_1', type: 'function', function: { name, arguments: args } }],
+		},
+		finishReason: 'tool_calls',
+		usage: {
+			promptTokens: 0,
+			completionTokens: 0,
+			totalTokens: 0,
+			cachedTokens: 0,
+			cacheWriteTokens: 0,
+		},
+	}
+}
+
+function makeExecutor(
+	registry: ToolRegistry,
+	extra: Partial<ToolExecutorConfig> = {},
+): { executor: ToolExecutor } {
+	const executor = new ToolExecutor(
+		{
+			tools: registry,
+			runId: RUN_ID,
+			workingDirectory: process.cwd(),
+			permissionMode: 'auto',
+			env: {},
+			abortSignal: new AbortController().signal,
+			...extra,
+		},
+		new ActivityStore(RUN_ID, { enabled: false, trackToolCalls: false, trackLlmTurns: false }),
+		() => Promise.resolve(),
+		makeLogger(),
+	)
+	return { executor }
+}
+
+/** A tool that fails `failures` times, then succeeds. */
+function flakyTool(opts: { failures: number; retryable: boolean; maxRetries?: number }) {
+	const attempts: number[] = []
+	let seen = 0
+	const tool = {
+		name: 'fetch_page',
+		description: 'Fetch a page',
+		inputSchema: z.object({ url: z.string() }),
+		...(opts.maxRetries !== undefined ? { maxRetries: opts.maxRetries } : {}),
+		execute: (): Promise<ToolResult> => {
+			seen++
+			attempts.push(seen)
+			if (seen <= opts.failures) {
+				return Promise.resolve({
+					success: false,
+					output: '',
+					error: 'connection reset',
+					retryable: opts.retryable,
+				})
+			}
+			return Promise.resolve({ success: true, output: `fetched on attempt ${seen}` })
+		},
+	} as unknown as ToolDefinition
+	return { tool, attempts }
+}
+
+let registry: ToolRegistry
+
+beforeEach(() => {
+	registry = new ToolRegistry()
+})
+
+describe('repairToolCall', () => {
+	const readFile = {
+		name: 'read_file',
+		description: 'Read a file',
+		inputSchema: z.object({ path: z.string() }),
+		execute: (input: unknown) =>
+			Promise.resolve({ success: true, output: `read ${(input as { path: string }).path}` }),
+	} as unknown as ToolDefinition
+
+	beforeEach(() => {
+		registry.register(readFile)
+	})
+
+	it('fixes unparseable arguments instead of spending a round trip on it', async () => {
+		const repair: RepairToolCall = ({ reason }) =>
+			reason === 'invalid_json' ? { arguments: '{"path":"a.ts"}' } : null
+
+		const { executor } = makeExecutor(registry, { repairToolCall: repair })
+		const batch = await executor.executeBatch(response('read_file', '{"path": "a.ts'))
+
+		expect(batch.results[0]?.isError).toBeFalsy()
+		expect(batch.results[0]?.output).toBe('read a.ts')
+	})
+
+	it('hands the repairer the schema to aim at', async () => {
+		const seen = vi.fn<RepairToolCall>(() => null)
+		const { executor } = makeExecutor(registry, { repairToolCall: seen })
+		await executor.executeBatch(response('read_file', '{bad'))
+
+		expect(seen).toHaveBeenCalledOnce()
+		const ctx = seen.mock.calls[0]?.[0]
+		expect(ctx?.reason).toBe('invalid_json')
+		expect(ctx?.jsonSchema).toMatchObject({ properties: { path: { type: 'string' } } })
+		expect(ctx?.availableTools).toContain('read_file')
+	})
+
+	it('can correct a near-miss tool name', async () => {
+		const repair: RepairToolCall = ({ reason, availableTools }) =>
+			reason === 'unknown_tool'
+				? { toolName: availableTools[0] as string, arguments: '{"path":"b.ts"}' }
+				: null
+
+		const { executor } = makeExecutor(registry, { repairToolCall: repair })
+		const batch = await executor.executeBatch(response('readFile', '{"path":"b.ts"}'))
+
+		expect(batch.results[0]?.output).toBe('read b.ts')
+		expect(batch.results[0]?.toolName).toBe('read_file')
+	})
+
+	it('repairs arguments that parse but violate the schema', async () => {
+		const repair: RepairToolCall = ({ reason }) =>
+			reason === 'schema_validation' ? { arguments: '{"path":"c.ts"}' } : null
+
+		const { executor } = makeExecutor(registry, { repairToolCall: repair })
+		const batch = await executor.executeBatch(response('read_file', '{"path": 42}'))
+
+		expect(batch.results[0]?.output).toBe('read c.ts')
+	})
+
+	it('declining is normal — the original error proceeds unchanged', async () => {
+		const { executor } = makeExecutor(registry, { repairToolCall: () => null })
+		const batch = await executor.executeBatch(response('read_file', '{bad'))
+
+		expect(batch.results[0]?.isError).toBe(true)
+		expect(batch.results[0]?.output).toContain('Invalid JSON')
+	})
+
+	it('tries exactly once — a still-broken repair does not loop', async () => {
+		// An unbounded repair loop is a hang, not a degradation.
+		const repair = vi.fn<RepairToolCall>(() => ({ arguments: 'still not json' }))
+		const { executor } = makeExecutor(registry, { repairToolCall: repair })
+		const batch = await executor.executeBatch(response('read_file', '{bad'))
+
+		expect(repair).toHaveBeenCalledOnce()
+		expect(batch.results[0]?.isError).toBe(true)
+	})
+
+	it('a throwing repairer does not take the run down', async () => {
+		const { executor } = makeExecutor(registry, {
+			repairToolCall: () => {
+				throw new Error('repair model unavailable')
+			},
+		})
+		const batch = await executor.executeBatch(response('read_file', '{bad'))
+
+		// The original error was always a perfectly good answer to give.
+		expect(batch.results[0]?.output).toContain('Invalid JSON')
+	})
+
+	it('with no repairer configured, behavior is exactly as before', async () => {
+		const { executor } = makeExecutor(registry)
+		const ok = await executor.executeBatch(response('read_file', '{"path":"d.ts"}'))
+		expect(ok.results[0]?.output).toBe('read d.ts')
+
+		const bad = await executor.executeBatch(response('read_file', '{bad'))
+		expect(bad.results[0]?.output).toContain('Invalid JSON')
+
+		// Schema errors still come from the registry, which has the better
+		// message — a "Required: ..." hint the model can self-correct from.
+		const invalid = await executor.executeBatch(response('read_file', '{"path": 42}'))
+		expect(invalid.results[0]?.isError).toBe(true)
+	})
+})
+
+describe('per-tool retry budget', () => {
+	it('retries a retryable failure in-loop and reports the success', async () => {
+		const { tool, attempts } = flakyTool({ failures: 2, retryable: true, maxRetries: 3 })
+		registry.register(tool)
+
+		const { executor } = makeExecutor(registry)
+		const batch = await executor.executeBatch(response('fetch_page', '{"url":"x"}'))
+
+		expect(attempts).toHaveLength(3)
+		expect(batch.results[0]?.output).toBe('fetched on attempt 3')
+		expect(batch.results[0]?.isError).toBeFalsy()
+	})
+
+	it('does NOT retry unless the tool opted in — the default is 0', async () => {
+		// The SDK cannot know a tool is idempotent. Silently re-running a
+		// write or a payment is worse than never retrying.
+		const { tool, attempts } = flakyTool({ failures: 1, retryable: true })
+		registry.register(tool)
+
+		const { executor } = makeExecutor(registry)
+		const batch = await executor.executeBatch(response('fetch_page', '{"url":"x"}'))
+
+		expect(attempts).toHaveLength(1)
+		expect(batch.results[0]?.isError).toBe(true)
+	})
+
+	it('does not spend the budget on a failure that will never succeed', async () => {
+		// A missing file does not appear on the second attempt; retrying
+		// only delays the error the model needs to see.
+		const { tool, attempts } = flakyTool({ failures: 5, retryable: false, maxRetries: 3 })
+		registry.register(tool)
+
+		const { executor } = makeExecutor(registry)
+		await executor.executeBatch(response('fetch_page', '{"url":"x"}'))
+
+		expect(attempts).toHaveLength(1)
+	})
+
+	it('gives up after the budget and hands the error to the model', async () => {
+		const { tool, attempts } = flakyTool({ failures: 99, retryable: true, maxRetries: 2 })
+		registry.register(tool)
+
+		const { executor } = makeExecutor(registry)
+		const batch = await executor.executeBatch(response('fetch_page', '{"url":"x"}'))
+
+		expect(attempts).toHaveLength(3) // the first try plus two retries
+		expect(batch.results[0]?.isError).toBe(true)
+		expect(batch.results[0]?.output).toContain('connection reset')
+	})
+})