@namzu/sdk 3.1.0 → 3.2.0

package/src/registry/tool/execute.ts

@@ -375,151 +375,155 @@ Executable tool names, descriptions, and JSON input schemas are attached through
 			: otelContext.active()
 
 		return tracer.startActiveSpan(toolSpanName(toolName), {}, parentCtx, async (span) => {
-			span.setAttributes({
-				[GENAI.TOOL_NAME]: toolName,
-				[GENAI.TOOL_TYPE]: 'function',
-			})
-
-			const tool = this.getOrThrow(toolName)
-
-			const availability = this.getAvailability(toolName)
-			if (availability !== 'active') {
-				const msg = `Tool "${toolName}" is ${availability} and cannot be executed`
-				this.log.warn(msg)
+			try {
 				span.setAttributes({
-					[NAMZU.TOOL_SUCCESS]: false,
-					[NAMZU.TOOL_ERROR]: msg,
+					[GENAI.TOOL_NAME]: toolName,
+					[GENAI.TOOL_TYPE]: 'function',
 				})
-				span.setStatus({ code: SpanStatusCode.ERROR, message: msg })
-				span.end()
-				return {
-					success: false,
-					output: '',
-					error: msg,
-				}
-			}
 
-			const mode = context.permissionContext?.mode ?? 'auto'
-			if (mode === 'plan') {
-				const isReadOnly = tool.isReadOnly ? tool.isReadOnly(rawInput) : false
-				if (!isReadOnly) {
-					const msg = `plan mode: non-read-only tool "${toolName}" blocked`
+				const tool = this.getOrThrow(toolName)
+
+				const availability = this.getAvailability(toolName)
+				if (availability !== 'active') {
+					const msg = `Tool "${toolName}" is ${availability} and cannot be executed`
+					this.log.warn(msg)
 					span.setAttributes({
 						[NAMZU.TOOL_SUCCESS]: false,
 						[NAMZU.TOOL_ERROR]: msg,
 					})
 					span.setStatus({ code: SpanStatusCode.ERROR, message: msg })
-					span.end()
 					return {
 						success: false,
 						output: '',
 						error: msg,
-						permissionDenied: true,
-						permissionMessage: msg,
 					}
 				}
-			}
 
-			const parseResult = tool.inputSchema.safeParse(rawInput)
-			if (!parseResult.success) {
-				const errorMessage = parseResult.error.issues
-					.map((i) => `${i.path.join('.')}: ${i.message}`)
-					.join('; ')
-
-				// Distinguish "model sent an empty/no-arg call" from
-				// "model sent partial args" — the first is most often a
-				// streaming hiccup or a definition-test ping (a provider
-				// occasionally pings tool surfaces with `{}` while the
-				// schema is still loading), the second is a genuine
-				// programming mistake by the model. The model self-
-				// corrects MUCH more reliably when the error tells it
-				// (a) which fields are required, (b) their types, and
-				// (c) a minimal example call. Without these hints the
-				// downstream UI just shows a red "Failed" row and the
-				// model rarely retries with the right args.
-				const isEmptyInput =
-					rawInput === null ||
-					rawInput === undefined ||
-					(typeof rawInput === 'object' &&
-						!Array.isArray(rawInput) &&
-						Object.keys(rawInput as Record<string, unknown>).length === 0)
-
-				const requiredHint = describeRequiredInput(tool.inputSchema)
-				// A conditional schema's required shape cannot be reconstructed
-				// from JSON Schema's top-level `required`, so the author gets to
-				// say what a valid retry looks like.
-				const recoveryHint = tool.validationErrorHint?.trim()
-					? ` ${tool.validationErrorHint.trim()}`
-					: ''
-
-				const enrichedMessage = isEmptyInput
-					? `Tool "${toolName}" was called with no arguments. ${requiredHint}${recoveryHint} Retry the call with the required parameters populated.`
-					: `Validation failed for "${toolName}": ${errorMessage}. ${requiredHint}${recoveryHint}`
-
-				this.log.error(`Tool input validation failed: ${toolName}`, {
-					errors: errorMessage,
-					empty: isEmptyInput,
-				})
+				const mode = context.permissionContext?.mode ?? 'auto'
+				if (mode === 'plan') {
+					const isReadOnly = tool.isReadOnly ? tool.isReadOnly(rawInput) : false
+					if (!isReadOnly) {
+						const msg = `plan mode: non-read-only tool "${toolName}" blocked`
+						span.setAttributes({
+							[NAMZU.TOOL_SUCCESS]: false,
+							[NAMZU.TOOL_ERROR]: msg,
+						})
+						span.setStatus({ code: SpanStatusCode.ERROR, message: msg })
+						return {
+							success: false,
+							output: '',
+							error: msg,
+							permissionDenied: true,
+							permissionMessage: msg,
+						}
+					}
+				}
 
-				span.setAttributes({
-					[NAMZU.TOOL_SUCCESS]: false,
-					[NAMZU.TOOL_ERROR]: `Validation: ${errorMessage}`,
-				})
-				span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage })
-				span.end()
+				const parseResult = tool.inputSchema.safeParse(rawInput)
+				if (!parseResult.success) {
+					const errorMessage = parseResult.error.issues
+						.map((i) => `${i.path.join('.')}: ${i.message}`)
+						.join('; ')
+
+					// Distinguish "model sent an empty/no-arg call" from
+					// "model sent partial args" — the first is most often a
+					// streaming hiccup or a definition-test ping (a provider
+					// occasionally pings tool surfaces with `{}` while the
+					// schema is still loading), the second is a genuine
+					// programming mistake by the model. The model self-
+					// corrects MUCH more reliably when the error tells it
+					// (a) which fields are required, (b) their types, and
+					// (c) a minimal example call. Without these hints the
+					// downstream UI just shows a red "Failed" row and the
+					// model rarely retries with the right args.
+					const isEmptyInput =
+						rawInput === null ||
+						rawInput === undefined ||
+						(typeof rawInput === 'object' &&
+							!Array.isArray(rawInput) &&
+							Object.keys(rawInput as Record<string, unknown>).length === 0)
+
+					const requiredHint = describeRequiredInput(tool.inputSchema)
+					// A conditional schema's required shape cannot be reconstructed
+					// from JSON Schema's top-level `required`, so the author gets to
+					// say what a valid retry looks like.
+					const recoveryHint = tool.validationErrorHint?.trim()
+						? ` ${tool.validationErrorHint.trim()}`
+						: ''
+
+					const enrichedMessage = isEmptyInput
+						? `Tool "${toolName}" was called with no arguments. ${requiredHint}${recoveryHint} Retry the call with the required parameters populated.`
+						: `Validation failed for "${toolName}": ${errorMessage}. ${requiredHint}${recoveryHint}`
+
+					this.log.error(`Tool input validation failed: ${toolName}`, {
+						errors: errorMessage,
+						empty: isEmptyInput,
+					})
 
-				return {
-					success: false,
-					output: '',
-					error: enrichedMessage,
+					span.setAttributes({
+						[NAMZU.TOOL_SUCCESS]: false,
+						[NAMZU.TOOL_ERROR]: `Validation: ${errorMessage}`,
+					})
+					span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage })
+
+					return {
+						success: false,
+						output: '',
+						error: enrichedMessage,
+					}
 				}
-			}
 
-			const finalInput = parseResult.data
+				const finalInput = parseResult.data
 
-			try {
-				this.log.debug(`Executing tool: ${toolName}`)
-				const startedAt = Date.now()
-				const result = await tool.execute(finalInput, context)
-				const durationMs = Date.now() - startedAt
-				this.log.debug(`Tool completed: ${toolName}`, {
-					success: result.success,
-				})
+				try {
+					this.log.debug(`Executing tool: ${toolName}`)
+					const startedAt = Date.now()
+					const result = await tool.execute(finalInput, context)
+					const durationMs = Date.now() - startedAt
+					this.log.debug(`Tool completed: ${toolName}`, {
+						success: result.success,
+					})
 
-				span.setAttribute(NAMZU.TOOL_SUCCESS, result.success)
-				recordToolCall(
-					toolName,
-					result.success,
-					result.success ? undefined : result.error,
-					durationMs,
-				)
-				if (!result.success && result.error) {
-					span.setAttribute(NAMZU.TOOL_ERROR, result.error)
-					span.setStatus({ code: SpanStatusCode.ERROR, message: result.error })
-				} else {
-					span.setStatus({ code: SpanStatusCode.OK })
-				}
+					span.setAttribute(NAMZU.TOOL_SUCCESS, result.success)
+					recordToolCall(
+						toolName,
+						result.success,
+						result.success ? undefined : result.error,
+						durationMs,
+					)
+					if (!result.success && result.error) {
+						span.setAttribute(NAMZU.TOOL_ERROR, result.error)
+						span.setStatus({ code: SpanStatusCode.ERROR, message: result.error })
+					} else {
+						span.setStatus({ code: SpanStatusCode.OK })
+					}
 
-				return result
-			} catch (err) {
-				const errorMessage = toErrorMessage(err)
-				this.log.error(`Tool execution error: ${toolName}`, {
-					error: errorMessage,
-				})
+					return result
+				} catch (err) {
+					const errorMessage = toErrorMessage(err)
+					this.log.error(`Tool execution error: ${toolName}`, {
+						error: errorMessage,
+					})
 
-				span.setAttributes({
-					[NAMZU.TOOL_SUCCESS]: false,
-					[NAMZU.TOOL_ERROR]: errorMessage,
-				})
-				span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage })
-				span.recordException(err instanceof Error ? err : new Error(errorMessage))
+					span.setAttributes({
+						[NAMZU.TOOL_SUCCESS]: false,
+						[NAMZU.TOOL_ERROR]: errorMessage,
+					})
+					span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage })
+					span.recordException(err instanceof Error ? err : new Error(errorMessage))
 
-				return {
-					success: false,
-					output: '',
-					error: `Tool "${toolName}" execution failed: ${errorMessage}`,
+					return {
+						success: false,
+						output: '',
+						error: `Tool "${toolName}" execution failed: ${errorMessage}`,
+					}
 				}
 			} finally {
+				// The only place this span ends. It used to be ended at three
+				// early returns and in a finally that opened below them, so
+				// anything throwing before that try — `getOrThrow` on a name the
+				// registry does not hold, for one — left the span open and the
+				// tool's trace unclosed.
 				span.end()
 			}
 		})

package/src/runtime/query/__tests__/per-step-skills.test.ts

@@ -0,0 +1,154 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+
+import { MockLLMProvider } from '../../../provider/mock.js'
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import type { SessionId, TenantId } from '../../../types/ids/index.js'
+import { createUserMessage } from '../../../types/message/index.js'
+import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
+import type { Skill } from '../../../types/skills/index.js'
+import { drainQuery } from '../index.js'
+
+/**
+ * A run's skills are fixed at `query()` time and rendered into the cached
+ * system prefix, so every skill a run might ever need is paid for on every
+ * turn. A phased agent rarely needs them all at once.
+ *
+ * A peer runtime resolves instructions, model, tools, skills and subagents
+ * from context at run time. namzu had the first three; this is the fourth.
+ * The fifth is deliberately absent — see the note at the bottom.
+ */
+
+let workdirs: string[] = []
+
+afterEach(async () => {
+	await Promise.all(workdirs.map((d) => rm(d, { recursive: true, force: true })))
+	workdirs = []
+})
+
+function skill(name: string, body?: string): Skill {
+	return {
+		metadata: { name, description: `${name} description` },
+		dirPath: `/skills/${name}`,
+		...(body !== undefined ? { body } : {}),
+	}
+}
+
+async function run(provider: MockLLMProvider, over: Record<string, unknown> = {}): Promise<void> {
+	const dir = await mkdtemp(join(tmpdir(), 'namzu-skills-'))
+	workdirs.push(dir)
+	await drainQuery({
+		provider,
+		tools: new ToolRegistry(),
+		runConfig: {
+			model: 'mock-model',
+			timeoutMs: 30_000,
+			tokenBudget: 100_000,
+			maxIterations: 4,
+			maxResponseTokens: 256,
+		},
+		agentId: 'agent_sk',
+		agentName: 'Skill Agent',
+		workingDirectory: dir,
+		sessionId: 'ses_sk' as SessionId,
+		threadId: 'thd_sk' as ThreadId,
+		projectId: 'prj_sk' as ProjectId,
+		tenantId: 'tnt_sk' as TenantId,
+		messages: [createUserMessage('go')],
+		...over,
+	} as never)
+}
+
+const sent = (provider: MockLLMProvider, index = 0): string =>
+	JSON.stringify(provider.requests.at(index)?.messages ?? [])
+
+describe('a step can put a skill in front of the model', () => {
+	it('renders the skill it named', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+
+		await run(provider, { prepareStep: () => ({ skills: [skill('search-the-web')] }) })
+
+		expect(sent(provider)).toContain('search-the-web')
+	})
+
+	it('sends nothing extra when the step names none', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+
+		await run(provider)
+
+		expect(sent(provider)).not.toContain('Available Skills')
+	})
+
+	it('treats an empty list as naming none', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+
+		await run(provider, { prepareStep: () => ({ skills: [] }) })
+
+		// An empty list is a caller saying "no skills this step", not a
+		// request for an empty section header.
+		expect(sent(provider)).not.toContain('Available Skills')
+	})
+
+	it('carries the step guidance alongside it', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+
+		await run(provider, {
+			prepareStep: () => ({ system: 'You are researching.', skills: [skill('search-the-web')] }),
+		})
+
+		// Both ride the same ephemeral trailing message; neither should
+		// displace the other.
+		const body = sent(provider)
+		expect(body).toContain('You are researching.')
+		expect(body).toContain('search-the-web')
+	})
+})
+
+describe('a step skill does not outlive its step', () => {
+	it('is absent from the next step that does not ask for it', async () => {
+		const provider = new MockLLMProvider({
+			turns: [{ toolCalls: [] as never, text: 'thinking' }, { text: 'done' }],
+		})
+
+		await run(provider, {
+			prepareStep: ({ stepNumber }: { stepNumber: number }) =>
+				stepNumber === 1 ? { skills: [skill('search-the-web')] } : {},
+		})
+
+		expect(sent(provider, 0)).toContain('search-the-web')
+		if (provider.requests.length > 1) {
+			expect(sent(provider, 1)).not.toContain('search-the-web')
+		}
+	})
+
+	it('does not accumulate across steps', async () => {
+		const provider = new MockLLMProvider({
+			turns: [{ text: 'still working' }, { text: 'done' }],
+		})
+
+		await run(provider, {
+			maxIterations: 2,
+			prepareStep: () => ({ skills: [skill('search-the-web')] }),
+		})
+
+		// Appended for the call, never retained. Were it written into the run's
+		// history the section would stack up turn after turn — the same
+		// message repeated, growing the prompt and invalidating the cached
+		// prefix every iteration.
+		const systemCounts = provider.requests.map(
+			(r) => (r.messages ?? []).filter((m) => m.role === 'system').length,
+		)
+		expect(new Set(systemCounts).size).toBe(1)
+	})
+})
+
+/**
+ * Sub-agents are deliberately NOT per-step, and this is the note rather than
+ * a test that would pretend otherwise. Which agents `create_task` can reach
+ * is baked into that tool's input schema, so varying it per step would
+ * rebuild the tool catalogue every turn — a worse prompt-cache trade than
+ * moving tools around, for a narrowing a step can already express by
+ * withholding `create_task` through `activeTools`.
+ */

package/src/runtime/query/__tests__/per-step-tool-choice.test.ts

@@ -0,0 +1,180 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import { MockLLMProvider } from '../../../provider/mock.js'
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import type { SessionId, TenantId } from '../../../types/ids/index.js'
+import { createUserMessage } from '../../../types/message/index.js'
+import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
+import { drainQuery } from '../index.js'
+
+/**
+ * Forcing a tool is the one step-shaping knob that can hang an agent.
+ *
+ * A forced choice that PERSISTS makes the model call a tool, read the
+ * result, and be forced again — forever. The peer SDK that puts
+ * `tool_choice` on persistent model settings carries three moving parts to
+ * undo that: a tool-use tracker, an opt-out flag, and a reset applied at
+ * two separate call sites. Its flag defaults to on precisely because
+ * turning it off hangs the agent.
+ *
+ * Putting the knob on `prepareStep` removes the failure instead of
+ * managing it: the next step is prepared from scratch, so a force cannot
+ * outlive the step that asked for it. There is no flag to get wrong.
+ */
+
+let workdirs: string[] = []
+
+afterEach(async () => {
+	await Promise.all(workdirs.map((d) => rm(d, { recursive: true, force: true })))
+	workdirs = []
+})
+
+async function mkWorkdir(): Promise<string> {
+	const dir = await mkdtemp(join(tmpdir(), 'namzu-tool-choice-'))
+	workdirs.push(dir)
+	return dir
+}
+
+function registerEcho(tools: ToolRegistry): void {
+	tools.register({
+		name: 'echo',
+		description: 'Echo the text back.',
+		inputSchema: z.object({ text: z.string().optional() }),
+		execute: async () => ({ success: true, output: 'ok' }),
+	})
+}
+
+async function baseParams(provider: MockLLMProvider, tools: ToolRegistry) {
+	return {
+		provider,
+		tools,
+		runConfig: {
+			model: 'mock-model',
+			timeoutMs: 30_000,
+			tokenBudget: 100_000,
+			maxIterations: 4,
+			maxResponseTokens: 256,
+		},
+		agentId: 'agent_tc',
+		agentName: 'Tool Choice Agent',
+		workingDirectory: await mkWorkdir(),
+		sessionId: 'ses_tc' as SessionId,
+		threadId: 'thd_tc' as ThreadId,
+		projectId: 'prj_tc' as ProjectId,
+		tenantId: 'tnt_tc' as TenantId,
+		messages: [createUserMessage('go')],
+	}
+}
+
+describe('a step can force the model to call a tool', () => {
+	it('puts the caller choice on that step request', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+		const tools = new ToolRegistry()
+		registerEcho(tools)
+
+		await drainQuery({
+			...(await baseParams(provider, tools)),
+			prepareStep: () => ({ toolChoice: 'required' as const }),
+		})
+
+		expect(provider.requests.at(0)?.toolChoice).toBe('required')
+	})
+
+	it('carries a named function through unchanged', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+		const tools = new ToolRegistry()
+		registerEcho(tools)
+
+		await drainQuery({
+			...(await baseParams(provider, tools)),
+			prepareStep: () => ({
+				toolChoice: { type: 'function' as const, function: { name: 'echo' } },
+			}),
+		})
+
+		expect(provider.requests.at(0)?.toolChoice).toEqual({
+			type: 'function',
+			function: { name: 'echo' },
+		})
+	})
+
+	it('leaves the request alone when no step asks', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+		const tools = new ToolRegistry()
+		registerEcho(tools)
+
+		await drainQuery(await baseParams(provider, tools))
+
+		expect(provider.requests.at(0)?.toolChoice).toBeUndefined()
+	})
+
+	it('says nothing when there are no tools to choose between', async () => {
+		const provider = new MockLLMProvider({ turns: [{ text: 'done' }] })
+
+		await drainQuery({
+			...(await baseParams(provider, new ToolRegistry())),
+			prepareStep: () => ({ toolChoice: 'required' as const }),
+		})
+
+		// `tool_choice` alongside an absent tool list is rejected by the
+		// providers, so a forced choice with nothing registered has to drop.
+		expect(provider.requests.at(0)?.toolChoice).toBeUndefined()
+	})
+})
+
+/**
+ * NOT covered here, and deliberately not faked: the forced-final turn takes
+ * precedence over a step's choice (`forceFinalize` wins in
+ * `iteration/index.ts`), so a budget-exhausted run can still stop asking
+ * for tools and answer. Reaching that branch needs the guard to raise a
+ * budget warning, which needs real usage the mock provider does not report.
+ * The precedence is implemented and read in review; it is not pinned.
+ */
+describe('a forced choice cannot outlive the step that asked for it', () => {
+	it('applies to the named step only, leaving later steps free', async () => {
+		const provider = new MockLLMProvider({
+			turns: [
+				{ toolCalls: [{ id: 'c1', name: 'echo', rawArguments: '{}' }] },
+				{ text: 'finished' },
+			],
+		})
+		const tools = new ToolRegistry()
+		registerEcho(tools)
+
+		await drainQuery({
+			...(await baseParams(provider, tools)),
+			// Only the first step is forced. Without per-step scoping this
+			// would keep forcing and the run would never reach a text answer.
+			prepareStep: ({ stepNumber }) =>
+				stepNumber === 1 ? { toolChoice: 'required' as const } : {},
+		})
+
+		expect(provider.requests.at(0)?.toolChoice).toBe('required')
+		expect(provider.requests.at(1)?.toolChoice).toBeUndefined()
+	})
+
+	it('a run whose every step forces a tool still ends, bounded by the loop', async () => {
+		const provider = new MockLLMProvider({
+			turns: [
+				{ toolCalls: [{ id: 'c1', name: 'echo', rawArguments: '{}' }] },
+				{ toolCalls: [{ id: 'c2', name: 'echo', rawArguments: '{}' }] },
+				{ text: 'finished' },
+			],
+		})
+		const tools = new ToolRegistry()
+		registerEcho(tools)
+
+		const run = await drainQuery({
+			...(await baseParams(provider, tools)),
+			prepareStep: () => ({ toolChoice: 'required' as const }),
+		})
+
+		// The knob cannot hang a run on its own: the iteration cap is still
+		// the backstop, and it settles rather than spinning.
+		expect(['completed', 'failed']).toContain(run.status)
+	})
+})