@namzu/sdk 0.6.0 → 1.1.0

package/src/registry/toolset/catalog.test.ts

@@ -0,0 +1,165 @@
+import { describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import type { ToolDefinition } from '../../types/tool/index.js'
+import { ToolRegistry } from '../tool/execute.js'
+import { ToolCatalog, createToolCatalogFromRegistry } from './catalog.js'
+
+function makeTool(name: string, description = `${name} tool`): ToolDefinition {
+	return {
+		name,
+		description,
+		inputSchema: z.object({ query: z.string().optional() }),
+		async execute() {
+			return { success: true, output: `${name}-ran` }
+		},
+	}
+}
+
+describe('ToolCatalog', () => {
+	it('keeps sources, toolsets, and tools as separate records', () => {
+		const catalog = new ToolCatalog()
+		catalog.registerSource({
+			id: 'mcp:microsoft-learn',
+			kind: 'mcp_server',
+			name: 'Microsoft Learn',
+			mcpServer: {
+				name: 'microsoft-learn',
+				url: 'https://learn.microsoft.com/api/mcp',
+				transport: 'streamable_http',
+			},
+		})
+		catalog.registerToolset({
+			id: 'mcp-toolset:microsoft-learn',
+			sourceId: 'mcp:microsoft-learn',
+			name: 'Microsoft Learn toolset',
+			defaultPolicy: {
+				enabled: true,
+				loading: 'deferred',
+				preferred: true,
+				permissionPolicy: 'always_allow',
+			},
+		})
+		catalog.registerTool({
+			name: 'microsoft_docs_search',
+			description: 'Search Microsoft documentation',
+			sourceId: 'mcp:microsoft-learn',
+			toolsetId: 'mcp-toolset:microsoft-learn',
+			policy: { enabled: true, loading: 'deferred', preferred: true },
+		})
+
+		expect(catalog.listSources()).toHaveLength(1)
+		expect(catalog.listToolsets()).toHaveLength(1)
+		expect(catalog.getToolsByLoading(['deferred']).map((t) => t.name)).toEqual([
+			'microsoft_docs_search',
+		])
+	})
+
+	it('searches deferred tools by name, description, source, and toolset', () => {
+		const registry = new ToolRegistry()
+		registry.register(makeTool('github_search_issues', 'Search repository issues'), 'deferred')
+		registry.register(makeTool('slack_search_messages', 'Search team messages'), 'deferred')
+		registry.register(makeTool('bash', 'Run a shell command'))
+
+		const catalog = createToolCatalogFromRegistry(registry, {
+			source: {
+				id: 'host',
+				kind: 'host_tool',
+				name: 'Host runtime',
+				description: 'Local shell and collaboration tools',
+			},
+			toolset: {
+				id: 'default-host',
+				name: 'Default host tools',
+				defaultPolicy: { enabled: true, loading: 'eager' },
+			},
+		})
+
+		expect(
+			catalog.searchTools('repository', { loading: ['deferred'] }).map((r) => r.tool.name),
+		).toEqual(['github_search_issues'])
+		expect(catalog.toLLMTools().map((t) => t.function.name)).toEqual(['bash'])
+	})
+
+	it('scores and reports the matched facets: partial name, source text, toolset text', () => {
+		const catalog = new ToolCatalog()
+		catalog.registerSource({
+			id: 'host',
+			kind: 'host_tool',
+			name: 'GitHub',
+			description: 'repository automation',
+		})
+		catalog.registerToolset({
+			id: 'issues',
+			sourceId: 'host',
+			name: 'Issue toolset',
+			defaultPolicy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'create_issue',
+			description: 'open a ticket',
+			sourceId: 'host',
+			toolsetId: 'issues',
+			policy: { enabled: true, loading: 'eager' },
+		})
+
+		// 'issue' hits the tool NAME (create_issue, partial) AND the TOOLSET name.
+		const byIssue = catalog.searchTools('issue')
+		expect(byIssue).toHaveLength(1)
+		expect([...(byIssue[0]?.matched ?? [])].sort()).toEqual(['name', 'toolset'])
+
+		// 'github' hits only the SOURCE text.
+		const bySource = catalog.searchTools('github')
+		expect(bySource[0]?.matched).toEqual(['source'])
+
+		// An exact name match outscores a partial one.
+		const exact = catalog.searchTools('create_issue')
+		expect(exact[0]?.matched).toContain('name')
+		expect(exact[0]?.score ?? 0).toBeGreaterThan(byIssue[0]?.score ?? 0)
+	})
+
+	it('adds the preferred boost so a preferred tool outranks an equal match', () => {
+		const catalog = new ToolCatalog()
+		catalog.registerSource({ id: 'host', kind: 'host_tool', name: 'Host' })
+		catalog.registerToolset({
+			id: 'ts',
+			sourceId: 'host',
+			name: 'Tools',
+			defaultPolicy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'search_plain',
+			description: 'search things',
+			sourceId: 'host',
+			toolsetId: 'ts',
+			policy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'search_pref',
+			description: 'search things',
+			sourceId: 'host',
+			toolsetId: 'ts',
+			policy: { enabled: true, loading: 'eager', preferred: true },
+		})
+
+		const results = catalog.searchTools('search')
+		expect(results.map((r) => r.tool.name)).toEqual(['search_pref', 'search_plain'])
+		const pref = results.find((r) => r.tool.name === 'search_pref')?.score ?? 0
+		const plain = results.find((r) => r.tool.name === 'search_plain')?.score ?? 0
+		expect(pref - plain).toBe(1)
+	})
+
+	it('preserves registry availability as catalog loading policy', () => {
+		const registry = new ToolRegistry()
+		registry.register(makeTool('read_file'))
+		registry.register(makeTool('web_search'), 'deferred')
+		registry.register(makeTool('write_file'), 'suspended')
+
+		const catalog = createToolCatalogFromRegistry(registry)
+
+		expect(catalog.getTool('read_file')?.policy.loading).toBe('eager')
+		expect(catalog.getTool('web_search')?.policy.loading).toBe('deferred')
+		expect(catalog.getTool('write_file')?.policy.loading).toBe('suspended')
+		expect(catalog.toLLMTools({ loading: ['suspended'] })).toEqual([])
+	})
+})

package/src/registry/toolset/catalog.ts

@@ -0,0 +1,283 @@
+import { zodToJsonSchema } from 'zod-to-json-schema'
+import type { ToolRegistryContract } from '../../types/tool/index.js'
+import type { LLMToolSchema, ToolAvailability, ToolDefinition } from '../../types/tool/index.js'
+import type {
+	ToolCatalogEntry,
+	ToolCatalogSearchResult,
+	ToolCatalogSnapshot,
+	ToolLoadingMode,
+	ToolSource,
+	ToolsetDefinition,
+	ToolsetPolicy,
+} from '../../types/toolset/index.js'
+
+export interface ToolCatalogSearchOptions {
+	readonly loading?: readonly ToolLoadingMode[]
+	readonly limit?: number
+}
+
+export interface ToolCatalogFromRegistryOptions {
+	readonly source?: ToolSource
+	readonly toolset?: Omit<ToolsetDefinition, 'sourceId'> & {
+		readonly sourceId?: string
+	}
+}
+
+const DEFAULT_HOST_SOURCE: ToolSource = {
+	id: 'host-tools',
+	kind: 'host_tool',
+	name: 'Host tools',
+	description: 'Tools executed by the host runtime.',
+}
+
+const DEFAULT_HOST_TOOLSET: ToolsetDefinition = {
+	id: 'host-tools',
+	sourceId: DEFAULT_HOST_SOURCE.id,
+	name: 'Host tools',
+	defaultPolicy: {
+		enabled: true,
+		loading: 'eager',
+		permissionPolicy: 'default',
+	},
+}
+
+export class ToolCatalog {
+	private sources = new Map<string, ToolSource>()
+	private toolsets = new Map<string, ToolsetDefinition>()
+	private tools = new Map<string, ToolCatalogEntry>()
+
+	registerSource(source: ToolSource): void {
+		this.sources.set(source.id, source)
+	}
+
+	registerToolset(toolset: ToolsetDefinition): void {
+		if (!this.sources.has(toolset.sourceId)) {
+			throw new Error(`Toolset "${toolset.id}" references unknown source "${toolset.sourceId}"`)
+		}
+		this.toolsets.set(toolset.id, toolset)
+	}
+
+	registerTool(tool: ToolCatalogEntry): void {
+		if (!this.sources.has(tool.sourceId)) {
+			throw new Error(`Tool "${tool.name}" references unknown source "${tool.sourceId}"`)
+		}
+		if (!this.toolsets.has(tool.toolsetId)) {
+			throw new Error(`Tool "${tool.name}" references unknown toolset "${tool.toolsetId}"`)
+		}
+		this.tools.set(tool.name, tool)
+	}
+
+	getSource(id: string): ToolSource | undefined {
+		return this.sources.get(id)
+	}
+
+	getToolset(id: string): ToolsetDefinition | undefined {
+		return this.toolsets.get(id)
+	}
+
+	getTool(name: string): ToolCatalogEntry | undefined {
+		return this.tools.get(name)
+	}
+
+	listSources(): ToolSource[] {
+		return [...this.sources.values()]
+	}
+
+	listToolsets(): ToolsetDefinition[] {
+		return [...this.toolsets.values()]
+	}
+
+	listTools(): ToolCatalogEntry[] {
+		return [...this.tools.values()]
+	}
+
+	snapshot(): ToolCatalogSnapshot {
+		return {
+			sources: this.listSources(),
+			toolsets: this.listToolsets(),
+			tools: this.listTools(),
+		}
+	}
+
+	getToolsByLoading(loading: readonly ToolLoadingMode[]): ToolCatalogEntry[] {
+		const wanted = new Set(loading)
+		return this.listTools().filter((tool) => wanted.has(resolveToolLoading(tool.policy)))
+	}
+
+	searchTools(query: string, options: ToolCatalogSearchOptions = {}): ToolCatalogSearchResult[] {
+		const normalized = query.trim().toLowerCase()
+		const terms = normalized.split(/\s+/).filter(Boolean)
+		const loading = options.loading ? new Set(options.loading) : null
+		const results: ToolCatalogSearchResult[] = []
+
+		for (const tool of this.listTools()) {
+			if (loading && !loading.has(resolveToolLoading(tool.policy))) continue
+			if (tool.policy.enabled === false) continue
+
+			const source = this.sources.get(tool.sourceId)
+			const toolset = this.toolsets.get(tool.toolsetId)
+			if (!source || !toolset) continue
+
+			const scored = scoreToolSearch({
+				terms,
+				tool,
+				source,
+				toolset,
+			})
+			if (scored.score <= 0) continue
+
+			results.push({
+				tool,
+				source,
+				toolset,
+				score: scored.score,
+				matched: scored.matched,
+			})
+		}
+
+		results.sort((a, b) => b.score - a.score || a.tool.name.localeCompare(b.tool.name))
+		return results.slice(0, options.limit ?? 5)
+	}
+
+	toLLMTools(options: { readonly loading?: readonly ToolLoadingMode[] } = {}): LLMToolSchema[] {
+		const loading = options.loading ?? ['eager']
+		return this.getToolsByLoading(loading)
+			.filter((tool) => tool.policy.enabled !== false)
+			.map((tool) => tool.llmSchema ?? toolDefinitionToLLMTool(tool.definition))
+			.filter((tool): tool is LLMToolSchema => Boolean(tool))
+	}
+}
+
+export function createToolCatalogFromRegistry(
+	registry: ToolRegistryContract,
+	options: ToolCatalogFromRegistryOptions = {},
+): ToolCatalog {
+	const source = options.source ?? DEFAULT_HOST_SOURCE
+	const toolset: ToolsetDefinition = {
+		...DEFAULT_HOST_TOOLSET,
+		...options.toolset,
+		sourceId: options.toolset?.sourceId ?? source.id,
+	}
+	const catalog = new ToolCatalog()
+	catalog.registerSource(source)
+	catalog.registerToolset(toolset)
+
+	for (const definition of registry.getAll()) {
+		const availability = registry.getAvailability(definition.name)
+		catalog.registerTool(
+			toolDefinitionToCatalogEntry(definition, {
+				availability,
+				sourceId: source.id,
+				toolsetId: toolset.id,
+				toolsetPolicy: toolset.defaultPolicy,
+			}),
+		)
+	}
+
+	return catalog
+}
+
+export function toolDefinitionToCatalogEntry(
+	definition: ToolDefinition,
+	input: {
+		readonly availability?: ToolAvailability
+		readonly sourceId: string
+		readonly toolsetId: string
+		readonly toolsetPolicy?: ToolsetPolicy
+	},
+): ToolCatalogEntry {
+	const loading = loadingFromAvailability(input.availability ?? 'active')
+	return {
+		name: definition.name,
+		description: definition.description,
+		sourceId: input.sourceId,
+		toolsetId: input.toolsetId,
+		definition,
+		permissions: definition.permissions,
+		category: definition.category,
+		policy: {
+			...input.toolsetPolicy,
+			enabled: input.toolsetPolicy?.enabled === false ? false : loading !== 'suspended',
+			loading,
+		},
+	}
+}
+
+export function loadingFromAvailability(availability: ToolAvailability): ToolLoadingMode {
+	switch (availability) {
+		case 'deferred':
+			return 'deferred'
+		case 'suspended':
+			return 'suspended'
+		default:
+			return 'eager'
+	}
+}
+
+function resolveToolLoading(policy: ToolsetPolicy): ToolLoadingMode {
+	if (policy.enabled === false) return 'disabled'
+	return policy.loading ?? 'eager'
+}
+
+function toolDefinitionToLLMTool(definition: ToolDefinition | undefined): LLMToolSchema | null {
+	if (!definition) return null
+	return {
+		type: 'function',
+		function: {
+			name: definition.name,
+			description: definition.description,
+			parameters: zodToJsonSchema(definition.inputSchema, {
+				target: 'jsonSchema7',
+				$refStrategy: 'none',
+			}) as Record<string, unknown>,
+		},
+	}
+}
+
+function scoreToolSearch(input: {
+	readonly terms: readonly string[]
+	readonly tool: ToolCatalogEntry
+	readonly source: ToolSource
+	readonly toolset: ToolsetDefinition
+}): { score: number; matched: string[] } {
+	const terms = input.terms.length > 0 ? input.terms : ['']
+	const matched = new Set<string>()
+	let score = 0
+
+	for (const term of terms) {
+		if (!term) continue
+
+		const toolName = input.tool.name.toLowerCase()
+		const description = input.tool.description.toLowerCase()
+		const sourceText =
+			`${input.source.name} ${input.source.description ?? ''} ${input.source.kind}`.toLowerCase()
+		const toolsetText = `${input.toolset.name} ${input.toolset.description ?? ''}`.toLowerCase()
+
+		if (toolName === term) {
+			score += 12
+			matched.add('name')
+		} else if (toolName.includes(term)) {
+			score += 8
+			matched.add('name')
+		}
+		if (description.includes(term)) {
+			score += 5
+			matched.add('description')
+		}
+		if (sourceText.includes(term)) {
+			score += 3
+			matched.add('source')
+		}
+		if (toolsetText.includes(term)) {
+			score += 2
+			matched.add('toolset')
+		}
+	}
+
+	if (input.tool.policy.preferred && score > 0) score += 1
+
+	return {
+		score,
+		matched: [...matched],
+	}
+}

package/src/runtime/query/__tests__/deferred-tools.test.ts

@@ -0,0 +1,230 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import { SearchToolsTool } from '../../../tools/builtins/search-tools.js'
+import type { RunId, SessionId, TenantId } from '../../../types/ids/index.js'
+import { createUserMessage } from '../../../types/message/index.js'
+import type {
+	ChatCompletionParams,
+	LLMProvider,
+	StreamChunk,
+} from '../../../types/provider/index.js'
+import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
+import { drainQuery } from '../index.js'
+
+const ZERO_USAGE = {
+	promptTokens: 0,
+	completionTokens: 0,
+	totalTokens: 0,
+	cachedTokens: 0,
+	cacheWriteTokens: 0,
+}
+
+class CapturingProvider implements LLMProvider {
+	readonly id = 'capturing'
+	readonly name = 'Capturing Provider'
+	lastParams?: ChatCompletionParams
+
+	async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
+		this.lastParams = params
+		yield {
+			id: 'msg_1',
+			delta: { content: 'done' },
+		}
+		yield {
+			id: 'msg_1',
+			delta: {},
+			finishReason: 'stop',
+			usage: ZERO_USAGE,
+		}
+	}
+}
+
+function registerDeferredDocumentTool(tools: ToolRegistry, name = 'generate_document'): void {
+	tools.register(
+		{
+			name,
+			description: 'Generate a project document by document id.',
+			inputSchema: z.object({
+				documentId: z.string(),
+			}),
+			execute: async () => ({ success: true, output: 'generated' }),
+		},
+		'deferred',
+	)
+}
+
+describe('query deferred tool discovery', () => {
+	let workdirs: string[] = []
+
+	afterEach(async () => {
+		await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
+		workdirs = []
+	})
+
+	it('auto-exposes search_tools when deferred tools are registered', async () => {
+		const provider = new CapturingProvider()
+		const tools = new ToolRegistry()
+		registerDeferredDocumentTool(tools)
+
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-deferred-tools-'))
+		workdirs.push(workingDirectory)
+
+		const run = await drainQuery({
+			provider,
+			tools,
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 5_000,
+				tokenBudget: 100_000,
+				maxIterations: 1,
+				maxResponseTokens: 256,
+			},
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('what tools can you use?')],
+			workingDirectory,
+			sessionId: 'ses_deferred_tools' as SessionId,
+			threadId: 'thd_deferred_tools' as ThreadId,
+			projectId: 'prj_deferred_tools' as ProjectId,
+			tenantId: 'tnt_deferred_tools' as TenantId,
+		})
+
+		expect(run.status).toBe('completed')
+		expect(tools.has(SearchToolsTool.name)).toBe(true)
+		expect(tools.getAvailability(SearchToolsTool.name)).toBe('active')
+		expect(tools.getAvailability('generate_document')).toBe('deferred')
+
+		const toolNames = provider.lastParams?.tools?.map((tool) => tool.function.name).sort() ?? []
+		expect(toolNames).toEqual([SearchToolsTool.name])
+
+		const systemPrompt = (provider.lastParams?.messages ?? [])
+			.filter((message) => message.role === 'system')
+			.map((message) => message.content)
+			.join('\n')
+		expect(systemPrompt).toContain('Use search_tools to load these before use:')
+		expect(systemPrompt).toContain('- generate_document')
+		expect(systemPrompt).not.toContain(
+			'Deferred tools are discoverable but not executable until the runtime activates them',
+		)
+	})
+
+	it('keeps search_tools executable when allowedTools names a deferred tool', async () => {
+		const provider = new CapturingProvider()
+		const tools = new ToolRegistry()
+		registerDeferredDocumentTool(tools)
+
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-deferred-tools-'))
+		workdirs.push(workingDirectory)
+
+		const run = await drainQuery({
+			provider,
+			tools,
+			allowedTools: ['generate_document'],
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 5_000,
+				tokenBudget: 100_000,
+				maxIterations: 1,
+				maxResponseTokens: 256,
+			},
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('generate D-01')],
+			workingDirectory,
+			sessionId: 'ses_deferred_allowed_tools' as SessionId,
+			threadId: 'thd_deferred_allowed_tools' as ThreadId,
+			projectId: 'prj_deferred_allowed_tools' as ProjectId,
+			tenantId: 'tnt_deferred_allowed_tools' as TenantId,
+		})
+
+		expect(run.status).toBe('completed')
+		expect(tools.getAvailability('generate_document')).toBe('deferred')
+
+		const toolNames = provider.lastParams?.tools?.map((tool) => tool.function.name).sort() ?? []
+		expect(toolNames).toEqual([SearchToolsTool.name])
+
+		const systemPrompt = (provider.lastParams?.messages ?? [])
+			.filter((message) => message.role === 'system')
+			.map((message) => message.content)
+			.join('\n')
+		expect(systemPrompt).toContain('Use search_tools to load these before use:')
+		expect(systemPrompt).toContain('- generate_document')
+	})
+
+	it('does not let search_tools reveal or activate deferred tools outside allowedTools', async () => {
+		const tools = new ToolRegistry()
+		registerDeferredDocumentTool(tools)
+		registerDeferredDocumentTool(tools, 'dangerous_purge_document')
+
+		// 'dangerous' matches only the out-of-allowlist tool ('delete'-style
+		// CRUD verbs are stop tokens and never match anything by themselves).
+		const result = await SearchToolsTool.execute(
+			{ query: 'dangerous' },
+			{
+				runId: 'run_deferred_allowed_tools' as RunId,
+				workingDirectory: '/tmp',
+				abortSignal: new AbortController().signal,
+				env: {},
+				log: () => undefined,
+				toolRegistry: tools,
+				allowedTools: ['generate_document', SearchToolsTool.name],
+			},
+		)
+
+		expect(result.success).toBe(true)
+		expect(result.output).toContain('No deferred tools matching "dangerous"')
+		expect(result.output).not.toContain('dangerous_purge_document')
+		expect(tools.getAvailability('generate_document')).toBe('deferred')
+		expect(tools.getAvailability('dangerous_purge_document')).toBe('deferred')
+	})
+
+	it('activates only the top-5 ranked matches and reports near-misses without activating', async () => {
+		const tools = new ToolRegistry()
+		// Eight deferred tools that all match "invoice" equally by name; the
+		// alphabetical tie-break makes the top-5 cut deterministic.
+		const names = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((s) => `invoice_${s}`)
+		for (const name of names) {
+			tools.register(
+				{
+					name,
+					description: `Billing helper ${name.slice(-1)}.`,
+					inputSchema: z.object({ id: z.string() }),
+					execute: async () => ({ success: true, output: 'ok' }),
+				},
+				'deferred',
+			)
+		}
+
+		const result = await SearchToolsTool.execute(
+			{ query: 'invoice' },
+			{
+				runId: 'run_deferred_top_k' as RunId,
+				workingDirectory: '/tmp',
+				abortSignal: new AbortController().signal,
+				env: {},
+				log: () => undefined,
+				toolRegistry: tools,
+			},
+		)
+
+		expect(result.success).toBe(true)
+		expect(result.output).toContain('Activated 5 tool(s)')
+		expect(result.output).toContain('NOT loaded')
+		expect(result.data).toMatchObject({
+			activated: ['invoice_a', 'invoice_b', 'invoice_c', 'invoice_d', 'invoice_e'],
+			count: 5,
+			nearMisses: ['invoice_f', 'invoice_g', 'invoice_h'],
+		})
+		for (const name of ['invoice_a', 'invoice_b', 'invoice_c', 'invoice_d', 'invoice_e']) {
+			expect(tools.getAvailability(name)).toBe('active')
+		}
+		for (const name of ['invoice_f', 'invoice_g', 'invoice_h']) {
+			expect(tools.getAvailability(name)).toBe('deferred')
+		}
+	})
+})