@namzu/sdk 1.0.0 → 1.2.0

package/src/registry/tool/execute.ts

@@ -19,7 +19,35 @@ export type { ToolExecutionResult }
 // Tokens too generic to identify a tool by name — ignored when matching a
 // batched `search_tools` query so they can't activate the whole catalog
 // (every bridged tool name shares the `clawtool` prefix, for instance).
-const SEARCH_STOP_TOKENS = new Set(['clawtool', 'tool', 'tools', 'mcp', 'the', 'and', 'for', 'use'])
+// Generic CRUD verbs are stopped too: a query like "list deals" must rank
+// by "deals", not token-match every `list_*` tool in the deferred catalog.
+const SEARCH_STOP_TOKENS = new Set([
+	'clawtool',
+	'tool',
+	'tools',
+	'mcp',
+	'the',
+	'and',
+	'for',
+	'use',
+	'list',
+	'read',
+	'create',
+	'update',
+	'get',
+	'find',
+	'delete',
+	'search',
+])
+
+// Weighted-scoring weights mirroring ToolCatalog.searchTools (the richer,
+// otherwise-unused catalog scorer): exact name 12, name substring 8,
+// description 5 — extended here with argument-name indexing (3), following
+// Anthropic's tool-search practice of searching argument names too.
+const SEARCH_WEIGHT_NAME_EXACT = 12
+const SEARCH_WEIGHT_NAME_PARTIAL = 8
+const SEARCH_WEIGHT_DESCRIPTION = 5
+const SEARCH_WEIGHT_ARGUMENT = 3
 
 export class ToolRegistry extends ManagedRegistry<ToolDefinition> {
 	private availability: Map<string, ToolAvailability> = new Map()
@@ -118,28 +146,58 @@ export class ToolRegistry extends ManagedRegistry<ToolDefinition> {
 		return this.availability.get(name) ?? 'active'
 	}
 
+	/**
+	 * Ranked lexical search over DEFERRED tools, score-descending (ties broken
+	 * by name) so callers can cap activation at a top-k. Each meaningful query
+	 * term (≥3 chars, not a stop token) is scored against the tool name
+	 * (exact/substring), description, and argument names; only tools with a
+	 * positive score are returned. Description matching is safe here precisely
+	 * because the result is RANKED — the `search_tools` builtin activates only
+	 * the top slice, so a shared word can no longer drag in the whole catalog.
+	 *
+	 * PARKED (phase 5 of the tool-loading plan): an embedding-backed semantic
+	 * upgrade was evaluated and deliberately NOT built — at ≤~35 deferred
+	 * in-house tools with distinct names, weighted lexical scoring sits inside
+	 * the literature's safe zone, and a weak retriever underperforms no
+	 * retriever at all. Revisit only when (a) the deferred catalog grows past
+	 * ~75-100 tools (realistic driver: connector-MCP growth), or (b) telemetry
+	 * shows a search_tools miss-rate above ~10%. Sticky activation is also
+	 * deliberate: activating inserts the schema into the tools array at its
+	 * registry position (a one-time prompt-cache prefix bust); re-defer/TTL
+	 * would churn that prefix repeatedly and is rejected.
+	 */
 	searchDeferred(query: string): ToolDefinition[] {
 		const q = query.toLowerCase().trim()
 		if (q.length === 0) return []
-		// Per-token matching exists only so a batched query naming several tools
-		// at once ("A2aCard PeerRegister PeerList") activates each. Restrict it
-		// to the tool NAME and drop short/generic tokens — matching tokens
-		// against descriptions (or letting a shared word like "clawtool"/"list"
-		// through) would activate the whole catalog and defeat deferral.
-		const tokens = q.split(/\s+/).filter((tok) => tok.length >= 3 && !SEARCH_STOP_TOKENS.has(tok))
-		// A bare generic token ("clawtool") identifies nothing specific — skip the
-		// broad whole-query match for it so it can't activate the whole catalog.
-		const wholeQueryUseful = q.length >= 3 && !SEARCH_STOP_TOKENS.has(q)
-		return this.getByAvailability(['deferred']).filter((t) => {
-			const name = t.name.toLowerCase()
-			// Whole-query match (single-term capability search) against name or
-			// description — the deliberate, narrow behaviour.
-			if (wholeQueryUseful && (name.includes(q) || t.description.toLowerCase().includes(q))) {
-				return true
+		const terms = q.split(/\s+/).filter((tok) => tok.length >= 3 && !SEARCH_STOP_TOKENS.has(tok))
+		if (terms.length === 0) return []
+
+		const scored: Array<{ tool: ToolDefinition; score: number }> = []
+		for (const tool of this.getByAvailability(['deferred'])) {
+			const name = tool.name.toLowerCase()
+			const description = tool.description.toLowerCase()
+			const argumentNames = listArgumentNames(tool)
+			let score = 0
+			for (const term of terms) {
+				if (name === term) {
+					score += SEARCH_WEIGHT_NAME_EXACT
+				} else if (name.includes(term)) {
+					score += SEARCH_WEIGHT_NAME_PARTIAL
+				}
+				if (description.includes(term)) {
+					score += SEARCH_WEIGHT_DESCRIPTION
+				}
+				if (argumentNames.some((arg) => arg.includes(term))) {
+					score += SEARCH_WEIGHT_ARGUMENT
+				}
 			}
-			// Batched multi-name query: any meaningful token, name only.
-			return tokens.some((tok) => name.includes(tok))
-		})
+			if (score > 0) {
+				scored.push({ tool, score })
+			}
+		}
+
+		scored.sort((a, b) => b.score - a.score || a.tool.name.localeCompare(b.tool.name))
+		return scored.map((entry) => entry.tool)
 	}
 
 	assignTiers(mapping: Record<string, string>): void {
@@ -176,12 +234,24 @@ Executable tool names, descriptions, and JSON input schemas are attached through
 </tool_runtime_contract>`
 
 		if (active.length > 0) {
-			const entries = active.map((t) => `- ${t.name}: ${t.description}`).join('\n')
+			// Name-only: every active tool's full description + JSON schema
+			// already rides the runtime tools parameter on each request —
+			// repeating descriptions here double-bills the same tokens.
+			const entries = active.map((t) => `- ${t.name}`).join('\n')
 			parts.push(`<available_tools>\n${entries}\n</available_tools>`)
 		}
 
 		if (deferred.length > 0) {
-			const entries = deferred.map((t) => `- ${t.name}`).join('\n')
+			// Name + one-line hint: deferred schemas stay off the wire, so the
+			// hint is the model's only signal of what a deferred tool does. A
+			// bare name list caused a real discovery failure in production
+			// (the agent never found read_document behind search_tools).
+			const entries = deferred
+				.map((t) => {
+					const hint = toolDiscoveryHint(t.description)
+					return hint.length > 0 ? `- ${t.name}: ${hint}` : `- ${t.name}`
+				})
+				.join('\n')
 			const deferredIntro =
 				this.has('search_tools') && this.getAvailability('search_tools') === 'active'
 					? 'Use search_tools to load these before use:'
@@ -372,6 +442,39 @@ Executable tool names, descriptions, and JSON input schemas are attached through
 	}
 }
 
+/**
+ * One-line discoverability hint for a deferred tool: the first sentence of
+ * its description, capped at ~100 chars. Used for the `<deferred_tools>`
+ * prompt listing and for `search_tools` near-miss suggestions, where the
+ * full description would re-import the token weight deferral avoids.
+ */
+export function toolDiscoveryHint(description: string, maxLength = 100): string {
+	const normalized = description.trim().replace(/\s+/g, ' ')
+	if (normalized.length === 0) return ''
+	const sentenceMatch = normalized.match(/^.*?[.!?](?=\s|$)/)
+	const sentence = sentenceMatch ? sentenceMatch[0] : normalized
+	if (sentence.length <= maxLength) return sentence
+	return `${sentence.slice(0, maxLength - 1).trimEnd()}…`
+}
+
+/**
+ * Lower-cased argument (property) names of a tool's input schema, used by
+ * `searchDeferred` ranking. Walks the JSON-Schema rendering (already a
+ * registration dependency) instead of Zod internals; opaque schemas simply
+ * contribute no argument matches.
+ */
+function listArgumentNames(tool: ToolDefinition): string[] {
+	try {
+		const json = zodToJsonSchema(tool.inputSchema, {
+			target: 'jsonSchema7',
+			$refStrategy: 'none',
+		}) as { properties?: Record<string, unknown> }
+		return Object.keys(json.properties ?? {}).map((key) => key.toLowerCase())
+	} catch {
+		return []
+	}
+}
+
 /**
  * Build a one-sentence "Required: <field>: <type>, <field>: <type>"
  * hint from a Zod schema, used to enrich tool-input validation

package/src/registry/toolset/catalog.test.ts

@@ -81,6 +81,74 @@ describe('ToolCatalog', () => {
 		expect(catalog.toLLMTools().map((t) => t.function.name)).toEqual(['bash'])
 	})
 
+	it('scores and reports the matched facets: partial name, source text, toolset text', () => {
+		const catalog = new ToolCatalog()
+		catalog.registerSource({
+			id: 'host',
+			kind: 'host_tool',
+			name: 'GitHub',
+			description: 'repository automation',
+		})
+		catalog.registerToolset({
+			id: 'issues',
+			sourceId: 'host',
+			name: 'Issue toolset',
+			defaultPolicy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'create_issue',
+			description: 'open a ticket',
+			sourceId: 'host',
+			toolsetId: 'issues',
+			policy: { enabled: true, loading: 'eager' },
+		})
+
+		// 'issue' hits the tool NAME (create_issue, partial) AND the TOOLSET name.
+		const byIssue = catalog.searchTools('issue')
+		expect(byIssue).toHaveLength(1)
+		expect([...(byIssue[0]?.matched ?? [])].sort()).toEqual(['name', 'toolset'])
+
+		// 'github' hits only the SOURCE text.
+		const bySource = catalog.searchTools('github')
+		expect(bySource[0]?.matched).toEqual(['source'])
+
+		// An exact name match outscores a partial one.
+		const exact = catalog.searchTools('create_issue')
+		expect(exact[0]?.matched).toContain('name')
+		expect(exact[0]?.score ?? 0).toBeGreaterThan(byIssue[0]?.score ?? 0)
+	})
+
+	it('adds the preferred boost so a preferred tool outranks an equal match', () => {
+		const catalog = new ToolCatalog()
+		catalog.registerSource({ id: 'host', kind: 'host_tool', name: 'Host' })
+		catalog.registerToolset({
+			id: 'ts',
+			sourceId: 'host',
+			name: 'Tools',
+			defaultPolicy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'search_plain',
+			description: 'search things',
+			sourceId: 'host',
+			toolsetId: 'ts',
+			policy: { enabled: true, loading: 'eager' },
+		})
+		catalog.registerTool({
+			name: 'search_pref',
+			description: 'search things',
+			sourceId: 'host',
+			toolsetId: 'ts',
+			policy: { enabled: true, loading: 'eager', preferred: true },
+		})
+
+		const results = catalog.searchTools('search')
+		expect(results.map((r) => r.tool.name)).toEqual(['search_pref', 'search_plain'])
+		const pref = results.find((r) => r.tool.name === 'search_pref')?.score ?? 0
+		const plain = results.find((r) => r.tool.name === 'search_plain')?.score ?? 0
+		expect(pref - plain).toBe(1)
+	})
+
 	it('preserves registry availability as catalog loading policy', () => {
 		const registry = new ToolRegistry()
 		registry.register(makeTool('read_file'))

package/src/run/reporter.ts

@@ -225,6 +225,15 @@ export function createRunReporter(parentLogger?: Logger): RunReporter {
 				})
 				break
 
+			case 'capability_warning':
+				log.warn('Provider capability mismatch', {
+					runId: event.runId,
+					capability: event.capability,
+					providerId: event.providerId,
+					message: event.message,
+				})
+				break
+
 			default: {
 				const _exhaustive: never = event
 				throw new Error(`Unhandled run event type: ${(_exhaustive as RunEvent).type}`)

package/src/runtime/query/__tests__/capability-negotiation.test.ts

@@ -0,0 +1,248 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import {
+	PERMISSIVE_PROVIDER_CAPABILITIES,
+	resolveProviderCapabilities,
+} from '../../../provider/capabilities.js'
+import { ToolRegistry } from '../../../registry/tool/execute.js'
+import type { SessionId, TenantId } from '../../../types/ids/index.js'
+import { createUserMessage } from '../../../types/message/index.js'
+import type {
+	ChatCompletionParams,
+	LLMProvider,
+	ProviderCapabilities,
+	StreamChunk,
+} from '../../../types/provider/index.js'
+import type { RunEvent } from '../../../types/run/index.js'
+import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
+import { drainQuery } from '../index.js'
+
+const ZERO_USAGE = {
+	promptTokens: 0,
+	completionTokens: 0,
+	totalTokens: 0,
+	cachedTokens: 0,
+	cacheWriteTokens: 0,
+}
+
+class CapturingProvider implements LLMProvider {
+	readonly id = 'capturing'
+	readonly name = 'Capturing Provider'
+	readonly capabilities?: ProviderCapabilities
+	lastParams?: ChatCompletionParams
+
+	constructor(capabilities?: ProviderCapabilities) {
+		this.capabilities = capabilities
+	}
+
+	async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
+		this.lastParams = params
+		yield { id: 'msg_1', delta: { content: 'done' } }
+		yield { id: 'msg_1', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
+	}
+}
+
+const NO_TOOLS_CAPABILITIES: ProviderCapabilities = {
+	supportsTools: false,
+	supportsStreaming: true,
+	supportsFunctionCalling: false,
+	supportsVision: true,
+}
+
+const NO_VISION_CAPABILITIES: ProviderCapabilities = {
+	supportsTools: true,
+	supportsStreaming: true,
+	supportsFunctionCalling: true,
+	supportsVision: false,
+}
+
+function registerEchoTool(tools: ToolRegistry): void {
+	tools.register({
+		name: 'echo',
+		description: 'Echo the text back.',
+		inputSchema: z.object({ text: z.string() }),
+		execute: async () => ({ success: true, output: 'ok' }),
+	})
+}
+
+function baseParams(provider: LLMProvider, tools: ToolRegistry, workingDirectory: string) {
+	return {
+		provider,
+		tools,
+		runConfig: {
+			model: 'mock-model',
+			timeoutMs: 5_000,
+			tokenBudget: 100_000,
+			maxIterations: 1,
+			maxResponseTokens: 256,
+		},
+		agentId: 'agent_test',
+		agentName: 'Test Agent',
+		workingDirectory,
+		sessionId: 'ses_capability' as SessionId,
+		threadId: 'thd_capability' as ThreadId,
+		projectId: 'prj_capability' as ProjectId,
+		tenantId: 'tnt_capability' as TenantId,
+	}
+}
+
+describe('resolveProviderCapabilities', () => {
+	it('resolves an undeclared provider to the permissive default', () => {
+		expect(resolveProviderCapabilities(new CapturingProvider())).toEqual(
+			PERMISSIVE_PROVIDER_CAPABILITIES,
+		)
+	})
+
+	it('fills a partial declaration (missing supportsVision) permissively per field', () => {
+		const resolved = resolveProviderCapabilities({
+			capabilities: {
+				supportsTools: false,
+				supportsStreaming: true,
+				supportsFunctionCalling: false,
+			},
+		})
+		expect(resolved.supportsTools).toBe(false)
+		expect(resolved.supportsVision).toBe(true)
+	})
+})
+
+describe('query() capability negotiation', () => {
+	let workdirs: string[] = []
+
+	afterEach(async () => {
+		await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
+		workdirs = []
+	})
+
+	async function mkWorkdir(): Promise<string> {
+		const dir = await mkdtemp(join(tmpdir(), 'namzu-capability-'))
+		workdirs.push(dir)
+		return dir
+	}
+
+	it('strips tool surfaces and emits a capability_warning for a no-tools provider', async () => {
+		const provider = new CapturingProvider(NO_TOOLS_CAPABILITIES)
+		const tools = new ToolRegistry()
+		registerEchoTool(tools)
+		const events: RunEvent[] = []
+
+		const run = await drainQuery(
+			{
+				...baseParams(provider, tools, await mkWorkdir()),
+				messages: [createUserMessage('hello')],
+			},
+			(event) => {
+				events.push(event)
+			},
+		)
+
+		expect(run.status).toBe('completed')
+
+		// The model was never told about tools: no tools param on the wire…
+		expect(provider.lastParams?.tools).toBeUndefined()
+		// …and no tool section in the system prompt.
+		const systemPrompt = (provider.lastParams?.messages ?? [])
+			.filter((m) => m.role === 'system')
+			.map((m) => m.content)
+			.join('\n')
+		expect(systemPrompt).not.toContain('<available_tools>')
+		expect(systemPrompt).not.toContain('echo')
+
+		// The host got the machine-readable warning.
+		const warning = events.find(
+			(e): e is Extract<RunEvent, { type: 'capability_warning' }> =>
+				e.type === 'capability_warning',
+		)
+		expect(warning?.capability).toBe('tools')
+		expect(warning?.providerId).toBe('capturing')
+		expect(warning?.message).toContain('stripped')
+	})
+
+	it('keeps tool surfaces intact for an undeclared (permissive-default) provider', async () => {
+		const provider = new CapturingProvider()
+		const tools = new ToolRegistry()
+		registerEchoTool(tools)
+
+		const run = await drainQuery({
+			...baseParams(provider, tools, await mkWorkdir()),
+			messages: [createUserMessage('hello')],
+		})
+
+		expect(run.status).toBe('completed')
+		expect(provider.lastParams?.tools?.map((t) => t.function.name)).toContain('echo')
+	})
+
+	it('emits a vision capability_warning when attachments hit a no-vision provider', async () => {
+		const provider = new CapturingProvider(NO_VISION_CAPABILITIES)
+		const events: RunEvent[] = []
+
+		const run = await drainQuery(
+			{
+				...baseParams(provider, new ToolRegistry(), await mkWorkdir()),
+				messages: [
+					createUserMessage('what is in this image?', [
+						{ data: 'aGVsbG8=', mediaType: 'image/png' },
+					]),
+				],
+			},
+			(event) => {
+				events.push(event)
+			},
+		)
+
+		expect(run.status).toBe('completed')
+		const warning = events.find(
+			(e): e is Extract<RunEvent, { type: 'capability_warning' }> =>
+				e.type === 'capability_warning',
+		)
+		expect(warning?.capability).toBe('vision')
+		expect(warning?.providerId).toBe('capturing')
+	})
+
+	it('does not warn when no attachments are present on a no-vision provider', async () => {
+		const provider = new CapturingProvider(NO_VISION_CAPABILITIES)
+		const events: RunEvent[] = []
+
+		await drainQuery(
+			{
+				...baseParams(provider, new ToolRegistry(), await mkWorkdir()),
+				messages: [createUserMessage('plain text')],
+			},
+			(event) => {
+				events.push(event)
+			},
+		)
+
+		expect(events.some((e) => e.type === 'capability_warning')).toBe(false)
+	})
+
+	it('strictCapabilities: true throws on a tools mismatch instead of degrading', async () => {
+		const provider = new CapturingProvider(NO_TOOLS_CAPABILITIES)
+		const tools = new ToolRegistry()
+		registerEchoTool(tools)
+
+		await expect(
+			drainQuery({
+				...baseParams(provider, tools, await mkWorkdir()),
+				messages: [createUserMessage('hello')],
+				strictCapabilities: true,
+			}),
+		).rejects.toThrow(/supportsTools: false/)
+	})
+
+	it('strictCapabilities: true throws on a vision mismatch instead of degrading', async () => {
+		const provider = new CapturingProvider(NO_VISION_CAPABILITIES)
+
+		await expect(
+			drainQuery({
+				...baseParams(provider, new ToolRegistry(), await mkWorkdir()),
+				messages: [createUserMessage('describe', [{ data: 'aGVsbG8=', mediaType: 'image/png' }])],
+				strictCapabilities: true,
+			}),
+		).rejects.toThrow(/supportsVision: false/)
+	})
+})