@namzu/sdk 3.1.0 → 3.2.0

package/src/telemetry/__tests__/model-call-span.test.ts

@@ -0,0 +1,189 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+/**
+ * There was no span around the model call at all.
+ *
+ * `chatSpanName` shipped in the telemetry attributes with zero call sites,
+ * so a run's traces carried no LLM latency whatsoever — and the one thing
+ * anybody opens a trace to find, which turn was slow and why, was the one
+ * thing missing from it. The token counts landed on the iteration span
+ * instead of the operation that produced them.
+ */
+
+interface Recorded {
+	name: string
+	attributes: Record<string, unknown>
+	ended: boolean
+	status?: number
+}
+
+const spans: Recorded[] = []
+
+vi.mock('../runtime-accessors.js', () => ({
+	getTracer: () => ({
+		startSpan: (name: string) => {
+			const rec: Recorded = { name, attributes: {}, ended: false }
+			spans.push(rec)
+			return {
+				setAttributes: (a: Record<string, unknown>) => Object.assign(rec.attributes, a),
+				setAttribute: (k: string, v: unknown) => {
+					rec.attributes[k] = v
+				},
+				setStatus: (s: { code: number }) => {
+					rec.status = s.code
+				},
+				recordException: () => undefined,
+				addEvent: () => undefined,
+				end: () => {
+					rec.ended = true
+				},
+				spanContext: () => ({ traceId: 't', spanId: 's', traceFlags: 1 }),
+				isRecording: () => true,
+				updateName: () => undefined,
+			}
+		},
+		startActiveSpan: (_n: string, _o: unknown, _c: unknown, fn: (s: unknown) => unknown) =>
+			fn({
+				setAttributes: () => undefined,
+				setStatus: () => undefined,
+				recordException: () => undefined,
+				end: () => undefined,
+				spanContext: () => ({ traceId: 't', spanId: 's', traceFlags: 1 }),
+				isRecording: () => true,
+			}),
+	}),
+	getMeter: () => ({
+		createCounter: () => ({ add: () => undefined }),
+		createHistogram: () => ({ record: () => undefined }),
+	}),
+}))
+
+let workdirs: string[] = []
+
+beforeEach(() => {
+	spans.length = 0
+})
+
+afterEach(async () => {
+	await Promise.all(workdirs.map((d) => rm(d, { recursive: true, force: true })))
+	workdirs = []
+})
+
+async function runOnce(turns: { text?: string }[]): Promise<void> {
+	const { MockLLMProvider } = await import('../../provider/mock.js')
+	const { ToolRegistry } = await import('../../registry/tool/execute.js')
+	const { drainQuery } = await import('../../runtime/query/index.js')
+	const { createUserMessage } = await import('../../types/message/index.js')
+
+	const dir = await mkdtemp(join(tmpdir(), 'namzu-chatspan-'))
+	workdirs.push(dir)
+
+	await drainQuery({
+		provider: new MockLLMProvider({ turns: turns as never }),
+		tools: new ToolRegistry(),
+		runConfig: {
+			model: 'mock-model',
+			timeoutMs: 30_000,
+			tokenBudget: 100_000,
+			maxIterations: 3,
+			maxResponseTokens: 256,
+			temperature: 0.3,
+		},
+		agentId: 'agent_cs',
+		agentName: 'Chat Span Agent',
+		workingDirectory: dir,
+		sessionId: 'ses_cs',
+		threadId: 'thd_cs',
+		projectId: 'prj_cs',
+		tenantId: 'tnt_cs',
+		messages: [createUserMessage('go')],
+	} as never)
+}
+
+const chatSpans = () => spans.filter((s) => s.name.startsWith('chat '))
+
+describe('the model call has a span of its own', () => {
+	it('opens one named for the model', async () => {
+		await runOnce([{ text: 'done' }])
+
+		// One per model call. A run makes at least one, and a forced-final
+		// turn makes another — both are model calls and both deserve a span.
+		expect(chatSpans().length).toBeGreaterThanOrEqual(1)
+		for (const s of chatSpans()) expect(s.name).toBe('chat mock-model')
+	})
+
+	it('closes it', async () => {
+		await runOnce([{ text: 'done' }])
+
+		for (const s of chatSpans()) expect(s.ended).toBe(true)
+	})
+
+	it('carries the request parameters that were actually used', async () => {
+		await runOnce([{ text: 'done' }])
+
+		const attrs = chatSpans()[0]?.attributes ?? {}
+		expect(attrs['gen_ai.operation.name']).toBe('chat')
+		expect(attrs['gen_ai.request.model']).toBe('mock-model')
+		expect(attrs['gen_ai.request.temperature']).toBe(0.3)
+		expect(attrs['gen_ai.request.max_tokens']).toBe(256)
+	})
+
+	it('carries the usage on the call that produced it', async () => {
+		await runOnce([{ text: 'done' }])
+
+		const attrs = chatSpans()[0]?.attributes ?? {}
+		// These previously landed only on the iteration span — one level up
+		// from the operation that spent the tokens.
+		expect(attrs).toHaveProperty('gen_ai.usage.input_tokens')
+		expect(attrs).toHaveProperty('gen_ai.usage.output_tokens')
+	})
+
+	it('records what the response said it was, not only what was asked for', async () => {
+		await runOnce([{ text: 'done' }])
+
+		const attrs = chatSpans()[0]?.attributes ?? {}
+		// A provider may answer on a different model than the alias asked
+		// for, and the response id is how a trace is matched to a provider's
+		// own logs. Both constants existed and neither was ever set.
+		expect(attrs['gen_ai.response.model']).toBe('mock-model')
+		expect(attrs['gen_ai.response.id']).toBeDefined()
+	})
+
+	it('records the finish reason as an array, per the convention', async () => {
+		await runOnce([{ text: 'done' }])
+
+		// One call can finish several ways when a provider returns more than
+		// one choice, which is why the convention makes it a list.
+		expect(Array.isArray(chatSpans()[0]?.attributes['gen_ai.response.finish_reasons'])).toBe(true)
+	})
+
+	it('reports cache tokens, which had constants and no producer', async () => {
+		await runOnce([{ text: 'done' }])
+
+		const attrs = chatSpans()[0]?.attributes ?? {}
+		expect(attrs).toHaveProperty('namzu.cache.read_tokens')
+		expect(attrs).toHaveProperty('namzu.cache.write_tokens')
+	})
+
+	it('opens one per turn, not one per run', async () => {
+		await runOnce([{ text: 'first' }, { text: 'second' }])
+
+		// A run with several turns is exactly the case a trace is opened for.
+		expect(chatSpans().length).toBeGreaterThanOrEqual(1)
+	})
+
+	it('nests under the iteration span rather than emitting as a root', async () => {
+		await runOnce([{ text: 'done' }])
+
+		// The iteration span is created first; the chat span is parented to it
+		// explicitly because this body is an async generator and the ambient
+		// context at resume time belongs to the consumer.
+		const iterationIndex = spans.findIndex((s) => s.name.includes('iteration'))
+		const chatIndex = spans.findIndex((s) => s.name.startsWith('chat '))
+		expect(iterationIndex).toBeGreaterThanOrEqual(0)
+		expect(chatIndex).toBeGreaterThan(iterationIndex)
+	})
+})

package/src/telemetry/__tests__/span-closure.test.ts

@@ -0,0 +1,153 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { z } from 'zod'
+
+/**
+ * A span that never ends is a trace that never closes, and the export is
+ * incomplete for exactly the run that failed.
+ *
+ * Both sites had the same shape: an `end()` call at every exit the author
+ * could see. The iteration loop had seventeen of them and the tool executor
+ * three plus a `finally` that opened below them. That is a rule every future
+ * edit has to remember, and it was already broken — anything throwing between
+ * the span's creation and the `try` left it open, and a generator abandoned
+ * by its consumer reached no exit at all.
+ */
+
+const ended: string[] = []
+const started: string[] = []
+
+function fakeSpan(name: string) {
+	return {
+		setAttributes: () => undefined,
+		setAttribute: () => undefined,
+		setStatus: () => undefined,
+		recordException: () => undefined,
+		addEvent: () => undefined,
+		end: () => ended.push(name),
+		spanContext: () => ({ traceId: 't', spanId: 's', traceFlags: 1 }),
+		isRecording: () => true,
+		updateName: () => undefined,
+	}
+}
+
+vi.mock('../runtime-accessors.js', () => ({
+	getTracer: () => ({
+		startSpan: (name: string) => {
+			started.push(name)
+			return fakeSpan(name)
+		},
+		startActiveSpan: (
+			name: string,
+			_opts: unknown,
+			_ctx: unknown,
+			fn: (span: ReturnType<typeof fakeSpan>) => unknown,
+		) => {
+			started.push(name)
+			return fn(fakeSpan(name))
+		},
+	}),
+	getMeter: () => ({
+		createCounter: () => ({ add: () => undefined }),
+		createHistogram: () => ({ record: () => undefined }),
+	}),
+}))
+
+function toolContext() {
+	return {
+		runId: 'run_span',
+		workingDirectory: '.',
+		abortSignal: new AbortController().signal,
+		env: {},
+		log: () => undefined,
+	} as never
+}
+
+beforeEach(() => {
+	started.length = 0
+	ended.length = 0
+})
+
+afterEach(() => {
+	vi.clearAllMocks()
+})
+
+describe('a tool span closes however the call leaves', () => {
+	it('closes on the ordinary path', async () => {
+		const { ToolRegistry } = await import('../../registry/tool/execute.js')
+		const tools = new ToolRegistry()
+		tools.register({
+			name: 'echo',
+			description: 'echo',
+			inputSchema: z.object({}),
+			execute: async () => ({ success: true, output: 'ok' }),
+		})
+
+		await tools.execute('echo', {}, toolContext())
+
+		expect(started).toHaveLength(1)
+		expect(ended).toEqual(started)
+	})
+
+	it('closes when the tool throws', async () => {
+		const { ToolRegistry } = await import('../../registry/tool/execute.js')
+		const tools = new ToolRegistry()
+		tools.register({
+			name: 'boom',
+			description: 'boom',
+			inputSchema: z.object({}),
+			execute: async () => {
+				throw new Error('kaboom')
+			},
+		})
+
+		await tools.execute('boom', {}, toolContext())
+
+		expect(ended).toEqual(started)
+	})
+
+	it('closes when input validation refuses the call', async () => {
+		const { ToolRegistry } = await import('../../registry/tool/execute.js')
+		const tools = new ToolRegistry()
+		tools.register({
+			name: 'strict',
+			description: 'strict',
+			inputSchema: z.object({ required: z.string() }),
+			execute: async () => ({ success: true, output: 'ok' }),
+		})
+
+		await tools.execute('strict', { wrong: 1 }, toolContext())
+
+		expect(ended).toEqual(started)
+	})
+
+	it('closes when the tool is not active', async () => {
+		const { ToolRegistry } = await import('../../registry/tool/execute.js')
+		const tools = new ToolRegistry()
+		tools.register(
+			{
+				name: 'later',
+				description: 'later',
+				inputSchema: z.object({}),
+				execute: async () => ({ success: true, output: 'ok' }),
+			},
+			'deferred',
+		)
+
+		await tools.execute('later', {}, toolContext())
+
+		expect(ended).toEqual(started)
+	})
+
+	it('closes when the registry does not hold the name at all', async () => {
+		const { ToolRegistry } = await import('../../registry/tool/execute.js')
+		const tools = new ToolRegistry()
+
+		// `getOrThrow` sat OUTSIDE the try that owned the finally, so this
+		// path — the one where the model invents a tool name — opened a span
+		// and never closed it.
+		await expect(tools.execute('nonexistent', {}, toolContext())).rejects.toThrow()
+
+		expect(started).toHaveLength(1)
+		expect(ended).toEqual(started)
+	})
+})

package/src/tools/advisory/index.ts

@@ -104,7 +104,7 @@ export function buildAdvisoryTools(opts: AdvisoryToolsOptions): ToolDefinition[]
 			const executionResult = await advisoryCtx.executor.consult(
 				advisor,
 				{ advisorId: advisor.id, question, domain, urgency, includeContext: include_context },
-				{ messages: [], iteration: 0 },
+				advisoryCtx.callContext(),
 			)
 
 			advisoryCtx.recordCall({

package/src/tools/coordinator/__tests__/plan-dependencies.test.ts

@@ -0,0 +1,186 @@
+import { describe, expect, it } from 'vitest'
+
+import { resolvePlanDependencies } from '../plan-dependencies.js'
+
+/**
+ * The model is shown `depends_on` on every plan step and told it means
+ * "Step descriptions this depends on". `approve_plan` then passed
+ * `dependsOn: []` for every step, so the ordering it declared was discarded
+ * at the only place it entered the system.
+ *
+ * The cost is not scheduling — the dependency gate in `PlanManager` has no
+ * callers — it is the approval. `dependsOn` is serialized into the
+ * `plan_approval` payload a human reads before saying yes, so the reviewer
+ * saw a plan whose steps all looked independent however carefully the model
+ * had ordered them.
+ */
+
+const id = (index: number) => `step_${index + 1}`
+
+describe('what the model described becomes what the plan holds', () => {
+	it('resolves a description to the step that carries it', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Read the config' },
+				{ description: 'Write the report', depends_on: ['Read the config'] },
+			],
+			id,
+		)
+
+		expect(result).toEqual({ ok: true, dependsOn: [[], ['step_1']] })
+	})
+
+	it('resolves several dependencies on one step', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Read the config' },
+				{ description: 'Read the code' },
+				{ description: 'Write the report', depends_on: ['Read the config', 'Read the code'] },
+			],
+			id,
+		)
+
+		expect(result.ok && result.dependsOn[2]).toEqual(['step_1', 'step_2'])
+	})
+
+	it('resolves a forward dependency, since order of declaration is not order of execution', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Write the report', depends_on: ['Read the config'] },
+				{ description: 'Read the config' },
+			],
+			id,
+		)
+
+		expect(result.ok && result.dependsOn[0]).toEqual(['step_2'])
+	})
+
+	it('leaves a step with no dependencies empty', () => {
+		const result = resolvePlanDependencies([{ description: 'Read the config' }], id)
+
+		expect(result).toEqual({ ok: true, dependsOn: [[]] })
+	})
+
+	it('forgives whitespace and casing a model did not keep identical', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Read  the config' },
+				{ description: 'Write', depends_on: ['read the   CONFIG '] },
+			],
+			id,
+		)
+
+		// Rejecting this would teach the model nothing, for a plan that was
+		// right.
+		expect(result.ok && result.dependsOn[1]).toEqual(['step_1'])
+	})
+
+	it('collapses the same dependency named twice', () => {
+		const result = resolvePlanDependencies(
+			[{ description: 'Read' }, { description: 'Write', depends_on: ['Read', 'Read'] }],
+			id,
+		)
+
+		expect(result.ok && result.dependsOn[1]).toEqual(['step_1'])
+	})
+})
+
+describe('a dependency that cannot mean anything is refused, not dropped', () => {
+	it('refuses a dependency naming no step', () => {
+		const result = resolvePlanDependencies(
+			[{ description: 'Write', depends_on: ['Read the config'] }],
+			id,
+		)
+
+		expect(result.ok).toBe(false)
+		// The model has to be able to act on it, so the offending text is named.
+		expect(!result.ok && result.error).toContain('Read the config')
+	})
+
+	it('refuses a dependency two steps could answer', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Review' },
+				{ description: 'Review' },
+				{ description: 'Report', depends_on: ['Review'] },
+			],
+			id,
+		)
+
+		// Picking either is a coin flip whose result a human then approves as
+		// if it were the model's intent.
+		expect(result.ok).toBe(false)
+		expect(!result.ok && result.error).toContain('2 steps share that description')
+	})
+
+	it('refuses a step that depends on itself', () => {
+		const result = resolvePlanDependencies([{ description: 'Loop', depends_on: ['Loop'] }], id)
+
+		expect(result.ok).toBe(false)
+		expect(!result.ok && result.error).toContain('depends on itself')
+	})
+
+	it('refuses a two-step cycle', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'A', depends_on: ['B'] },
+				{ description: 'B', depends_on: ['A'] },
+			],
+			id,
+		)
+
+		expect(result.ok).toBe(false)
+		expect(!result.ok && result.error).toContain('loop')
+	})
+
+	it('refuses a cycle several steps long', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'A', depends_on: ['C'] },
+				{ description: 'B', depends_on: ['A'] },
+				{ description: 'C', depends_on: ['B'] },
+			],
+			id,
+		)
+
+		// This is the failure worth catching hardest: no step in a loop can
+		// start, so the plan does not error — it simply stops.
+		expect(result.ok).toBe(false)
+		expect(!result.ok && result.error).toContain('loop')
+	})
+
+	it('names every step in the loop so it can be broken', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Alpha', depends_on: ['Gamma'] },
+				{ description: 'Beta' },
+				{ description: 'Gamma', depends_on: ['Alpha'] },
+			],
+			id,
+		)
+
+		expect(result.ok).toBe(false)
+		if (result.ok) return
+		expect(result.error).toContain('Alpha')
+		expect(result.error).toContain('Gamma')
+		// Beta is not in the loop and must not be blamed for it.
+		expect(result.error).not.toContain('Beta')
+	})
+
+	it('accepts a diamond, which is not a cycle', () => {
+		const result = resolvePlanDependencies(
+			[
+				{ description: 'Start' },
+				{ description: 'Left', depends_on: ['Start'] },
+				{ description: 'Right', depends_on: ['Start'] },
+				{ description: 'Join', depends_on: ['Left', 'Right'] },
+			],
+			id,
+		)
+
+		// Two paths reaching one step is an ordinary plan shape, and a naive
+		// visited-set cycle check calls it a loop.
+		expect(result.ok).toBe(true)
+		expect(result.ok && result.dependsOn[3]).toEqual(['step_2', 'step_3'])
+	})
+})

package/src/tools/coordinator/index.ts

@@ -8,6 +8,7 @@ import type { RunId, TaskId } from '../../types/ids/index.js'
 import type { TaskStore } from '../../types/task/index.js'
 import type { ToolDefinition } from '../../types/tool/index.js'
 import { defineTool } from '../defineTool.js'
+import { resolvePlanDependencies } from './plan-dependencies.js'
 
 export type TaskLaunchedCallback = (
 	agentTaskId: TaskId,
@@ -437,6 +438,15 @@ export function buildCoordinatorTools(opts: CoordinatorToolsOptions): ToolDefini
 					}
 				}
 
+				// Resolve BEFORE touching the plan manager. A refusal here has to
+				// leave no half-built plan behind: `startGenerating` replaces the
+				// current plan, so failing after it would discard a plan that was
+				// fine in favour of one that never completes.
+				const dependencies = resolvePlanDependencies(steps, (index) => `step_${index + 1}`)
+				if (!dependencies.ok) {
+					return { success: false, output: '', error: dependencies.error }
+				}
+
 				pm.startGenerating(title)
 				for (let i = 0; i < steps.length; i++) {
 					const step = steps[i]
@@ -445,7 +455,10 @@ export function buildCoordinatorTools(opts: CoordinatorToolsOptions): ToolDefini
 						id: `step_${i + 1}`,
 						description: step.description,
 						toolName: step.agent_id ? 'create_task' : undefined,
-						dependsOn: [],
+						// Was `[]` unconditionally, which dropped every ordering
+						// constraint the model was invited to express — and put an
+						// empty dependency list in front of the human approving it.
+						dependsOn: [...(dependencies.dependsOn[i] ?? [])],
 						order: i + 1,
 					})
 				}