@namzu/sdk 6.2.0 → 8.0.0

package/src/runtime/query/__tests__/completion-notification.test.ts

@@ -10,7 +10,12 @@ import { defineTool } from '../../../tools/defineTool.js'
 import type { TaskHandle } from '../../../types/agent/gateway.js'
 import type { SessionId, TaskId, TenantId } from '../../../types/ids/index.js'
 import { createUserMessage } from '../../../types/message/index.js'
-import type { LLMProvider, StreamChunk } from '../../../types/provider/index.js'
+import type { Message } from '../../../types/message/index.js'
+import type {
+	ChatCompletionParams,
+	LLMProvider,
+	StreamChunk,
+} from '../../../types/provider/index.js'
 import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
 import { drainQuery } from '../index.js'
 
@@ -39,9 +44,19 @@ class ToolThenAnswerProvider implements LLMProvider {
 	readonly id = 'tool-then-answer'
 	readonly name = 'Tool Then Answer Provider'
 	calls = 0
+	/**
+	 * What the model was actually SHOWN, turn by turn.
+	 *
+	 * `run.messages` proves a notification exists somewhere in the transcript.
+	 * It does not prove the model ever saw it — a delivery that only happens on
+	 * the way out satisfies the transcript and leaves the model with nothing to
+	 * act on, which is the whole point of injecting mid-run.
+	 */
+	readonly requests: Message[][] = []
 
-	async *chatStream(): AsyncIterable<StreamChunk> {
+	async *chatStream(params: ChatCompletionParams): AsyncIterable<StreamChunk> {
 		this.calls += 1
+		this.requests.push([...params.messages])
 
 		if (this.calls === 1) {
 			yield {
@@ -71,6 +86,43 @@ class ToolThenAnswerProvider implements LLMProvider {
 	}
 }
 
+/** The same two turns, aimed at the tool the exit tests register. */
+class CallsFinisherProvider implements LLMProvider {
+	readonly id = 'calls-finisher'
+	readonly name = 'Calls Finisher Provider'
+	calls = 0
+
+	async *chatStream(): AsyncIterable<StreamChunk> {
+		this.calls += 1
+
+		if (this.calls === 1) {
+			yield {
+				id: 'msg_1',
+				delta: {
+					toolCalls: [
+						{
+							index: 0,
+							id: 'toolu_finisher_1',
+							type: 'function',
+							function: { name: 'finisher', arguments: '{}' },
+						},
+					],
+				},
+			}
+			yield {
+				id: 'msg_1',
+				delta: {},
+				finishReason: 'tool_calls',
+				usage: ZERO_USAGE,
+			}
+			return
+		}
+
+		yield { id: 'msg_2', delta: { content: 'Done.' } }
+		yield { id: 'msg_2', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
+	}
+}
+
 const noop = defineTool({
 	name: 'noop',
 	description: 'does nothing',
@@ -102,15 +154,18 @@ afterEach(async () => {
 	workdirs.length = 0
 })
 
-async function runWith(inbox: CompletionInbox | undefined): Promise<string[]> {
+async function runWith(
+	inbox: CompletionInbox | undefined,
+): Promise<{ userMessages: string[]; provider: ToolThenAnswerProvider }> {
 	const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-completion-'))
 	workdirs.push(workingDirectory)
 
 	const tools = new ToolRegistry()
 	tools.register(noop)
 
+	const provider = new ToolThenAnswerProvider()
 	const run = await drainQuery({
-		provider: new ToolThenAnswerProvider(),
+		provider,
 		tools,
 		...(inbox ? { completionInbox: inbox } : {}),
 		agentId: 'agent_test',
@@ -130,24 +185,36 @@ async function runWith(inbox: CompletionInbox | undefined): Promise<string[]> {
 		tenantId: 'tnt_completion' as TenantId,
 	})
 
-	return run.messages
-		.filter((m) => m.role === 'user')
-		.map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)))
+	return {
+		userMessages: run.messages
+			.filter((m) => m.role === 'user')
+			.map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content))),
+		provider,
+	}
+}
+
+/** Settled before the turn ended, with nothing waiting on it. */
+function inboxHolding(taskId: string, result: string): CompletionInbox {
+	const inbox = new CompletionInbox()
+	// Said before the announcement because `create_task` says it on every
+	// launch: an inbox only hears about tasks its own run started, so a
+	// gateway shared between two supervisors cannot cross-deliver.
+	inbox.launched(taskId as TaskId)
+	inbox.attach({
+		onTaskCompleted: (cb: (h: TaskHandle) => void) => {
+			cb(completed(taskId, result))
+			return () => {}
+		},
+		getTask: () => undefined,
+	} as never)
+	return inbox
 }
 
 describe('an unclaimed completion reaches the transcript', () => {
 	it('injects the notification as a user message the next turn can read', async () => {
-		const inbox = new CompletionInbox()
-		// Settled by a gateway before the turn ended, with nothing waiting on
-		// it — the abandoned-launch case.
-		inbox.attach({
-			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
-				cb(completed('tsk_late', 'the worker finished after the wait was abandoned'))
-				return () => {}
-			},
-		} as never)
+		const inbox = inboxHolding('tsk_late', 'the worker finished after the wait was abandoned')
 
-		const userMessages = await runWith(inbox)
+		const { userMessages } = await runWith(inbox)
 		const notification = userMessages.find((m) => m.includes('task-notification'))
 
 		expect(notification).toBeDefined()
@@ -155,16 +222,36 @@ describe('an unclaimed completion reaches the transcript', () => {
 		expect(notification).toContain('the worker finished after the wait was abandoned')
 	})
 
+	it('shows it to the model on the very next turn, not eventually', async () => {
+		// Three sites now drain this inbox — the post-tool-batch injection, the
+		// bounded hold on the final-answer turn, and the exit-time delivery —
+		// and all three put the same text in `run.messages`. So an assertion on
+		// the transcript, or on the LAST model request, passes whichever of the
+		// three did the work, and none of them is pinned.
+		//
+		// The one that matters is the injection right after the tool batch,
+		// because it is the only one that reaches the model while there is
+		// still work to redirect. Hence the SECOND request specifically: with
+		// that injection gone the notification still arrives, one turn later,
+		// via the hold — later is a different behaviour, and this is the
+		// assertion that can tell them apart.
+		const inbox = inboxHolding('tsk_late', 'THE WORKER SAID THIS')
+
+		const { provider } = await runWith(inbox)
+
+		const turnAfterTheToolBatch = provider.requests[1] ?? []
+		expect(
+			turnAfterTheToolBatch.some(
+				(m) => typeof m.content === 'string' && m.content.includes('THE WORKER SAID THIS'),
+			),
+			'the model was not shown the completion on the turn after the tool batch',
+		).toBe(true)
+	})
+
 	it('drains it exactly once, however many turns follow', async () => {
-		const inbox = new CompletionInbox()
-		inbox.attach({
-			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
-				cb(completed('tsk_late', 'only once'))
-				return () => {}
-			},
-		} as never)
+		const inbox = inboxHolding('tsk_late', 'only once')
 
-		const userMessages = await runWith(inbox)
+		const { userMessages } = await runWith(inbox)
 
 		expect(userMessages.filter((m) => m.includes('task-notification'))).toHaveLength(1)
 	})
@@ -172,16 +259,10 @@ describe('an unclaimed completion reaches the transcript', () => {
 	it('says nothing when every completion was already delivered', async () => {
 		// The `dc16d58` regression at the loop level: a blocking `create_task`
 		// claims its own completion, so the transcript must stay clean.
-		const inbox = new CompletionInbox()
-		inbox.attach({
-			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
-				cb(completed('tsk_awaited', 'delivered as a tool_result'))
-				return () => {}
-			},
-		} as never)
+		const inbox = inboxHolding('tsk_awaited', 'delivered as a tool_result')
 		inbox.claim('tsk_awaited' as TaskId)
 
-		const userMessages = await runWith(inbox)
+		const { userMessages } = await runWith(inbox)
 
 		expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
 	})
@@ -189,8 +270,379 @@ describe('an unclaimed completion reaches the transcript', () => {
 	it('runs unchanged with no inbox at all', async () => {
 		// The kernel must not require one: a host on the old wiring keeps
 		// working, it just never hears about abandoned completions.
-		const userMessages = await runWith(undefined)
+		const { userMessages } = await runWith(undefined)
 
 		expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
 	})
 })
+
+/**
+ * The exits that are not the ordinary final answer.
+ *
+ * The inbox was consulted at exactly one site, inside the no-tool-calls branch.
+ * The loop leaves by eight other routes, and three of them are ways a run
+ * legitimately ENDS: a tool the author declared terminal, a captured structured
+ * output, and the host's `stopWhen`. A worker that finished while any of those
+ * was deciding had its output dropped on the floor — the gateway held the
+ * result, the run closed, and nothing ever read it.
+ *
+ * These drive `drainQuery` rather than the helper directly, and that is the
+ * point: the delivery happens in the loop, so a unit test on the inbox proves
+ * nothing about whether the loop reaches it.
+ */
+describe('a run that ends some other way still hands over what finished', () => {
+	async function runEndingWith(options: {
+		terminal?: boolean
+		stopWhen?: boolean
+	}): Promise<{ userMessages: string[]; run: Awaited<ReturnType<typeof drainQuery>> }> {
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-completion-exit-'))
+		workdirs.push(workingDirectory)
+
+		const inbox = new CompletionInbox()
+		inbox.launched('tsk_bg' as TaskId)
+		let announce: ((h: TaskHandle) => void) | undefined
+		inbox.attach({
+			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
+				announce = cb
+				return () => {
+					announce = undefined
+				}
+			},
+			getTask: () => undefined,
+		} as never)
+
+		// Settles from inside the tool call, so the completion is in hand
+		// BEFORE the exit is decided. The question under test is which exits
+		// hand it over, not whether the wait works.
+		const finisher = defineTool({
+			name: 'finisher',
+			description: 'the worker finishes while this runs',
+			inputSchema: z.object({}),
+			category: 'analysis',
+			permissions: [],
+			readOnly: true,
+			destructive: false,
+			concurrencySafe: true,
+			...(options.terminal ? { terminal: true } : {}),
+			async execute() {
+				announce?.(completed('tsk_bg', 'THE BACKGROUND WORKER RESULT'))
+				return { success: true, output: 'this call is the answer' }
+			},
+		})
+
+		const tools = new ToolRegistry()
+		tools.register(finisher)
+
+		const run = await drainQuery({
+			provider: new CallsFinisherProvider(),
+			tools,
+			completionInbox: inbox,
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('delegate and report')],
+			workingDirectory,
+			...(options.stopWhen ? { stopWhen: () => true } : {}),
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 10_000,
+				tokenBudget: 100_000,
+				maxIterations: 4,
+				maxResponseTokens: 256,
+			},
+			sessionId: 'ses_completion' as SessionId,
+			threadId: 'thd_completion' as ThreadId,
+			projectId: 'prj_completion' as ProjectId,
+			tenantId: 'tnt_completion' as TenantId,
+		})
+
+		return {
+			userMessages: run.messages
+				.filter((m) => m.role === 'user')
+				.map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content))),
+			run,
+		}
+	}
+
+	it('a terminal tool settles the run without discarding the worker', async () => {
+		const { userMessages } = await runEndingWith({ terminal: true })
+
+		expect(
+			userMessages.some((m) => m.includes('THE BACKGROUND WORKER RESULT')),
+			'the terminal-tool exit dropped a finished completion',
+		).toBe(true)
+	})
+
+	it("the host's stopWhen ends the run without discarding it either", async () => {
+		const { userMessages } = await runEndingWith({ stopWhen: true })
+
+		expect(
+			userMessages.some((m) => m.includes('THE BACKGROUND WORKER RESULT')),
+			'the stopWhen exit dropped a finished completion',
+		).toBe(true)
+	})
+
+	it('lets a worker that has not arrived yet outrank the host stop predicate, once', async () => {
+		// A chosen precedence, not something `stopWhen` implies. A stop
+		// predicate is a programmable halt that says nothing about whether the
+		// answer is complete — unlike a terminal tool or a captured structured
+		// output, which decide the result and leave no turn to use anything in.
+		// So this one waits, delivers, and gives the model the turn; the
+		// predicate fires again next turn with nothing pending and stops. One
+		// extra turn, bounded.
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-stopwhen-hold-'))
+		workdirs.push(workingDirectory)
+
+		const inbox = new CompletionInbox()
+		let announce: ((h: TaskHandle) => void) | undefined
+		inbox.attach({
+			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
+				announce = cb
+				return () => {
+					announce = undefined
+				}
+			},
+			getTask: () => undefined,
+		} as never)
+		// OUTSTANDING, not arrived: the exit-time delivery has nothing to hand
+		// over, so only a hold can produce this result.
+		inbox.expect('tsk_slow' as TaskId)
+
+		const tools = new ToolRegistry()
+		tools.register(
+			defineTool({
+				name: 'finisher',
+				description: 'the worker is still going when this returns',
+				inputSchema: z.object({}),
+				category: 'analysis',
+				permissions: [],
+				readOnly: true,
+				destructive: false,
+				concurrencySafe: true,
+				async execute() {
+					setTimeout(() => announce?.(completed('tsk_slow', 'LATE WORKER RESULT')), 20).unref?.()
+					return { success: true, output: 'launched' }
+				},
+			}),
+		)
+
+		const provider = new CallsFinisherProvider()
+		const run = await drainQuery({
+			provider,
+			tools,
+			completionInbox: inbox,
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('go')],
+			workingDirectory,
+			stopWhen: () => true,
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 20_000,
+				tokenBudget: 100_000,
+				maxIterations: 4,
+				maxResponseTokens: 256,
+			},
+			sessionId: 'ses_completion' as SessionId,
+			threadId: 'thd_completion' as ThreadId,
+			projectId: 'prj_completion' as ProjectId,
+			tenantId: 'tnt_completion' as TenantId,
+		} as never)
+
+		const userText = (run.messages as { role: string; content: unknown }[])
+			.filter((m) => m.role === 'user')
+			.map((m) => (typeof m.content === 'string' ? m.content : ''))
+
+		expect(
+			userText.some((m) => m.includes('LATE WORKER RESULT')),
+			'the stop predicate discarded a worker that had not landed yet',
+		).toBe(true)
+		// The extra turn is the point: the model was asked again, once.
+		expect(provider.calls, 'the model never got a turn to use the result').toBe(2)
+
+		// And the run says WHY it is over. The extra turn is prose, and
+		// `stopWhen` is consulted only after a tool batch — deliberately, so a
+		// predicate can see what the tools returned — so the predicate is never
+		// asked again and the run leaves by the ordinary route. Reporting
+		// `end_turn` there would name the shape of the last message rather than
+		// the host's decision, and this repo carries thirteen `StopReason`
+		// values precisely so a run that ends for a nameable reason names it.
+		expect(run.stopReason).toBe('stop_condition')
+	}, 60_000)
+
+	it('forgets the deferral if the extra turn did not end the run', async () => {
+		// The lifetime is the whole safety of the flag. It is set at the end of
+		// one iteration and read by the next, and the next clears it whether or
+		// not anything read it — so a deferral cannot colour a stop reason two
+		// turns later, when the predicate has since been asked again and said
+		// no. A flag that outlived its continuation would put `stop_condition`
+		// on a run the host had stopped asking to stop.
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-defer-life-'))
+		workdirs.push(workingDirectory)
+
+		const inbox = new CompletionInbox()
+		let announce: ((h: TaskHandle) => void) | undefined
+		inbox.attach({
+			onTaskCompleted: (cb: (h: TaskHandle) => void) => {
+				announce = cb
+				return () => {
+					announce = undefined
+				}
+			},
+			getTask: () => undefined,
+		} as never)
+		inbox.expect('tsk_slow' as TaskId)
+
+		const tools = new ToolRegistry()
+		tools.register(
+			defineTool({
+				name: 'finisher',
+				description: 'launches, then does nothing',
+				inputSchema: z.object({}),
+				category: 'analysis',
+				permissions: [],
+				readOnly: true,
+				destructive: false,
+				concurrencySafe: true,
+				async execute() {
+					setTimeout(() => announce?.(completed('tsk_slow', 'LATE RESULT')), 20).unref?.()
+					return { success: true, output: 'ok' }
+				},
+			}),
+		)
+
+		/** Tool, tool, prose — so the predicate is asked twice. */
+		class ToolToolProse implements LLMProvider {
+			readonly id = 'tool-tool-prose'
+			readonly name = 'Tool Tool Prose'
+			calls = 0
+			async *chatStream(): AsyncIterable<StreamChunk> {
+				this.calls += 1
+				if (this.calls <= 2) {
+					yield {
+						id: `m${this.calls}`,
+						delta: {
+							toolCalls: [
+								{
+									index: 0,
+									id: `toolu_${this.calls}`,
+									type: 'function',
+									function: { name: 'finisher', arguments: '{}' },
+								},
+							],
+						},
+					}
+					yield { id: `m${this.calls}`, delta: {}, finishReason: 'tool_calls', usage: ZERO_USAGE }
+					return
+				}
+				yield { id: 'm3', delta: { content: 'Done.' } }
+				yield { id: 'm3', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
+			}
+		}
+
+		// True once — the deferral — then never again.
+		let asked = 0
+		const run = await drainQuery({
+			provider: new ToolToolProse(),
+			tools,
+			completionInbox: inbox,
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('go')],
+			workingDirectory,
+			stopWhen: () => ++asked === 1,
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 20_000,
+				tokenBudget: 100_000,
+				maxIterations: 6,
+				maxResponseTokens: 256,
+			},
+			sessionId: 'ses_completion' as SessionId,
+			threadId: 'thd_completion' as ThreadId,
+			projectId: 'prj_completion' as ProjectId,
+			tenantId: 'tnt_completion' as TenantId,
+		} as never)
+
+		expect(asked, 'the predicate was never asked a second time').toBeGreaterThan(1)
+		expect(run.stopReason, 'a spent deferral coloured a later stop reason').toBe('end_turn')
+	}, 60_000)
+
+	it('names the work it walked away from, without cancelling it', async () => {
+		// The other half of the principle. Delivering what arrived is honest;
+		// staying silent about what did not is the false impression the whole
+		// mechanism exists to prevent. Naming rather than cancelling, because
+		// "the parent answered early" is a weaker warrant for killing a child
+		// than "the clock ran out" — and this subsystem already ruled that the
+		// clock does not license it.
+		const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-abandoned-'))
+		workdirs.push(workingDirectory)
+
+		const inbox = new CompletionInbox()
+		inbox.attach({ onTaskCompleted: () => () => {}, getTask: () => undefined } as never)
+		// Launched, never settles, and the terminal tool ends the run over it.
+		inbox.expect('tsk_still_running' as TaskId)
+
+		const tools = new ToolRegistry()
+		tools.register(
+			defineTool({
+				name: 'finisher',
+				description: 'ends the run',
+				inputSchema: z.object({}),
+				category: 'analysis',
+				permissions: [],
+				readOnly: true,
+				destructive: false,
+				concurrencySafe: true,
+				terminal: true,
+				async execute() {
+					return { success: true, output: 'this call is the answer' }
+				},
+			} as never),
+		)
+
+		const run = await drainQuery({
+			provider: new CallsFinisherProvider(),
+			tools,
+			completionInbox: inbox,
+			agentId: 'agent_test',
+			agentName: 'Test Agent',
+			messages: [createUserMessage('go')],
+			workingDirectory,
+			runConfig: {
+				model: 'mock-model',
+				timeoutMs: 10_000,
+				tokenBudget: 100_000,
+				maxIterations: 4,
+				maxResponseTokens: 256,
+			},
+			sessionId: 'ses_completion' as SessionId,
+			threadId: 'thd_completion' as ThreadId,
+			projectId: 'prj_completion' as ProjectId,
+			tenantId: 'tnt_completion' as TenantId,
+		} as never)
+
+		expect(run.abandonedTaskIds, 'the run said nothing about the worker it left running').toEqual([
+			'tsk_still_running',
+		])
+		// Named, not stopped: the inbox still counts it as outstanding, and
+		// nothing asked the gateway to cancel anything.
+		expect(inbox.outstandingTaskIds).toEqual(['tsk_still_running'])
+	})
+
+	it('leaves the field absent when the run walked away from nothing', async () => {
+		// A field that is always present says nothing; one that appears only
+		// when there is something to report is a signal a host can branch on.
+		const { run } = await runEndingWith({ terminal: true })
+
+		expect(run.abandonedTaskIds).toBeUndefined()
+	})
+
+	it('hands it over exactly once when the ordinary exit already did', async () => {
+		// The `dc16d58` failure in a new place: an exit-time delivery that
+		// re-sends what the in-loop drain already sent is the duplicate bug
+		// again. `drain()` empties, so the second call finds nothing.
+		const { userMessages } = await runEndingWith({})
+
+		expect(userMessages.filter((m) => m.includes('THE BACKGROUND WORKER RESULT'))).toHaveLength(1)
+	})
+})