@namzu/sdk 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +254 -0
- package/dist/agents/SupervisorAgent.d.ts.map +1 -1
- package/dist/agents/SupervisorAgent.js +18 -0
- package/dist/agents/SupervisorAgent.js.map +1 -1
- package/dist/agents/__tests__/supervisor-coordinator-registration.test.js +31 -0
- package/dist/agents/__tests__/supervisor-coordinator-registration.test.js.map +1 -1
- package/dist/gateway/__tests__/completion-inbox.test.d.ts +2 -0
- package/dist/gateway/__tests__/completion-inbox.test.d.ts.map +1 -0
- package/dist/gateway/__tests__/completion-inbox.test.js +275 -0
- package/dist/gateway/__tests__/completion-inbox.test.js.map +1 -0
- package/dist/gateway/completion-inbox.d.ts +113 -0
- package/dist/gateway/completion-inbox.d.ts.map +1 -0
- package/dist/gateway/completion-inbox.js +225 -0
- package/dist/gateway/completion-inbox.js.map +1 -0
- package/dist/public-runtime.d.ts +1 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +4 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/registry/tool/__tests__/dialect.test.js +8 -5
- package/dist/registry/tool/__tests__/dialect.test.js.map +1 -1
- package/dist/registry/tool/dialect.js +11 -4
- package/dist/registry/tool/dialect.js.map +1 -1
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +34 -0
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/runtime/query/__tests__/completion-notification.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/completion-notification.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/completion-notification.test.js +168 -0
- package/dist/runtime/query/__tests__/completion-notification.test.js.map +1 -0
- package/dist/runtime/query/__tests__/run-survives-its-own-park.proc-test.d.ts +2 -0
- package/dist/runtime/query/__tests__/run-survives-its-own-park.proc-test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/run-survives-its-own-park.proc-test.js +122 -0
- package/dist/runtime/query/__tests__/run-survives-its-own-park.proc-test.js.map +1 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.js +171 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.js.map +1 -0
- package/dist/runtime/query/executor.d.ts +14 -0
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +20 -1
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/index.d.ts +9 -0
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +1 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +74 -1
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/context.d.ts +9 -0
- package/dist/runtime/query/iteration/phases/context.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/context.js +43 -13
- package/dist/runtime/query/iteration/phases/context.js.map +1 -1
- package/dist/tools/builtins/__tests__/bash.proc-test.d.ts +2 -0
- package/dist/tools/builtins/__tests__/bash.proc-test.d.ts.map +1 -0
- package/dist/tools/builtins/__tests__/bash.proc-test.js +115 -0
- package/dist/tools/builtins/__tests__/bash.proc-test.js.map +1 -0
- package/dist/tools/builtins/__tests__/bash.test.d.ts +2 -0
- package/dist/tools/builtins/__tests__/bash.test.d.ts.map +1 -0
- package/dist/tools/builtins/__tests__/bash.test.js +65 -0
- package/dist/tools/builtins/__tests__/bash.test.js.map +1 -0
- package/dist/tools/builtins/bash.d.ts.map +1 -1
- package/dist/tools/builtins/bash.js +89 -17
- package/dist/tools/builtins/bash.js.map +1 -1
- package/dist/tools/coordinator/__tests__/allow-delegation.test.d.ts +2 -0
- package/dist/tools/coordinator/__tests__/allow-delegation.test.d.ts.map +1 -0
- package/dist/tools/coordinator/__tests__/allow-delegation.test.js +94 -0
- package/dist/tools/coordinator/__tests__/allow-delegation.test.js.map +1 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.d.ts +2 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.d.ts.map +1 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js +255 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js.map +1 -0
- package/dist/tools/coordinator/__tests__/task-list.test.js +13 -7
- package/dist/tools/coordinator/__tests__/task-list.test.js.map +1 -1
- package/dist/tools/coordinator/index.d.ts +44 -0
- package/dist/tools/coordinator/index.d.ts.map +1 -1
- package/dist/tools/coordinator/index.js +239 -16
- package/dist/tools/coordinator/index.js.map +1 -1
- package/dist/types/agent/supervisor.d.ts +27 -0
- package/dist/types/agent/supervisor.d.ts.map +1 -1
- package/dist/types/tool/index.d.ts +13 -0
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/package.json +3 -2
- package/src/agents/SupervisorAgent.ts +19 -0
- package/src/agents/__tests__/supervisor-coordinator-registration.test.ts +37 -0
- package/src/gateway/__tests__/completion-inbox.test.ts +343 -0
- package/src/gateway/completion-inbox.ts +241 -0
- package/src/public-runtime.ts +4 -0
- package/src/registry/tool/__tests__/dialect.test.ts +8 -5
- package/src/registry/tool/dialect.ts +11 -4
- package/src/registry/tool/execute.ts +35 -0
- package/src/runtime/query/__tests__/completion-notification.test.ts +196 -0
- package/src/runtime/query/__tests__/run-survives-its-own-park.proc-test.ts +127 -0
- package/src/runtime/query/__tests__/step-allow-list.test.ts +205 -0
- package/src/runtime/query/executor.ts +21 -1
- package/src/runtime/query/index.ts +11 -0
- package/src/runtime/query/iteration/index.ts +77 -1
- package/src/runtime/query/iteration/phases/context.ts +51 -15
- package/src/tools/builtins/__tests__/bash.proc-test.ts +142 -0
- package/src/tools/builtins/__tests__/bash.test.ts +79 -0
- package/src/tools/builtins/bash.ts +100 -18
- package/src/tools/coordinator/__tests__/allow-delegation.test.ts +120 -0
- package/src/tools/coordinator/__tests__/completion-delivery.test.ts +347 -0
- package/src/tools/coordinator/__tests__/task-list.test.ts +13 -7
- package/src/tools/coordinator/index.ts +278 -21
- package/src/types/agent/supervisor.ts +28 -0
- package/src/types/tool/index.ts +13 -0
|
@@ -116,10 +116,17 @@ function to2020(value: unknown): unknown {
|
|
|
116
116
|
continue
|
|
117
117
|
}
|
|
118
118
|
if (key === 'additionalItems') {
|
|
119
|
-
// Only meaningful alongside an array-form `items`,
|
|
120
|
-
// calls the same thing `items`.
|
|
121
|
-
//
|
|
122
|
-
|
|
119
|
+
// Only meaningful alongside an array-form `items`, and 2020-12
|
|
120
|
+
// calls the same thing `items`.
|
|
121
|
+
//
|
|
122
|
+
// `false` is carried across, and the first version of this dropped
|
|
123
|
+
// it on the reasoning that a closed tuple is 2020-12's default. It
|
|
124
|
+
// is not: with `prefixItems` set and no `items`, elements past the
|
|
125
|
+
// tuple are UNCONSTRAINED. Dropping the `false` therefore turned a
|
|
126
|
+
// closed tuple into an open one — a schema the author wrote to
|
|
127
|
+
// forbid a third element silently began to allow any. The wire
|
|
128
|
+
// accepts `items: false`, measured, so nothing was gained by it.
|
|
129
|
+
if (Array.isArray(node.items)) out.items = to2020(child)
|
|
123
130
|
continue
|
|
124
131
|
}
|
|
125
132
|
out[key] = to2020(child)
|
|
@@ -438,6 +438,41 @@ Executable tool names, descriptions, and JSON input schemas are attached through
|
|
|
438
438
|
}
|
|
439
439
|
}
|
|
440
440
|
|
|
441
|
+
// A turn that was narrowed may only call what it was narrowed to.
|
|
442
|
+
//
|
|
443
|
+
// This used to be enforced nowhere. `allowedTools` decided which
|
|
444
|
+
// schemas went into the request and was then carried into this
|
|
445
|
+
// context and read by nothing, so the restriction was a statement
|
|
446
|
+
// about the menu rather than about the kitchen: a model that named
|
|
447
|
+
// a withheld tool — from earlier context, from a gateway holding
|
|
448
|
+
// its own tool list, from a replayed cache prefix — had it run.
|
|
449
|
+
// The type says "restrict which tools the model may call"; this is
|
|
450
|
+
// the line that makes that true.
|
|
451
|
+
//
|
|
452
|
+
// Absent means unrestricted. An EMPTY list does not: it is a turn
|
|
453
|
+
// that may call nothing, and treating it as "no restriction" is
|
|
454
|
+
// the fail-open reading this codebase has already been bitten by
|
|
455
|
+
// once, in the delegate roster.
|
|
456
|
+
const allowed = context.allowedTools
|
|
457
|
+
if (allowed !== undefined && !allowed.includes(toolName)) {
|
|
458
|
+
const msg = `Tool "${toolName}" is not available on this step. Available: ${allowed.length > 0 ? allowed.join(', ') : '(none)'}`
|
|
459
|
+
this.log.warn('Blocked a tool outside the step allow-list', {
|
|
460
|
+
tool: toolName,
|
|
461
|
+
allowed: allowed.length,
|
|
462
|
+
})
|
|
463
|
+
span.setAttributes({
|
|
464
|
+
[NAMZU.TOOL_SUCCESS]: false,
|
|
465
|
+
[NAMZU.TOOL_ERROR]: msg,
|
|
466
|
+
})
|
|
467
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: msg })
|
|
468
|
+
return {
|
|
469
|
+
success: false,
|
|
470
|
+
output: '',
|
|
471
|
+
error: msg,
|
|
472
|
+
permissionDenied: true,
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
441
476
|
const mode = context.permissionContext?.mode ?? 'auto'
|
|
442
477
|
if (mode === 'plan') {
|
|
443
478
|
const isReadOnly = tool.isReadOnly ? tool.isReadOnly(rawInput) : false
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
5
|
+
import { z } from 'zod'
|
|
6
|
+
|
|
7
|
+
import { CompletionInbox } from '../../../gateway/completion-inbox.js'
|
|
8
|
+
import { ToolRegistry } from '../../../registry/tool/execute.js'
|
|
9
|
+
import { defineTool } from '../../../tools/defineTool.js'
|
|
10
|
+
import type { TaskHandle } from '../../../types/agent/gateway.js'
|
|
11
|
+
import type { SessionId, TaskId, TenantId } from '../../../types/ids/index.js'
|
|
12
|
+
import { createUserMessage } from '../../../types/message/index.js'
|
|
13
|
+
import type { LLMProvider, StreamChunk } from '../../../types/provider/index.js'
|
|
14
|
+
import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
|
|
15
|
+
import { drainQuery } from '../index.js'
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The last link, and the one most likely to be built and never wired.
|
|
19
|
+
*
|
|
20
|
+
* `CompletionInbox` can queue an unclaimed completion and format it, and the
|
|
21
|
+
* coordinator tools can decline to claim one — but none of that reaches the
|
|
22
|
+
* model unless the iteration loop actually drains the inbox into the
|
|
23
|
+
* transcript. A mechanism that is declared, threaded through types, and
|
|
24
|
+
* driven by nothing is precisely the shape of the defect this whole change
|
|
25
|
+
* exists to fix, so the drain gets its own test at the loop level rather
|
|
26
|
+
* than being assumed from its parts.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const ZERO_USAGE = {
|
|
30
|
+
promptTokens: 0,
|
|
31
|
+
completionTokens: 0,
|
|
32
|
+
totalTokens: 0,
|
|
33
|
+
cachedTokens: 0,
|
|
34
|
+
cacheWriteTokens: 0,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Calls a tool once, then answers. Two turns is all the drain needs. */
|
|
38
|
+
class ToolThenAnswerProvider implements LLMProvider {
|
|
39
|
+
readonly id = 'tool-then-answer'
|
|
40
|
+
readonly name = 'Tool Then Answer Provider'
|
|
41
|
+
calls = 0
|
|
42
|
+
|
|
43
|
+
async *chatStream(): AsyncIterable<StreamChunk> {
|
|
44
|
+
this.calls += 1
|
|
45
|
+
|
|
46
|
+
if (this.calls === 1) {
|
|
47
|
+
yield {
|
|
48
|
+
id: 'msg_1',
|
|
49
|
+
delta: {
|
|
50
|
+
toolCalls: [
|
|
51
|
+
{
|
|
52
|
+
index: 0,
|
|
53
|
+
id: 'toolu_noop_1',
|
|
54
|
+
type: 'function',
|
|
55
|
+
function: { name: 'noop', arguments: '{}' },
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
yield {
|
|
61
|
+
id: 'msg_1',
|
|
62
|
+
delta: {},
|
|
63
|
+
finishReason: 'tool_calls',
|
|
64
|
+
usage: ZERO_USAGE,
|
|
65
|
+
}
|
|
66
|
+
return
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
yield { id: 'msg_2', delta: { content: 'Done.' } }
|
|
70
|
+
yield { id: 'msg_2', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const noop = defineTool({
|
|
75
|
+
name: 'noop',
|
|
76
|
+
description: 'does nothing',
|
|
77
|
+
inputSchema: z.object({}),
|
|
78
|
+
category: 'analysis',
|
|
79
|
+
permissions: [],
|
|
80
|
+
readOnly: true,
|
|
81
|
+
destructive: false,
|
|
82
|
+
concurrencySafe: true,
|
|
83
|
+
async execute() {
|
|
84
|
+
return { success: true, output: 'ok' }
|
|
85
|
+
},
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
function completed(taskId: string, result: string): TaskHandle {
|
|
89
|
+
return {
|
|
90
|
+
taskId: taskId as TaskId,
|
|
91
|
+
agentId: 'reviewer',
|
|
92
|
+
state: 'completed',
|
|
93
|
+
createdAt: 1_000,
|
|
94
|
+
completedAt: 4_000,
|
|
95
|
+
result: { status: 'completed', result } as TaskHandle['result'],
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const workdirs: string[] = []
|
|
100
|
+
afterEach(async () => {
|
|
101
|
+
await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
|
|
102
|
+
workdirs.length = 0
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
async function runWith(inbox: CompletionInbox | undefined): Promise<string[]> {
|
|
106
|
+
const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-completion-'))
|
|
107
|
+
workdirs.push(workingDirectory)
|
|
108
|
+
|
|
109
|
+
const tools = new ToolRegistry()
|
|
110
|
+
tools.register(noop)
|
|
111
|
+
|
|
112
|
+
const run = await drainQuery({
|
|
113
|
+
provider: new ToolThenAnswerProvider(),
|
|
114
|
+
tools,
|
|
115
|
+
...(inbox ? { completionInbox: inbox } : {}),
|
|
116
|
+
agentId: 'agent_test',
|
|
117
|
+
agentName: 'Test Agent',
|
|
118
|
+
messages: [createUserMessage('delegate and report')],
|
|
119
|
+
workingDirectory,
|
|
120
|
+
runConfig: {
|
|
121
|
+
model: 'mock-model',
|
|
122
|
+
timeoutMs: 10_000,
|
|
123
|
+
tokenBudget: 100_000,
|
|
124
|
+
maxIterations: 4,
|
|
125
|
+
maxResponseTokens: 256,
|
|
126
|
+
},
|
|
127
|
+
sessionId: 'ses_completion' as SessionId,
|
|
128
|
+
threadId: 'thd_completion' as ThreadId,
|
|
129
|
+
projectId: 'prj_completion' as ProjectId,
|
|
130
|
+
tenantId: 'tnt_completion' as TenantId,
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
return run.messages
|
|
134
|
+
.filter((m) => m.role === 'user')
|
|
135
|
+
.map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)))
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
describe('an unclaimed completion reaches the transcript', () => {
|
|
139
|
+
it('injects the notification as a user message the next turn can read', async () => {
|
|
140
|
+
const inbox = new CompletionInbox()
|
|
141
|
+
// Settled by a gateway before the turn ended, with nothing waiting on
|
|
142
|
+
// it — the abandoned-launch case.
|
|
143
|
+
inbox.attach({
|
|
144
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
145
|
+
cb(completed('tsk_late', 'the worker finished after the wait was abandoned'))
|
|
146
|
+
return () => {}
|
|
147
|
+
},
|
|
148
|
+
} as never)
|
|
149
|
+
|
|
150
|
+
const userMessages = await runWith(inbox)
|
|
151
|
+
const notification = userMessages.find((m) => m.includes('task-notification'))
|
|
152
|
+
|
|
153
|
+
expect(notification).toBeDefined()
|
|
154
|
+
expect(notification).toContain('tsk_late')
|
|
155
|
+
expect(notification).toContain('the worker finished after the wait was abandoned')
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
it('drains it exactly once, however many turns follow', async () => {
|
|
159
|
+
const inbox = new CompletionInbox()
|
|
160
|
+
inbox.attach({
|
|
161
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
162
|
+
cb(completed('tsk_late', 'only once'))
|
|
163
|
+
return () => {}
|
|
164
|
+
},
|
|
165
|
+
} as never)
|
|
166
|
+
|
|
167
|
+
const userMessages = await runWith(inbox)
|
|
168
|
+
|
|
169
|
+
expect(userMessages.filter((m) => m.includes('task-notification'))).toHaveLength(1)
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
it('says nothing when every completion was already delivered', async () => {
|
|
173
|
+
// The `dc16d58` regression at the loop level: a blocking `create_task`
|
|
174
|
+
// claims its own completion, so the transcript must stay clean.
|
|
175
|
+
const inbox = new CompletionInbox()
|
|
176
|
+
inbox.attach({
|
|
177
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
178
|
+
cb(completed('tsk_awaited', 'delivered as a tool_result'))
|
|
179
|
+
return () => {}
|
|
180
|
+
},
|
|
181
|
+
} as never)
|
|
182
|
+
inbox.claim('tsk_awaited' as TaskId)
|
|
183
|
+
|
|
184
|
+
const userMessages = await runWith(inbox)
|
|
185
|
+
|
|
186
|
+
expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('runs unchanged with no inbox at all', async () => {
|
|
190
|
+
// The kernel must not require one: a host on the old wiring keeps
|
|
191
|
+
// working, it just never hears about abandoned completions.
|
|
192
|
+
const userMessages = await runWith(undefined)
|
|
193
|
+
|
|
194
|
+
expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
|
|
195
|
+
})
|
|
196
|
+
})
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process'
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { fileURLToPath } from 'node:url'
|
|
6
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A run has to survive its own waiting.
|
|
10
|
+
*
|
|
11
|
+
* Every HITL park went through a timer that was deliberately `unref`'d, so a
|
|
12
|
+
* pending park-recorder could never hold a process open after the run
|
|
13
|
+
* settled. The intent was right and the scope was wrong: the run AWAITS that
|
|
14
|
+
* timer, mid-turn, on every park. An unref'd timer does not keep Node's loop
|
|
15
|
+
* alive — so once the decision resolved and the run sat out the rest of the
|
|
16
|
+
* delay, the loop had nothing ref'd left in it and the process exited. Mid
|
|
17
|
+
* turn. Exit code 0. No terminal event, no error, and nothing done.
|
|
18
|
+
*
|
|
19
|
+
* The headless surfaces could not finish a turn at all: the first tool call
|
|
20
|
+
* completed and the process ended.
|
|
21
|
+
*
|
|
22
|
+
* **This test spawns a child process on purpose.** In-process it is
|
|
23
|
+
* unwritable — a test runner holds the event loop open for the whole file,
|
|
24
|
+
* which is the exact prop that hid this for as long as it was hidden. Every
|
|
25
|
+
* existing test passed throughout. If you are tempted to rewrite this as an
|
|
26
|
+
* ordinary `drainQuery` call because spawning is slow, that rewrite is the
|
|
27
|
+
* bug coming back.
|
|
28
|
+
*
|
|
29
|
+
* It loads the BUILT entry point, because that is what a consumer loads and
|
|
30
|
+
* because a child process cannot resolve TypeScript. The `test:proc` script
|
|
31
|
+
* therefore builds first — a stale `dist` reports a failure that has nothing
|
|
32
|
+
* to do with the code under test, and would report a pass just as
|
|
33
|
+
* confidently.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
const workdirs: string[] = []
|
|
37
|
+
afterEach(() => {
|
|
38
|
+
for (const dir of workdirs) rmSync(dir, { recursive: true, force: true })
|
|
39
|
+
workdirs.length = 0
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* A scripted provider and a run, in a file with no test runner under it.
|
|
44
|
+
*
|
|
45
|
+
* Two turns: one tool call, then an answer. The park happens between them,
|
|
46
|
+
* which is where the process used to die.
|
|
47
|
+
*/
|
|
48
|
+
const SCRIPT = `
|
|
49
|
+
import { pathToFileURL } from 'node:url'
|
|
50
|
+
const sdk = await import(pathToFileURL(process.argv[2]).href)
|
|
51
|
+
const { ToolRegistry, drainQuery, defineTool } = sdk
|
|
52
|
+
const { z } = await import(pathToFileURL(process.argv[3]).href)
|
|
53
|
+
|
|
54
|
+
const ZERO = { promptTokens: 0, completionTokens: 0, totalTokens: 0, cachedTokens: 0, cacheWriteTokens: 0 }
|
|
55
|
+
|
|
56
|
+
let calls = 0
|
|
57
|
+
const provider = {
|
|
58
|
+
id: 'scripted',
|
|
59
|
+
name: 'Scripted',
|
|
60
|
+
async *chatStream() {
|
|
61
|
+
calls += 1
|
|
62
|
+
if (calls === 1) {
|
|
63
|
+
yield { id: 'm1', delta: { toolCalls: [{ index: 0, id: 't1', type: 'function', function: { name: 'ping', arguments: '{}' } }] } }
|
|
64
|
+
yield { id: 'm1', delta: {}, finishReason: 'tool_calls', usage: ZERO }
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
yield { id: 'm2', delta: { content: 'answered' } }
|
|
68
|
+
yield { id: 'm2', delta: {}, finishReason: 'stop', usage: ZERO }
|
|
69
|
+
},
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const tools = new ToolRegistry()
|
|
73
|
+
tools.register(defineTool({
|
|
74
|
+
name: 'ping',
|
|
75
|
+
description: 'pings',
|
|
76
|
+
inputSchema: z.object({}),
|
|
77
|
+
category: 'analysis',
|
|
78
|
+
permissions: [],
|
|
79
|
+
readOnly: true,
|
|
80
|
+
destructive: false,
|
|
81
|
+
concurrencySafe: true,
|
|
82
|
+
async execute() { return { success: true, output: 'pong' } },
|
|
83
|
+
}))
|
|
84
|
+
|
|
85
|
+
let last = '(none)'
|
|
86
|
+
drainQuery({
|
|
87
|
+
provider,
|
|
88
|
+
tools,
|
|
89
|
+
agentId: 'a',
|
|
90
|
+
agentName: 'A',
|
|
91
|
+
messages: [{ role: 'user', content: 'go', timestamp: Date.now() }],
|
|
92
|
+
workingDirectory: process.argv[4],
|
|
93
|
+
runConfig: { model: 'm', timeoutMs: 20000, tokenBudget: 10000, maxIterations: 4, maxResponseTokens: 128 },
|
|
94
|
+
sessionId: 'ses_x', threadId: 'thd_x', projectId: 'prj_x', tenantId: 'tnt_x',
|
|
95
|
+
}, (e) => { last = e.type }).then(
|
|
96
|
+
(run) => console.log('RESULT ' + JSON.stringify({ status: run.status, stop: run.stopReason, last })),
|
|
97
|
+
(err) => console.log('THREW ' + (err?.message ?? err)),
|
|
98
|
+
)
|
|
99
|
+
process.on('exit', () => console.log('LAST ' + last))
|
|
100
|
+
`
|
|
101
|
+
|
|
102
|
+
describe('a run outlives its own HITL park', () => {
|
|
103
|
+
it('finishes in a process with nothing else holding the event loop open', () => {
|
|
104
|
+
const dir = mkdtempSync(join(tmpdir(), 'namzu-park-'))
|
|
105
|
+
workdirs.push(dir)
|
|
106
|
+
const script = join(dir, 'run.mjs')
|
|
107
|
+
writeFileSync(script, SCRIPT)
|
|
108
|
+
|
|
109
|
+
// `fileURLToPath`, not `pathname` with the leading slash stripped. That
|
|
110
|
+
// stripping is right on Windows, where `pathname` is `/C:/…`, and wrong
|
|
111
|
+
// everywhere else, where it turns an absolute POSIX path into a
|
|
112
|
+
// relative one — which is how this passed locally and failed in CI with
|
|
113
|
+
// the repo root pasted in front of itself.
|
|
114
|
+
const sdkEntry = fileURLToPath(new URL('../../../../dist/index.js', import.meta.url))
|
|
115
|
+
const zodEntry = require.resolve('zod')
|
|
116
|
+
|
|
117
|
+
const out = execFileSync(process.execPath, [script, sdkEntry, zodEntry, dir], {
|
|
118
|
+
encoding: 'utf8',
|
|
119
|
+
timeout: 60_000,
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
// The run reached its own end rather than the process reaching it first.
|
|
123
|
+
expect(out, `the run did not complete:\n${out}`).toContain('RESULT ')
|
|
124
|
+
expect(out).toContain('"status":"completed"')
|
|
125
|
+
expect(out).toContain('"last":"run_completed"')
|
|
126
|
+
}, 90_000)
|
|
127
|
+
})
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
5
|
+
import { z } from 'zod'
|
|
6
|
+
|
|
7
|
+
import { ToolRegistry } from '../../../registry/tool/execute.js'
|
|
8
|
+
import { defineTool } from '../../../tools/defineTool.js'
|
|
9
|
+
import type { SessionId, TenantId } from '../../../types/ids/index.js'
|
|
10
|
+
import { createUserMessage } from '../../../types/message/index.js'
|
|
11
|
+
import type { LLMProvider, StreamChunk } from '../../../types/provider/index.js'
|
|
12
|
+
import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
|
|
13
|
+
import { drainQuery } from '../index.js'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Narrowing a step has to narrow what RUNS, not only what is offered.
|
|
17
|
+
*
|
|
18
|
+
* `prepareStep.activeTools` says "restrict which tools the model may call
|
|
19
|
+
* this step, by name", and for a while it did nothing of the sort: it chose
|
|
20
|
+
* which schemas went into the request, was copied into the tool context, and
|
|
21
|
+
* was read by nobody on the execution path. A model that named a withheld
|
|
22
|
+
* tool had it run — and a model does name one, from a repeated call earlier
|
|
23
|
+
* in the context, from a gateway carrying its own tool list, or from a
|
|
24
|
+
* replayed cache prefix.
|
|
25
|
+
*
|
|
26
|
+
* These tests call the withheld tool deliberately, which is the only way to
|
|
27
|
+
* tell an enforced restriction from a presentational one. Every existing test
|
|
28
|
+
* asked the model nicely and so could not have caught this.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const ZERO_USAGE = {
|
|
32
|
+
promptTokens: 0,
|
|
33
|
+
completionTokens: 0,
|
|
34
|
+
totalTokens: 0,
|
|
35
|
+
cachedTokens: 0,
|
|
36
|
+
cacheWriteTokens: 0,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
let danger = 0
|
|
40
|
+
|
|
41
|
+
const readOnly = defineTool({
|
|
42
|
+
name: 'read_only',
|
|
43
|
+
description: 'observes',
|
|
44
|
+
inputSchema: z.object({}),
|
|
45
|
+
category: 'analysis',
|
|
46
|
+
permissions: [],
|
|
47
|
+
readOnly: true,
|
|
48
|
+
destructive: false,
|
|
49
|
+
concurrencySafe: true,
|
|
50
|
+
async execute() {
|
|
51
|
+
return { success: true, output: 'observed' }
|
|
52
|
+
},
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
const dangerous = defineTool({
|
|
56
|
+
name: 'danger',
|
|
57
|
+
description: 'mutates',
|
|
58
|
+
inputSchema: z.object({}),
|
|
59
|
+
category: 'custom',
|
|
60
|
+
permissions: [],
|
|
61
|
+
readOnly: false,
|
|
62
|
+
destructive: true,
|
|
63
|
+
concurrencySafe: false,
|
|
64
|
+
async execute() {
|
|
65
|
+
danger += 1
|
|
66
|
+
return { success: true, output: 'the withheld tool ran' }
|
|
67
|
+
},
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
/** Calls whichever tool it is told to, then answers. */
|
|
71
|
+
class NamesTool implements LLMProvider {
|
|
72
|
+
readonly id = 'names-tool'
|
|
73
|
+
readonly name = 'Names Tool Provider'
|
|
74
|
+
calls = 0
|
|
75
|
+
offered: string[][] = []
|
|
76
|
+
|
|
77
|
+
constructor(private readonly toolName: string) {}
|
|
78
|
+
|
|
79
|
+
async *chatStream(params: {
|
|
80
|
+
tools?: { function: { name: string } }[]
|
|
81
|
+
}): AsyncIterable<StreamChunk> {
|
|
82
|
+
this.calls += 1
|
|
83
|
+
this.offered.push((params.tools ?? []).map((t) => t.function.name))
|
|
84
|
+
|
|
85
|
+
if (this.calls === 1) {
|
|
86
|
+
yield {
|
|
87
|
+
id: 'msg_1',
|
|
88
|
+
delta: {
|
|
89
|
+
toolCalls: [
|
|
90
|
+
{
|
|
91
|
+
index: 0,
|
|
92
|
+
id: 'toolu_1',
|
|
93
|
+
type: 'function',
|
|
94
|
+
function: { name: this.toolName, arguments: '{}' },
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
}
|
|
99
|
+
yield { id: 'msg_1', delta: {}, finishReason: 'tool_calls', usage: ZERO_USAGE }
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
yield { id: 'msg_2', delta: { content: 'done' } }
|
|
104
|
+
yield { id: 'msg_2', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const workdirs: string[] = []
|
|
109
|
+
afterEach(async () => {
|
|
110
|
+
await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
|
|
111
|
+
workdirs.length = 0
|
|
112
|
+
danger = 0
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
async function run(opts: {
|
|
116
|
+
names: string
|
|
117
|
+
allowedTools?: string[]
|
|
118
|
+
activeTools?: string[]
|
|
119
|
+
}): Promise<{ offered: string[][]; output: string }> {
|
|
120
|
+
const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-allow-'))
|
|
121
|
+
workdirs.push(workingDirectory)
|
|
122
|
+
|
|
123
|
+
const tools = new ToolRegistry()
|
|
124
|
+
tools.register(readOnly)
|
|
125
|
+
tools.register(dangerous)
|
|
126
|
+
|
|
127
|
+
const provider = new NamesTool(opts.names)
|
|
128
|
+
let output = ''
|
|
129
|
+
|
|
130
|
+
await drainQuery(
|
|
131
|
+
{
|
|
132
|
+
provider,
|
|
133
|
+
tools,
|
|
134
|
+
...(opts.allowedTools ? { allowedTools: opts.allowedTools } : {}),
|
|
135
|
+
...(opts.activeTools
|
|
136
|
+
? { prepareStep: () => ({ activeTools: opts.activeTools as string[] }) }
|
|
137
|
+
: {}),
|
|
138
|
+
agentId: 'agent_test',
|
|
139
|
+
agentName: 'Test Agent',
|
|
140
|
+
messages: [createUserMessage('go')],
|
|
141
|
+
workingDirectory,
|
|
142
|
+
runConfig: {
|
|
143
|
+
model: 'mock-model',
|
|
144
|
+
timeoutMs: 10_000,
|
|
145
|
+
tokenBudget: 100_000,
|
|
146
|
+
maxIterations: 3,
|
|
147
|
+
maxResponseTokens: 256,
|
|
148
|
+
},
|
|
149
|
+
sessionId: 'ses_allow' as SessionId,
|
|
150
|
+
threadId: 'thd_allow' as ThreadId,
|
|
151
|
+
projectId: 'prj_allow' as ProjectId,
|
|
152
|
+
tenantId: 'tnt_allow' as TenantId,
|
|
153
|
+
},
|
|
154
|
+
(event) => {
|
|
155
|
+
if (event.type === 'tool_completed') output += event.result ?? ''
|
|
156
|
+
},
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
return { offered: provider.offered, output }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
describe('a narrowed step narrows what can run, not just what is shown', () => {
|
|
163
|
+
it('refuses a tool the step withheld, even when the model names it', async () => {
|
|
164
|
+
const { offered, output } = await run({ names: 'danger', activeTools: ['read_only'] })
|
|
165
|
+
|
|
166
|
+
// The request really was narrowed — otherwise this test proves nothing
|
|
167
|
+
// about enforcement, only about the model's manners.
|
|
168
|
+
expect(offered[0]).toEqual(['read_only'])
|
|
169
|
+
// And the call was refused rather than executed.
|
|
170
|
+
expect(danger, 'the withheld tool executed').toBe(0)
|
|
171
|
+
expect(output).toContain('not available on this step')
|
|
172
|
+
})
|
|
173
|
+
|
|
174
|
+
it('refuses a tool outside the run-level list too', async () => {
|
|
175
|
+
const { output } = await run({ names: 'danger', allowedTools: ['read_only'] })
|
|
176
|
+
|
|
177
|
+
expect(danger).toBe(0)
|
|
178
|
+
expect(output).toContain('not available on this step')
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
it('still runs a tool that is on the list', async () => {
|
|
182
|
+
const { output } = await run({ names: 'read_only', activeTools: ['read_only'] })
|
|
183
|
+
|
|
184
|
+
expect(output).toContain('observed')
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('leaves an unnarrowed run alone', async () => {
|
|
188
|
+
// Absent means unrestricted. A run that never narrows anything must not
|
|
189
|
+
// suddenly start refusing its own tools.
|
|
190
|
+
const { output } = await run({ names: 'danger' })
|
|
191
|
+
|
|
192
|
+
expect(danger).toBe(1)
|
|
193
|
+
expect(output).toContain('the withheld tool ran')
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('treats an empty list as "nothing", not as "no restriction"', async () => {
|
|
197
|
+
// The fail-open reading this codebase has already been bitten by once,
|
|
198
|
+
// in the delegate roster: an empty allow-list IS the answer, and
|
|
199
|
+
// degrading it to "unrestricted" is how a closed list becomes open.
|
|
200
|
+
const { output } = await run({ names: 'danger', activeTools: [] })
|
|
201
|
+
|
|
202
|
+
expect(danger).toBe(0)
|
|
203
|
+
expect(output).toContain('(none)')
|
|
204
|
+
})
|
|
205
|
+
})
|
|
@@ -205,6 +205,8 @@ export class ToolExecutor {
|
|
|
205
205
|
private workingStateManager?: WorkingStateManager
|
|
206
206
|
private probes: ProbeRegistry
|
|
207
207
|
private parentSpan?: Span
|
|
208
|
+
/** Set per turn by the orchestrator; see {@link setStepAllowedTools}. */
|
|
209
|
+
private stepAllowedTools?: readonly string[]
|
|
208
210
|
private readonly readPaths: Set<string> = new Set()
|
|
209
211
|
private readonly readFingerprints: Map<string, string> = new Map()
|
|
210
212
|
private readonly fileReadTracker: FileReadTracker = {
|
|
@@ -251,6 +253,21 @@ export class ToolExecutor {
|
|
|
251
253
|
this.parentSpan = span
|
|
252
254
|
}
|
|
253
255
|
|
|
256
|
+
/**
|
|
257
|
+
* Narrow what this turn may call, or clear the narrowing.
|
|
258
|
+
*
|
|
259
|
+
* Re-set each turn by the orchestrator for the same reason the parent span
|
|
260
|
+
* is: `prepareStep` can hand a different list to every step, and the run's
|
|
261
|
+
* own `allowedTools` is only the default when a step names none.
|
|
262
|
+
*
|
|
263
|
+
* Without this the executor could only ever see the RUN-level list, so a
|
|
264
|
+
* per-step narrowing reached the request that was sent and nothing else —
|
|
265
|
+
* the model was shown fewer tools and could still call all of them.
|
|
266
|
+
*/
|
|
267
|
+
setStepAllowedTools(names: readonly string[] | undefined): void {
|
|
268
|
+
this.stepAllowedTools = names
|
|
269
|
+
}
|
|
270
|
+
|
|
254
271
|
/**
|
|
255
272
|
* Answer every `tool_use` block in `response` with exactly one
|
|
256
273
|
* `tool_result`.
|
|
@@ -424,7 +441,10 @@ export class ToolExecutor {
|
|
|
424
441
|
},
|
|
425
442
|
invocationState: this.config.invocationState,
|
|
426
443
|
toolRegistry: this.config.tools,
|
|
427
|
-
|
|
444
|
+
// The step's list wins where it has one; the run's is the default.
|
|
445
|
+
// Same precedence the request already uses when it decides which
|
|
446
|
+
// schemas to send, so the menu and the kitchen agree.
|
|
447
|
+
allowedTools: this.stepAllowedTools ?? this.config.allowedTools,
|
|
428
448
|
sandbox: this.config.sandbox,
|
|
429
449
|
fileReadTracker: this.fileReadTracker,
|
|
430
450
|
...(this.parentSpan ? { parentSpan: this.parentSpan } : {}),
|
|
@@ -410,6 +410,16 @@ export interface QueryParams {
|
|
|
410
410
|
|
|
411
411
|
taskGateway?: import('../../types/agent/gateway.js').TaskGateway
|
|
412
412
|
|
|
413
|
+
/**
|
|
414
|
+
* Where a worker completion goes when no tool call is waiting for it.
|
|
415
|
+
*
|
|
416
|
+
* Supplied by whoever built the coordinator tools, because the tools and
|
|
417
|
+
* this loop have to share one inbox: the tools claim what they deliver,
|
|
418
|
+
* and the loop delivers what is left. Omitted, the loop drains nothing and
|
|
419
|
+
* the behaviour is exactly what it was before the inbox existed.
|
|
420
|
+
*/
|
|
421
|
+
completionInbox?: import('../../gateway/completion-inbox.js').CompletionInbox
|
|
422
|
+
|
|
413
423
|
launchedTasks?: Map<
|
|
414
424
|
import('../../types/ids/index.js').TaskId,
|
|
415
425
|
import('./iteration/phases/context.js').LaunchedTaskMeta
|
|
@@ -842,6 +852,7 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
842
852
|
checkpointMgr,
|
|
843
853
|
planManager: ctx.planManager,
|
|
844
854
|
taskGateway: params.taskGateway,
|
|
855
|
+
completionInbox: params.completionInbox,
|
|
845
856
|
taskStore: params.taskStore,
|
|
846
857
|
launchedTasks: params.launchedTasks ?? new Map(),
|
|
847
858
|
// Run-scoped. An approval is a statement about this run's work;
|