@namzu/sdk 5.2.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +237 -0
- package/dist/agents/SupervisorAgent.d.ts.map +1 -1
- package/dist/agents/SupervisorAgent.js +13 -0
- package/dist/agents/SupervisorAgent.js.map +1 -1
- package/dist/gateway/__tests__/completion-inbox.test.d.ts +2 -0
- package/dist/gateway/__tests__/completion-inbox.test.d.ts.map +1 -0
- package/dist/gateway/__tests__/completion-inbox.test.js +258 -0
- package/dist/gateway/__tests__/completion-inbox.test.js.map +1 -0
- package/dist/gateway/completion-inbox.d.ts +113 -0
- package/dist/gateway/completion-inbox.d.ts.map +1 -0
- package/dist/gateway/completion-inbox.js +211 -0
- package/dist/gateway/completion-inbox.js.map +1 -0
- package/dist/provider/__tests__/strict-schema.test.js +50 -2
- package/dist/provider/__tests__/strict-schema.test.js.map +1 -1
- package/dist/provider/__tests__/vendor-detail.test.d.ts +2 -0
- package/dist/provider/__tests__/vendor-detail.test.d.ts.map +1 -0
- package/dist/provider/__tests__/vendor-detail.test.js +89 -0
- package/dist/provider/__tests__/vendor-detail.test.js.map +1 -0
- package/dist/provider/errors.d.ts +38 -5
- package/dist/provider/errors.d.ts.map +1 -1
- package/dist/provider/errors.js +107 -5
- package/dist/provider/errors.js.map +1 -1
- package/dist/provider/strict-schema.d.ts.map +1 -1
- package/dist/provider/strict-schema.js +64 -8
- package/dist/provider/strict-schema.js.map +1 -1
- package/dist/public-runtime.d.ts +4 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +10 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/registry/tool/__tests__/dialect.test.d.ts +2 -0
- package/dist/registry/tool/__tests__/dialect.test.d.ts.map +1 -0
- package/dist/registry/tool/__tests__/dialect.test.js +146 -0
- package/dist/registry/tool/__tests__/dialect.test.js.map +1 -0
- package/dist/registry/tool/dialect.d.ts +50 -0
- package/dist/registry/tool/dialect.d.ts.map +1 -0
- package/dist/registry/tool/dialect.js +138 -0
- package/dist/registry/tool/dialect.js.map +1 -0
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +34 -0
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/registry/toolset/catalog.d.ts.map +1 -1
- package/dist/registry/toolset/catalog.js +10 -5
- package/dist/registry/toolset/catalog.js.map +1 -1
- package/dist/runtime/query/__tests__/completion-notification.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/completion-notification.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/completion-notification.test.js +168 -0
- package/dist/runtime/query/__tests__/completion-notification.test.js.map +1 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.d.ts +2 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.d.ts.map +1 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.js +171 -0
- package/dist/runtime/query/__tests__/step-allow-list.test.js.map +1 -0
- package/dist/runtime/query/__tests__/stream-recovery.test.js +6 -0
- package/dist/runtime/query/__tests__/stream-recovery.test.js.map +1 -1
- package/dist/runtime/query/executor.d.ts +14 -0
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +20 -1
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/index.d.ts +9 -0
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +1 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/iteration/index.d.ts.map +1 -1
- package/dist/runtime/query/iteration/index.js +74 -1
- package/dist/runtime/query/iteration/index.js.map +1 -1
- package/dist/runtime/query/iteration/phases/context.d.ts +9 -0
- package/dist/runtime/query/iteration/phases/context.d.ts.map +1 -1
- package/dist/runtime/query/iteration/phases/context.js.map +1 -1
- package/dist/runtime/query/result.d.ts.map +1 -1
- package/dist/runtime/query/result.js +6 -0
- package/dist/runtime/query/result.js.map +1 -1
- package/dist/tools/coordinator/__tests__/completion-delivery.test.d.ts +2 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.d.ts.map +1 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js +255 -0
- package/dist/tools/coordinator/__tests__/completion-delivery.test.js.map +1 -0
- package/dist/tools/coordinator/__tests__/task-list.test.js +13 -7
- package/dist/tools/coordinator/__tests__/task-list.test.js.map +1 -1
- package/dist/tools/coordinator/index.d.ts +35 -0
- package/dist/tools/coordinator/index.d.ts.map +1 -1
- package/dist/tools/coordinator/index.js +222 -16
- package/dist/tools/coordinator/index.js.map +1 -1
- package/dist/types/provider/error.d.ts +20 -4
- package/dist/types/provider/error.d.ts.map +1 -1
- package/dist/types/tool/index.d.ts +13 -0
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/package.json +1 -1
- package/src/agents/SupervisorAgent.ts +14 -0
- package/src/gateway/__tests__/completion-inbox.test.ts +320 -0
- package/src/gateway/completion-inbox.ts +227 -0
- package/src/provider/__tests__/strict-schema.test.ts +58 -2
- package/src/provider/__tests__/vendor-detail.test.ts +107 -0
- package/src/provider/errors.ts +106 -5
- package/src/provider/strict-schema.ts +65 -8
- package/src/public-runtime.ts +11 -0
- package/src/registry/tool/__tests__/dialect.test.ts +200 -0
- package/src/registry/tool/dialect.ts +143 -0
- package/src/registry/tool/execute.ts +35 -0
- package/src/registry/toolset/catalog.ts +10 -5
- package/src/runtime/query/__tests__/completion-notification.test.ts +196 -0
- package/src/runtime/query/__tests__/step-allow-list.test.ts +205 -0
- package/src/runtime/query/__tests__/stream-recovery.test.ts +6 -0
- package/src/runtime/query/executor.ts +21 -1
- package/src/runtime/query/index.ts +11 -0
- package/src/runtime/query/iteration/index.ts +77 -1
- package/src/runtime/query/iteration/phases/context.ts +10 -0
- package/src/runtime/query/result.ts +6 -0
- package/src/tools/coordinator/__tests__/completion-delivery.test.ts +347 -0
- package/src/tools/coordinator/__tests__/task-list.test.ts +13 -7
- package/src/tools/coordinator/index.ts +250 -20
- package/src/types/provider/error.ts +20 -4
- package/src/types/tool/index.ts +13 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The dialect a wire speaks, and how to say the same schema in it.
|
|
3
|
+
*
|
|
4
|
+
* A tool has one Zod schema. What changes between providers is not the tool —
|
|
5
|
+
* it is the JSON Schema dialect the wire parses, which is a property of the
|
|
6
|
+
* wire. So the shape is rendered once, canonically, and each driver converts
|
|
7
|
+
* at the boundary where it knows which wire it is about to talk to.
|
|
8
|
+
*
|
|
9
|
+
* This exists because that layering was missing and it cost a production
|
|
10
|
+
* outage. `renderToolSchema` emits draft-07 (zod-to-json-schema's
|
|
11
|
+
* `jsonSchema7` target), every driver forwarded it verbatim, and one of the
|
|
12
|
+
* wires namzu speaks requires draft 2020-12. Measured against that live
|
|
13
|
+
* endpoint:
|
|
14
|
+
*
|
|
15
|
+
* | tool schema | result |
|
|
16
|
+
* |--------------------------------|-----------------------------------------|
|
|
17
|
+
* | `items: [a, b]` (draft-07) | 400 — "must match JSON Schema draft 2020-12" |
|
|
18
|
+
* | `prefixItems: [a, b]` (2020-12)| accepted |
|
|
19
|
+
* | `items: { a }` | accepted |
|
|
20
|
+
*
|
|
21
|
+
* The failure is NOT about strict tool use. It fires with strict validation
|
|
22
|
+
* unset, and with it on the dialect error arrives *before* the strict-subset
|
|
23
|
+
* error — so a guard scoped to strict misses it entirely. One was, and did.
|
|
24
|
+
*
|
|
25
|
+
* Which wires want which dialect is the drivers' knowledge, not this file's:
|
|
26
|
+
* the vocabulary lives beside the wire that speaks it, and only the mechanism
|
|
27
|
+
* lives here.
|
|
28
|
+
*
|
|
29
|
+
* Only the conversions namzu can actually emit are implemented. The renderer
|
|
30
|
+
* runs with `$refStrategy: 'none'`, so there are no `$ref`/`definitions` to
|
|
31
|
+
* rewrite; a construct that cannot appear is not worth code that cannot be
|
|
32
|
+
* tested.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** Which spelling of JSON Schema a wire accepts. */
|
|
36
|
+
export type JsonSchemaDialect = 'draft-07' | '2020-12'
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Rendered schemas are memoized and deeply frozen, so their identity is
|
|
40
|
+
* stable for the life of the tool — which makes them a sound `WeakMap` key.
|
|
41
|
+
*
|
|
42
|
+
* Converting per request would re-walk every tool's tree on every iteration,
|
|
43
|
+
* which is the waste `renderToolSchema`'s own cache exists to remove, and it
|
|
44
|
+
* would hand a fresh object to the wire each time. The tools block renders at
|
|
45
|
+
* position 0 of the prompt-cache prefix, so a differently-ordered but equal
|
|
46
|
+
* object still invalidates the cache for the whole run. Caching the conversion
|
|
47
|
+
* keeps the bytes identical across iterations.
|
|
48
|
+
*/
|
|
49
|
+
const CONVERTED = new Map<JsonSchemaDialect, WeakMap<object, Record<string, unknown>>>()
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Say this schema in the dialect the wire parses.
|
|
53
|
+
*
|
|
54
|
+
* Returns the input unchanged — same reference — when nothing needs saying
|
|
55
|
+
* differently, so the common case costs one map lookup and no allocation.
|
|
56
|
+
*/
|
|
57
|
+
export function toSchemaDialect(
|
|
58
|
+
schema: Record<string, unknown>,
|
|
59
|
+
dialect: JsonSchemaDialect,
|
|
60
|
+
): Record<string, unknown> {
|
|
61
|
+
if (dialect === 'draft-07') return schema
|
|
62
|
+
|
|
63
|
+
let cache = CONVERTED.get(dialect)
|
|
64
|
+
if (!cache) {
|
|
65
|
+
cache = new WeakMap()
|
|
66
|
+
CONVERTED.set(dialect, cache)
|
|
67
|
+
}
|
|
68
|
+
const hit = cache.get(schema)
|
|
69
|
+
if (hit) return hit
|
|
70
|
+
|
|
71
|
+
const converted = to2020(schema) as Record<string, unknown>
|
|
72
|
+
const frozen = deepFreeze(converted)
|
|
73
|
+
cache.set(schema, frozen)
|
|
74
|
+
return frozen
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Whether a schema still carries a construct the 2020-12 wire refuses.
|
|
79
|
+
*
|
|
80
|
+
* Exported so a test can sweep every shipped tool without reaching into the
|
|
81
|
+
* conversion, and so a driver can assert rather than hope.
|
|
82
|
+
*/
|
|
83
|
+
export function findDraft07Only(schema: unknown, path = ''): string[] {
|
|
84
|
+
if (Array.isArray(schema)) {
|
|
85
|
+
return schema.flatMap((item, i) => findDraft07Only(item, `${path}[${i}]`))
|
|
86
|
+
}
|
|
87
|
+
if (typeof schema !== 'object' || schema === null) return []
|
|
88
|
+
|
|
89
|
+
const node = schema as Record<string, unknown>
|
|
90
|
+
const found: string[] = []
|
|
91
|
+
// A tuple. In draft-07 the positional schemas live in `items`; 2020-12
|
|
92
|
+
// moved them to `prefixItems` and kept `items` for the rest, so the array
|
|
93
|
+
// form is not merely old — it means something else now, and the wire
|
|
94
|
+
// rejects the whole request rather than one field.
|
|
95
|
+
if (Array.isArray(node.items)) found.push(`${path ? `${path}.` : ''}items`)
|
|
96
|
+
// `additionalItems` only ever qualified an array-form `items`; 2020-12
|
|
97
|
+
// spells that `items`.
|
|
98
|
+
if ('additionalItems' in node) found.push(`${path ? `${path}.` : ''}additionalItems`)
|
|
99
|
+
|
|
100
|
+
for (const [key, value] of Object.entries(node)) {
|
|
101
|
+
if (key === 'items' && Array.isArray(value)) continue
|
|
102
|
+
found.push(...findDraft07Only(value, path ? `${path}.${key}` : key))
|
|
103
|
+
}
|
|
104
|
+
return found
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function to2020(value: unknown): unknown {
|
|
108
|
+
if (Array.isArray(value)) return value.map(to2020)
|
|
109
|
+
if (typeof value !== 'object' || value === null) return value
|
|
110
|
+
|
|
111
|
+
const node = value as Record<string, unknown>
|
|
112
|
+
const out: Record<string, unknown> = {}
|
|
113
|
+
for (const [key, child] of Object.entries(node)) {
|
|
114
|
+
if (key === 'items' && Array.isArray(child)) {
|
|
115
|
+
out.prefixItems = child.map(to2020)
|
|
116
|
+
continue
|
|
117
|
+
}
|
|
118
|
+
if (key === 'additionalItems') {
|
|
119
|
+
// Only meaningful alongside an array-form `items`, and 2020-12
|
|
120
|
+
// calls the same thing `items`.
|
|
121
|
+
//
|
|
122
|
+
// `false` is carried across, and the first version of this dropped
|
|
123
|
+
// it on the reasoning that a closed tuple is 2020-12's default. It
|
|
124
|
+
// is not: with `prefixItems` set and no `items`, elements past the
|
|
125
|
+
// tuple are UNCONSTRAINED. Dropping the `false` therefore turned a
|
|
126
|
+
// closed tuple into an open one — a schema the author wrote to
|
|
127
|
+
// forbid a third element silently began to allow any. The wire
|
|
128
|
+
// accepts `items: false`, measured, so nothing was gained by it.
|
|
129
|
+
if (Array.isArray(node.items)) out.items = to2020(child)
|
|
130
|
+
continue
|
|
131
|
+
}
|
|
132
|
+
out[key] = to2020(child)
|
|
133
|
+
}
|
|
134
|
+
return out
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function deepFreeze<T>(value: T): T {
|
|
138
|
+
if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value
|
|
139
|
+
for (const key of Object.keys(value as Record<string, unknown>)) {
|
|
140
|
+
deepFreeze((value as Record<string, unknown>)[key])
|
|
141
|
+
}
|
|
142
|
+
return Object.freeze(value)
|
|
143
|
+
}
|
|
@@ -438,6 +438,41 @@ Executable tool names, descriptions, and JSON input schemas are attached through
|
|
|
438
438
|
}
|
|
439
439
|
}
|
|
440
440
|
|
|
441
|
+
// A turn that was narrowed may only call what it was narrowed to.
|
|
442
|
+
//
|
|
443
|
+
// This used to be enforced nowhere. `allowedTools` decided which
|
|
444
|
+
// schemas went into the request and was then carried into this
|
|
445
|
+
// context and read by nothing, so the restriction was a statement
|
|
446
|
+
// about the menu rather than about the kitchen: a model that named
|
|
447
|
+
// a withheld tool — from earlier context, from a gateway holding
|
|
448
|
+
// its own tool list, from a replayed cache prefix — had it run.
|
|
449
|
+
// The type says "restrict which tools the model may call"; this is
|
|
450
|
+
// the line that makes that true.
|
|
451
|
+
//
|
|
452
|
+
// Absent means unrestricted. An EMPTY list does not: it is a turn
|
|
453
|
+
// that may call nothing, and treating it as "no restriction" is
|
|
454
|
+
// the fail-open reading this codebase has already been bitten by
|
|
455
|
+
// once, in the delegate roster.
|
|
456
|
+
const allowed = context.allowedTools
|
|
457
|
+
if (allowed !== undefined && !allowed.includes(toolName)) {
|
|
458
|
+
const msg = `Tool "${toolName}" is not available on this step. Available: ${allowed.length > 0 ? allowed.join(', ') : '(none)'}`
|
|
459
|
+
this.log.warn('Blocked a tool outside the step allow-list', {
|
|
460
|
+
tool: toolName,
|
|
461
|
+
allowed: allowed.length,
|
|
462
|
+
})
|
|
463
|
+
span.setAttributes({
|
|
464
|
+
[NAMZU.TOOL_SUCCESS]: false,
|
|
465
|
+
[NAMZU.TOOL_ERROR]: msg,
|
|
466
|
+
})
|
|
467
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: msg })
|
|
468
|
+
return {
|
|
469
|
+
success: false,
|
|
470
|
+
output: '',
|
|
471
|
+
error: msg,
|
|
472
|
+
permissionDenied: true,
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
441
476
|
const mode = context.permissionContext?.mode ?? 'auto'
|
|
442
477
|
if (mode === 'plan') {
|
|
443
478
|
const isReadOnly = tool.isReadOnly ? tool.isReadOnly(rawInput) : false
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { zodToJsonSchema } from 'zod-to-json-schema'
|
|
2
1
|
import type { ToolRegistryContract } from '../../types/tool/index.js'
|
|
3
2
|
import type { LLMToolSchema, ToolAvailability, ToolDefinition } from '../../types/tool/index.js'
|
|
4
3
|
import type {
|
|
@@ -10,6 +9,7 @@ import type {
|
|
|
10
9
|
ToolsetDefinition,
|
|
11
10
|
ToolsetPolicy,
|
|
12
11
|
} from '../../types/toolset/index.js'
|
|
12
|
+
import { renderToolSchema } from '../tool/schema.js'
|
|
13
13
|
|
|
14
14
|
export interface ToolCatalogSearchOptions {
|
|
15
15
|
readonly loading?: readonly ToolLoadingMode[]
|
|
@@ -245,12 +245,17 @@ function toolDefinitionToLLMTool(definition: ToolDefinition | undefined): LLMToo
|
|
|
245
245
|
function: {
|
|
246
246
|
name: definition.name,
|
|
247
247
|
description: definition.description,
|
|
248
|
+
// Through `renderToolSchema`, not a second inline conversion. The
|
|
249
|
+
// options were already identical, so this was not a different
|
|
250
|
+
// rendering — it was the same rendering without the guarantees:
|
|
251
|
+
// no `$schema` stripping (that key rides in the tools block, which
|
|
252
|
+
// sits at position 0 inside the prompt-cache prefix), no memoization,
|
|
253
|
+
// and no freeze. Two paths that agree today are two paths that can
|
|
254
|
+
// disagree tomorrow, and a tool reaching the wire through the catalog
|
|
255
|
+
// rather than the registry is not a different tool.
|
|
248
256
|
parameters:
|
|
249
257
|
(definition.modelInputSchema ? structuredClone(definition.modelInputSchema) : undefined) ??
|
|
250
|
-
(
|
|
251
|
-
target: 'jsonSchema7',
|
|
252
|
-
$refStrategy: 'none',
|
|
253
|
-
}) as Record<string, unknown>),
|
|
258
|
+
renderToolSchema(definition.inputSchema),
|
|
254
259
|
},
|
|
255
260
|
}
|
|
256
261
|
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
5
|
+
import { z } from 'zod'
|
|
6
|
+
|
|
7
|
+
import { CompletionInbox } from '../../../gateway/completion-inbox.js'
|
|
8
|
+
import { ToolRegistry } from '../../../registry/tool/execute.js'
|
|
9
|
+
import { defineTool } from '../../../tools/defineTool.js'
|
|
10
|
+
import type { TaskHandle } from '../../../types/agent/gateway.js'
|
|
11
|
+
import type { SessionId, TaskId, TenantId } from '../../../types/ids/index.js'
|
|
12
|
+
import { createUserMessage } from '../../../types/message/index.js'
|
|
13
|
+
import type { LLMProvider, StreamChunk } from '../../../types/provider/index.js'
|
|
14
|
+
import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
|
|
15
|
+
import { drainQuery } from '../index.js'
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The last link, and the one most likely to be built and never wired.
|
|
19
|
+
*
|
|
20
|
+
* `CompletionInbox` can queue an unclaimed completion and format it, and the
|
|
21
|
+
* coordinator tools can decline to claim one — but none of that reaches the
|
|
22
|
+
* model unless the iteration loop actually drains the inbox into the
|
|
23
|
+
* transcript. A mechanism that is declared, threaded through types, and
|
|
24
|
+
* driven by nothing is precisely the shape of the defect this whole change
|
|
25
|
+
* exists to fix, so the drain gets its own test at the loop level rather
|
|
26
|
+
* than being assumed from its parts.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const ZERO_USAGE = {
|
|
30
|
+
promptTokens: 0,
|
|
31
|
+
completionTokens: 0,
|
|
32
|
+
totalTokens: 0,
|
|
33
|
+
cachedTokens: 0,
|
|
34
|
+
cacheWriteTokens: 0,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Calls a tool once, then answers. Two turns is all the drain needs. */
|
|
38
|
+
class ToolThenAnswerProvider implements LLMProvider {
|
|
39
|
+
readonly id = 'tool-then-answer'
|
|
40
|
+
readonly name = 'Tool Then Answer Provider'
|
|
41
|
+
calls = 0
|
|
42
|
+
|
|
43
|
+
async *chatStream(): AsyncIterable<StreamChunk> {
|
|
44
|
+
this.calls += 1
|
|
45
|
+
|
|
46
|
+
if (this.calls === 1) {
|
|
47
|
+
yield {
|
|
48
|
+
id: 'msg_1',
|
|
49
|
+
delta: {
|
|
50
|
+
toolCalls: [
|
|
51
|
+
{
|
|
52
|
+
index: 0,
|
|
53
|
+
id: 'toolu_noop_1',
|
|
54
|
+
type: 'function',
|
|
55
|
+
function: { name: 'noop', arguments: '{}' },
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
yield {
|
|
61
|
+
id: 'msg_1',
|
|
62
|
+
delta: {},
|
|
63
|
+
finishReason: 'tool_calls',
|
|
64
|
+
usage: ZERO_USAGE,
|
|
65
|
+
}
|
|
66
|
+
return
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
yield { id: 'msg_2', delta: { content: 'Done.' } }
|
|
70
|
+
yield { id: 'msg_2', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const noop = defineTool({
|
|
75
|
+
name: 'noop',
|
|
76
|
+
description: 'does nothing',
|
|
77
|
+
inputSchema: z.object({}),
|
|
78
|
+
category: 'analysis',
|
|
79
|
+
permissions: [],
|
|
80
|
+
readOnly: true,
|
|
81
|
+
destructive: false,
|
|
82
|
+
concurrencySafe: true,
|
|
83
|
+
async execute() {
|
|
84
|
+
return { success: true, output: 'ok' }
|
|
85
|
+
},
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
function completed(taskId: string, result: string): TaskHandle {
|
|
89
|
+
return {
|
|
90
|
+
taskId: taskId as TaskId,
|
|
91
|
+
agentId: 'reviewer',
|
|
92
|
+
state: 'completed',
|
|
93
|
+
createdAt: 1_000,
|
|
94
|
+
completedAt: 4_000,
|
|
95
|
+
result: { status: 'completed', result } as TaskHandle['result'],
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const workdirs: string[] = []
|
|
100
|
+
afterEach(async () => {
|
|
101
|
+
await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
|
|
102
|
+
workdirs.length = 0
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
async function runWith(inbox: CompletionInbox | undefined): Promise<string[]> {
|
|
106
|
+
const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-completion-'))
|
|
107
|
+
workdirs.push(workingDirectory)
|
|
108
|
+
|
|
109
|
+
const tools = new ToolRegistry()
|
|
110
|
+
tools.register(noop)
|
|
111
|
+
|
|
112
|
+
const run = await drainQuery({
|
|
113
|
+
provider: new ToolThenAnswerProvider(),
|
|
114
|
+
tools,
|
|
115
|
+
...(inbox ? { completionInbox: inbox } : {}),
|
|
116
|
+
agentId: 'agent_test',
|
|
117
|
+
agentName: 'Test Agent',
|
|
118
|
+
messages: [createUserMessage('delegate and report')],
|
|
119
|
+
workingDirectory,
|
|
120
|
+
runConfig: {
|
|
121
|
+
model: 'mock-model',
|
|
122
|
+
timeoutMs: 10_000,
|
|
123
|
+
tokenBudget: 100_000,
|
|
124
|
+
maxIterations: 4,
|
|
125
|
+
maxResponseTokens: 256,
|
|
126
|
+
},
|
|
127
|
+
sessionId: 'ses_completion' as SessionId,
|
|
128
|
+
threadId: 'thd_completion' as ThreadId,
|
|
129
|
+
projectId: 'prj_completion' as ProjectId,
|
|
130
|
+
tenantId: 'tnt_completion' as TenantId,
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
return run.messages
|
|
134
|
+
.filter((m) => m.role === 'user')
|
|
135
|
+
.map((m) => (typeof m.content === 'string' ? m.content : JSON.stringify(m.content)))
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
describe('an unclaimed completion reaches the transcript', () => {
|
|
139
|
+
it('injects the notification as a user message the next turn can read', async () => {
|
|
140
|
+
const inbox = new CompletionInbox()
|
|
141
|
+
// Settled by a gateway before the turn ended, with nothing waiting on
|
|
142
|
+
// it — the abandoned-launch case.
|
|
143
|
+
inbox.attach({
|
|
144
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
145
|
+
cb(completed('tsk_late', 'the worker finished after the wait was abandoned'))
|
|
146
|
+
return () => {}
|
|
147
|
+
},
|
|
148
|
+
} as never)
|
|
149
|
+
|
|
150
|
+
const userMessages = await runWith(inbox)
|
|
151
|
+
const notification = userMessages.find((m) => m.includes('task-notification'))
|
|
152
|
+
|
|
153
|
+
expect(notification).toBeDefined()
|
|
154
|
+
expect(notification).toContain('tsk_late')
|
|
155
|
+
expect(notification).toContain('the worker finished after the wait was abandoned')
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
it('drains it exactly once, however many turns follow', async () => {
|
|
159
|
+
const inbox = new CompletionInbox()
|
|
160
|
+
inbox.attach({
|
|
161
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
162
|
+
cb(completed('tsk_late', 'only once'))
|
|
163
|
+
return () => {}
|
|
164
|
+
},
|
|
165
|
+
} as never)
|
|
166
|
+
|
|
167
|
+
const userMessages = await runWith(inbox)
|
|
168
|
+
|
|
169
|
+
expect(userMessages.filter((m) => m.includes('task-notification'))).toHaveLength(1)
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
it('says nothing when every completion was already delivered', async () => {
|
|
173
|
+
// The `dc16d58` regression at the loop level: a blocking `create_task`
|
|
174
|
+
// claims its own completion, so the transcript must stay clean.
|
|
175
|
+
const inbox = new CompletionInbox()
|
|
176
|
+
inbox.attach({
|
|
177
|
+
onTaskCompleted: (cb: (h: TaskHandle) => void) => {
|
|
178
|
+
cb(completed('tsk_awaited', 'delivered as a tool_result'))
|
|
179
|
+
return () => {}
|
|
180
|
+
},
|
|
181
|
+
} as never)
|
|
182
|
+
inbox.claim('tsk_awaited' as TaskId)
|
|
183
|
+
|
|
184
|
+
const userMessages = await runWith(inbox)
|
|
185
|
+
|
|
186
|
+
expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('runs unchanged with no inbox at all', async () => {
|
|
190
|
+
// The kernel must not require one: a host on the old wiring keeps
|
|
191
|
+
// working, it just never hears about abandoned completions.
|
|
192
|
+
const userMessages = await runWith(undefined)
|
|
193
|
+
|
|
194
|
+
expect(userMessages.some((m) => m.includes('task-notification'))).toBe(false)
|
|
195
|
+
})
|
|
196
|
+
})
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { mkdtemp, rm } from 'node:fs/promises'
|
|
2
|
+
import { tmpdir } from 'node:os'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
import { afterEach, describe, expect, it } from 'vitest'
|
|
5
|
+
import { z } from 'zod'
|
|
6
|
+
|
|
7
|
+
import { ToolRegistry } from '../../../registry/tool/execute.js'
|
|
8
|
+
import { defineTool } from '../../../tools/defineTool.js'
|
|
9
|
+
import type { SessionId, TenantId } from '../../../types/ids/index.js'
|
|
10
|
+
import { createUserMessage } from '../../../types/message/index.js'
|
|
11
|
+
import type { LLMProvider, StreamChunk } from '../../../types/provider/index.js'
|
|
12
|
+
import type { ProjectId, ThreadId } from '../../../types/session/ids.js'
|
|
13
|
+
import { drainQuery } from '../index.js'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Narrowing a step has to narrow what RUNS, not only what is offered.
|
|
17
|
+
*
|
|
18
|
+
* `prepareStep.activeTools` says "restrict which tools the model may call
|
|
19
|
+
* this step, by name", and for a while it did nothing of the sort: it chose
|
|
20
|
+
* which schemas went into the request, was copied into the tool context, and
|
|
21
|
+
* was read by nobody on the execution path. A model that named a withheld
|
|
22
|
+
* tool had it run — and a model does name one, from a repeated call earlier
|
|
23
|
+
* in the context, from a gateway carrying its own tool list, or from a
|
|
24
|
+
* replayed cache prefix.
|
|
25
|
+
*
|
|
26
|
+
* These tests call the withheld tool deliberately, which is the only way to
|
|
27
|
+
* tell an enforced restriction from a presentational one. Every existing test
|
|
28
|
+
* asked the model nicely and so could not have caught this.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const ZERO_USAGE = {
|
|
32
|
+
promptTokens: 0,
|
|
33
|
+
completionTokens: 0,
|
|
34
|
+
totalTokens: 0,
|
|
35
|
+
cachedTokens: 0,
|
|
36
|
+
cacheWriteTokens: 0,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
let danger = 0
|
|
40
|
+
|
|
41
|
+
const readOnly = defineTool({
|
|
42
|
+
name: 'read_only',
|
|
43
|
+
description: 'observes',
|
|
44
|
+
inputSchema: z.object({}),
|
|
45
|
+
category: 'analysis',
|
|
46
|
+
permissions: [],
|
|
47
|
+
readOnly: true,
|
|
48
|
+
destructive: false,
|
|
49
|
+
concurrencySafe: true,
|
|
50
|
+
async execute() {
|
|
51
|
+
return { success: true, output: 'observed' }
|
|
52
|
+
},
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
const dangerous = defineTool({
|
|
56
|
+
name: 'danger',
|
|
57
|
+
description: 'mutates',
|
|
58
|
+
inputSchema: z.object({}),
|
|
59
|
+
category: 'custom',
|
|
60
|
+
permissions: [],
|
|
61
|
+
readOnly: false,
|
|
62
|
+
destructive: true,
|
|
63
|
+
concurrencySafe: false,
|
|
64
|
+
async execute() {
|
|
65
|
+
danger += 1
|
|
66
|
+
return { success: true, output: 'the withheld tool ran' }
|
|
67
|
+
},
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
/** Calls whichever tool it is told to, then answers. */
|
|
71
|
+
class NamesTool implements LLMProvider {
|
|
72
|
+
readonly id = 'names-tool'
|
|
73
|
+
readonly name = 'Names Tool Provider'
|
|
74
|
+
calls = 0
|
|
75
|
+
offered: string[][] = []
|
|
76
|
+
|
|
77
|
+
constructor(private readonly toolName: string) {}
|
|
78
|
+
|
|
79
|
+
async *chatStream(params: {
|
|
80
|
+
tools?: { function: { name: string } }[]
|
|
81
|
+
}): AsyncIterable<StreamChunk> {
|
|
82
|
+
this.calls += 1
|
|
83
|
+
this.offered.push((params.tools ?? []).map((t) => t.function.name))
|
|
84
|
+
|
|
85
|
+
if (this.calls === 1) {
|
|
86
|
+
yield {
|
|
87
|
+
id: 'msg_1',
|
|
88
|
+
delta: {
|
|
89
|
+
toolCalls: [
|
|
90
|
+
{
|
|
91
|
+
index: 0,
|
|
92
|
+
id: 'toolu_1',
|
|
93
|
+
type: 'function',
|
|
94
|
+
function: { name: this.toolName, arguments: '{}' },
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
}
|
|
99
|
+
yield { id: 'msg_1', delta: {}, finishReason: 'tool_calls', usage: ZERO_USAGE }
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
yield { id: 'msg_2', delta: { content: 'done' } }
|
|
104
|
+
yield { id: 'msg_2', delta: {}, finishReason: 'stop', usage: ZERO_USAGE }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const workdirs: string[] = []
|
|
109
|
+
afterEach(async () => {
|
|
110
|
+
await Promise.all(workdirs.map((dir) => rm(dir, { recursive: true, force: true })))
|
|
111
|
+
workdirs.length = 0
|
|
112
|
+
danger = 0
|
|
113
|
+
})
|
|
114
|
+
|
|
115
|
+
async function run(opts: {
|
|
116
|
+
names: string
|
|
117
|
+
allowedTools?: string[]
|
|
118
|
+
activeTools?: string[]
|
|
119
|
+
}): Promise<{ offered: string[][]; output: string }> {
|
|
120
|
+
const workingDirectory = await mkdtemp(join(tmpdir(), 'namzu-allow-'))
|
|
121
|
+
workdirs.push(workingDirectory)
|
|
122
|
+
|
|
123
|
+
const tools = new ToolRegistry()
|
|
124
|
+
tools.register(readOnly)
|
|
125
|
+
tools.register(dangerous)
|
|
126
|
+
|
|
127
|
+
const provider = new NamesTool(opts.names)
|
|
128
|
+
let output = ''
|
|
129
|
+
|
|
130
|
+
await drainQuery(
|
|
131
|
+
{
|
|
132
|
+
provider,
|
|
133
|
+
tools,
|
|
134
|
+
...(opts.allowedTools ? { allowedTools: opts.allowedTools } : {}),
|
|
135
|
+
...(opts.activeTools
|
|
136
|
+
? { prepareStep: () => ({ activeTools: opts.activeTools as string[] }) }
|
|
137
|
+
: {}),
|
|
138
|
+
agentId: 'agent_test',
|
|
139
|
+
agentName: 'Test Agent',
|
|
140
|
+
messages: [createUserMessage('go')],
|
|
141
|
+
workingDirectory,
|
|
142
|
+
runConfig: {
|
|
143
|
+
model: 'mock-model',
|
|
144
|
+
timeoutMs: 10_000,
|
|
145
|
+
tokenBudget: 100_000,
|
|
146
|
+
maxIterations: 3,
|
|
147
|
+
maxResponseTokens: 256,
|
|
148
|
+
},
|
|
149
|
+
sessionId: 'ses_allow' as SessionId,
|
|
150
|
+
threadId: 'thd_allow' as ThreadId,
|
|
151
|
+
projectId: 'prj_allow' as ProjectId,
|
|
152
|
+
tenantId: 'tnt_allow' as TenantId,
|
|
153
|
+
},
|
|
154
|
+
(event) => {
|
|
155
|
+
if (event.type === 'tool_completed') output += event.result ?? ''
|
|
156
|
+
},
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
return { offered: provider.offered, output }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
describe('a narrowed step narrows what can run, not just what is shown', () => {
|
|
163
|
+
it('refuses a tool the step withheld, even when the model names it', async () => {
|
|
164
|
+
const { offered, output } = await run({ names: 'danger', activeTools: ['read_only'] })
|
|
165
|
+
|
|
166
|
+
// The request really was narrowed — otherwise this test proves nothing
|
|
167
|
+
// about enforcement, only about the model's manners.
|
|
168
|
+
expect(offered[0]).toEqual(['read_only'])
|
|
169
|
+
// And the call was refused rather than executed.
|
|
170
|
+
expect(danger, 'the withheld tool executed').toBe(0)
|
|
171
|
+
expect(output).toContain('not available on this step')
|
|
172
|
+
})
|
|
173
|
+
|
|
174
|
+
it('refuses a tool outside the run-level list too', async () => {
|
|
175
|
+
const { output } = await run({ names: 'danger', allowedTools: ['read_only'] })
|
|
176
|
+
|
|
177
|
+
expect(danger).toBe(0)
|
|
178
|
+
expect(output).toContain('not available on this step')
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
it('still runs a tool that is on the list', async () => {
|
|
182
|
+
const { output } = await run({ names: 'read_only', activeTools: ['read_only'] })
|
|
183
|
+
|
|
184
|
+
expect(output).toContain('observed')
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('leaves an unnarrowed run alone', async () => {
|
|
188
|
+
// Absent means unrestricted. A run that never narrows anything must not
|
|
189
|
+
// suddenly start refusing its own tools.
|
|
190
|
+
const { output } = await run({ names: 'danger' })
|
|
191
|
+
|
|
192
|
+
expect(danger).toBe(1)
|
|
193
|
+
expect(output).toContain('the withheld tool ran')
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
it('treats an empty list as "nothing", not as "no restriction"', async () => {
|
|
197
|
+
// The fail-open reading this codebase has already been bitten by once,
|
|
198
|
+
// in the delegate roster: an empty allow-list IS the answer, and
|
|
199
|
+
// degrading it to "unrestricted" is how a closed list becomes open.
|
|
200
|
+
const { output } = await run({ names: 'danger', activeTools: [] })
|
|
201
|
+
|
|
202
|
+
expect(danger).toBe(0)
|
|
203
|
+
expect(output).toContain('(none)')
|
|
204
|
+
})
|
|
205
|
+
})
|
|
@@ -84,6 +84,7 @@ class ClassifiedFailureProvider implements LLMProvider {
|
|
|
84
84
|
providerId: 'classified-failure',
|
|
85
85
|
status: 429,
|
|
86
86
|
retryAfterMs: 2000,
|
|
87
|
+
detail: 'rate limit reached for this organization',
|
|
87
88
|
}),
|
|
88
89
|
)
|
|
89
90
|
}
|
|
@@ -210,11 +211,16 @@ describe('query stream recovery', () => {
|
|
|
210
211
|
)
|
|
211
212
|
|
|
212
213
|
expect(run.status).toBe('failed')
|
|
214
|
+
// `detail` rides along with the classification. Without it a host
|
|
215
|
+
// rendering this metadata knows a request was rejected but not why, and
|
|
216
|
+
// has to go re-parse the message string — which is the re-parsing this
|
|
217
|
+
// structured field exists to avoid.
|
|
213
218
|
expect(run.lastProviderError).toEqual({
|
|
214
219
|
kind: 'throttle',
|
|
215
220
|
providerId: 'classified-failure',
|
|
216
221
|
status: 429,
|
|
217
222
|
retryAfterMs: 2000,
|
|
223
|
+
detail: 'rate limit reached for this organization',
|
|
218
224
|
})
|
|
219
225
|
expect(events.find((event) => event.type === 'run_failed')).toMatchObject({
|
|
220
226
|
type: 'run_failed',
|
|
@@ -205,6 +205,8 @@ export class ToolExecutor {
|
|
|
205
205
|
private workingStateManager?: WorkingStateManager
|
|
206
206
|
private probes: ProbeRegistry
|
|
207
207
|
private parentSpan?: Span
|
|
208
|
+
/** Set per turn by the orchestrator; see {@link setStepAllowedTools}. */
|
|
209
|
+
private stepAllowedTools?: readonly string[]
|
|
208
210
|
private readonly readPaths: Set<string> = new Set()
|
|
209
211
|
private readonly readFingerprints: Map<string, string> = new Map()
|
|
210
212
|
private readonly fileReadTracker: FileReadTracker = {
|
|
@@ -251,6 +253,21 @@ export class ToolExecutor {
|
|
|
251
253
|
this.parentSpan = span
|
|
252
254
|
}
|
|
253
255
|
|
|
256
|
+
/**
|
|
257
|
+
* Narrow what this turn may call, or clear the narrowing.
|
|
258
|
+
*
|
|
259
|
+
* Re-set each turn by the orchestrator for the same reason the parent span
|
|
260
|
+
* is: `prepareStep` can hand a different list to every step, and the run's
|
|
261
|
+
* own `allowedTools` is only the default when a step names none.
|
|
262
|
+
*
|
|
263
|
+
* Without this the executor could only ever see the RUN-level list, so a
|
|
264
|
+
* per-step narrowing reached the request that was sent and nothing else —
|
|
265
|
+
* the model was shown fewer tools and could still call all of them.
|
|
266
|
+
*/
|
|
267
|
+
setStepAllowedTools(names: readonly string[] | undefined): void {
|
|
268
|
+
this.stepAllowedTools = names
|
|
269
|
+
}
|
|
270
|
+
|
|
254
271
|
/**
|
|
255
272
|
* Answer every `tool_use` block in `response` with exactly one
|
|
256
273
|
* `tool_result`.
|
|
@@ -424,7 +441,10 @@ export class ToolExecutor {
|
|
|
424
441
|
},
|
|
425
442
|
invocationState: this.config.invocationState,
|
|
426
443
|
toolRegistry: this.config.tools,
|
|
427
|
-
|
|
444
|
+
// The step's list wins where it has one; the run's is the default.
|
|
445
|
+
// Same precedence the request already uses when it decides which
|
|
446
|
+
// schemas to send, so the menu and the kitchen agree.
|
|
447
|
+
allowedTools: this.stepAllowedTools ?? this.config.allowedTools,
|
|
428
448
|
sandbox: this.config.sandbox,
|
|
429
449
|
fileReadTracker: this.fileReadTracker,
|
|
430
450
|
...(this.parentSpan ? { parentSpan: this.parentSpan } : {}),
|