@b4run/testing 0.8.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +51 -0
  3. package/dist/aimock-runner.d.ts +48 -0
  4. package/dist/aimock-runner.d.ts.map +1 -0
  5. package/dist/aimock-runner.js +81 -0
  6. package/dist/checkpointer-conformance.d.ts +43 -0
  7. package/dist/checkpointer-conformance.d.ts.map +1 -0
  8. package/dist/checkpointer-conformance.js +381 -0
  9. package/dist/fake-embedder.d.ts +9 -0
  10. package/dist/fake-embedder.d.ts.map +1 -0
  11. package/dist/fake-embedder.js +38 -0
  12. package/dist/fixture-builder.d.ts +29 -0
  13. package/dist/fixture-builder.d.ts.map +1 -0
  14. package/dist/fixture-builder.js +43 -0
  15. package/dist/fixture-file.d.ts +14 -0
  16. package/dist/fixture-file.d.ts.map +1 -0
  17. package/dist/fixture-file.js +49 -0
  18. package/dist/harness.d.ts +35 -0
  19. package/dist/harness.d.ts.map +1 -0
  20. package/dist/harness.js +211 -0
  21. package/dist/http-inject.d.ts +22 -0
  22. package/dist/http-inject.d.ts.map +1 -0
  23. package/dist/http-inject.js +55 -0
  24. package/dist/index.d.ts +20 -0
  25. package/dist/index.d.ts.map +1 -0
  26. package/dist/index.js +19 -0
  27. package/dist/matchers.d.ts +56 -0
  28. package/dist/matchers.d.ts.map +1 -0
  29. package/dist/matchers.js +262 -0
  30. package/dist/memory-conformance.d.ts +13 -0
  31. package/dist/memory-conformance.d.ts.map +1 -0
  32. package/dist/memory-conformance.js +1333 -0
  33. package/dist/memory.d.ts +11 -0
  34. package/dist/memory.d.ts.map +1 -0
  35. package/dist/memory.js +20 -0
  36. package/dist/middleware-harness.d.ts +11 -0
  37. package/dist/middleware-harness.d.ts.map +1 -0
  38. package/dist/middleware-harness.js +35 -0
  39. package/dist/permissions-conformance.d.ts +24 -0
  40. package/dist/permissions-conformance.d.ts.map +1 -0
  41. package/dist/permissions-conformance.js +156 -0
  42. package/dist/record-fixtures.d.ts +21 -0
  43. package/dist/record-fixtures.d.ts.map +1 -0
  44. package/dist/record-fixtures.js +31 -0
  45. package/dist/record.d.ts +12 -0
  46. package/dist/record.d.ts.map +1 -0
  47. package/dist/record.js +22 -0
  48. package/dist/regex-safety.d.ts +2 -0
  49. package/dist/regex-safety.d.ts.map +1 -0
  50. package/dist/regex-safety.js +66 -0
  51. package/dist/run-result.d.ts +111 -0
  52. package/dist/run-result.d.ts.map +1 -0
  53. package/dist/run-result.js +287 -0
  54. package/dist/subprocess.d.ts +25 -0
  55. package/dist/subprocess.d.ts.map +1 -0
  56. package/dist/subprocess.js +223 -0
  57. package/dist/thread-access-harness.d.ts +40 -0
  58. package/dist/thread-access-harness.d.ts.map +1 -0
  59. package/dist/thread-access-harness.js +42 -0
  60. package/dist/threads-conformance.d.ts +18 -0
  61. package/dist/threads-conformance.d.ts.map +1 -0
  62. package/dist/threads-conformance.js +315 -0
  63. package/dist/tool-harness.d.ts +16 -0
  64. package/dist/tool-harness.d.ts.map +1 -0
  65. package/dist/tool-harness.js +25 -0
  66. package/dist/tsconfig.tsbuildinfo +1 -0
  67. package/dist/workspace-harness.d.ts +15 -0
  68. package/dist/workspace-harness.d.ts.map +1 -0
  69. package/dist/workspace-harness.js +43 -0
  70. package/package.json +71 -0
@@ -0,0 +1,11 @@
1
+ import type { MemoryRecord, MemoryStore } from "@b4run/memory";
2
+ type SeedInput = Partial<MemoryRecord> & Pick<MemoryRecord, "id" | "namespace" | "content"> & {
3
+ embedding?: Float32Array;
4
+ embeddingModel?: string;
5
+ };
6
+ /** Seed memory rows into a store (or a sqlite path) with sensible defaults for tests. */
7
+ export declare function seedMemory(target: MemoryStore | {
8
+ readonly path: string;
9
+ }, records: readonly SeedInput[]): Promise<MemoryStore>;
10
+ export {};
11
+ //# sourceMappingURL=memory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../src/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAG9D,KAAK,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,GACpC,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,WAAW,GAAG,SAAS,CAAC,GAAG;IACnD,SAAS,CAAC,EAAE,YAAY,CAAA;IACxB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB,CAAA;AAEH,yFAAyF;AACzF,wBAAsB,UAAU,CAC9B,MAAM,EAAE,WAAW,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAC/C,OAAO,EAAE,SAAS,SAAS,EAAE,GAC5B,OAAO,CAAC,WAAW,CAAC,CAoBtB"}
package/dist/memory.js ADDED
@@ -0,0 +1,20 @@
1
+ import { sqliteMemoryStore } from "@b4run/memory";
2
+ /** Seed memory rows into a store (or a sqlite path) with sensible defaults for tests. */
3
+ export async function seedMemory(target, records) {
4
+ const store = "put" in target ? target : sqliteMemoryStore(target);
5
+ for (const r of records) {
6
+ const { embedding, embeddingModel, ...record } = r;
7
+ await store.put({
8
+ kind: "semantic",
9
+ data: {},
10
+ source: { type: "eval", id: "seed" },
11
+ confidence: 1,
12
+ tags: [],
13
+ status: "active",
14
+ createdAt: "2026-01-01T00:00:00.000Z",
15
+ updatedAt: "2026-01-01T00:00:00.000Z",
16
+ ...record,
17
+ }, embedding && embeddingModel ? { embedding, embeddingModel } : undefined);
18
+ }
19
+ return store;
20
+ }
@@ -0,0 +1,11 @@
1
+ import type { BackendContext, FilesystemBackend, FilesystemMiddleware } from "@b4run/workspace";
2
+ export interface MiddlewareHarness {
3
+ readonly backend: FilesystemBackend;
4
+ readonly ctx: BackendContext;
5
+ readonly dir: string;
6
+ assertForwardsAll(): void;
7
+ close(): Promise<void>;
8
+ [Symbol.asyncDispose](): Promise<void>;
9
+ }
10
+ export declare function createMiddlewareHarness(middleware: FilesystemMiddleware): Promise<MiddlewareHarness>;
11
+ //# sourceMappingURL=middleware-harness.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"middleware-harness.d.ts","sourceRoot":"","sources":["../src/middleware-harness.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAG/F,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAA;IACnC,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAA;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,iBAAiB,IAAI,IAAI,CAAA;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvC;AAED,wBAAsB,uBAAuB,CAC3C,UAAU,EAAE,oBAAoB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAkC5B"}
@@ -0,0 +1,35 @@
1
+ import { realpathSync } from "node:fs";
2
+ import { mkdtemp, rm } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { localFilesystem } from "@b4run/workspace/node";
6
+ export async function createMiddlewareHarness(middleware) {
7
+ const dir = realpathSync(await mkdtemp(join(tmpdir(), "b4-mw-harness-")));
8
+ const base = localFilesystem();
9
+ const backend = middleware(base);
10
+ const controller = new AbortController();
11
+ const ctx = { signal: controller.signal, workspaceRoot: dir };
12
+ let closed = false;
13
+ const close = async () => {
14
+ if (closed)
15
+ return;
16
+ closed = true;
17
+ controller.abort();
18
+ await rm(dir, { force: true, recursive: true });
19
+ };
20
+ return {
21
+ backend,
22
+ ctx,
23
+ dir,
24
+ assertForwardsAll() {
25
+ const baseMethods = Object.keys(base);
26
+ const missing = baseMethods.filter((method) => typeof base[method] === "function" && typeof backend[method] !== "function");
27
+ if (missing.length > 0) {
28
+ throw new Error(`Middleware dropped backend method(s) the base provides: ${missing.join(", ")}. ` +
29
+ "A FilesystemMiddleware must forward every method (required and optional) it does not intercept.");
30
+ }
31
+ },
32
+ close,
33
+ [Symbol.asyncDispose]: close,
34
+ };
35
+ }
@@ -0,0 +1,24 @@
1
+ import type { PermissionMode, PermissionsFile, PermissionsStore } from "@b4run/permissions";
2
+ /** What a backend needs in order to build a store: the config-seeded lists plus the resolved mode. */
3
+ export interface PermissionsStoreInit {
4
+ readonly config: PermissionsFile | undefined;
5
+ readonly mode: PermissionMode;
6
+ }
7
+ /**
8
+ * The contract every PermissionsStore must satisfy. Run against the file-backed
9
+ * node store (tmpdir, always) and any other backend (e.g. Postgres, gated) so
10
+ * they cannot drift. Pass vitest's `describe`; `makeStore` returns a FRESH
11
+ * store — no config entries beyond `init.config`, no prior runtime grants.
12
+ *
13
+ * Runtime grants are seeded through `addAllow` rather than by writing the
14
+ * backing store directly: that is the only portable way to reach the "runtime"
15
+ * half of the state machine, since where those entries live (a JSON file, a
16
+ * table) is exactly what differs between backends.
17
+ */
18
+ export declare function runPermissionsStoreConformance(opts: {
19
+ readonly name: string;
20
+ readonly makeStore: (init: PermissionsStoreInit) => Promise<PermissionsStore> | PermissionsStore;
21
+ readonly describe: (name: string, fn: () => void) => void;
22
+ readonly close?: (store: PermissionsStore) => Promise<void> | void;
23
+ }): void;
24
+ //# sourceMappingURL=permissions-conformance.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"permissions-conformance.d.ts","sourceRoot":"","sources":["../src/permissions-conformance.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAG3F,sGAAsG;AACtG,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,SAAS,CAAA;IAC5C,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;CAC9B;AAOD;;;;;;;;;;GAUG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,OAAO,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,CAAA;IAChG,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,IAAI,KAAK,IAAI,CAAA;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CACnE,GAAG,IAAI,CAmKP"}
@@ -0,0 +1,156 @@
1
+ import { expect, test } from "vitest";
2
+ const file = (allow, deny = {}) => ({ version: 1, allow, deny });
3
+ /**
4
+ * The contract every PermissionsStore must satisfy. Run against the file-backed
5
+ * node store (tmpdir, always) and any other backend (e.g. Postgres, gated) so
6
+ * they cannot drift. Pass vitest's `describe`; `makeStore` returns a FRESH
7
+ * store — no config entries beyond `init.config`, no prior runtime grants.
8
+ *
9
+ * Runtime grants are seeded through `addAllow` rather than by writing the
10
+ * backing store directly: that is the only portable way to reach the "runtime"
11
+ * half of the state machine, since where those entries live (a JSON file, a
12
+ * table) is exactly what differs between backends.
13
+ */
14
+ export function runPermissionsStoreConformance(opts) {
15
+ const { name, makeStore, describe, close } = opts;
16
+ describe(`PermissionsStore conformance: ${name}`, () => {
17
+ async function withStore(init, fn) {
18
+ const store = await makeStore(init);
19
+ try {
20
+ await store.load();
21
+ await fn(store);
22
+ }
23
+ finally {
24
+ await close?.(store);
25
+ }
26
+ }
27
+ test("exposes the resolved mode", async () => {
28
+ await withStore({ config: undefined, mode: "non-interactive" }, async (s) => {
29
+ expect(s.mode).toBe("non-interactive");
30
+ });
31
+ });
32
+ test("no entries for a tool → unknown", async () => {
33
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
34
+ expect(s.match("bash", "npm install")).toBe("unknown");
35
+ expect(s.match("unknownTool", "anything")).toBe("unknown");
36
+ });
37
+ });
38
+ test("config allow entries match by prefix", async () => {
39
+ await withStore({ config: file({ bash: ["npm install"] }), mode: "interactive" }, async (s) => {
40
+ expect(s.match("bash", "npm install react")).toBe("allow");
41
+ expect(s.match("bash", "npm install")).toBe("allow");
42
+ // The candidate must start with the pattern, not the other way round.
43
+ expect(s.match("bash", "npm")).toBe("unknown");
44
+ expect(s.match("bash", "rm -rf /")).toBe("unknown");
45
+ });
46
+ });
47
+ test("patterns are scoped per tool key", async () => {
48
+ await withStore({ config: file({ bash: ["ls"] }), mode: "interactive" }, async (s) => {
49
+ expect(s.match("bash", "ls -la")).toBe("allow");
50
+ expect(s.match("readFile", "ls -la")).toBe("unknown");
51
+ });
52
+ });
53
+ test("deny wins over allow", async () => {
54
+ await withStore({ config: file({ bash: ["rm"] }, { bash: ["rm -rf"] }), mode: "interactive" }, async (s) => {
55
+ expect(s.match("bash", "rm -rf /tmp")).toBe("deny");
56
+ expect(s.match("bash", "rm ./file")).toBe("allow");
57
+ });
58
+ });
59
+ test('the reserved "tool" key matches EXACTLY, never by prefix', async () => {
60
+ await withStore({ config: file({ tool: ["deploy"] }), mode: "interactive" }, async (s) => {
61
+ expect(s.match("tool", "deploy")).toBe("allow");
62
+ // Tool names must not prefix-match, or approving "deploy" would
63
+ // silently approve "deployProd".
64
+ expect(s.match("tool", "deployProd")).toBe("unknown");
65
+ });
66
+ await withStore({ config: file({}, { tool: ["deploy"] }), mode: "interactive" }, async (s) => {
67
+ expect(s.match("tool", "deploy")).toBe("deny");
68
+ expect(s.match("tool", "deployProd")).toBe("unknown");
69
+ });
70
+ });
71
+ test("bypass mode: match is always unknown, even with config allow AND deny", async () => {
72
+ await withStore({ config: file({ bash: ["ls"] }, { bash: ["rm"] }), mode: "bypass" }, async (s) => {
73
+ expect(s.match("bash", "ls -la")).toBe("unknown");
74
+ expect(s.match("bash", "rm -rf /")).toBe("unknown");
75
+ await s.addAllow("bash", "cat");
76
+ expect(s.match("bash", "cat x")).toBe("unknown");
77
+ });
78
+ });
79
+ test("non-interactive mode: config applies, runtime grants are ignored", async () => {
80
+ await withStore({ config: file({ bash: ["ls"] }, { bash: ["rm"] }), mode: "non-interactive" }, async (s) => {
81
+ expect(s.match("bash", "ls -la")).toBe("allow");
82
+ expect(s.match("bash", "rm -rf /")).toBe("deny");
83
+ await s.addAllow("bash", "npm install");
84
+ expect(s.match("bash", "npm install react")).toBe("unknown");
85
+ });
86
+ });
87
+ test("interactive mode: config and runtime grants are concatenated per tool key", async () => {
88
+ await withStore({ config: file({ bash: ["ls"] }), mode: "interactive" }, async (s) => {
89
+ await s.addAllow("bash", "npm install");
90
+ await s.addAllow("readFile", "/tmp/");
91
+ // Same key: both lists contribute. Different key: independent.
92
+ expect(s.match("bash", "ls -la")).toBe("allow");
93
+ expect(s.match("bash", "npm install react")).toBe("allow");
94
+ expect(s.match("readFile", "/tmp/x.txt")).toBe("allow");
95
+ });
96
+ });
97
+ test("interactive mode: a config deny beats a runtime allow", async () => {
98
+ await withStore({ config: file({}, { bash: ["rm -rf"] }), mode: "interactive" }, async (s) => {
99
+ await s.addAllow("bash", "rm");
100
+ expect(s.match("bash", "rm -rf /tmp")).toBe("deny");
101
+ expect(s.match("bash", "rm ./file")).toBe("allow");
102
+ });
103
+ });
104
+ test("addAllow is visible to match() as soon as it resolves", async () => {
105
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
106
+ expect(s.match("bash", "npm install")).toBe("unknown");
107
+ await s.addAllow("bash", "npm install");
108
+ expect(s.match("bash", "npm install react")).toBe("allow");
109
+ });
110
+ });
111
+ test("addAllow is idempotent — repeating a grant changes nothing", async () => {
112
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
113
+ await s.addAllow("bash", "ls");
114
+ await s.addAllow("bash", "ls");
115
+ // Whether the backend de-duplicates its rows is storage-internal; what
116
+ // the contract pins is that a repeat neither throws nor flips the
117
+ // decision.
118
+ expect(s.match("bash", "ls -la")).toBe("allow");
119
+ });
120
+ });
121
+ test("concurrent addAllow calls all land", async () => {
122
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
123
+ await Promise.all([
124
+ s.addAllow("bash", "ls"),
125
+ s.addAllow("bash", "pwd"),
126
+ s.addAllow("bash", "cat"),
127
+ ]);
128
+ expect(s.match("bash", "ls -la")).toBe("allow");
129
+ expect(s.match("bash", "pwd")).toBe("allow");
130
+ expect(s.match("bash", "cat f")).toBe("allow");
131
+ });
132
+ });
133
+ test('addAllow under the "tool" key stays exact', async () => {
134
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
135
+ await s.addAllow("tool", "deploy");
136
+ expect(s.match("tool", "deploy")).toBe("allow");
137
+ expect(s.match("tool", "deployProd")).toBe("unknown");
138
+ });
139
+ });
140
+ test("load() is re-callable and preserves config plus runtime grants", async () => {
141
+ await withStore({ config: file({ bash: ["ls"] }), mode: "interactive" }, async (s) => {
142
+ await s.addAllow("bash", "npm install");
143
+ await s.load();
144
+ await s.load();
145
+ expect(s.match("bash", "ls -la")).toBe("allow");
146
+ expect(s.match("bash", "npm install react")).toBe("allow");
147
+ });
148
+ });
149
+ test("load() on a store with no config and no prior grants leaves it empty", async () => {
150
+ await withStore({ config: undefined, mode: "interactive" }, async (s) => {
151
+ await s.load();
152
+ expect(s.match("bash", "anything")).toBe("unknown");
153
+ });
154
+ });
155
+ });
156
+ }
@@ -0,0 +1,21 @@
1
+ import type { AimockResponse, FixtureSet } from "./fixture-builder.js";
2
+ /** One captured real-model exchange: the request the agent sent + the response to bake. */
3
+ export interface Recording {
4
+ readonly request: {
5
+ readonly messages?: ReadonlyArray<{
6
+ readonly role: string;
7
+ readonly content: unknown;
8
+ }>;
9
+ };
10
+ readonly response: AimockResponse;
11
+ }
12
+ /**
13
+ * Convert ordered recordings (one per LLM call within a single case/thread) into
14
+ * a replay FixtureSet, keyed with the SAME `{userMessage,turnIndex,hasToolResult}`
15
+ * convention `script()` produces — so the recorded file replays through the same
16
+ * aimock matcher with no drift. `turnIndex` is the 0-based ordinal of the call
17
+ * within the thread; `userMessage` is the first user message (stable across the
18
+ * thread); `hasToolResult` is whether a tool-role message is already present.
19
+ */
20
+ export declare function recordingsToFixtures(recordings: readonly Recording[]): FixtureSet;
21
+ //# sourceMappingURL=record-fixtures.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"record-fixtures.d.ts","sourceRoot":"","sources":["../src/record-fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,cAAc,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAErF,2FAA2F;AAC3F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;YAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;SAAE,CAAC,CAAA;KACxF,CAAA;IACD,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;CAClC;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,CAYjF"}
@@ -0,0 +1,31 @@
1
+ function firstUserMessage(req) {
2
+ for (const m of req.messages ?? []) {
3
+ if (m.role === "user" && typeof m.content === "string")
4
+ return m.content;
5
+ }
6
+ return undefined;
7
+ }
8
+ function hasToolResult(req) {
9
+ return (req.messages ?? []).some((m) => m.role === "tool");
10
+ }
11
+ /**
12
+ * Convert ordered recordings (one per LLM call within a single case/thread) into
13
+ * a replay FixtureSet, keyed with the SAME `{userMessage,turnIndex,hasToolResult}`
14
+ * convention `script()` produces — so the recorded file replays through the same
15
+ * aimock matcher with no drift. `turnIndex` is the 0-based ordinal of the call
16
+ * within the thread; `userMessage` is the first user message (stable across the
17
+ * thread); `hasToolResult` is whether a tool-role message is already present.
18
+ */
19
+ export function recordingsToFixtures(recordings) {
20
+ return recordings.map((rec, turnIndex) => {
21
+ const userMessage = firstUserMessage(rec.request);
22
+ return {
23
+ match: {
24
+ ...(userMessage !== undefined ? { userMessage } : {}),
25
+ turnIndex,
26
+ hasToolResult: hasToolResult(rec.request),
27
+ },
28
+ response: rec.response,
29
+ };
30
+ });
31
+ }
@@ -0,0 +1,12 @@
1
+ export interface RecordOptions {
2
+ readonly out: string;
3
+ /** Upstream provider base, default OpenAI. */
4
+ readonly provider?: string;
5
+ }
6
+ /**
7
+ * Records a real provider interaction into an aimock fixture file. LOCAL ONLY —
8
+ * requires a real OPENAI_API_KEY in env. Never run in CI (CI replays strict
9
+ * read-only). Throws on a non-zero recorder exit.
10
+ */
11
+ export declare function record(opts: RecordOptions): void;
12
+ //# sourceMappingURL=record.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"record.d.ts","sourceRoot":"","sources":["../src/record.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,8CAA8C;IAC9C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,IAAI,CAmBhD"}
package/dist/record.js ADDED
@@ -0,0 +1,22 @@
1
+ import { spawnSync } from "node:child_process";
2
+ /**
3
+ * Records a real provider interaction into an aimock fixture file. LOCAL ONLY —
4
+ * requires a real OPENAI_API_KEY in env. Never run in CI (CI replays strict
5
+ * read-only). Throws on a non-zero recorder exit.
6
+ */
7
+ export function record(opts) {
8
+ const provider = opts.provider ?? "https://api.openai.com";
9
+ const result = spawnSync("npx", [
10
+ "-p",
11
+ "@copilotkit/aimock",
12
+ "llmock",
13
+ "--record",
14
+ "--provider-openai",
15
+ provider,
16
+ "--out",
17
+ opts.out,
18
+ ], { stdio: "inherit", env: process.env });
19
+ if (result.status !== 0) {
20
+ throw new Error(`aimock recorder exited with status ${result.status ?? "null"}`);
21
+ }
22
+ }
@@ -0,0 +1,2 @@
1
+ export declare function createSafeRegexTester(expression: RegExp): (input: string) => boolean;
2
+ //# sourceMappingURL=regex-safety.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"regex-safety.d.ts","sourceRoot":"","sources":["../src/regex-safety.ts"],"names":[],"mappings":"AAmDA,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CA8BpF"}
@@ -0,0 +1,66 @@
1
+ import { Script } from "node:vm";
2
+ const MAX_REGEX_SOURCE_CODE_UNITS = 4_096;
3
+ const MAX_REGEX_INPUT_CODE_UNITS = 65_536;
4
+ const REGEX_EXECUTION_TIMEOUT_MS = 100;
5
+ const OVERSIZED_REGEX_SOURCE_MESSAGE = "Regular expression source exceeds 4096 UTF-16 code units";
6
+ const OVERSIZED_REGEX_INPUT_MESSAGE = "Regular expression input exceeds 65536 UTF-16 code units";
7
+ const REGEX_TIMEOUT_MESSAGE = "Regular expression evaluation exceeded 100ms execution limit";
8
+ const REGEX_TEST_SCRIPT = new Script("new RegExp(source, flags).test(input)", {
9
+ filename: "b4-regex-evaluation.vm",
10
+ });
11
+ function intrinsicRegExpGetter(property) {
12
+ const getter = Object.getOwnPropertyDescriptor(RegExp.prototype, property)?.get;
13
+ if (getter === undefined) {
14
+ throw new Error(`RegExp.prototype.${property} is unavailable`);
15
+ }
16
+ return getter;
17
+ }
18
+ const REGEXP_SOURCE_GETTER = intrinsicRegExpGetter("source");
19
+ const REGEXP_FLAG_GETTERS = [
20
+ [intrinsicRegExpGetter("hasIndices"), "d"],
21
+ [intrinsicRegExpGetter("global"), "g"],
22
+ [intrinsicRegExpGetter("ignoreCase"), "i"],
23
+ [intrinsicRegExpGetter("multiline"), "m"],
24
+ [intrinsicRegExpGetter("dotAll"), "s"],
25
+ [intrinsicRegExpGetter("unicode"), "u"],
26
+ [intrinsicRegExpGetter("unicodeSets"), "v"],
27
+ [intrinsicRegExpGetter("sticky"), "y"],
28
+ ];
29
+ function snapshotFlags(expression) {
30
+ let flags = "";
31
+ for (const [getter, flag] of REGEXP_FLAG_GETTERS) {
32
+ if (Reflect.apply(getter, expression, []) === true)
33
+ flags += flag;
34
+ }
35
+ return flags;
36
+ }
37
+ function isScriptExecutionTimeout(error) {
38
+ return (typeof error === "object" &&
39
+ error !== null &&
40
+ "code" in error &&
41
+ error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT");
42
+ }
43
+ export function createSafeRegexTester(expression) {
44
+ const source = Reflect.apply(REGEXP_SOURCE_GETTER, expression, []);
45
+ const flags = snapshotFlags(expression);
46
+ if (source.length > MAX_REGEX_SOURCE_CODE_UNITS) {
47
+ throw new RangeError(OVERSIZED_REGEX_SOURCE_MESSAGE);
48
+ }
49
+ return (input) => {
50
+ if (input.length > MAX_REGEX_INPUT_CODE_UNITS) {
51
+ throw new RangeError(OVERSIZED_REGEX_INPUT_MESSAGE);
52
+ }
53
+ try {
54
+ return (REGEX_TEST_SCRIPT.runInNewContext({ flags, input, source }, {
55
+ contextCodeGeneration: { strings: false, wasm: false },
56
+ timeout: REGEX_EXECUTION_TIMEOUT_MS,
57
+ }) === true);
58
+ }
59
+ catch (error) {
60
+ if (isScriptExecutionTimeout(error)) {
61
+ throw new RangeError(REGEX_TIMEOUT_MESSAGE);
62
+ }
63
+ throw error;
64
+ }
65
+ };
66
+ }
@@ -0,0 +1,111 @@
1
+ import type { StreamChunk } from "@b4run/cli/runtime";
2
+ export interface ObservedToolCall {
3
+ readonly name: string;
4
+ readonly args: unknown;
5
+ readonly id?: string;
6
+ }
7
+ export interface ObservedToolResult {
8
+ readonly name: string;
9
+ /** LangChain ToolMessage status, when present. */
10
+ readonly status?: "error" | "success";
11
+ /** The tool result content (string when the tool returned text/JSON). */
12
+ readonly content: unknown;
13
+ /** True when the tool reported an error (status === "error"). */
14
+ readonly isError: boolean;
15
+ }
16
+ /** Extract tool results from final conversation messages.
17
+ *
18
+ * Handles two shapes:
19
+ * - Serialized LangChain format: `{ lc:1, type:"constructor", id:[...,"ToolMessage"], kwargs:{name,status,content} }`
20
+ * - Live LangChain instance format: `{ type:"tool", name, status, content, ... }`
21
+ */
22
+ export declare function deriveToolResults(messages: ReadonlyArray<Record<string, unknown>>): ObservedToolResult[];
23
+ export interface CommandInterruptDetail {
24
+ readonly command: string;
25
+ readonly suggestedPattern: string;
26
+ }
27
+ export interface PathInterruptDetail {
28
+ readonly path: string;
29
+ readonly operation: "readFile" | "writeFile" | "listDir";
30
+ readonly suggestedPattern: string;
31
+ }
32
+ export interface ToolInterruptDetail {
33
+ readonly toolName: string;
34
+ readonly argsPreview: string;
35
+ readonly suggestedPattern: string;
36
+ }
37
+ export interface MemoryInterruptDetail {
38
+ readonly namespace: string;
39
+ readonly identity: string;
40
+ readonly oldId: string;
41
+ readonly oldContent: string;
42
+ readonly newContent: string;
43
+ readonly suggestedPattern: string;
44
+ }
45
+ export interface SubagentInterruptDetail {
46
+ readonly parentRouteId: string;
47
+ readonly subagentName: string;
48
+ readonly subagentRouteId: string;
49
+ readonly inputPreview: string;
50
+ readonly reason?: string;
51
+ readonly suggestedPattern: string;
52
+ }
53
+ interface InterruptInfoBase {
54
+ readonly interruptId: string;
55
+ readonly callId?: string;
56
+ }
57
+ export type InterruptInfo = InterruptInfoBase & ({
58
+ readonly kind: "command";
59
+ readonly detail: CommandInterruptDetail;
60
+ } | {
61
+ readonly kind: "path";
62
+ readonly detail: PathInterruptDetail;
63
+ } | {
64
+ readonly kind: "tool";
65
+ readonly detail: ToolInterruptDetail;
66
+ } | {
67
+ readonly kind: "memory";
68
+ readonly detail: MemoryInterruptDetail;
69
+ } | {
70
+ readonly kind: "subagent";
71
+ readonly detail: SubagentInterruptDetail;
72
+ });
73
+ export interface Todo {
74
+ readonly content: string;
75
+ readonly status: string;
76
+ }
77
+ export interface SubagentToolCall {
78
+ readonly name: string;
79
+ readonly args: unknown;
80
+ }
81
+ export interface SubagentRun {
82
+ readonly callId: string;
83
+ readonly name: string;
84
+ readonly toolCalls: ReadonlyArray<SubagentToolCall>;
85
+ readonly finalMessage?: string;
86
+ readonly error?: string;
87
+ }
88
+ export interface SubagentEvent {
89
+ readonly type: string;
90
+ readonly data: Record<string, unknown>;
91
+ }
92
+ export interface AgentRunResult {
93
+ readonly finalMessage: string;
94
+ readonly messages: ReadonlyArray<Record<string, unknown>>;
95
+ readonly toolCalls: ReadonlyArray<ObservedToolCall>;
96
+ readonly toolResults: ReadonlyArray<ObservedToolResult>;
97
+ readonly tokens: ReadonlyArray<string>;
98
+ readonly state: Record<string, unknown>;
99
+ readonly threadId: string;
100
+ readonly interrupts: ReadonlyArray<InterruptInfo>;
101
+ readonly planUpdates: ReadonlyArray<{
102
+ todos: ReadonlyArray<Todo>;
103
+ }>;
104
+ readonly todos: ReadonlyArray<Todo>;
105
+ readonly subagents: ReadonlyArray<SubagentRun>;
106
+ readonly subagentEvents: ReadonlyArray<SubagentEvent>;
107
+ readonly systemPrompt: string;
108
+ }
109
+ export declare function collectRunResult(stream: AsyncIterable<StreamChunk>, threadId: string): Promise<AgentRunResult>;
110
+ export {};
111
+ //# sourceMappingURL=run-result.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run-result.d.ts","sourceRoot":"","sources":["../src/run-result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAErD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,kDAAkD;IAClD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACrC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,iEAAiE;IACjE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAC/C,kBAAkB,EAAE,CAiCtB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,GAAG,SAAS,CAAA;IACxD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;CAClC;AAED,UAAU,iBAAiB;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAC3C,CACI;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,sBAAsB,CAAA;CAAE,GACrE;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAA;CAAE,GAC/D;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,mBAAmB,CAAA;CAAE,GAC/D;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,qBAAqB,CAAA;CAAE,GACnE;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,uBAAuB,CAAA;CAAE,CAC1E,CAAA;AAEH,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,gBAAgB,CAAC,CAAA;IACnD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IACzD,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,gBAAgB,CAAC,CAAA;IACnD,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAA;IACvD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACvC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,aAAa,CAAC,CAAA;IACjD,QAAQ,CAAC,WAAW,EAAE,aAAa,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC,CAAA;IACnE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAA;IACnC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;IAC9C,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,aAAa,CAAC,CAAA;IACrD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAC9B;AAkJD,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,aAAa,CAAC,WAAW,CAAC,EAClC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,CAAC,CAkJzB"}