@b4run/testing 0.8.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +51 -0
  3. package/dist/aimock-runner.d.ts +48 -0
  4. package/dist/aimock-runner.d.ts.map +1 -0
  5. package/dist/aimock-runner.js +81 -0
  6. package/dist/checkpointer-conformance.d.ts +43 -0
  7. package/dist/checkpointer-conformance.d.ts.map +1 -0
  8. package/dist/checkpointer-conformance.js +381 -0
  9. package/dist/fake-embedder.d.ts +9 -0
  10. package/dist/fake-embedder.d.ts.map +1 -0
  11. package/dist/fake-embedder.js +38 -0
  12. package/dist/fixture-builder.d.ts +29 -0
  13. package/dist/fixture-builder.d.ts.map +1 -0
  14. package/dist/fixture-builder.js +43 -0
  15. package/dist/fixture-file.d.ts +14 -0
  16. package/dist/fixture-file.d.ts.map +1 -0
  17. package/dist/fixture-file.js +49 -0
  18. package/dist/harness.d.ts +35 -0
  19. package/dist/harness.d.ts.map +1 -0
  20. package/dist/harness.js +211 -0
  21. package/dist/http-inject.d.ts +22 -0
  22. package/dist/http-inject.d.ts.map +1 -0
  23. package/dist/http-inject.js +55 -0
  24. package/dist/index.d.ts +20 -0
  25. package/dist/index.d.ts.map +1 -0
  26. package/dist/index.js +19 -0
  27. package/dist/matchers.d.ts +56 -0
  28. package/dist/matchers.d.ts.map +1 -0
  29. package/dist/matchers.js +262 -0
  30. package/dist/memory-conformance.d.ts +13 -0
  31. package/dist/memory-conformance.d.ts.map +1 -0
  32. package/dist/memory-conformance.js +1333 -0
  33. package/dist/memory.d.ts +11 -0
  34. package/dist/memory.d.ts.map +1 -0
  35. package/dist/memory.js +20 -0
  36. package/dist/middleware-harness.d.ts +11 -0
  37. package/dist/middleware-harness.d.ts.map +1 -0
  38. package/dist/middleware-harness.js +35 -0
  39. package/dist/permissions-conformance.d.ts +24 -0
  40. package/dist/permissions-conformance.d.ts.map +1 -0
  41. package/dist/permissions-conformance.js +156 -0
  42. package/dist/record-fixtures.d.ts +21 -0
  43. package/dist/record-fixtures.d.ts.map +1 -0
  44. package/dist/record-fixtures.js +31 -0
  45. package/dist/record.d.ts +12 -0
  46. package/dist/record.d.ts.map +1 -0
  47. package/dist/record.js +22 -0
  48. package/dist/regex-safety.d.ts +2 -0
  49. package/dist/regex-safety.d.ts.map +1 -0
  50. package/dist/regex-safety.js +66 -0
  51. package/dist/run-result.d.ts +111 -0
  52. package/dist/run-result.d.ts.map +1 -0
  53. package/dist/run-result.js +287 -0
  54. package/dist/subprocess.d.ts +25 -0
  55. package/dist/subprocess.d.ts.map +1 -0
  56. package/dist/subprocess.js +223 -0
  57. package/dist/thread-access-harness.d.ts +40 -0
  58. package/dist/thread-access-harness.d.ts.map +1 -0
  59. package/dist/thread-access-harness.js +42 -0
  60. package/dist/threads-conformance.d.ts +18 -0
  61. package/dist/threads-conformance.d.ts.map +1 -0
  62. package/dist/threads-conformance.js +315 -0
  63. package/dist/tool-harness.d.ts +16 -0
  64. package/dist/tool-harness.d.ts.map +1 -0
  65. package/dist/tool-harness.js +25 -0
  66. package/dist/tsconfig.tsbuildinfo +1 -0
  67. package/dist/workspace-harness.d.ts +15 -0
  68. package/dist/workspace-harness.d.ts.map +1 -0
  69. package/dist/workspace-harness.js +43 -0
  70. package/package.json +71 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) Brian Love
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,51 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/cacheplane/b4run/main/docs/brand/b4-logo-horizontal-black-on-white.png" alt="B4.run" width="180" />
3
+ </p>
4
+
5
+ # @b4run/testing
6
+
7
+ Supported harnesses, fixtures, matchers, deterministic embedders, and runtime helpers for testing B4.run applications.
8
+
9
+ **Use this when:** You want to test B4.run agent behavior without making live model calls.
10
+
11
+ ## Install
12
+
13
+ ```bash
14
+ pnpm add -D @b4run/testing vitest
15
+ ```
16
+
17
+ ## Example
18
+
19
+ ```ts
20
+ import { createAgentHarness, expectFinalMessage, script } from "@b4run/testing"
21
+
22
+ await using harness = await createAgentHarness({
23
+ appRoot: process.cwd(),
24
+ route: "/support#agent",
25
+ })
26
+ const result = await harness.run({
27
+ input: "Say hello",
28
+ fixtures: script().user("Say hello").replies("Hello!"),
29
+ })
30
+
31
+ expectFinalMessage(result).toContain("Hello")
32
+ ```
33
+
34
+ ## Runtime and stability
35
+
36
+ `@b4run/testing` is a supported node-only testing surface. `createAgentHarness()` temporarily changes process-wide model environment variables and runtime caches; await `close()` and do not run concurrent harnesses in one process. Fixture replay is deterministic and CI-safe; live recording requires model credentials.
37
+
38
+ ## Related
39
+
40
+ - [Testing API reference](https://b4.run/docs/api/testing) — exact harness and lifecycle contracts.
41
+ - [Agent Test Harness](https://b4.run/docs/testing-agents) — end-to-end agent testing.
42
+ - [Fixtures and Recording](https://b4.run/docs/testing-agents/fixtures) — deterministic replay and optional live recording.
43
+ - [`@b4run/evals`](https://www.npmjs.com/package/@b4run/evals) — repeatable datasets, scorers, and release gates.
44
+
45
+ ## Maturity and support
46
+
47
+ B4.run is pre-1.0, and its public surface can change. All publishable B4.run packages release together as a fixed group; review the [`@b4run/testing` changelog](https://github.com/cacheplane/b4run/blob/main/packages/testing/CHANGELOG.md) and [upgrading guide](https://b4.run/docs/upgrading) before upgrading. For support, use [GitHub Discussions](https://github.com/cacheplane/b4run/discussions); report defects in [GitHub Issues](https://github.com/cacheplane/b4run/issues).
48
+
49
+ ## License
50
+
51
+ MIT. See the [repository license](https://github.com/cacheplane/b4run/blob/main/LICENSE).
@@ -0,0 +1,48 @@
1
+ import type { AimockFixture } from "./fixture-builder.js";
2
+ import type { Recording } from "./record-fixtures.js";
3
+ export interface Aimock {
4
+ readonly port: number;
5
+ /** Base URL with the `/v1` suffix the OpenAI SDK expects. */
6
+ readonly baseUrl: string;
7
+ /** Append more fixtures onto the live mock without restarting it. */
8
+ addFixtures(fixtures: readonly AimockFixture[]): void;
9
+ /** Remove all registered fixtures (the mock keeps running). */
10
+ clearFixtures(): void;
11
+ /** All requests the mock has received (aimock's journal). */
12
+ getRequests(): ReadonlyArray<{
13
+ body: {
14
+ messages?: Array<{
15
+ role: string;
16
+ content: unknown;
17
+ }>;
18
+ tools?: Array<{
19
+ type?: string;
20
+ function?: {
21
+ name?: string;
22
+ };
23
+ }>;
24
+ } | null;
25
+ }>;
26
+ /** Current count of registered fixtures (snapshot point for getRecordingsSince). */
27
+ getFixtureCount(): number;
28
+ /**
29
+ * Recordings captured since the given journal length and fixture count — pairs
30
+ * proxied journal entries (request) with newly-recorded fixtures (response),
31
+ * both windowed to a single run so multi-run reuse can't cross-align.
32
+ */
33
+ getRecordingsSince(journalStart: number, fixtureStart: number): readonly Recording[];
34
+ /** Ordered recordings (request + baked response) for proxied calls captured in record mode. */
35
+ getRecordings(): readonly Recording[];
36
+ close(): Promise<void>;
37
+ [Symbol.asyncDispose](): Promise<void>;
38
+ }
39
+ export declare function createAimock(opts: {
40
+ readonly fixtures: readonly AimockFixture[];
41
+ /** When set, proxy unmatched requests to the given upstream providers. */
42
+ readonly proxy?: {
43
+ openai: string;
44
+ };
45
+ /** With proxy: capture (record) proxied responses so getRecordings() returns them. */
46
+ readonly record?: boolean;
47
+ }): Promise<Aimock>;
48
+ //# sourceMappingURL=aimock-runner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"aimock-runner.d.ts","sourceRoot":"","sources":["../src/aimock-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAkB,MAAM,sBAAsB,CAAA;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAErD,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,qEAAqE;IACrE,WAAW,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,GAAG,IAAI,CAAA;IACrD,+DAA+D;IAC/D,aAAa,IAAI,IAAI,CAAA;IACrB,6DAA6D;IAC7D,WAAW,IAAI,aAAa,CAAC;QAC3B,IAAI,EAAE;YACJ,QAAQ,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,OAAO,CAAA;aAAE,CAAC,CAAA;YACpD,KAAK,CAAC,EAAE,KAAK,CAAC;gBAAE,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAC,QAAQ,CAAC,EAAE;oBAAE,IAAI,CAAC,EAAE,MAAM,CAAA;iBAAE,CAAA;aAAE,CAAC,CAAA;SAC/D,GAAG,IAAI,CAAA;KACT,CAAC,CAAA;IACF,oFAAoF;IACpF,eAAe,IAAI,MAAM,CAAA;IACzB;;;;OAIG;IACH,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;IACpF,+FAA+F;IAC/F,aAAa,IAAI,SAAS,SAAS,EAAE,CAAA;IACrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvC;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE;IACvC,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAA;IAC3C,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IACnC,sFAAsF;IACtF,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAC1B,GAAG,OAAO,CAAC,MAAM,CAAC,CA8FlB"}
@@ -0,0 +1,81 @@
1
+ import * as fs from "node:fs";
2
+ import * as os from "node:os";
3
+ import * as path from "node:path";
4
+ import { LLMock } from "@copilotkit/aimock";
5
+ export async function createAimock(opts) {
6
+ // When recording, use a private temp dir so aimock writes fixtures there
7
+ // instead of ./fixtures/recorded in the caller's CWD.
8
+ const recordTmpDir = opts.proxy && opts.record === true
9
+ ? fs.mkdtempSync(path.join(os.tmpdir(), "b4-runmock-record-"))
10
+ : null;
11
+ const mock = new LLMock(opts.proxy
12
+ ? {
13
+ port: 0,
14
+ chunkSize: 4096,
15
+ record: {
16
+ providers: { openai: opts.proxy.openai },
17
+ proxyOnly: opts.record !== true,
18
+ ...(recordTmpDir !== null ? { fixturePath: recordTmpDir } : {}),
19
+ },
20
+ }
21
+ : { port: 0, chunkSize: 4096 });
22
+ if (opts.fixtures.length > 0) {
23
+ mock.addFixturesFromJSON(opts.fixtures);
24
+ }
25
+ await mock.start();
26
+ // Capture the fixture count at start so getRecordings() can diff against it.
27
+ const initialFixtureCount = mock.getFixtures().length;
28
+ // Local helper — windowed to [journalStart, fixtureStart) so multi-run reuse
29
+ // can't cross-align recorded responses with the wrong requests.
30
+ const getRecordingsSince = (journalStart, fixtureStart) => {
31
+ const newFixtures = mock.getFixtures().slice(fixtureStart);
32
+ if (newFixtures.length === 0)
33
+ return [];
34
+ const proxyEntries = mock.getRequests()
35
+ .slice(journalStart)
36
+ .filter((e) => e.response?.source === "proxy");
37
+ return newFixtures.map((fixture, i) => {
38
+ const messages = proxyEntries[i]?.body?.messages;
39
+ const response = fixture.response;
40
+ return { request: messages !== undefined ? { messages } : {}, response };
41
+ });
42
+ };
43
+ let stopped = false;
44
+ const handle = {
45
+ port: mock.port,
46
+ baseUrl: `${mock.url}/v1`,
47
+ addFixtures(fixtures) {
48
+ if (fixtures.length > 0) {
49
+ mock.addFixturesFromJSON(fixtures);
50
+ }
51
+ },
52
+ clearFixtures() {
53
+ mock.clearFixtures();
54
+ },
55
+ getRequests() {
56
+ return mock.getRequests();
57
+ },
58
+ getFixtureCount() {
59
+ return mock.getFixtures().length;
60
+ },
61
+ getRecordingsSince(journalStart, fixtureStart) {
62
+ return getRecordingsSince(journalStart, fixtureStart);
63
+ },
64
+ getRecordings() {
65
+ return getRecordingsSince(0, initialFixtureCount);
66
+ },
67
+ async close() {
68
+ if (stopped)
69
+ return;
70
+ stopped = true;
71
+ await mock.stop();
72
+ if (recordTmpDir !== null) {
73
+ fs.rmSync(recordTmpDir, { recursive: true, force: true });
74
+ }
75
+ },
76
+ [Symbol.asyncDispose]() {
77
+ return this.close();
78
+ },
79
+ };
80
+ return handle;
81
+ }
@@ -0,0 +1,43 @@
1
+ import type { BaseCheckpointSaver } from "@langchain/langgraph-checkpoint";
2
+ /**
3
+ * The contract every checkpointer must satisfy. Run against B4.run's SQLite saver
4
+ * (in-process, always) and any other backend (e.g. Postgres, gated) so they
5
+ * cannot drift. Pass vitest's `describe`; `makeSaver` returns a FRESH empty
6
+ * saver per call.
7
+ *
8
+ * Asserted through the PUBLIC saver API only — never the tables. The backends
9
+ * genuinely differ underneath (SQLite stores the whole Checkpoint as one BLOB;
10
+ * another saver may split payloads across columns or externalize channel values
11
+ * keyed by version), and none of that is contract.
12
+ *
13
+ * Two behaviors are CAPABILITY FLAGS rather than assertions, because SQLite
14
+ * deliberately does less than a backend legitimately can:
15
+ * - `listPendingWrites` — SQLite's `list()` yields lightweight tuples with
16
+ * `pendingWrites: []` and expects callers needing writes to re-fetch via
17
+ * `getTuple`.
18
+ * - `listFilter` — SQLite ignores `options.filter` entirely.
19
+ * Asserting SQLite's shortcuts as contract would fail a richer backend for
20
+ * being better, so each backend declares what it actually promises instead.
21
+ *
22
+ * The two hostile round-trips (a NUL byte and a lone surrogate, in metadata AND
23
+ * in a channel name and value) are not edge-case trivia: they are exactly where
24
+ * a jsonb-backed store diverges from the BLOB-backed incumbent, and B4.run
25
+ * produces such payloads for real — sandbox stdout flows unmodified into tool
26
+ * results and from there into checkpoint metadata. They are deliberately NOT
27
+ * asserted for a pending write's channel column, which is plain text in every
28
+ * backend (SQLite transcodes a lone surrogate there to U+FFFD) and is fed by
29
+ * the graph definition rather than by model or tool output.
30
+ */
31
+ export declare function runCheckpointerConformance(opts: {
32
+ readonly name: string;
33
+ readonly makeSaver: () => Promise<BaseCheckpointSaver> | BaseCheckpointSaver;
34
+ readonly describe: (name: string, fn: () => void) => void;
35
+ readonly close?: (saver: BaseCheckpointSaver) => Promise<void> | void;
36
+ readonly supports?: {
37
+ /** `list()` hydrates `pendingWrites` (SQLite: no). */
38
+ readonly listPendingWrites?: boolean;
39
+ /** `list({ filter })` narrows by metadata (SQLite: no). */
40
+ readonly listFilter?: boolean;
41
+ };
42
+ }): void;
43
+ //# sourceMappingURL=checkpointer-conformance.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"checkpointer-conformance.d.ts","sourceRoot":"","sources":["../src/checkpointer-conformance.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EAKpB,MAAM,iCAAiC,CAAA;AAgExC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAC/C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,OAAO,CAAC,mBAAmB,CAAC,GAAG,mBAAmB,CAAA;IAC5E,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,IAAI,KAAK,IAAI,CAAA;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACrE,QAAQ,CAAC,QAAQ,CAAC,EAAE;QAClB,sDAAsD;QACtD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;QACpC,2DAA2D;QAC3D,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAA;KAC9B,CAAA;CACF,GAAG,IAAI,CAqTP"}
@@ -0,0 +1,381 @@
1
+ import { expect, test } from "vitest";
2
+ /** A payload SQLite's BLOB round-trips but a jsonb column rejects (SQLSTATE 22P05). */
3
+ const NUL = "before\u0000after";
4
+ /** Unpaired high surrogate — rejected by jsonb columns (SQLSTATE 22P02). */
5
+ const LONE_SURROGATE = "lone\ud800end";
6
+ /**
7
+ * A checkpoint plus the `newVersions` LangGraph would pass alongside it.
8
+ *
9
+ * `newVersions` is REALISTIC (one entry per written channel) on purpose: a
10
+ * backend that externalizes `channel_values` keyed by version silently drops
11
+ * them when `newVersions` is empty, so an empty map would let that divergence
12
+ * pass unnoticed.
13
+ */
14
+ function mk(id, channelValues, version = 2) {
15
+ const channelVersions = {};
16
+ for (const key of Object.keys(channelValues))
17
+ channelVersions[key] = version;
18
+ return {
19
+ checkpoint: {
20
+ v: 4,
21
+ id,
22
+ ts: "2026-08-07T00:00:00.000Z",
23
+ channel_values: channelValues,
24
+ channel_versions: channelVersions,
25
+ versions_seen: { agent: { ...channelVersions } },
26
+ },
27
+ newVersions: { ...channelVersions },
28
+ };
29
+ }
30
+ const meta = (over = {}) => ({
31
+ source: over.source ?? "loop",
32
+ step: over.step ?? 1,
33
+ parents: {},
34
+ });
35
+ const cfg = (threadId, ns = "", checkpointId) => ({
36
+ configurable: {
37
+ thread_id: threadId,
38
+ checkpoint_ns: ns,
39
+ ...(checkpointId === undefined ? {} : { checkpoint_id: checkpointId }),
40
+ },
41
+ });
42
+ async function collect(gen) {
43
+ const out = [];
44
+ for await (const tuple of gen)
45
+ out.push(tuple);
46
+ return out;
47
+ }
48
+ const noteOf = (metadata) => metadata?.note;
49
+ /**
50
+ * The contract every checkpointer must satisfy. Run against B4.run's SQLite saver
51
+ * (in-process, always) and any other backend (e.g. Postgres, gated) so they
52
+ * cannot drift. Pass vitest's `describe`; `makeSaver` returns a FRESH empty
53
+ * saver per call.
54
+ *
55
+ * Asserted through the PUBLIC saver API only — never the tables. The backends
56
+ * genuinely differ underneath (SQLite stores the whole Checkpoint as one BLOB;
57
+ * another saver may split payloads across columns or externalize channel values
58
+ * keyed by version), and none of that is contract.
59
+ *
60
+ * Two behaviors are CAPABILITY FLAGS rather than assertions, because SQLite
61
+ * deliberately does less than a backend legitimately can:
62
+ * - `listPendingWrites` — SQLite's `list()` yields lightweight tuples with
63
+ * `pendingWrites: []` and expects callers needing writes to re-fetch via
64
+ * `getTuple`.
65
+ * - `listFilter` — SQLite ignores `options.filter` entirely.
66
+ * Asserting SQLite's shortcuts as contract would fail a richer backend for
67
+ * being better, so each backend declares what it actually promises instead.
68
+ *
69
+ * The two hostile round-trips (a NUL byte and a lone surrogate, in metadata AND
70
+ * in a channel name and value) are not edge-case trivia: they are exactly where
71
+ * a jsonb-backed store diverges from the BLOB-backed incumbent, and B4.run
72
+ * produces such payloads for real — sandbox stdout flows unmodified into tool
73
+ * results and from there into checkpoint metadata. They are deliberately NOT
74
+ * asserted for a pending write's channel column, which is plain text in every
75
+ * backend (SQLite transcodes a lone surrogate there to U+FFFD) and is fed by
76
+ * the graph definition rather than by model or tool output.
77
+ */
78
+ export function runCheckpointerConformance(opts) {
79
+ const { name, makeSaver, describe, close, supports } = opts;
80
+ describe(`Checkpointer conformance: ${name}`, () => {
81
+ test("put + getTuple round-trips the checkpoint, channel values and metadata", async () => {
82
+ const s = await makeSaver();
83
+ try {
84
+ const { checkpoint, newVersions } = mk("ckpt-1", { messages: ["hi"], count: 3 });
85
+ const metadata = meta({ step: 7 });
86
+ const returned = await s.put(cfg("t1"), checkpoint, metadata, newVersions);
87
+ expect(returned.configurable?.thread_id).toBe("t1");
88
+ expect(returned.configurable?.checkpoint_ns).toBe("");
89
+ expect(returned.configurable?.checkpoint_id).toBe("ckpt-1");
90
+ const tuple = await s.getTuple(cfg("t1", "", "ckpt-1"));
91
+ expect(tuple?.checkpoint.id).toBe("ckpt-1");
92
+ expect(tuple?.checkpoint.ts).toBe(checkpoint.ts);
93
+ expect(tuple?.checkpoint.channel_values).toEqual({ messages: ["hi"], count: 3 });
94
+ expect(tuple?.checkpoint.channel_versions).toEqual(checkpoint.channel_versions);
95
+ expect(tuple?.checkpoint.versions_seen).toEqual(checkpoint.versions_seen);
96
+ expect(tuple?.metadata).toEqual(metadata);
97
+ expect(tuple?.config.configurable?.thread_id).toBe("t1");
98
+ expect(tuple?.config.configurable?.checkpoint_ns).toBe("");
99
+ expect(tuple?.config.configurable?.checkpoint_id).toBe("ckpt-1");
100
+ }
101
+ finally {
102
+ await close?.(s);
103
+ }
104
+ });
105
+ test("getTuple without a checkpoint_id returns the newest checkpoint", async () => {
106
+ const s = await makeSaver();
107
+ try {
108
+ const a = mk("ckpt-a", { messages: ["a"] });
109
+ const b = mk("ckpt-b", { messages: ["b"] });
110
+ await s.put(cfg("t1"), a.checkpoint, meta({ step: 0 }), a.newVersions);
111
+ await s.put(cfg("t1"), b.checkpoint, meta({ step: 1 }), b.newVersions);
112
+ expect((await s.getTuple(cfg("t1")))?.checkpoint.id).toBe("ckpt-b");
113
+ }
114
+ finally {
115
+ await close?.(s);
116
+ }
117
+ });
118
+ test("getTuple returns undefined for an unknown thread or checkpoint", async () => {
119
+ const s = await makeSaver();
120
+ try {
121
+ expect(await s.getTuple(cfg("t-missing"))).toBeUndefined();
122
+ const a = mk("ckpt-a", { messages: ["a"] });
123
+ await s.put(cfg("t1"), a.checkpoint, meta(), a.newVersions);
124
+ expect(await s.getTuple(cfg("t1", "", "ckpt-nope"))).toBeUndefined();
125
+ }
126
+ finally {
127
+ await close?.(s);
128
+ }
129
+ });
130
+ test("put replaces a checkpoint written under the same id", async () => {
131
+ const s = await makeSaver();
132
+ try {
133
+ const first = mk("ckpt-1", { messages: ["v1"] });
134
+ const second = mk("ckpt-1", { messages: ["v2"] }, 3);
135
+ await s.put(cfg("t1"), first.checkpoint, meta({ step: 1 }), first.newVersions);
136
+ await s.put(cfg("t1"), second.checkpoint, meta({ step: 2 }), second.newVersions);
137
+ const tuple = await s.getTuple(cfg("t1", "", "ckpt-1"));
138
+ expect(tuple?.checkpoint.channel_values).toEqual({ messages: ["v2"] });
139
+ expect(tuple?.metadata?.step).toBe(2);
140
+ expect(await collect(s.list(cfg("t1")))).toHaveLength(1);
141
+ }
142
+ finally {
143
+ await close?.(s);
144
+ }
145
+ });
146
+ test("the parent chain is recorded: root has none, children point back", async () => {
147
+ const s = await makeSaver();
148
+ try {
149
+ const root = mk("ckpt-1", { messages: ["a"] });
150
+ const rootConfig = await s.put(cfg("t1"), root.checkpoint, meta({ step: 0 }), root.newVersions);
151
+ const child = mk("ckpt-2", { messages: ["a", "b"] }, 3);
152
+ await s.put(rootConfig, child.checkpoint, meta({ step: 1 }), child.newVersions);
153
+ expect((await s.getTuple(cfg("t1", "", "ckpt-1")))?.parentConfig).toBeUndefined();
154
+ const childTuple = await s.getTuple(cfg("t1", "", "ckpt-2"));
155
+ expect(childTuple?.parentConfig?.configurable?.thread_id).toBe("t1");
156
+ expect(childTuple?.parentConfig?.configurable?.checkpoint_ns).toBe("");
157
+ expect(childTuple?.parentConfig?.configurable?.checkpoint_id).toBe("ckpt-1");
158
+ }
159
+ finally {
160
+ await close?.(s);
161
+ }
162
+ });
163
+ test("putWrites is visible in getTuple().pendingWrites, in call order", async () => {
164
+ const s = await makeSaver();
165
+ try {
166
+ const a = mk("ckpt-1", { messages: ["a"] });
167
+ const config = await s.put(cfg("t1"), a.checkpoint, meta(), a.newVersions);
168
+ await s.putWrites(config, [
169
+ ["messages", { role: "ai", text: "one" }],
170
+ ["counter", 2],
171
+ ], "task-1");
172
+ expect((await s.getTuple(cfg("t1", "", "ckpt-1")))?.pendingWrites).toEqual([
173
+ ["task-1", "messages", { role: "ai", text: "one" }],
174
+ ["task-1", "counter", 2],
175
+ ]);
176
+ }
177
+ finally {
178
+ await close?.(s);
179
+ }
180
+ });
181
+ test("putWrites is idempotent on (task_id, idx)", async () => {
182
+ const s = await makeSaver();
183
+ try {
184
+ const a = mk("ckpt-1", { messages: ["a"] });
185
+ const config = await s.put(cfg("t1"), a.checkpoint, meta(), a.newVersions);
186
+ await s.putWrites(config, [["messages", "a"]], "task-1");
187
+ await s.putWrites(config, [["messages", "a"]], "task-1");
188
+ expect((await s.getTuple(cfg("t1", "", "ckpt-1")))?.pendingWrites).toEqual([
189
+ ["task-1", "messages", "a"],
190
+ ]);
191
+ }
192
+ finally {
193
+ await close?.(s);
194
+ }
195
+ });
196
+ test("writes are scoped to their own checkpoint", async () => {
197
+ const s = await makeSaver();
198
+ try {
199
+ const a = mk("ckpt-1", { messages: ["a"] });
200
+ const b = mk("ckpt-2", { messages: ["b"] }, 3);
201
+ const configA = await s.put(cfg("t1"), a.checkpoint, meta({ step: 0 }), a.newVersions);
202
+ await s.put(configA, b.checkpoint, meta({ step: 1 }), b.newVersions);
203
+ await s.putWrites(configA, [["messages", "only-for-1"]], "task-1");
204
+ expect((await s.getTuple(cfg("t1", "", "ckpt-2")))?.pendingWrites).toEqual([]);
205
+ }
206
+ finally {
207
+ await close?.(s);
208
+ }
209
+ });
210
+ test("list yields checkpoints newest-first", async () => {
211
+ const s = await makeSaver();
212
+ try {
213
+ for (const id of ["ckpt-a", "ckpt-b", "ckpt-c"]) {
214
+ const c = mk(id, { messages: [id] });
215
+ await s.put(cfg("t1"), c.checkpoint, meta(), c.newVersions);
216
+ }
217
+ expect((await collect(s.list(cfg("t1")))).map((t) => t.checkpoint.id)).toEqual([
218
+ "ckpt-c",
219
+ "ckpt-b",
220
+ "ckpt-a",
221
+ ]);
222
+ }
223
+ finally {
224
+ await close?.(s);
225
+ }
226
+ });
227
+ test("list honors limit", async () => {
228
+ const s = await makeSaver();
229
+ try {
230
+ for (const id of ["ckpt-a", "ckpt-b", "ckpt-c"]) {
231
+ const c = mk(id, { messages: [id] });
232
+ await s.put(cfg("t1"), c.checkpoint, meta(), c.newVersions);
233
+ }
234
+ expect((await collect(s.list(cfg("t1"), { limit: 2 }))).map((t) => t.checkpoint.id)).toEqual(["ckpt-c", "ckpt-b"]);
235
+ }
236
+ finally {
237
+ await close?.(s);
238
+ }
239
+ });
240
+ test("list honors before, exclusively", async () => {
241
+ const s = await makeSaver();
242
+ try {
243
+ for (const id of ["ckpt-a", "ckpt-b", "ckpt-c"]) {
244
+ const c = mk(id, { messages: [id] });
245
+ await s.put(cfg("t1"), c.checkpoint, meta(), c.newVersions);
246
+ }
247
+ const tuples = await collect(s.list(cfg("t1"), { before: cfg("t1", "", "ckpt-c") }));
248
+ expect(tuples.map((t) => t.checkpoint.id)).toEqual(["ckpt-b", "ckpt-a"]);
249
+ }
250
+ finally {
251
+ await close?.(s);
252
+ }
253
+ });
254
+ test("list on an unknown thread yields nothing", async () => {
255
+ const s = await makeSaver();
256
+ try {
257
+ expect(await collect(s.list(cfg("t-missing")))).toEqual([]);
258
+ }
259
+ finally {
260
+ await close?.(s);
261
+ }
262
+ });
263
+ test("checkpoint_ns isolates checkpoints, writes and listing", async () => {
264
+ const s = await makeSaver();
265
+ try {
266
+ const root = mk("ckpt-1", { messages: ["root"] });
267
+ const sub = mk("ckpt-1", { messages: ["sub"] });
268
+ const rootConfig = await s.put(cfg("t1"), root.checkpoint, meta(), root.newVersions);
269
+ await s.put(cfg("t1", "child"), sub.checkpoint, meta(), sub.newVersions);
270
+ await s.putWrites(rootConfig, [["messages", "root-write"]], "task-1");
271
+ expect((await s.getTuple(cfg("t1", "", "ckpt-1")))?.checkpoint.channel_values).toEqual({
272
+ messages: ["root"],
273
+ });
274
+ const subTuple = await s.getTuple(cfg("t1", "child", "ckpt-1"));
275
+ expect(subTuple?.checkpoint.channel_values).toEqual({ messages: ["sub"] });
276
+ expect(subTuple?.config.configurable?.checkpoint_ns).toBe("child");
277
+ expect(subTuple?.pendingWrites).toEqual([]);
278
+ expect(await collect(s.list(cfg("t1", "child")))).toHaveLength(1);
279
+ }
280
+ finally {
281
+ await close?.(s);
282
+ }
283
+ });
284
+ test("deleteThread removes that thread's checkpoints and writes, leaving others", async () => {
285
+ const s = await makeSaver();
286
+ try {
287
+ const doomed = mk("ckpt-1", { messages: ["doomed"] });
288
+ const keeper = mk("ckpt-1", { messages: ["keeper"] });
289
+ const doomedConfig = await s.put(cfg("t1"), doomed.checkpoint, meta(), doomed.newVersions);
290
+ const keeperConfig = await s.put(cfg("t2"), keeper.checkpoint, meta(), keeper.newVersions);
291
+ await s.putWrites(doomedConfig, [["messages", "gone"]], "task-1");
292
+ await s.putWrites(keeperConfig, [["messages", "kept"]], "task-1");
293
+ await s.put(cfg("t1", "child"), doomed.checkpoint, meta(), doomed.newVersions);
294
+ await s.deleteThread("t1");
295
+ expect(await s.getTuple(cfg("t1", "", "ckpt-1"))).toBeUndefined();
296
+ expect(await s.getTuple(cfg("t1", "child", "ckpt-1"))).toBeUndefined();
297
+ expect(await collect(s.list(cfg("t1")))).toEqual([]);
298
+ const kept = await s.getTuple(cfg("t2", "", "ckpt-1"));
299
+ expect(kept?.checkpoint.channel_values).toEqual({ messages: ["keeper"] });
300
+ expect(kept?.pendingWrites).toEqual([["task-1", "messages", "kept"]]);
301
+ // Re-putting the same id proves the old writes really went away rather
302
+ // than lingering to re-attach.
303
+ await s.put(cfg("t1"), doomed.checkpoint, meta(), doomed.newVersions);
304
+ expect((await s.getTuple(cfg("t1", "", "ckpt-1")))?.pendingWrites).toEqual([]);
305
+ }
306
+ finally {
307
+ await close?.(s);
308
+ }
309
+ });
310
+ test("round-trips a NUL byte in metadata, a channel name and a channel value", async () => {
311
+ const s = await makeSaver();
312
+ try {
313
+ // Hostile bytes go everywhere the payload travels: a channel NAME and a
314
+ // channel VALUE inside the serialized checkpoint (which also carries
315
+ // them into channel_versions and versions_seen), an extra metadata
316
+ // field, and a pending-write value. NOT into the write's channel
317
+ // column: that is a plain text column in every backend, and channel
318
+ // names come from the graph definition, never from model or tool output.
319
+ const { checkpoint, newVersions } = mk("ckpt-1", { [NUL]: NUL, messages: [NUL] });
320
+ const metadata = { ...meta(), note: NUL };
321
+ const config = await s.put(cfg("t1"), checkpoint, metadata, newVersions);
322
+ await s.putWrites(config, [["messages", NUL]], "task-1");
323
+ const tuple = await s.getTuple(cfg("t1", "", "ckpt-1"));
324
+ expect(tuple?.checkpoint.channel_values).toEqual({ [NUL]: NUL, messages: [NUL] });
325
+ expect(tuple?.checkpoint.channel_versions).toEqual(checkpoint.channel_versions);
326
+ expect(tuple?.checkpoint.versions_seen).toEqual(checkpoint.versions_seen);
327
+ expect(noteOf(tuple?.metadata)).toBe(NUL);
328
+ expect(tuple?.pendingWrites).toEqual([["task-1", "messages", NUL]]);
329
+ }
330
+ finally {
331
+ await close?.(s);
332
+ }
333
+ });
334
+ test("round-trips a lone surrogate in metadata, a channel name and a channel value", async () => {
335
+ const s = await makeSaver();
336
+ try {
337
+ const values = { [LONE_SURROGATE]: LONE_SURROGATE, messages: [LONE_SURROGATE] };
338
+ const { checkpoint, newVersions } = mk("ckpt-1", values);
339
+ const metadata = { ...meta(), note: LONE_SURROGATE };
340
+ const config = await s.put(cfg("t1"), checkpoint, metadata, newVersions);
341
+ await s.putWrites(config, [["messages", LONE_SURROGATE]], "task-1");
342
+ const tuple = await s.getTuple(cfg("t1", "", "ckpt-1"));
343
+ expect(tuple?.checkpoint.channel_values).toEqual(values);
344
+ expect(tuple?.checkpoint.channel_versions).toEqual(checkpoint.channel_versions);
345
+ expect(tuple?.checkpoint.versions_seen).toEqual(checkpoint.versions_seen);
346
+ expect(noteOf(tuple?.metadata)).toBe(LONE_SURROGATE);
347
+ expect(tuple?.pendingWrites).toEqual([["task-1", "messages", LONE_SURROGATE]]);
348
+ }
349
+ finally {
350
+ await close?.(s);
351
+ }
352
+ });
353
+ test.skipIf(supports?.listPendingWrites !== true)("list hydrates pendingWrites (declared capability)", async () => {
354
+ const s = await makeSaver();
355
+ try {
356
+ const a = mk("ckpt-1", { messages: ["a"] });
357
+ const config = await s.put(cfg("t1"), a.checkpoint, meta(), a.newVersions);
358
+ await s.putWrites(config, [["messages", "w"]], "task-1");
359
+ const [tuple] = await collect(s.list(cfg("t1")));
360
+ expect(tuple?.pendingWrites).toEqual([["task-1", "messages", "w"]]);
361
+ }
362
+ finally {
363
+ await close?.(s);
364
+ }
365
+ });
366
+ test.skipIf(supports?.listFilter !== true)("list narrows by metadata filter (declared capability)", async () => {
367
+ const s = await makeSaver();
368
+ try {
369
+ const a = mk("ckpt-a", { messages: ["a"] });
370
+ const b = mk("ckpt-b", { messages: ["b"] });
371
+ await s.put(cfg("t1"), a.checkpoint, meta({ source: "input", step: 0 }), a.newVersions);
372
+ await s.put(cfg("t1"), b.checkpoint, meta({ source: "loop", step: 1 }), b.newVersions);
373
+ const tuples = await collect(s.list(cfg("t1"), { filter: { source: "loop" } }));
374
+ expect(tuples.map((t) => t.checkpoint.id)).toEqual(["ckpt-b"]);
375
+ }
376
+ finally {
377
+ await close?.(s);
378
+ }
379
+ });
380
+ });
381
+ }
@@ -0,0 +1,9 @@
1
+ import type { Embedder } from "@b4run/core";
2
+ /**
3
+ * Deterministic, network-free embedder for tests: a normalized bag-of-token-hash
4
+ * vector, so strings sharing tokens are nearer in cosine. NOT for production.
5
+ */
6
+ export declare function fakeEmbedder(opts?: {
7
+ readonly dims?: number;
8
+ }): Embedder;
9
+ //# sourceMappingURL=fake-embedder.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fake-embedder.d.ts","sourceRoot":"","sources":["../src/fake-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAW3C;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAuBxE"}