@niwa-labs/harness-mcode 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,722 @@
1
+ // The MiniMax Code reading side against the durable history: a session whose journal is empty
2
+ // (it is not adopted yet) keeps its history in `durable-history/sessions.g1.db`.
3
+ //
4
+ // The database is built here the way the harness builds it: batches of entries, each payload an
5
+ // `MZS2` header, a codec byte and a compressed JSON array.
6
+
7
+ import { Database } from "bun:sqlite";
8
+ import { afterEach, beforeEach, expect, test } from "bun:test";
9
+ import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
10
+ import { join } from "node:path";
11
+ import { gzipSync } from "node:zlib";
12
+
13
+ import { harnessId, sessionId } from "@niwa-labs/harness";
14
+ import type { Locator, SessionRecord } from "@niwa-labs/harness";
15
+
16
+ import { McodeReading } from "../src/mcode/reading.ts";
17
+ import { tempDirSync } from "@niwa-labs/testing";
18
+
19
+ let home: string;
20
+ let db: Database;
21
+ let nextBatch = 1;
22
+ let nextSeq = 1;
23
+ let uid = 0;
24
+ const reading = () => new McodeReading({ harness: harnessId("mcode-test"), home });
25
+
26
+ beforeEach(() => {
27
+ home = tempDirSync("mcode-durable");
28
+ mkdirSync(join(home, "durable-history"), { recursive: true });
29
+ db = new Database(join(home, "durable-history", "sessions.g1.db"));
30
+ db.run("create table batches (batch_id integer primary key, created_ms integer not null, compressed integer not null, payload blob not null)");
31
+ db.run("create table entries (entry_id text primary key, seq integer not null, ordinal integer not null, batch_id integer not null)");
32
+ nextBatch = 1;
33
+ nextSeq = 1;
34
+ });
35
+ afterEach(() => {
36
+ db.close();
37
+ rmSync(home, { recursive: true, force: true });
38
+ });
39
+
40
+ interface Entry {
41
+ readonly id: string;
42
+ readonly kind: string;
43
+ readonly payload: object;
44
+ }
45
+
46
+ /** A batch of entries, encoded as the harness encodes it. */
47
+ function putBatch(entries: Entry[], codec: 1 | 2 = 1): void {
48
+ const json = Buffer.from(JSON.stringify(entries.map((e) => ({ ...e, parentId: null }))));
49
+ const frame = codec === 1 ? Buffer.from(Bun.zstdCompressSync(json)) : gzipSync(json);
50
+ const header = Buffer.alloc(21);
51
+ header.write("MZS2", 0, "latin1");
52
+ header[4] = codec;
53
+ header.writeBigUInt64BE(BigInt(json.length), 5);
54
+ header.writeBigUInt64BE(BigInt(frame.length), 13);
55
+ const batch = nextBatch++;
56
+ db.run("insert into batches values (?, ?, 1, ?)", [batch, 0, Buffer.concat([header, frame])]);
57
+ for (const [ordinal, entry] of entries.entries()) {
58
+ db.run("insert into entries values (?, ?, ?, ?)", [entry.id, nextSeq++, ordinal, batch]);
59
+ }
60
+ }
61
+
62
+ const envelope = (messageId: string, message: object) => ({ message_id: messageId, turn_id: "t1", message });
63
+ const user = (text: string) => ({ role: "user", content: [{ type: "text", text }], timestamp: 1000 });
64
+ const assistant = (text: string, call?: { id: string; name: string }) => ({
65
+ role: "assistant",
66
+ content: [
67
+ { type: "text", text },
68
+ ...(call === undefined ? [] : [{ type: "toolCall", id: call.id, name: call.name, arguments: { a: 1 } }]),
69
+ ],
70
+ timestamp: 2000,
71
+ });
72
+ const toolResult = (callId: string, text: string) => ({
73
+ role: "toolResult",
74
+ toolCallId: callId,
75
+ toolName: "bash",
76
+ content: [{ type: "text", text }],
77
+ timestamp: 3000,
78
+ });
79
+
80
+ const full = (native: string, reason: string, envelopes: object[]): Entry => ({
81
+ id: `${native}|full|${reason}|sha256:${uid++}`,
82
+ kind: "history.v1.full",
83
+ payload: { sessionId: native, reason, envelopes },
84
+ });
85
+ const append = (native: string, envelopes: object[]): Entry => ({
86
+ id: `${native}|turn:history:${uid++}|x`,
87
+ kind: "history.v1.append",
88
+ payload: { sessionId: native, turnId: "t1", operationId: "op", envelopes },
89
+ });
90
+
91
+ /** A session directory with the manifest and an empty journal. */
92
+ function sessionDir(native: string, journal?: string): string {
93
+ const encoded = Buffer.from(native).toString("base64url");
94
+ const dir = join(home, "v2", "sessions", "2026", "11", "01", `12-00-00-000-session_${encoded}`);
95
+ mkdirSync(dir, { recursive: true });
96
+ writeFileSync(join(dir, "manifest.json"), JSON.stringify({ sessionId: native, createdAtMs: 1000, updatedAtMs: 2000 }));
97
+ if (journal !== undefined) writeFileSync(join(dir, "messages.jsonl"), journal);
98
+ return dir;
99
+ }
100
+
101
+ const id = (native: string) => sessionId(harnessId("mcode-test"), native);
102
+
103
+ async function all(native: string, from?: Locator): Promise<SessionRecord[]> {
104
+ const out: SessionRecord[] = [];
105
+ for await (const record of reading().read(id(native), from)) out.push(record);
106
+ return out;
107
+ }
108
+
109
+ test("a session with an empty journal is read from the durable history: full, then appends, by role", async () => {
110
+ sessionDir("mvs_a", "");
111
+ putBatch([
112
+ full("mvs_a", "seed", [envelope("m1", user("hello"))]),
113
+ append("mvs_a", [envelope("m2", assistant("calling", { id: "c1", name: "bash" }))]),
114
+ ]);
115
+ putBatch([append("mvs_a", [envelope("m3", toolResult("c1", "done"))])], 2);
116
+ const records = await all("mvs_a");
117
+ expect(records.map((r) => r.kind)).toEqual(["user", "assistant", "tool-call", "tool-result"]);
118
+ expect(records[0]).toMatchObject({ kind: "user", text: "hello" });
119
+ expect(records[2]).toMatchObject({ kind: "tool-call", callId: "c1", name: "bash" });
120
+ expect(records[3]).toMatchObject({ kind: "tool-result", callId: "c1", output: "done", ok: true });
121
+ });
122
+
123
+ test("the active view is the last full and the appends after it, a message once", async () => {
124
+ sessionDir("mvs_b", "");
125
+ putBatch([
126
+ full("mvs_b", "seed", [envelope("old", user("before the adopt"))]),
127
+ append("mvs_b", [envelope("old2", user("also before"))]),
128
+ full("mvs_b", "adopt", [envelope("m1", user("one")), envelope("m2", assistant("two"))]),
129
+ append("mvs_b", [envelope("m2", assistant("two")), envelope("m3", user("three"))]),
130
+ ]);
131
+ const records = await all("mvs_b");
132
+ expect(records.map((r) => (r.kind === "user" || r.kind === "assistant" ? r.text : r.kind))).toEqual([
133
+ "one",
134
+ "two",
135
+ "three",
136
+ ]);
137
+ });
138
+
139
+ test("a non-empty journal wins over the durable history", async () => {
140
+ const line = JSON.stringify(envelope("j1", user("from the journal")));
141
+ sessionDir("mvs_c", `${line}\n`);
142
+ putBatch([full("mvs_c", "adopt", [envelope("m1", user("from the db"))])]);
143
+ const records = await all("mvs_c");
144
+ expect(records).toHaveLength(1);
145
+ expect(records[0]).toMatchObject({ text: "from the journal" });
146
+ });
147
+
148
+ test("a read resumes after a locator; a locator of a replaced history starts over", async () => {
149
+ sessionDir("mvs_d", "");
150
+ putBatch([
151
+ full("mvs_d", "seed", [envelope("m1", user("one"))]),
152
+ append("mvs_d", [envelope("m2", assistant("two", { id: "c1", name: "bash" }))]),
153
+ ]);
154
+ const first = await all("mvs_d");
155
+ expect(first.map((r) => r.kind)).toEqual(["user", "assistant", "tool-call"]);
156
+ expect(first[0]?.locator.startsWith("mcode-db:v1:")).toBe(true);
157
+
158
+ // After the first record: the rest. After the assistant text: the call of that envelope.
159
+ expect((await all("mvs_d", first[0]!.locator)).map((r) => r.kind)).toEqual(["assistant", "tool-call"]);
160
+ expect((await all("mvs_d", first[1]!.locator)).map((r) => r.kind)).toEqual(["tool-call"]);
161
+ expect(await all("mvs_d", first[2]!.locator)).toEqual([]);
162
+
163
+ // New messages arrive: only they come after the last locator.
164
+ putBatch([append("mvs_d", [envelope("m3", user("three"))])]);
165
+ const fresh = await all("mvs_d", first[2]!.locator);
166
+ expect(fresh.map((r) => r.kind)).toEqual(["user"]);
167
+
168
+ // The history is replaced and no longer holds that message: the read starts over.
169
+ putBatch([full("mvs_d", "adopt", [envelope("n1", user("new"))])]);
170
+ const started = await all("mvs_d", first[2]!.locator);
171
+ expect(started.map((r) => r.kind)).toEqual(["user"]);
172
+ // The first record of a read that started over says so; a read that goes on does not.
173
+ expect(started[0]?.reanchored).toBe(true);
174
+ expect(fresh.some((r) => r.reanchored === true)).toBe(false);
175
+ expect(first.some((r) => r.reanchored === true)).toBe(false);
176
+ });
177
+
178
+ test("raw gives the envelope whole, and coverage follows the locator", async () => {
179
+ sessionDir("mvs_e", "");
180
+ putBatch([full("mvs_e", "seed", [envelope("m1", user("one"))])]);
181
+ const [record] = await all("mvs_e");
182
+ const raw = (await reading().raw(id("mvs_e"), record!.locator)) as { message_id: string };
183
+ expect(raw.message_id).toBe("m1");
184
+ expect(await reading().coverage(id("mvs_e"))).toMatchObject({ status: "not-scanned" });
185
+ expect(await reading().coverage(id("mvs_e"), record!.locator)).toMatchObject({ status: "empty" });
186
+ await expect(reading().raw(id("mvs_e"), "mcode-db:v1:1:0:0:1:nope" as Locator)).rejects.toThrow("no record");
187
+ });
188
+
189
+ test("the listing finds a session by its manifest, with its working directory from the preamble", async () => {
190
+ sessionDir("mvs_f", "");
191
+ const preamble = "<agent-context>\nYOUR WORKSPACE DIRECTORY: /work/db\n</agent-context>\nhi";
192
+ putBatch([full("mvs_f", "seed", [envelope("m1", user(preamble))])]);
193
+ const listed: Array<{ native: string; cwd?: string }> = [];
194
+ for await (const info of reading().list()) listed.push({ native: info.id.native, ...(info.cwd === undefined ? {} : { cwd: info.cwd }) });
195
+ expect(listed).toEqual([{ native: "mvs_f", cwd: "/work/db" }]);
196
+ });
197
+
198
+ test("a session only in the database is listed and read; a filter leaves it out", async () => {
199
+ putBatch([full("mvs_only", "seed", [envelope("m1", user("lonely"))])]);
200
+ const listed: string[] = [];
201
+ for await (const info of reading().list()) listed.push(info.id.native);
202
+ expect(listed).toEqual(["mvs_only"]);
203
+ expect((await all("mvs_only")).map((r) => r.kind)).toEqual(["user"]);
204
+ const filtered: string[] = [];
205
+ for await (const info of reading().list({ cwd: "/x" })) filtered.push(info.id.native);
206
+ expect(filtered).toEqual([]);
207
+ });
208
+
209
+ test("without a database nothing changes: an empty journal and no ledger holds nothing", async () => {
210
+ db.close();
211
+ rmSync(join(home, "durable-history"), { recursive: true });
212
+ sessionDir("mvs_g", "");
213
+ expect(await all("mvs_g")).toEqual([]);
214
+ db = new Database(":memory:");
215
+ });
216
+
217
+ test("a home without durable-history reads exactly as before: the ledger when there is one, else nothing", async () => {
218
+ db.close();
219
+ rmSync(join(home, "durable-history"), { recursive: true });
220
+ const dir = sessionDir("mvs_h", "");
221
+ writeFileSync(
222
+ join(dir, "ledger.jsonl"),
223
+ `${JSON.stringify({
224
+ kind: "message.pi_history_appended",
225
+ sessionId: "mvs_h",
226
+ messages: [user("from the ledger")],
227
+ seq: 1,
228
+ })}\n`,
229
+ );
230
+ const records = await all("mvs_h");
231
+ expect(records.map((r) => r.kind)).toEqual(["user"]);
232
+ expect(records[0]?.locator.startsWith("mcode-ledger:")).toBe(true);
233
+ db = new Database(":memory:");
234
+ });
235
+
236
+ test("a corrupt database file does not throw out of the reader: the session reads as it did without it", async () => {
237
+ db.close();
238
+ writeFileSync(join(home, "durable-history", "sessions.g1.db"), "this is not a sqlite database at all".repeat(40));
239
+ sessionDir("mvs_i", "");
240
+ const warn = console.warn;
241
+ const told: string[] = [];
242
+ console.warn = (...args: unknown[]) => void told.push(args.join(" "));
243
+ try {
244
+ expect(await all("mvs_i")).toEqual([]);
245
+ const listed: string[] = [];
246
+ for await (const info of reading().list()) listed.push(info.id.native);
247
+ expect(listed).toEqual(["mvs_i"]);
248
+ // Nothing could be read: that is not an empty session, it is a scan that failed.
249
+ expect(await reading().coverage(id("mvs_i"))).toMatchObject({ status: "failed" });
250
+ } finally {
251
+ console.warn = warn;
252
+ }
253
+ expect(told.length).toBeLessThanOrEqual(1);
254
+ db = new Database(":memory:");
255
+ });
256
+
257
+ test("a database with the wrong tables, or a batch that cannot be decoded, falls back to the ledger", async () => {
258
+ db.close();
259
+ const path = join(home, "durable-history", "sessions.g1.db");
260
+ rmSync(path);
261
+ const bad = new Database(path);
262
+ bad.run("create table entries (entry_id text primary key, seq integer, ordinal integer, batch_id integer)");
263
+ bad.run("insert into entries values ('mvs_j|full|seed|x', 1, 0, 1)");
264
+ bad.close();
265
+ const dir = sessionDir("mvs_j", "");
266
+ writeFileSync(
267
+ join(dir, "ledger.jsonl"),
268
+ `${JSON.stringify({ kind: "message.pi_history_appended", sessionId: "mvs_j", messages: [user("ledger")], seq: 1 })}\n`,
269
+ );
270
+ const warn = console.warn;
271
+ console.warn = () => undefined;
272
+ try {
273
+ expect((await all("mvs_j")).map((r) => r.kind)).toEqual(["user"]);
274
+ } finally {
275
+ console.warn = warn;
276
+ }
277
+
278
+ // A batch with an unknown codec: the table is there, the payload is not readable.
279
+ rmSync(path);
280
+ const odd = new Database(path);
281
+ odd.run("create table batches (batch_id integer primary key, created_ms integer, compressed integer, payload blob)");
282
+ odd.run("create table entries (entry_id text primary key, seq integer, ordinal integer, batch_id integer)");
283
+ const header = Buffer.alloc(21);
284
+ header.write("MZS2", 0, "latin1");
285
+ header[4] = 9;
286
+ odd.run("insert into batches values (1, 0, 1, ?)", [Buffer.concat([header, Buffer.from("zz")])]);
287
+ odd.run("insert into entries values ('mvs_j|full|seed|y', 1, 0, 1)");
288
+ odd.close();
289
+ console.warn = () => undefined;
290
+ try {
291
+ expect((await all("mvs_j")).map((r) => r.kind)).toEqual(["user"]);
292
+ } finally {
293
+ console.warn = warn;
294
+ }
295
+ db = new Database(":memory:");
296
+ });
297
+
298
+ /** Silences (and counts) what the reader tells about a database it cannot use. */
299
+ async function quietly<T>(run: () => Promise<T>): Promise<{ result: T; told: string[] }> {
300
+ const warn = console.warn;
301
+ const told: string[] = [];
302
+ console.warn = (...args: unknown[]) => void told.push(args.join(" "));
303
+ try {
304
+ return { result: await run(), told };
305
+ } finally {
306
+ console.warn = warn;
307
+ }
308
+ }
309
+
310
+ const texts = (records: SessionRecord[]) =>
311
+ records.map((r) => (r.kind === "user" || r.kind === "assistant" ? r.text : r.kind));
312
+
313
+ test("an adopt that keeps the message ids replaces the full: a locator given before it resumes with no repeats", async () => {
314
+ sessionDir("mvs_k", "");
315
+ putBatch([
316
+ full("mvs_k", "seed", [envelope("m1", user("one")), envelope("m2", assistant("two"))]),
317
+ append("mvs_k", [envelope("m3", user("three"))]),
318
+ ]);
319
+ const before = await all("mvs_k");
320
+ const last = before[before.length - 1]!.locator;
321
+ // The adopt states the history again, with the same ids, and a message after it.
322
+ putBatch([
323
+ full("mvs_k", "adopt", [envelope("m1", user("one")), envelope("m2", assistant("two")), envelope("m3", user("three"))]),
324
+ append("mvs_k", [envelope("m4", assistant("four"))]),
325
+ ]);
326
+ expect(texts(await all("mvs_k", last))).toEqual(["four"]);
327
+ expect(texts(await all("mvs_k", before[0]!.locator))).toEqual(["two", "three", "four"]);
328
+ });
329
+
330
+ test("a session in the database only, and its journal that appears later: db locators carry over without repeats", async () => {
331
+ const dir = sessionDir("mvs_l", "");
332
+ putBatch([
333
+ full("mvs_l", "seed", [envelope("m1", user("one"))]),
334
+ append("mvs_l", [envelope("m2", assistant("two", { id: "c1", name: "bash" }))]),
335
+ ]);
336
+ const before = await all("mvs_l");
337
+ expect(before.map((r) => r.kind)).toEqual(["user", "assistant", "tool-call"]);
338
+
339
+ // The session is adopted: its journal now holds the history, and one message more.
340
+ const lines = [
341
+ envelope("m1", user("one")),
342
+ envelope("m2", assistant("two", { id: "c1", name: "bash" })),
343
+ envelope("m3", toolResult("c1", "done")),
344
+ ];
345
+ writeFileSync(join(dir, "messages.jsonl"), lines.map((l) => JSON.stringify(l)).join("\n") + "\n");
346
+
347
+ const afterText = await all("mvs_l", before[1]!.locator); // the text of m2 was delivered, its call was not
348
+ expect(afterText.map((r) => r.kind)).toEqual(["tool-call", "tool-result"]);
349
+ expect(afterText[0]!.locator.startsWith("mcode:v1:")).toBe(true);
350
+ expect((await all("mvs_l", before[2]!.locator)).map((r) => r.kind)).toEqual(["tool-result"]);
351
+
352
+ // The same locator still gives the envelope whole, and coverage counts what is not delivered.
353
+ const raw = (await reading().raw(id("mvs_l"), before[0]!.locator)) as { message_id: string };
354
+ expect(raw.message_id).toBe("m1");
355
+ expect(await reading().coverage(id("mvs_l"), before[2]!.locator)).toMatchObject({ status: "not-scanned" });
356
+ const done = (await all("mvs_l")).pop()!;
357
+ expect(await reading().coverage(id("mvs_l"), done.locator)).toMatchObject({ status: "empty" });
358
+
359
+ // A message the journal does not hold starts the read over, as a rewritten file does.
360
+ const unknown = "mcode-db:v1:9:0:0:1:nowhere" as Locator;
361
+ const startedOver = await all("mvs_l", unknown);
362
+ expect(texts(startedOver)).toEqual(["one", "two", "tool-call", "tool-result"]);
363
+ expect(startedOver.map((r) => r.reanchored)).toEqual([true, undefined, undefined, undefined]);
364
+ });
365
+
366
+ test("the active view does not take the entries of a session whose id starts the same", async () => {
367
+ sessionDir("mvs_n", "");
368
+ putBatch([
369
+ full("mvs_n", "seed", [envelope("m1", user("n"))]),
370
+ full("mvs_nn", "seed", [envelope("x1", user("nn"))]),
371
+ full("mvs_n0", "seed", [envelope("y1", user("n0"))]),
372
+ ]);
373
+ expect(texts(await all("mvs_n"))).toEqual(["n"]);
374
+ expect(texts(await all("mvs_nn"))).toEqual(["nn"]);
375
+ const listed: string[] = [];
376
+ for await (const info of reading().list()) listed.push(info.id.native);
377
+ expect(listed.sort()).toEqual(["mvs_n", "mvs_n0", "mvs_nn"]);
378
+ });
379
+
380
+ test("unknown kinds, entries that are not objects and envelopes of no shape do not throw", async () => {
381
+ sessionDir("mvs_o", "");
382
+ const json = Buffer.from(
383
+ JSON.stringify([
384
+ { id: "mvs_o|odd|1", parentId: null, kind: "tool.offload.v1", payload: { blob: "x".repeat(10) } },
385
+ null,
386
+ { id: "mvs_o|full|seed|z", parentId: null, kind: "history.v1.full", payload: { envelopes: [null, 7, "s", { message_id: "a" }, envelope("m1", user("ok"))] } },
387
+ { id: "mvs_o|next|1", parentId: null, kind: "history.v1.append", payload: null },
388
+ { id: "mvs_o|next|2", parentId: null, kind: "history.v1.append", payload: { envelopes: "no" } },
389
+ ]),
390
+ );
391
+ const frame = Buffer.from(Bun.zstdCompressSync(json));
392
+ const header = Buffer.alloc(21);
393
+ header.write("MZS2", 0, "latin1");
394
+ header[4] = 1;
395
+ header.writeBigUInt64BE(BigInt(json.length), 5);
396
+ header.writeBigUInt64BE(BigInt(frame.length), 13);
397
+ db.run("insert into batches values (1, 0, 1, ?)", [Buffer.concat([header, frame])]);
398
+ for (let ordinal = 0; ordinal < 5; ordinal++) {
399
+ db.run("insert into entries values (?, ?, ?, 1)", [`mvs_o|e${ordinal}`, ordinal + 1, ordinal]);
400
+ }
401
+ const { result, told } = await quietly(() => all("mvs_o"));
402
+ expect(told).toEqual([]);
403
+ expect(result.map((r) => r.kind)).toEqual(["unknown", "unknown", "unknown", "unknown", "user"]);
404
+ });
405
+
406
+ test("a zstd batch is read with the bounded decoder of node:zlib, whether or not Bun has one of its own", async () => {
407
+ sessionDir("mvs_p", "");
408
+ putBatch([full("mvs_p", "seed", [envelope("m1", user("zstd"))])]);
409
+ const bun = Bun as unknown as { zstdDecompressSync?: unknown };
410
+ const saved = bun.zstdDecompressSync;
411
+ bun.zstdDecompressSync = undefined;
412
+ try {
413
+ expect(texts(await all("mvs_p"))).toEqual(["zstd"]);
414
+ } finally {
415
+ bun.zstdDecompressSync = saved;
416
+ }
417
+ });
418
+
419
+ test("a batch that claims to be enormous is refused, not inflated: the session reads as it did without the database", async () => {
420
+ const dir = sessionDir("mvs_q", "");
421
+ writeFileSync(
422
+ join(dir, "ledger.jsonl"),
423
+ `${JSON.stringify({ kind: "message.pi_history_appended", sessionId: "mvs_q", messages: [user("ledger")], seq: 1 })}\n`,
424
+ );
425
+ const header = Buffer.alloc(21);
426
+ header.write("MZS2", 0, "latin1");
427
+ header[4] = 1;
428
+ header.writeBigUInt64BE(1n << 40n, 5);
429
+ db.run("insert into batches values (1, 0, 1, ?)", [Buffer.concat([header, Buffer.from("not a frame")])]);
430
+ db.run("insert into entries values ('mvs_q|full|seed|h', 1, 0, 1)");
431
+ const { result, told } = await quietly(() => all("mvs_q"));
432
+ expect(result.map((r) => r.kind)).toEqual(["user"]);
433
+ expect(result[0]!.locator.startsWith("mcode-ledger:")).toBe(true);
434
+ expect(told.join()).toContain("larger than it may be");
435
+ });
436
+
437
+ test("a batch whose header claims little but inflates past the cap is refused, zstd and gzip alike", async () => {
438
+ const dir = sessionDir("mvs_lie", "");
439
+ writeFileSync(
440
+ join(dir, "ledger.jsonl"),
441
+ `${JSON.stringify({ kind: "message.pi_history_appended", sessionId: "mvs_lie", messages: [user("ledger")], seq: 1 })}\n`,
442
+ );
443
+ // Valid JSON (an empty array) padded with blanks: without the cap it would read as an empty history.
444
+ const huge = Buffer.alloc(256 * 1024 * 1024 + 16, 0x20);
445
+ huge[0] = 0x5b;
446
+ huge[huge.length - 1] = 0x5d;
447
+ for (const [codec, frame] of [
448
+ [1, Buffer.from(Bun.zstdCompressSync(huge))],
449
+ [2, gzipSync(huge)],
450
+ ] as const) {
451
+ const header = Buffer.alloc(21);
452
+ header.write("MZS2", 0, "latin1");
453
+ header[4] = codec;
454
+ header.writeBigUInt64BE(10n, 5);
455
+ db.run("delete from entries");
456
+ db.run("delete from batches");
457
+ db.run("insert into batches values (1, 0, 1, ?)", [Buffer.concat([header, frame])]);
458
+ db.run("insert into entries values ('mvs_lie|full|seed|h', 1, 0, 1)");
459
+ const { result } = await quietly(() => all("mvs_lie"));
460
+ expect(result.map((r) => r.kind)).toEqual(["user"]);
461
+ expect(result[0]!.locator.startsWith("mcode-ledger:")).toBe(true);
462
+ }
463
+ });
464
+
465
+ test("a database locked by a writer is not a failure of the reader: it reads as without the database", async () => {
466
+ sessionDir("mvs_r", "");
467
+ putBatch([full("mvs_r", "seed", [envelope("m1", user("held"))])]);
468
+ expect(texts(await all("mvs_r"))).toEqual(["held"]);
469
+ db.run("begin exclusive");
470
+ try {
471
+ const { result } = await quietly(async () => {
472
+ const records = await all("mvs_r");
473
+ const listed: string[] = [];
474
+ for await (const info of reading().list()) listed.push(info.id.native);
475
+ return { records, listed, coverage: await reading().coverage(id("mvs_r")) };
476
+ });
477
+ expect(result.records).toEqual([]);
478
+ expect(result.listed).toEqual(["mvs_r"]);
479
+ expect(result.coverage).toMatchObject({ status: "failed" });
480
+ } finally {
481
+ db.run("rollback");
482
+ }
483
+ expect(texts(await all("mvs_r"))).toEqual(["held"]);
484
+ });
485
+
486
+ test("a database in WAL mode, with a write in flight, shows the committed state and takes no lock", async () => {
487
+ sessionDir("mvs_s", "");
488
+ db.run("pragma journal_mode = wal");
489
+ putBatch([full("mvs_s", "seed", [envelope("m1", user("committed"))])]);
490
+ db.run("begin");
491
+ putBatch([append("mvs_s", [envelope("m2", user("in flight"))])]);
492
+ expect(texts(await all("mvs_s"))).toEqual(["committed"]);
493
+ // The writer is not held up by the reader: it commits, and the next read sees it.
494
+ db.run("commit");
495
+ expect(texts(await all("mvs_s"))).toEqual(["committed", "in flight"]);
496
+ expect(db.query("pragma wal_checkpoint(passive)").get()).toBeDefined();
497
+ });
498
+
499
+ test("the listing takes the working directory from the head of the history, not by decoding all of it", async () => {
500
+ sessionDir("mvs_t", "");
501
+ const preamble = "<agent-context>\nYOUR WORKSPACE DIRECTORY: /work/head\n</agent-context>\nhi";
502
+ putBatch([full("mvs_t", "seed", [envelope("m1", user(preamble))])]);
503
+ // A later batch that could not be decoded: the listing never reaches it.
504
+ const header = Buffer.alloc(21);
505
+ header.write("MZS2", 0, "latin1");
506
+ header[4] = 9;
507
+ db.run("insert into batches values (50, 0, 1, ?)", [header]);
508
+ db.run("insert into entries values ('mvs_t|turn:history:late|x', 99, 0, 50)");
509
+ const { result, told } = await quietly(async () => {
510
+ const listed: Array<{ native: string; cwd?: string }> = [];
511
+ for await (const info of reading().list()) listed.push({ native: info.id.native, ...(info.cwd === undefined ? {} : { cwd: info.cwd }) });
512
+ return listed;
513
+ });
514
+ expect(result).toEqual([{ native: "mvs_t", cwd: "/work/head" }]);
515
+ expect(told).toEqual([]);
516
+ });
517
+
518
+ test("reading never writes to the database", async () => {
519
+ sessionDir("mvs_u", "");
520
+ putBatch([full("mvs_u", "seed", [envelope("m1", user("same"))])]);
521
+ const path = join(home, "durable-history", "sessions.g1.db");
522
+ const hash = () => new Bun.CryptoHasher("sha256").update(readFileSync(path)).digest("hex");
523
+ const before = hash();
524
+ await all("mvs_u");
525
+ for await (const _ of reading().list()) void _;
526
+ await reading().coverage(id("mvs_u"));
527
+ expect(hash()).toBe(before);
528
+ });
529
+
530
+ test("the look at the head ends at the first human message: a history with no preamble is not decoded to its end", async () => {
531
+ sessionDir("mvs_v", "");
532
+ putBatch([full("mvs_v", "seed", [envelope("m1", user("no preamble here"))])]);
533
+ const header = Buffer.alloc(21);
534
+ header.write("MZS2", 0, "latin1");
535
+ header[4] = 9;
536
+ db.run("insert into batches values (50, 0, 1, ?)", [header]);
537
+ db.run("insert into entries values ('mvs_v|turn:history:late|x', 99, 0, 50)");
538
+ const { result, told } = await quietly(async () => {
539
+ const listed: string[] = [];
540
+ for await (const info of reading().list()) listed.push(info.id.native);
541
+ return listed;
542
+ });
543
+ expect(result).toEqual(["mvs_v"]);
544
+ expect(told).toEqual([]);
545
+ });
546
+
547
+ test("a database that cannot be read after a db locator was given: the session falls back with no locator error", async () => {
548
+ const dir = sessionDir("mvs_w", "");
549
+ writeFileSync(
550
+ join(dir, "ledger.jsonl"),
551
+ `${JSON.stringify({ kind: "message.pi_history_appended", sessionId: "mvs_w", messages: [user("ledger")], seq: 1 })}\n`,
552
+ );
553
+ putBatch([full("mvs_w", "seed", [envelope("m1", user("one"))])]);
554
+ const [first] = await all("mvs_w");
555
+ db.close();
556
+ writeFileSync(join(home, "durable-history", "sessions.g1.db"), "garbage".repeat(500));
557
+ const { result } = await quietly(async () => ({
558
+ records: await all("mvs_w", first!.locator),
559
+ coverage: await reading().coverage(id("mvs_w"), first!.locator),
560
+ }));
561
+ expect(result.records.map((r) => r.kind)).toEqual(["user"]);
562
+ expect(result.coverage).toMatchObject({ status: "not-scanned" });
563
+ db = new Database(":memory:");
564
+ });
565
+
566
+ test("a checkpoint taken from the ledger while the database was unreadable still works once it is back", async () => {
567
+ const dir = sessionDir("mvs_move", "");
568
+ writeFileSync(
569
+ join(dir, "ledger.jsonl"),
570
+ `${JSON.stringify({ kind: "message.pi_history_appended", sessionId: "mvs_move", messages: [user("one")], seq: 1 })}\n`,
571
+ );
572
+ putBatch([full("mvs_move", "seed", [envelope("m1", user("one")), envelope("m2", user("two"))])]);
573
+ // The database is there but a writer holds it: the session is read from its ledger.
574
+ db.run("begin exclusive");
575
+ let checkpoint: Locator;
576
+ try {
577
+ const { result } = await quietly(() => all("mvs_move"));
578
+ checkpoint = result[0]!.locator;
579
+ expect(checkpoint.startsWith("mcode-ledger:")).toBe(true);
580
+ } finally {
581
+ db.run("rollback");
582
+ }
583
+ // The database answers again: a locator of the ledger names no record in it, so the read starts over.
584
+ const moved = await all("mvs_move", checkpoint);
585
+ expect(texts(moved)).toEqual(["one", "two"]);
586
+ expect(moved.map((r) => r.reanchored)).toEqual([true, undefined]);
587
+ expect(await reading().coverage(id("mvs_move"), checkpoint)).toEqual({ status: "not-scanned" });
588
+ });
589
+
590
+ test("a zstd batch that inflates past the cap is refused while it is decoded, not after it was allocated", async () => {
591
+ const dirOfBatch = tempDirSync("mcode-bomb");
592
+ try {
593
+ // 300 MB of blanks in a few kilobytes, under a header that claims 10 bytes.
594
+ const frame = Buffer.from(Bun.zstdCompressSync(Buffer.alloc(300 * 1024 * 1024, 0x20)));
595
+ const header = Buffer.alloc(21);
596
+ header.write("MZS2", 0, "latin1");
597
+ header[4] = 1;
598
+ header.writeBigUInt64BE(10n, 5);
599
+ const batch = join(dirOfBatch, "batch.bin");
600
+ writeFileSync(batch, Buffer.concat([header, frame]));
601
+ const durable = new URL("../src/mcode/durable.ts", import.meta.url).pathname;
602
+ // A process of its own: the peak of its memory is what is measured.
603
+ const script = `
604
+ import { readFileSync } from "node:fs";
605
+ import { decodeBatch } from ${JSON.stringify(durable)};
606
+ const payload = readFileSync(process.argv[1]);
607
+ const before = process.resourceUsage().maxRSS;
608
+ let refused = "";
609
+ try { decodeBatch(payload); } catch (error) { refused = error.message; }
610
+ console.log(JSON.stringify({ refused, grewKb: process.resourceUsage().maxRSS - before }));
611
+ `;
612
+ const ran = Bun.spawnSync(["bun", "-e", script, batch]);
613
+ const out = JSON.parse(ran.stdout.toString()) as { refused: string; grewKb: number };
614
+ expect(out.refused).toContain("larger than it may be");
615
+ // The cap is 256 MB: what is allocated before the refusal stays within it, not the size the frame claims.
616
+ expect(out.grewKb).toBeLessThan(400 * 1024);
617
+ } finally {
618
+ rmSync(dirOfBatch, { recursive: true, force: true });
619
+ }
620
+ });
621
+
622
+ test("the fork reads the active generation in the session's own database through its native id", async () => {
623
+ const native = "mvs_fork";
624
+ const storage = join(home, "durable-history");
625
+ const directory = join(storage, "sessions", native);
626
+ mkdirSync(directory, { recursive: true });
627
+ const reader = new McodeReading({ harness: harnessId("ncode-test"), home, perSessionHistory: true });
628
+ const target = sessionId(harnessId("ncode-test"), native);
629
+ const old = envelope("outdated", assistant("earlier"));
630
+ putBatch([full(native, "seed", [old])]);
631
+ db.run("vacuum into ?", [join(directory, "sessions.g1.db")]);
632
+ sessionDir(native, JSON.stringify(old) + "\n");
633
+ putBatch([full(native, "adopt", [envelope("question", user("ping"))]),
634
+ append(native, [envelope("answer", assistant("PONG"))]),
635
+ append("mvs_other", [envelope("other", assistant("wrong session"))])], 2);
636
+ db.run("vacuum into ?", [join(directory, "sessions.g2.db")]);
637
+ const records = await Array.fromAsync(reader.read(target));
638
+ expect(texts(records)).toEqual(["ping", "PONG"]);
639
+ expect(texts(await Array.fromAsync(reader.read(target, records[0]!.locator)))).toEqual(["PONG"]);
640
+ expect(await reader.raw(target, records[1]!.locator)).toMatchObject({ message_id: "answer" });
641
+ expect(await reader.coverage(target, records[1]!.locator)).toEqual({ status: "empty" });
642
+ expect(texts(await all(native))).toEqual(["earlier"]);
643
+ await expect(Array.fromAsync(reader.read(id(native)))).rejects.toThrow("another harness installation");
644
+ });
645
+
646
+ test("Ncode lists a database-only session by its stored native identity", async () => {
647
+ const native = "mvs_example:import";
648
+ const storage = join(home, "durable-history");
649
+ const directory = join(storage, "sessions", "mvs_example-import");
650
+ mkdirSync(directory, { recursive: true });
651
+ putBatch([full(native, "seed", [envelope("question", user("per-session only"))])]);
652
+ db.run("vacuum into ?", [join(directory, "sessions.g2.db")]);
653
+ const misplaced = join(storage, "sessions", "unrelated");
654
+ mkdirSync(misplaced);
655
+ db.run("vacuum into ?", [join(misplaced, "sessions.g1.db")]);
656
+ db.close();
657
+ rmSync(join(storage, "sessions.g1.db"));
658
+ db = new Database(":memory:");
659
+ symlinkSync(directory, join(storage, "sessions", "linked"));
660
+ const stale = join(directory, "sessions.g1.db");
661
+ writeFileSync(stale, "an older generation is not active");
662
+ const reader = new McodeReading({ harness: harnessId("ncode-test"), home, perSessionHistory: true });
663
+ const result = await Array.fromAsync(reader.list());
664
+ expect(result.map((item) => item.id.native)).toEqual([native]);
665
+ expect(texts(await Array.fromAsync(reader.read(result[0]!.id)))).toEqual(["per-session only"]);
666
+ expect(await Array.fromAsync(reader.list({ natives: [sessionId(harnessId("ncode-test"), "other").native] }))).toEqual([]);
667
+ expect(await Array.fromAsync(reader.list({ natives: [result[0]!.id.native] }))).toEqual(result);
668
+ expect(await Array.fromAsync(reading().list())).toEqual([]);
669
+ sessionDir(native);
670
+ expect((await Array.fromAsync(reader.list())).map((item) => item.id.native)).toEqual([native]);
671
+ });
672
+
673
+ test("a matching Ncode fold checkpoint supplies the compacted view and later appends", async () => {
674
+ const native = "mvs_compacted";
675
+ const storage = join(home, "durable-history");
676
+ const directory = join(storage, "sessions", native);
677
+ const cache = join(storage, "fold-cache");
678
+ mkdirSync(directory, { recursive: true });
679
+ mkdirSync(cache);
680
+ const anchor = native + "|compact";
681
+ putBatch([full(native, "seed", [envelope("old", user("superseded"))])]);
682
+ putBatch([{ id: anchor, kind: "history.v2.compaction.replace", payload: { sessionId: native } }]);
683
+ putBatch([append(native, [envelope("reply", assistant("next"))])]);
684
+ db.run("vacuum into ?", [join(directory, "sessions.g1.db")]);
685
+ const reader = new McodeReading({ harness: harnessId("ncode-test"), home, perSessionHistory: true });
686
+ const target = sessionId(harnessId("ncode-test"), native);
687
+ const checkpoint = {
688
+ schema: 1, sessionId: native, backendDirectory: directory, backendGeneration: 1, backendSeq: 2,
689
+ fold: { headSeq: 2, generation: 1, segmentCounts: [1], segmentSeqs: [2], lastEntryId: anchor,
690
+ envelopes: [envelope("summary", user("current summary"))] },
691
+ };
692
+ const path = join(cache, native + ".json");
693
+ for (const fields of [{ sessionId: "foreign" }, { backendGeneration: 2 }, { backendDirectory: storage },
694
+ { backendSeq: 100 }, { backendSeq: 3 }]) {
695
+ writeFileSync(path, JSON.stringify({ ...checkpoint, ...fields }));
696
+ expect(await reader.coverage(target)).toMatchObject({ status: "failed" });
697
+ await expect(Array.fromAsync(reader.read(target))).rejects.toThrow("cannot materialize");
698
+ }
699
+ writeFileSync(path, JSON.stringify(checkpoint));
700
+ const records = await Array.fromAsync(reader.read(target));
701
+ expect(texts(records)).toEqual(["current summary", "next"]);
702
+ expect(texts(await Array.fromAsync(reader.read(target, records[0]!.locator)))).toEqual(["next"]);
703
+ expect(await reader.raw(target, records[0]!.locator)).toMatchObject({ message_id: "summary" });
704
+ });
705
+
706
+ test("Ncode refuses a foreign envelope or symlink in the per-session store", async () => {
707
+ const native = "mvs_isolated";
708
+ const storage = join(home, "durable-history");
709
+ const directory = join(storage, "sessions", native);
710
+ mkdirSync(directory, { recursive: true });
711
+ const target = sessionId(harnessId("ncode-test"), native);
712
+ const reader = new McodeReading({ harness: target.harness, home, perSessionHistory: true });
713
+ const path = join(directory, "sessions.g1.db");
714
+ symlinkSync(join(storage, "sessions.g1.db"), path);
715
+ await expect(Array.fromAsync(reader.read(target))).rejects.toThrow("invalid session history file");
716
+ rmSync(path);
717
+ const entry = full(native, "seed", [envelope("wrong", assistant("private"))]);
718
+ putBatch([{ ...entry, payload: { ...entry.payload, sessionId: "elsewhere" } }]);
719
+ db.run("vacuum into ?", [path]);
720
+ await expect(Array.fromAsync(reader.read(target))).rejects.toThrow("identity mismatch");
721
+ expect(await reader.coverage(target)).toMatchObject({ status: "failed" });
722
+ });