@astrosheep/pi-context 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +10 -10
  2. package/dist/src/budget.js +63 -0
  3. package/dist/src/dream/apply.js +87 -0
  4. package/dist/src/dream/cli.js +82 -0
  5. package/dist/src/dream/gates.js +21 -0
  6. package/dist/src/dream/lock.js +58 -0
  7. package/dist/src/dream/manifest.js +16 -0
  8. package/dist/src/dream/runner.js +56 -0
  9. package/dist/src/history-tools.js +105 -0
  10. package/dist/src/history.js +210 -0
  11. package/dist/src/index.js +99 -0
  12. package/dist/src/memory/frontmatter.js +134 -0
  13. package/dist/src/memory/paths.js +54 -0
  14. package/dist/src/memory/store.js +297 -0
  15. package/dist/src/memory/tools.js +175 -0
  16. package/dist/src/notes.js +101 -0
  17. package/dist/src/prompts.js +79 -0
  18. package/dist/src/protocol.js +52 -0
  19. package/dist/src/reset-lifecycle.js +101 -0
  20. package/dist/src/session-reader.js +1 -0
  21. package/dist/src/thresholds.js +72 -0
  22. package/dist/src/tool-output.js +172 -0
  23. package/dist/src/tool-schema.js +26 -0
  24. package/dist/src/warning.js +44 -0
  25. package/dist/test/agent-loop.test.js +212 -0
  26. package/dist/test/coherence.test.js +371 -0
  27. package/dist/test/dream.test.js +43 -0
  28. package/dist/test/history.test.js +21 -0
  29. package/dist/test/integration.test.js +1716 -0
  30. package/dist/test/memory.test.js +370 -0
  31. package/dist/test/pagination.property.test.js +476 -0
  32. package/dist/test/reset-lifecycle.test.js +199 -0
  33. package/docs/reset-lifecycle.md +1 -1
  34. package/package.json +9 -3
  35. package/playbook.md +5 -0
  36. package/src/dream/apply.ts +47 -0
  37. package/src/dream/cli.ts +33 -0
  38. package/src/dream/gates.ts +19 -0
  39. package/src/dream/lock.ts +39 -0
  40. package/src/dream/manifest.ts +21 -0
  41. package/src/dream/runner.ts +53 -0
  42. package/src/history-tools.ts +6 -6
  43. package/src/history.ts +1 -1
  44. package/src/index.ts +2 -2
  45. package/src/memory/frontmatter.ts +153 -0
  46. package/src/memory/paths.ts +60 -0
  47. package/src/memory/store.ts +310 -0
  48. package/src/memory/tools.ts +175 -0
  49. package/src/prompts.ts +28 -17
  50. package/src/protocol.ts +7 -7
  51. package/src/note-tools.ts +0 -171
@@ -0,0 +1,212 @@
1
+ import assert from "node:assert/strict";
2
+ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import test from "node:test";
6
+ import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
7
+ import { createAgentSession, DefaultResourceLoader, ModelRuntime, SessionManager, SettingsManager } from "@earendil-works/pi-coding-agent";
8
+ import piContext from "../src/index.js";
9
+ import { WARNING_TYPE, GUIDANCE_TYPE } from "../src/protocol.js";
10
+ for (const mode of ["golden", "write-error", "ignored-warning", "explicit", "uncompactable", "followup", "steering", "repeat", "abort"]) {
11
+ test(`real Pi loop: ${mode} reset preserves history and handles completion`, { timeout: 15000 }, async () => {
12
+ const dir = mkdtempSync(join(tmpdir(), "pi-context-loop-"));
13
+ const previousDir = process.env.PI_CODING_AGENT_DIR;
14
+ const previousNotesRoot = process.env.PI_NOTES_HOME;
15
+ process.env.PI_CODING_AGENT_DIR = dir;
16
+ const notesRoot = mkdtempSync(join(tmpdir(), "pi-context-loop-notes-"));
17
+ process.env.PI_NOTES_HOME = notesRoot;
18
+ let session;
19
+ try {
20
+ const runtime = await ModelRuntime.create({ authPath: join(dir, "auth.json"), modelsPath: null, modelsStorePath: join(dir, "models"), refreshOnCreate: false });
21
+ await runtime.setRuntimeApiKey("openai", "scripted-test-key");
22
+ const base = runtime.getModels("openai")[0];
23
+ assert.ok(base);
24
+ const model = { ...base, contextWindow: 100000, maxTokens: 4096 };
25
+ const usageMode = mode === "golden" || mode === "write-error" || mode === "ignored-warning";
26
+ const expectedResets = mode === "abort" || mode === "uncompactable" ? 0 : 1;
27
+ const settings = { compaction: { enabled: usageMode, reserveTokens: 32768, keepRecentTokens: mode === "uncompactable" ? 1 : 200 }, retry: { enabled: false } };
28
+ writeFileSync(join(dir, "settings.json"), JSON.stringify(settings));
29
+ const settingsManager = SettingsManager.create(dir, dir);
30
+ let resets = 0;
31
+ let settled = 0;
32
+ let targetResets = expectedResets;
33
+ let activeSentinel = "OLD_CONTEXT_SENTINEL";
34
+ let finish;
35
+ let failFinish;
36
+ const finished = new Promise((resolve, reject) => { finish = resolve; failFinish = reject; });
37
+ const finishTimeout = setTimeout(() => failFinish(new Error(`timed out waiting for ${mode} agent settlement`)), 5000);
38
+ const loader = new DefaultResourceLoader({ cwd: dir, agentDir: dir, settingsManager,
39
+ noExtensions: true, noSkills: true, noThemes: true, noPromptTemplates: true,
40
+ systemPromptOverride: () => "Use the tools as requested.", agentsFilesOverride: () => ({ agentsFiles: [] }),
41
+ extensionFactories: [piContext, (pi) => {
42
+ pi.on("session_compact", () => { resets++; });
43
+ pi.on("agent_settled", () => {
44
+ settled++;
45
+ if (mode === "abort") {
46
+ if (settled === 1)
47
+ finish();
48
+ return;
49
+ }
50
+ if (resets >= targetResets && requests.length > 0 && !requests.at(-1).includes(activeSentinel))
51
+ finish();
52
+ });
53
+ pi.on("tool_result", () => {
54
+ if (mode === "abort")
55
+ void session.abort();
56
+ });
57
+ pi.on("tool_call", async () => {
58
+ if (mode === "followup")
59
+ await session.followUp("QUEUED_INPUT_SENTINEL");
60
+ if (mode === "steering")
61
+ await session.steer("QUEUED_INPUT_SENTINEL");
62
+ });
63
+ }],
64
+ });
65
+ await loader.reload();
66
+ const sm = SessionManager.inMemory(dir);
67
+ sm.appendMessage({ role: "user", content: "Earlier work to retain in durable history.", timestamp: Date.now() });
68
+ sm.appendMessage({ role: "assistant", api: model.api, provider: model.provider, model: model.id,
69
+ content: [{ type: "text", text: "Earlier result. ".repeat(100) }], stopReason: "stop", timestamp: Date.now(),
70
+ usage: { input: 100, output: 100, cacheRead: 0, cacheWrite: 0, totalTokens: 200, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } });
71
+ ({ session } = await createAgentSession({ cwd: dir, agentDir: dir, modelRuntime: runtime, model, settingsManager, sessionManager: sm, resourceLoader: loader, tools: ["new_context", "notes_write", "get_context_remaining"] }));
72
+ const requests = [];
73
+ let checkpointed = false;
74
+ let freshTurns = 0;
75
+ session.agent.streamFunction = (_model, context) => {
76
+ requests.push(JSON.stringify(context.messages));
77
+ const n = requests.length;
78
+ const request = requests[n - 1];
79
+ const fresh = !request.includes("OLD_CONTEXT_SENTINEL");
80
+ if (fresh)
81
+ freshTurns++;
82
+ const sawWarning = request.includes("Your memory is about to be erased");
83
+ const sawGuidance = request.includes("Your brain is almost out of room");
84
+ const explicitReset = (n === 1 && !usageMode && mode !== "uncompactable") || (mode === "repeat" && (n === 1 || n === 3));
85
+ const checkpoint = usageMode && sawWarning && !checkpointed && mode !== "ignored-warning";
86
+ if (checkpoint)
87
+ checkpointed = true;
88
+ // The warning is chosen from the previous turn's usage, so a scripted run has to
89
+ // keep taking turns until it sees the warning (first window) or the next
90
+ // window's reminder. "ignored-warning" keeps working instead of checkpointing.
91
+ const probe = usageMode && !checkpoint && ((!fresh && !sawWarning) || (mode === "ignored-warning" && sawWarning) || (fresh && freshTurns === 2 && !sawGuidance));
92
+ const tokens = usageMode ? (fresh ? (freshTurns === 1 ? 100 : 50000) : sawWarning ? 70000 : n === 1 ? 50000 : 60000) : 100;
93
+ const tool = explicitReset || (mode === "uncompactable" && n === 1);
94
+ const call = probe ? "get_context_remaining" : checkpoint ? "notes_write" : tool ? "new_context" : undefined;
95
+ const message = { role: "assistant", api: model.api, provider: model.provider, model: model.id,
96
+ content: probe ? [{ type: "toolCall", id: "probe-call", name: "get_context_remaining", arguments: {} }]
97
+ : checkpoint ? [{ type: "toolCall", id: "checkpoint-call", name: "notes_write", arguments: { path: mode === "write-error" ? "../invalid.md" : "checkpoint.md", content: "CHECKPOINT_SENTINEL" } }]
98
+ : tool ? [{ type: "toolCall", id: "reset-call", name: "new_context", arguments: {} }]
99
+ : [{ type: "text", text: fresh ? "Resumed." : "Working." }],
100
+ stopReason: call ? "toolUse" : "stop", timestamp: Date.now(),
101
+ usage: { input: tokens, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: tokens + 1, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
102
+ };
103
+ const stream = createAssistantMessageEventStream();
104
+ stream.push({ type: "done", reason: message.stopReason, message });
105
+ stream.end();
106
+ return stream;
107
+ };
108
+ await session.bindExtensions({});
109
+ let failure;
110
+ session.subscribe((event) => {
111
+ if (mode === "uncompactable" && event.type === "compaction_end" && event.errorMessage) {
112
+ failure = event.errorMessage;
113
+ finish();
114
+ }
115
+ });
116
+ await session.prompt("OLD_CONTEXT_SENTINEL: save progress and continue the task.");
117
+ await finished;
118
+ clearTimeout(finishTimeout);
119
+ await session.waitForIdle();
120
+ if (mode === "abort") {
121
+ assert.equal(resets, 0, "user cancellation clears pending rollover");
122
+ assert.equal(requests.length, 1, "no continuation resurrects the cancelled run");
123
+ await session.prompt("Resume explicitly after cancellation.");
124
+ assert.equal(requests.length, 2);
125
+ assert.ok(requests[1].includes("Resume explicitly after cancellation."));
126
+ assert.ok(requests[1].includes("OLD_CONTEXT_SENTINEL"), "no reset happened: history is intact");
127
+ return;
128
+ }
129
+ if (mode === "uncompactable") {
130
+ assert.match(failure ?? "", /Nothing to compact/);
131
+ assert.equal(resets, 0);
132
+ assert.equal(requests.length, 1, "failed reset does not loop or resume automatically");
133
+ await session.prompt("Continue after the failed reset.");
134
+ assert.equal(requests.length, 2);
135
+ assert.ok(requests[1].includes("OLD_CONTEXT_SENTINEL"));
136
+ return;
137
+ }
138
+ assert.equal(resets, expectedResets);
139
+ if (mode === "golden" || mode === "write-error" || mode === "ignored-warning") {
140
+ const branch = sm.getBranch();
141
+ const guidanceIndices = branch.flatMap((entry, i) => entry.type === "custom_message" && entry.customType === GUIDANCE_TYPE ? [i] : []);
142
+ const warningIndices = branch.flatMap((entry, i) => entry.type === "custom_message" && entry.customType === WARNING_TYPE ? [i] : []);
143
+ const noteFile = join(notesRoot, "pi", "session", sm.getSessionId(), "checkpoint.md");
144
+ const resetIndex = branch.findIndex((entry) => entry.type === "compaction");
145
+ assert.equal(guidanceIndices.length, 1, "one early reminder");
146
+ assert.equal(warningIndices.length, 1, "one final warning steer");
147
+ assert.ok(warningIndices[0] > guidanceIndices[0], "the reminder precedes the warning");
148
+ if (mode === "ignored-warning") {
149
+ assert.equal(existsSync(noteFile), false, "an ignored warning leaves no checkpoint");
150
+ }
151
+ else if (mode === "write-error") {
152
+ assert.equal(existsSync(noteFile), false, "failed write creates no checkpoint");
153
+ assert.ok(branch.some((entry) => entry.type === "message" && entry.message.role === "toolResult" && entry.message.toolName === "notes_write" && entry.message.isError));
154
+ assert.ok(!requests.at(-1).includes("CHECKPOINT_SENTINEL"), "fresh context must not invent a saved note");
155
+ }
156
+ else {
157
+ assert.ok(existsSync(noteFile), "the checkpoint is a real file on disk");
158
+ assert.ok(resetIndex > warningIndices[0], "the warning precedes the wipe");
159
+ assert.ok(requests.at(-1).includes("CHECKPOINT_SENTINEL"), "fresh boot carries the saved checkpoint");
160
+ }
161
+ assert.ok(!requests.at(-1).includes("Your brain is almost out of room"), "new window excludes old guidance");
162
+ }
163
+ if (mode === "followup" || mode === "steering") {
164
+ assert.ok(requests[1].includes("QUEUED_INPUT_SENTINEL"), "queued user work is delivered before rollover");
165
+ assert.ok(requests[1].includes("OLD_CONTEXT_SENTINEL"), "queue drains in the existing window");
166
+ assert.ok(!requests[2].includes("QUEUED_INPUT_SENTINEL"), "queue is not replayed after rollover");
167
+ const queuedEntries = sm.getBranch().filter((entry) => entry.type === "message" && JSON.stringify(entry.message).includes("QUEUED_INPUT_SENTINEL"));
168
+ assert.equal(queuedEntries.length, 1, "one durable user input");
169
+ }
170
+ if (mode === "repeat") {
171
+ const nextFinished = new Promise((resolve) => { finish = resolve; });
172
+ targetResets = 2;
173
+ activeSentinel = "SECOND_WINDOW_SENTINEL";
174
+ await session.prompt("SECOND_WINDOW_SENTINEL: " + "Additional work. ".repeat(100));
175
+ await nextFinished;
176
+ await session.waitForIdle();
177
+ assert.equal(resets, 2);
178
+ assert.ok(!requests.at(-1).includes("SECOND_WINDOW_SENTINEL"));
179
+ const boundaries = sm.getBranch().filter((entry) => entry.type === "compaction");
180
+ assert.equal(new Set(boundaries.map((entry) => JSON.stringify(entry.details))).size, 2);
181
+ }
182
+ assert.ok(requests[0].includes("OLD_CONTEXT_SENTINEL"));
183
+ assert.ok(!requests.at(-1).includes("OLD_CONTEXT_SENTINEL"));
184
+ assert.ok(requests.at(-1).includes("context_window"));
185
+ assert.ok(JSON.stringify(session.sessionManager.getBranch()).includes("OLD_CONTEXT_SENTINEL"));
186
+ if (mode === "golden") {
187
+ await session.prompt("Keep working in the new window until its reminder threshold.");
188
+ assert.equal(resets, 1);
189
+ const branch = sm.getBranch();
190
+ const boundary = branch.findIndex((entry) => entry.type === "compaction");
191
+ const reminders = branch.flatMap((entry, i) => entry.type === "custom_message" && entry.customType === GUIDANCE_TYPE ? [i] : []);
192
+ assert.equal(reminders.length, 2, "the next window gets its own reminder");
193
+ assert.ok(reminders[1] > boundary, "old messages cannot suppress a new window's reminder");
194
+ const warnings = branch.flatMap((entry, i) => entry.type === "custom_message" && entry.customType === WARNING_TYPE ? [i] : []);
195
+ assert.equal(warnings.length, 1, "the next window has no warning yet");
196
+ }
197
+ }
198
+ finally {
199
+ session?.dispose();
200
+ if (previousDir === undefined)
201
+ delete process.env.PI_CODING_AGENT_DIR;
202
+ else
203
+ process.env.PI_CODING_AGENT_DIR = previousDir;
204
+ if (previousNotesRoot === undefined)
205
+ delete process.env.PI_NOTES_HOME;
206
+ else
207
+ process.env.PI_NOTES_HOME = previousNotesRoot;
208
+ rmSync(dir, { recursive: true, force: true });
209
+ rmSync(notesRoot, { recursive: true, force: true });
210
+ }
211
+ });
212
+ }
@@ -0,0 +1,371 @@
1
+ /**
2
+ * OWNER: pi-context (adopted).
3
+ * STATUS: tracked acceptance spec for the history and notes read/search tools. No skip.
4
+ * CLAIM: following the cursors these tools return must reconstruct the original text exactly,
5
+ * or the result must name the skipped range. Both read tools are one character window over two
6
+ * stores (notes_read and history_read share the cursor walk below). In v2 this failed
7
+ * at 13 sites; the rows that demanded an over-budget line in a single call are rebuilt as
8
+ * cursor-walking rows below (the CLAIM explicitly licenses that: reconstruct exactly by
9
+ * following cursors, or name the skipped range).
10
+ * HERMETIC: this file reads only its own in-memory session. Corpus replays of real sessions
11
+ * are NOT hermetic, must be single-pass, and belong in a dev script, not npm test.
12
+ */
13
+ import assert from "node:assert/strict";
14
+ import { mkdtempSync } from "node:fs";
15
+ import { tmpdir } from "node:os";
16
+ import { join } from "node:path";
17
+ import test from "node:test";
18
+ import { SessionManager } from "@earendil-works/pi-coding-agent";
19
+ import piContext, { historyFromSession } from "../src/index.js";
20
+ import { TOOL_OUTPUT_MAX_BYTES } from "../src/tool-output.js";
21
+ import { stripLeadingFrontmatter } from "../src/memory/frontmatter.js";
22
+ process.env.PI_CODING_AGENT_DIR = mkdtempSync(join(tmpdir(), "pc-coherence-agent-"));
23
+ process.env.PI_NOTES_HOME = mkdtempSync(join(tmpdir(), "pc-coherence-notes-"));
24
+ function makeExtension(sessionManager) {
25
+ const captured = { tools: new Map() };
26
+ const api = {
27
+ registerFlag() { },
28
+ registerTool(tool) { captured.tools.set(tool.name, tool); },
29
+ registerCommand() { },
30
+ on() { },
31
+ appendEntry(customType, data) { sessionManager.appendCustomEntry(customType, data); },
32
+ sendMessage(message) {
33
+ sessionManager.appendCustomMessageEntry(message.customType, message.content, message.display, message.details);
34
+ },
35
+ };
36
+ piContext(api);
37
+ return captured;
38
+ }
39
+ function context(sessionManager) {
40
+ const fake = {
41
+ sessionManager,
42
+ getContextUsage: () => undefined,
43
+ compact: () => { },
44
+ isIdle: () => true,
45
+ hasPendingMessages: () => false,
46
+ cwd: "/private/tmp/pi-context-test",
47
+ isProjectTrusted: () => true,
48
+ ui: { notify: () => { } },
49
+ };
50
+ return fake;
51
+ }
52
+ async function call(captured, name, params, ctx) {
53
+ const tool = captured.tools.get(name);
54
+ assert.ok(tool, `registered ${name}`);
55
+ return tool.execute("call-1", params, new AbortController().signal, () => { }, ctx);
56
+ }
57
+ function resultJson(result) {
58
+ const text = result.content[0];
59
+ assert.ok(text && text.type === "text");
60
+ return JSON.parse(text.text);
61
+ }
62
+ function assertWithinBudget(result, label) {
63
+ const bytes = Buffer.byteLength(result.content[0] && result.content[0].type === "text" ? result.content[0].text : "", "utf8");
64
+ if (bytes > TOOL_OUTPUT_MAX_BYTES)
65
+ failures.push(`${label}: response is ${bytes} bytes, over the ${TOOL_OUTPUT_MAX_BYTES}-byte budget`);
66
+ }
67
+ function appendText(sessionManager, text) {
68
+ return sessionManager.appendMessage({ role: "user", content: [{ type: "text", text }], timestamp: Date.now() });
69
+ }
70
+ /**
71
+ * Decode a raw read (notes_read / history_read): a one-line bracketed header, then
72
+ * the payload verbatim (which may itself contain newlines), so split on the first newline only.
73
+ */
74
+ function resultRead(result) {
75
+ const text = result.content[0];
76
+ assert.ok(text && text.type === "text", "read result carries text");
77
+ const newline = text.text.indexOf("\n");
78
+ assert.ok(newline !== -1, "raw read carries a header line and a payload");
79
+ const header = text.text.slice(0, newline);
80
+ const content = text.text.slice(newline + 1);
81
+ assert.match(header, /^\[/, "the header is bracketed");
82
+ assert.match(header, /\]$/, "the header closes its bracket");
83
+ const match = header.match(/ · chars (\d+)-(\d+) of (\d+) · (end|continue at offset_chars=(\d+))/);
84
+ assert.ok(match, `read header names the char range and resume cursor: ${header}`);
85
+ const offset_chars = Number(match[1]);
86
+ const total_chars = Number(match[3]);
87
+ const next_offset_chars = match[4] === "end" ? null : Number(match[5]);
88
+ assert.equal([...content].length, Number(match[2]) - offset_chars, "the header range matches the delivered payload");
89
+ return { header, content, offset_chars, total_chars, next_offset_chars };
90
+ }
91
+ const PROFILES = [
92
+ { name: "cjk", unit: "历" },
93
+ { name: "emoji", unit: "🕷" },
94
+ { name: "ascii", unit: "x" },
95
+ ];
96
+ const failures = [];
97
+ const report = [];
98
+ const codePoints = (text) => [...text].length;
99
+ const codePointSlice = (text, start, end) => [...text].slice(start, end).join("");
100
+ /**
101
+ * Follow either read tool's cursor exactly as the protocol tells the model to, asserting the
102
+ * cursor law on every page. Both tools are the same character window over two stores, so one
103
+ * walker serves both; `address` carries the tool's own identity parameters, and `options.start`
104
+ * lets a walk begin at a resolved address (a search hit's offset, or a negative tail read).
105
+ */
106
+ async function walkWindow(captured, ctx, tool, address, label, options = {}) {
107
+ const parts = [];
108
+ let offset = options.start ?? 0;
109
+ let next = 0;
110
+ let calls = 0;
111
+ let total = 0;
112
+ while (next !== null && calls < 400) {
113
+ const params = { ...address, offset_chars: offset };
114
+ if (options.limitChars !== undefined)
115
+ params.limit_chars = options.limitChars;
116
+ const result = await call(captured, tool, params, ctx);
117
+ assertWithinBudget(result, `${label} offset=${offset}`);
118
+ const page = resultRead(result);
119
+ total = page.total_chars;
120
+ const delivered = codePoints(page.content);
121
+ if (page.offset_chars !== offset)
122
+ failures.push(`cursor law: ${label} echoed offset_chars=${page.offset_chars} for request offset ${offset}`);
123
+ if (page.next_offset_chars !== null && page.next_offset_chars !== offset + delivered) {
124
+ failures.push(`cursor law: ${label} next_offset_chars=${page.next_offset_chars} but offset ${offset} + delivered ${delivered}`);
125
+ }
126
+ if (page.next_offset_chars === null && offset + delivered !== page.total_chars) {
127
+ failures.push(`false exhaustion: ${label} returned next_offset_chars=null with ${page.total_chars - (offset + delivered)} characters undelivered`);
128
+ }
129
+ if (page.content.includes("…") || page.content.includes("[truncated"))
130
+ failures.push(`honest payload: ${label} appended a marker at offset ${offset}`);
131
+ parts.push(page.content);
132
+ next = page.next_offset_chars;
133
+ if (next !== null)
134
+ offset = next;
135
+ calls++;
136
+ }
137
+ if (offset !== total && calls >= 400)
138
+ failures.push(`${label} never terminated`);
139
+ return parts.join("");
140
+ }
141
+ const walkHistory = (captured, ctx, windowId, itemId, limitChars) => walkWindow(captured, ctx, "history_read", { window_id: windowId, item_id: itemId }, `history_read ${itemId}`, { limitChars });
142
+ const walkNote = (captured, ctx, path, start = 0) => walkWindow(captured, ctx, "notes_read", { path }, `notes_read ${path}`, { start });
143
+ test("coherence: following the returned cursors reconstructs the original text exactly", async () => {
144
+ const session = SessionManager.inMemory("/private/tmp/pi-context-test");
145
+ const captured = makeExtension(session);
146
+ const ctx = context(session);
147
+ const windowId = historyFromSession(ctx)[0].windowId;
148
+ // --- history_read: every profile x length reconstructs; cursor law holds per page ---
149
+ for (const profile of PROFILES) {
150
+ for (const length of [12_000, 12_001, 20_000, 30_000]) {
151
+ const original = profile.unit.repeat(length);
152
+ const itemId = appendText(session, original);
153
+ const reconstructed = await walkHistory(captured, ctx, windowId, itemId);
154
+ const missing = codePoints(original) - codePoints(reconstructed);
155
+ const line = `history_read default: ${profile.name} ${length} chars (${Buffer.byteLength(original, "utf8")} bytes) -> delivered ${codePoints(reconstructed)} chars, missing ${missing}, marker=${reconstructed.includes("[truncated")}`;
156
+ report.push(line);
157
+ if (reconstructed !== original)
158
+ failures.push(line);
159
+ session.appendMessage({ role: "assistant", content: [{ type: "text", text: `ack ${length}` }], timestamp: Date.now() });
160
+ }
161
+ }
162
+ // --- notes_read: a 40,000-code-point single line plus a tail, three profiles ---
163
+ for (const profile of PROFILES) {
164
+ const hugeLine = profile.unit.repeat(40_000);
165
+ const text = `${hugeLine}\ntail line`;
166
+ const path = `huge-${profile.name}.md`;
167
+ await call(captured, "notes_write", { path, content: text }, ctx);
168
+ const reconstructed = stripLeadingFrontmatter(await walkNote(captured, ctx, path));
169
+ const missing = codePoints(text) - codePoints(reconstructed);
170
+ const line = `notes_read: ${profile.name} single line ${codePoints(hugeLine)} chars (${Buffer.byteLength(hugeLine, "utf8")} bytes) -> delivered ${codePoints(reconstructed)} chars, missing ${missing}, marker=${reconstructed.includes("[truncated")}`;
171
+ report.push(line);
172
+ if (reconstructed !== text)
173
+ failures.push(line);
174
+ }
175
+ // --- The terminator itself can lie. A single line with no trailing newline is exactly what
176
+ // notes_write { content } produces; the returned cursor must not be null while text remains.
177
+ for (const profile of PROFILES) {
178
+ const hugeLine = profile.unit.repeat(40_000);
179
+ const path = `solo-${profile.name}.md`;
180
+ await call(captured, "notes_write", { path, content: hugeLine }, ctx);
181
+ const first = resultRead(await call(captured, "notes_read", { path }, ctx));
182
+ const deliveredBody = first.content.startsWith("---\n") ? stripLeadingFrontmatter(first.content) : first.content;
183
+ const undelivered = codePoints(hugeLine) - codePoints(deliveredBody);
184
+ report.push(`notes_read no trailing newline: ${profile.name} ${codePoints(hugeLine)} chars -> first page delivered ${codePoints(deliveredBody)} body chars, undelivered ${undelivered}, offset_chars=${first.offset_chars}, next_offset_chars=${String(first.next_offset_chars)}, marker=${first.content.includes("[truncated")}`);
185
+ if (first.offset_chars !== 0)
186
+ failures.push(`window echo: ${profile.name} first page echoed offset_chars=${first.offset_chars}, expected 0`);
187
+ if (first.total_chars < codePoints(hugeLine))
188
+ failures.push(`window total: ${profile.name} total_chars=${first.total_chars}, expected at least the body length`);
189
+ if (!first.content.startsWith("---\n"))
190
+ failures.push(`frontmatter first: ${profile.name} first page does not open with frontmatter`);
191
+ if (first.next_offset_chars === null && undelivered > 0)
192
+ failures.push(`false exhaustion: ${profile.name} returns next_offset_chars=null while ${undelivered} characters were never delivered`);
193
+ if (!hugeLine.startsWith(deliveredBody))
194
+ failures.push(`prefix law: ${profile.name} first page body is not a prefix of the line`);
195
+ const reconstructed = stripLeadingFrontmatter(await walkNote(captured, ctx, path));
196
+ if (reconstructed !== hugeLine)
197
+ failures.push(`notes_read single line is not reconstructible: ${profile.name}, ${codePoints(hugeLine) - codePoints(reconstructed)} chars missing`);
198
+ }
199
+ // --- The empty note must terminate: no self-feeding cursor, and a frontmatter-only window from 0.
200
+ await call(captured, "notes_write", { path: "empty.md", content: "" }, ctx);
201
+ const empty = resultRead(await call(captured, "notes_read", { path: "empty.md" }, ctx));
202
+ report.push(`notes_read empty note: offset_chars=${empty.offset_chars} total_chars=${empty.total_chars} next_offset_chars=${String(empty.next_offset_chars)} content=${JSON.stringify(empty.content)}`);
203
+ if (empty.offset_chars !== 0 || !empty.content.startsWith("---\n") || !empty.content.endsWith("---\n\n"))
204
+ failures.push("the empty note is not a frontmatter-only window from 0");
205
+ if (empty.next_offset_chars !== null)
206
+ failures.push("pagination hole: the empty note is never exhausted");
207
+ // --- notes_search: an over-budget matched line is named, then read back with cursors ---
208
+ const hugeCjkLine = "历".repeat(40_000);
209
+ const searched = resultJson(await call(captured, "notes_search", { query: "历", pattern: "huge-cjk.md" }, ctx));
210
+ const matchedFile = searched.files[0];
211
+ const matched = matchedFile?.matches[0];
212
+ report.push(`notes_search: matches_total=${String(matchedFile?.matches_total)} returned=${String(matchedFile?.matches.length)} first match delivered ${codePoints(matched?.text ?? "")} of ${String(matched?.total_chars)} chars, truncated=${String(matched?.truncated)}`);
213
+ if (!matched)
214
+ failures.push("notes_search dropped the over-budget matched line entirely");
215
+ else {
216
+ if (matched.truncated !== true)
217
+ failures.push("notes_search does not flag the over-budget matched line as truncated");
218
+ if (matched.total_chars !== codePoints(hugeCjkLine))
219
+ failures.push(`notes_search match total_chars=${matched.total_chars}, expected ${codePoints(hugeCjkLine)}`);
220
+ if (!hugeCjkLine.startsWith(matched.text))
221
+ failures.push("notes_search delivered a non-prefix of the matched line");
222
+ if (codePoints(matched.text) >= matched.total_chars)
223
+ failures.push("notes_search claims the over-budget line fits in one response");
224
+ const walked = stripLeadingFrontmatter(await walkNote(captured, ctx, "huge-cjk.md"));
225
+ if (walked !== `${hugeCjkLine}\ntail line`)
226
+ failures.push(`notes_search match line is not reconstructible from the note read: missing ${codePoints(`${hugeCjkLine}\ntail line`) - codePoints(walked)} chars`);
227
+ }
228
+ // --- history_search: a hit's visible text may be cut, but the offset it carries
229
+ // must resolve to the query through history_read (addresses-only mode).
230
+ const searchItemContent = `${"padding ".repeat(400)}历史内容${" trailing".repeat(400)}`;
231
+ const searchItemId = appendText(session, searchItemContent);
232
+ const hit = resultJson(await call(captured, "history_search", { query: "历史内容", max_chars_per_item: 400, window_id: windowId }, ctx));
233
+ const first = hit.items.find((item) => item.item_id === searchItemId);
234
+ report.push(`history_search: hit present=${Boolean(first)}, fields=${JSON.stringify(Object.keys(first ?? {}))}, match_offset_chars=${String(first?.match_offset_chars)}, truncated=${String(first?.truncated)}`);
235
+ if (!first)
236
+ failures.push("history_search did not return the matching item");
237
+ else {
238
+ if (first.truncated !== true)
239
+ failures.push("history_search does not flag the capped item as truncated");
240
+ if (first.total_chars !== codePoints(searchItemContent))
241
+ failures.push(`history_search total_chars=${first.total_chars}, expected ${codePoints(searchItemContent)}`);
242
+ if (!searchItemContent.startsWith(first.truncated_content))
243
+ failures.push("history_search delivered a non-prefix of the item");
244
+ if (first.truncated_content.includes("…"))
245
+ failures.push("history_search appended a marker to the payload");
246
+ if (!Number.isInteger(first.match_offset_chars))
247
+ failures.push("history_search carries no match_offset_chars");
248
+ else {
249
+ const at = resultRead(await call(captured, "history_read", { window_id: windowId, item_id: searchItemId, offset_chars: first.match_offset_chars, limit_chars: 8 }, ctx));
250
+ if (!at.content.includes("历史内容"))
251
+ failures.push(`history_read at match_offset_chars=${first.match_offset_chars} does not show the query`);
252
+ }
253
+ }
254
+ // max_chars_per_item: 1 is a real address page for both history tools.
255
+ const addresses = resultJson(await call(captured, "history_search", { query: "历史内容", max_chars_per_item: 1, window_id: windowId }, ctx));
256
+ const address = addresses.items.find((item) => item.item_id === searchItemId);
257
+ report.push(`history_search max_chars_per_item=1: address=${JSON.stringify(address)}`);
258
+ if (!address)
259
+ failures.push("history_search max_chars_per_item=1 dropped the hit");
260
+ else {
261
+ if (codePoints(address.truncated_content) !== 1)
262
+ failures.push(`max_chars_per_item=1 delivered ${codePoints(address.truncated_content)} code points`);
263
+ if (address.truncated !== true || address.total_chars !== codePoints(searchItemContent))
264
+ failures.push("max_chars_per_item=1 does not name the full length");
265
+ if (!Number.isInteger(address.match_offset_chars))
266
+ failures.push("max_chars_per_item=1 carries no address");
267
+ }
268
+ const listed = resultJson(await call(captured, "history_list", { window_id: windowId, max_chars_per_item: 1, recent_first: false, limit: 500 }, ctx));
269
+ const listedAddress = listed.items.find((item) => item.item_id === searchItemId);
270
+ report.push(`history_list max_chars_per_item=1: ${JSON.stringify(listedAddress)}`);
271
+ if (!listedAddress)
272
+ failures.push("history_list max_chars_per_item=1 dropped the item");
273
+ else if (codePoints(listedAddress.truncated_content) !== 1 || listedAddress.truncated !== true || listedAddress.total_chars !== codePoints(searchItemContent)) {
274
+ failures.push("history_list max_chars_per_item=1 is not an honest address page");
275
+ }
276
+ // --- brain-04: capping a file's matches to fit the wire budget must be named, never silent.
277
+ const manyLines = Array.from({ length: 8_000 }, (_, index) => `needle ${index}`);
278
+ await call(captured, "notes_write", { path: "many.md", content: manyLines.join("\n") }, ctx);
279
+ const many = resultJson(await call(captured, "notes_search", { query: "needle", pattern: "many.md" }, ctx));
280
+ const manyEntry = many.files[0];
281
+ report.push(`brain-04: matches_total=${String(manyEntry?.matches_total)} returned=${String(manyEntry?.matches.length)}, next_cursor=${String(many.next_cursor)}`);
282
+ if (!manyEntry)
283
+ failures.push("brain-04: the many-match file is absent from the search result");
284
+ else {
285
+ if (manyEntry.matches_total !== manyLines.length)
286
+ failures.push(`brain-04: matches_total=${manyEntry.matches_total}, expected ${manyLines.length}`);
287
+ if (!(manyEntry.matches_total > manyEntry.matches.length))
288
+ failures.push("brain-04: budget-capped matches were silently dropped (matches_total === matches.length)");
289
+ if (manyEntry.matches_total - manyEntry.matches.length <= 0)
290
+ failures.push("brain-04: the response names no dropped matches");
291
+ }
292
+ // A file whose first match alone is over budget with more matches behind it: dropping trailing
293
+ // matches and cutting the kept line must still leave the whole response inside the wire budget.
294
+ const manyHugeLines = Array.from({ length: 4 }, (_, index) => `needle ${index} ${"w".repeat(45_000)}`);
295
+ await call(captured, "notes_write", { path: "huge-many.md", content: manyHugeLines.join("\n") }, ctx);
296
+ const hugeManyResult = await call(captured, "notes_search", { query: "needle", pattern: "huge-many.md" }, ctx);
297
+ assertWithinBudget(hugeManyResult, "notes_search huge-many");
298
+ const hugeEntry = resultJson(hugeManyResult).files[0];
299
+ report.push(`huge-many: matches_total=${String(hugeEntry?.matches_total)} returned=${String(hugeEntry?.matches.length)} truncated=${String(hugeEntry?.matches[0]?.truncated)}`);
300
+ if (!hugeEntry)
301
+ failures.push("huge-many: the many-huge-match file is absent from the search result");
302
+ else {
303
+ if (!(hugeEntry.matches_total > hugeEntry.matches.length))
304
+ failures.push("huge-many: dropped matches are not named");
305
+ if (hugeEntry.matches[0]?.truncated !== true)
306
+ failures.push("huge-many: the kept match is not flagged as a prefix");
307
+ }
308
+ // --- notes_search addresses: a match's offset_chars is the body-absolute code-point
309
+ // position of the earliest query occurrence in its line, so search → read composes exactly like
310
+ // history's match_offset_chars two-stage.
311
+ const addressLine1 = "pad ".repeat(50);
312
+ const addressLine3 = `${"历".repeat(20)}needle-address here`;
313
+ const addressLine4 = "zeta 历 needle-address";
314
+ await call(captured, "notes_write", { path: "address.md", content: `${addressLine1}\nsecond\n${addressLine3}\n${addressLine4}` }, ctx);
315
+ const expectedAddress = codePoints(addressLine1) + 1 + codePoints("second") + 1 + 20;
316
+ const addressHit = resultJson(await call(captured, "notes_search", { query: "needle-address", pattern: "address.md" }, ctx)).files[0]?.matches.find((match) => match.line === 3);
317
+ report.push(`notes_search address: line=${String(addressHit?.line)} offset_chars=${String(addressHit?.offset_chars)} expected=${expectedAddress}`);
318
+ const addressOffset = addressHit?.offset_chars;
319
+ if (typeof addressOffset !== "number")
320
+ failures.push("notes_search carries no offset_chars");
321
+ else if (addressOffset !== expectedAddress)
322
+ failures.push(`notes_search offset_chars=${addressOffset}, expected ${expectedAddress} (body-absolute, at the query)`);
323
+ // Multi-query OR: a line's address is the earliest occurrence of any query inside that line.
324
+ const line4Base = expectedAddress - 20 + codePoints(addressLine3) + 1;
325
+ const orLine4 = resultJson(await call(captured, "notes_search", { query: ["needle-address", "zeta"], pattern: "address.md" }, ctx)).files[0]?.matches.find((match) => match.line === 4);
326
+ report.push(`notes_search OR address: offset_chars=${String(orLine4?.offset_chars)} expected=${line4Base}`);
327
+ if (orLine4?.offset_chars !== line4Base)
328
+ failures.push(`notes_search OR offset_chars=${String(orLine4?.offset_chars)}, expected ${line4Base} (earliest of any query)`);
329
+ // --- Negative offsets on both stores: a tail read reaches the end in one call, the response
330
+ // echoes the resolved absolute offset, N >= total_chars reads from the start, and the cursor
331
+ // law still holds when a negative-start page is cut short.
332
+ for (const profile of PROFILES) {
333
+ const tailText = `head${profile.unit.repeat(20)}TAIL${profile.unit.repeat(20)}`;
334
+ const path = `tail-${profile.name}.md`;
335
+ await call(captured, "notes_write", { path, content: tailText }, ctx);
336
+ const full = resultRead(await call(captured, "notes_read", { path }, ctx));
337
+ const total = full.total_chars;
338
+ const bodyChars = codePoints(tailText);
339
+ const tail = resultRead(await call(captured, "notes_read", { path, offset_chars: -10 }, ctx));
340
+ report.push(`notes_read negative offset: ${profile.name} total=${total} -> offset_chars=${tail.offset_chars} next=${String(tail.next_offset_chars)} content=${JSON.stringify(tail.content)}`);
341
+ if (tail.offset_chars !== total - 10)
342
+ failures.push(`negative offset: notes_read ${profile.name} echoed ${tail.offset_chars}, expected ${total - 10}`);
343
+ if (tail.content !== codePointSlice(tailText, bodyChars - 10))
344
+ failures.push(`negative offset: notes_read ${profile.name} did not reach the body tail in one call`);
345
+ if (tail.next_offset_chars !== null)
346
+ failures.push(`negative offset: notes_read ${profile.name} tail read is not exhausted`);
347
+ const fromStart = resultRead(await call(captured, "notes_read", { path, offset_chars: -(total + 5), limit_chars: 8 }, ctx));
348
+ if (fromStart.offset_chars !== 0)
349
+ failures.push(`negative offset: notes_read ${profile.name} with N >= total_chars echoed ${fromStart.offset_chars}, expected 0`);
350
+ if (fromStart.content !== codePointSlice(full.content, 0, 8))
351
+ failures.push(`negative offset: notes_read ${profile.name} with N >= total_chars did not read from the start`);
352
+ const cut = resultRead(await call(captured, "notes_read", { path, offset_chars: -15, limit_chars: 4 }, ctx));
353
+ if (cut.next_offset_chars !== cut.offset_chars + codePoints(cut.content))
354
+ failures.push(`negative offset: notes_read ${profile.name} cut a negative-start read off the cursor law`);
355
+ const resumed = resultRead(await call(captured, "notes_read", { path, offset_chars: cut.next_offset_chars }, ctx));
356
+ if (resumed.offset_chars !== cut.next_offset_chars)
357
+ failures.push(`negative offset: notes_read ${profile.name} resume echoed ${resumed.offset_chars}, expected ${String(cut.next_offset_chars)}`);
358
+ }
359
+ // history_read gains the identical sugar over a durable item.
360
+ const historyTailText = `${"h".repeat(50)}END`;
361
+ const historyTailId = appendText(session, historyTailText);
362
+ const historyTail = resultRead(await call(captured, "history_read", { window_id: windowId, item_id: historyTailId, offset_chars: -3 }, ctx));
363
+ report.push(`history_read negative offset: offset_chars=${historyTail.offset_chars} next=${String(historyTail.next_offset_chars)} content=${JSON.stringify(historyTail.content)}`);
364
+ if (historyTail.offset_chars !== 50 || historyTail.content !== "END" || historyTail.next_offset_chars !== null)
365
+ failures.push(`negative offset: history_read returned ${JSON.stringify(historyTail)}`);
366
+ const historyFromStart = resultRead(await call(captured, "history_read", { window_id: windowId, item_id: historyTailId, offset_chars: -500, limit_chars: 4 }, ctx));
367
+ if (historyFromStart.offset_chars !== 0 || historyFromStart.content !== "hhhh")
368
+ failures.push(`negative offset: history_read with N >= total_chars returned ${JSON.stringify(historyFromStart)}`);
369
+ console.log(report.map((line) => ` ${line}`).join("\n"));
370
+ assert.deepEqual(failures, [], `cursor-following lost text at ${failures.length} site(s)`);
371
+ });