@howaboua/pi-notes-compaction 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/LICENSE +21 -0
  3. package/README.md +17 -2
  4. package/index.ts +8 -0
  5. package/package.json +16 -13
  6. package/src/board/context-briefing.ts +245 -0
  7. package/src/board/delivery.ts +85 -0
  8. package/src/guidance.ts +121 -0
  9. package/src/notes/AGENTS.md +27 -0
  10. package/src/{activation.ts → notes/activation.ts} +8 -10
  11. package/src/{attachment-sharing.ts → notes/attachment-sharing.ts} +1 -1
  12. package/src/{attachment-snapshot.ts → notes/attachment-snapshot.ts} +33 -20
  13. package/src/{context-sharing.ts → notes/context-sharing.ts} +7 -2
  14. package/src/{extension-hooks.ts → notes/extension-hooks.ts} +1 -12
  15. package/src/{extension-sharing.ts → notes/extension-sharing.ts} +1 -1
  16. package/src/notes/feature.ts +234 -0
  17. package/src/notes/history-search-pruning.mjs +154 -0
  18. package/src/{history-search-worker.mjs → notes/history-search-worker.mjs} +1 -1
  19. package/src/{history-windows.ts → notes/history-windows.ts} +10 -22
  20. package/src/{history.ts → notes/history.ts} +6 -5
  21. package/src/{index.ts → notes/index.ts} +20 -11
  22. package/src/{integrations.ts → notes/integrations.ts} +19 -3
  23. package/src/{lifecycle.ts → notes/lifecycle.ts} +0 -2
  24. package/src/{note-query.ts → notes/note-query.ts} +3 -5
  25. package/src/notes/note-result.ts +119 -0
  26. package/src/{note-save-marker.ts → notes/note-save-marker.ts} +3 -1
  27. package/src/{notes-lookup.ts → notes/notes-lookup.ts} +1 -1
  28. package/src/{query-executor.ts → notes/query-executor.ts} +7 -2
  29. package/src/{query-input.ts → notes/query-input.ts} +1 -0
  30. package/src/{remote.ts → notes/remote.ts} +1 -1
  31. package/src/{saved-session-access.ts → notes/saved-session-access.ts} +14 -10
  32. package/src/{saved-session.ts → notes/saved-session.ts} +31 -15
  33. package/src/notes/saved-sharing-records.ts +170 -0
  34. package/src/notes/session-pruning-cleanup.mjs +163 -0
  35. package/src/notes/session-pruning-exit.mjs +312 -0
  36. package/src/{session-pruning-screen.ts → notes/session-pruning-screen.ts} +1 -1
  37. package/src/notes/session-pruning-sparse.mjs +133 -0
  38. package/src/notes/session-pruning-state.mjs +123 -0
  39. package/src/{session-pruning.ts → notes/session-pruning.ts} +39 -6
  40. package/src/{settings-command.ts → notes/settings-command.ts} +29 -42
  41. package/src/{settings-controls.ts → notes/settings-controls.ts} +61 -75
  42. package/src/{settings-screen.ts → notes/settings-screen.ts} +25 -11
  43. package/src/{settings.ts → notes/settings.ts} +37 -6
  44. package/src/{shared-notes.ts → notes/shared-notes.ts} +21 -13
  45. package/src/{shared-session-reader.ts → notes/shared-session-reader.ts} +8 -11
  46. package/src/{sharing-service.ts → notes/sharing-service.ts} +1 -1
  47. package/src/{tool-contracts.ts → notes/tool-contracts.ts} +71 -44
  48. package/src/{tools.ts → notes/tools.ts} +2 -2
  49. package/src/{tree-handoff-read.ts → notes/tree-handoff-read.ts} +47 -32
  50. package/src/{window-message.ts → notes/window-message.ts} +8 -25
  51. package/src/settings-navigation.ts +11 -0
  52. package/src/timestamps.ts +3 -0
  53. package/tsconfig.json +9 -0
  54. package/src/history-search-pruning.mjs +0 -120
  55. package/src/note-result.ts +0 -70
  56. package/src/saved-sharing-records.ts +0 -135
  57. /package/src/{agent-identity.ts → notes/agent-identity.ts} +0 -0
  58. /package/src/{binding-startup.ts → notes/binding-startup.ts} +0 -0
  59. /package/src/{bridge.ts → notes/bridge.ts} +0 -0
  60. /package/src/{compaction-projection.ts → notes/compaction-projection.ts} +0 -0
  61. /package/src/{context-owner.ts → notes/context-owner.ts} +0 -0
  62. /package/src/{fresh-checkpoint.ts → notes/fresh-checkpoint.ts} +0 -0
  63. /package/src/{fresh-notes.ts → notes/fresh-notes.ts} +0 -0
  64. /package/src/{history-search-protocol.ts → notes/history-search-protocol.ts} +0 -0
  65. /package/src/{history-search.ts → notes/history-search.ts} +0 -0
  66. /package/src/{idle-admission.ts → notes/idle-admission.ts} +0 -0
  67. /package/src/{model-role-config.ts → notes/model-role-config.ts} +0 -0
  68. /package/src/{model-role-screen.ts → notes/model-role-screen.ts} +0 -0
  69. /package/src/{model-roles.ts → notes/model-roles.ts} +0 -0
  70. /package/src/{notes.ts → notes/notes.ts} +0 -0
  71. /package/src/{rollover.ts → notes/rollover.ts} +0 -0
  72. /package/src/{saved-session-index.mjs → notes/saved-session-index.mjs} +0 -0
  73. /package/src/{saved-session.mjs → notes/saved-session.mjs} +0 -0
  74. /package/src/{session-pruning-content.ts → notes/session-pruning-content.ts} +0 -0
  75. /package/src/{session-pruning-plan.ts → notes/session-pruning-plan.ts} +0 -0
  76. /package/src/{session-pruning-policy.ts → notes/session-pruning-policy.ts} +0 -0
  77. /package/src/{session-pruning-source.ts → notes/session-pruning-source.ts} +0 -0
  78. /package/src/{session-pruning-transaction.ts → notes/session-pruning-transaction.ts} +0 -0
  79. /package/src/{sqlite.mjs → notes/sqlite.mjs} +0 -0
  80. /package/src/{store.ts → notes/store.ts} +0 -0
  81. /package/src/{task-boundary.ts → notes/task-boundary.ts} +0 -0
  82. /package/src/{tree-archive-manifest.mjs → notes/tree-archive-manifest.mjs} +0 -0
  83. /package/src/{tree-archive.ts → notes/tree-archive.ts} +0 -0
  84. /package/src/{tree-checkpoint.ts → notes/tree-checkpoint.ts} +0 -0
  85. /package/src/{window-checkpoint.ts → notes/window-checkpoint.ts} +0 -0
  86. /package/src/{windows.ts → notes/windows.ts} +0 -0
@@ -1,18 +1,16 @@
1
- import { join } from "node:path";
2
1
  import type {
3
2
  ExtensionAPI,
4
3
  ExtensionContext,
5
4
  } from "@earendil-works/pi-coding-agent";
6
- import { CONFIG_DIR_NAME, getAgentDir } from "@earendil-works/pi-coding-agent";
7
5
  import { type NotesBridge, TOOL_NAMES } from "./bridge.js";
8
6
  import { indexHistory } from "./history.js";
9
7
  import type { HistorySearch } from "./history-search.js";
10
8
  import type { NotesLifecycle } from "./lifecycle.js";
11
9
  import { sessionRef } from "./notes.js";
12
10
  import {
11
+ loadNotesSettings,
13
12
  modelIsActive,
14
13
  type NotesSettings,
15
- readSettings,
16
14
  writeSettings,
17
15
  } from "./settings.js";
18
16
  import type { NotesStore } from "./store.js";
@@ -35,6 +33,7 @@ export function createNotesActivation(
35
33
  if (lifecycle.active) lifecycle.reset(ctx);
36
34
  lifecycle.active = false;
37
35
  search.setSession(undefined);
36
+ store.close();
38
37
  pi.setActiveTools(
39
38
  pi
40
39
  .getActiveTools()
@@ -99,19 +98,18 @@ export function createNotesActivation(
99
98
  lifecycle.idleMinutes = value.idleMinutes;
100
99
  lifecycle.taskBoundaryReminder = value.taskBoundaryReminder;
101
100
  }
102
- function start(ctx: ExtensionContext, family: NotesBridge): void {
101
+ function start(
102
+ ctx: ExtensionContext,
103
+ family: NotesBridge,
104
+ initial?: NotesSettings,
105
+ ): void {
103
106
  search.setSession(undefined);
104
107
  lifecycle.reset(ctx);
105
108
  lifecycle.active = false;
106
109
  ready = false;
107
110
  unavailable = undefined;
108
111
  try {
109
- settings = readSettings(settingsPath, [
110
- join(getAgentDir(), "pi-codex-conversion.json"),
111
- ...(ctx.isProjectTrusted()
112
- ? [join(ctx.cwd, CONFIG_DIR_NAME, "pi-codex-conversion.json")]
113
- : []),
114
- ]);
112
+ settings = initial ?? loadNotesSettings(ctx);
115
113
  writeSettings(settingsPath, settings);
116
114
  applySettings(settings);
117
115
  lifecycle.bridge = family;
@@ -150,7 +150,7 @@ export function createAttachmentSharing(
150
150
  const details = readAttachmentSnapshot(snapshot, params);
151
151
  return {
152
152
  ...noteResult({ ...details, ...annotation }),
153
- details: { codexHistoryNotes: details },
153
+ details: { codexHistoryNotes: { ...details, ...annotation } },
154
154
  };
155
155
  },
156
156
  } satisfies Pick<
@@ -1,35 +1,48 @@
1
- import { Type } from "typebox";
2
- import { Check } from "typebox/value";
1
+ import {
2
+ arr,
3
+ compile,
4
+ lit,
5
+ num,
6
+ obj,
7
+ opt,
8
+ str,
9
+ union,
10
+ } from "@earendil-works/pi-schema";
3
11
  import { contextTargetAgent, isContextAgentName } from "./agent-identity.js";
4
12
  import type { RemoteNoteReference } from "./context-sharing.js";
5
13
  import { normalizeNoteParams, readNotes } from "./notes.js";
6
14
  import type { NoteRevision } from "./store.js";
7
15
 
8
- const Snapshot = Type.Object({
9
- protocol: Type.Literal(1),
10
- timestamp: Type.Number({ minimum: 0 }),
11
- files: Type.Array(
12
- Type.Object({
13
- path: Type.String(),
14
- text: Type.String(),
15
- createdAt: Type.Number(),
16
- updatedAt: Type.Number(),
17
- operation: Type.Optional(
18
- Type.Union([
19
- Type.Literal("local_append_overlay"),
20
- Type.Literal("local_replacement"),
21
- ]),
16
+ const Snapshot = obj(
17
+ {
18
+ protocol: lit(1),
19
+ timestamp: num({ minimum: 0 }),
20
+ files: arr(
21
+ obj(
22
+ {
23
+ path: str(),
24
+ text: str(),
25
+ createdAt: num(),
26
+ updatedAt: num(),
27
+ operation: opt(
28
+ union([lit("local_append_overlay"), lit("local_replacement")]),
29
+ ),
30
+ },
31
+ { open: true },
22
32
  ),
23
- }),
24
- ),
25
- });
33
+ ),
34
+ },
35
+ { open: true },
36
+ );
37
+
38
+ const snapshotValidator = compile(Snapshot);
26
39
 
27
40
  /** Retained checkpoints remain readable without a running owner or database. */
28
41
  export function readAttachmentSnapshot(
29
42
  snapshot: unknown,
30
43
  params: Record<string, unknown>,
31
44
  ): Record<string, unknown> {
32
- if (!Check(Snapshot, snapshot))
45
+ if (!snapshotValidator.check(snapshot))
33
46
  throw new Error("Invalid retained note snapshot");
34
47
  if (
35
48
  !["list_files_by_prefix", "read_file", "search_contents"].includes(
@@ -141,7 +141,10 @@ export const CONTEXT_SHARING_REQUEST =
141
141
  "pi-notes-compaction:context-sharing:request";
142
142
 
143
143
  /** Optional, load-order-independent integration. No provider or tool schema changes. */
144
- export function connectNotesContextSharing(pi: ExtensionAPI): {
144
+ export function connectNotesContextSharing(
145
+ pi: ExtensionAPI,
146
+ onAvailable?: () => void,
147
+ ): {
145
148
  readonly service: ContextSharingService | undefined;
146
149
  dispose(): void;
147
150
  } {
@@ -157,8 +160,10 @@ export function connectNotesContextSharing(pi: ExtensionAPI): {
157
160
  typeof candidate.bind === "function" &&
158
161
  typeof candidate.execute === "function" &&
159
162
  typeof candidate.registerRouter === "function"
160
- )
163
+ ) {
161
164
  service = candidate as ContextSharingService;
165
+ onAvailable?.();
166
+ }
162
167
  });
163
168
  pi.events.emit(CONTEXT_SHARING_REQUEST, { protocol: 1 });
164
169
  return {
@@ -1,6 +1,6 @@
1
1
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
2
  import type { NotesActivation } from "./activation.js";
3
- import type { NotesBridge, NotesOwner } from "./bridge.js";
3
+ import type { NotesOwner } from "./bridge.js";
4
4
  import { NESTED_NOTE_OUTCOME, type NestedNoteOutcome } from "./fresh-notes.js";
5
5
  import { registerNotesCompactionIntegration } from "./integrations.js";
6
6
  import type { NotesLifecycle } from "./lifecycle.js";
@@ -20,18 +20,7 @@ export async function registerNotesHooks(
20
20
  owner: NotesOwner,
21
21
  checkpoints: WindowCheckpoints,
22
22
  markers: ReturnType<typeof registerNoteSaveMarkers>,
23
- family: NotesBridge,
24
23
  ) {
25
- pi.on("session_start", async (_event, ctx) => {
26
- activation.start(ctx, family);
27
- if (activation.available) {
28
- try {
29
- await checkpoints.restore(ctx);
30
- } catch (error) {
31
- ctx.ui.notify(`Context checkpoint failed: ${String(error)}`, "error");
32
- }
33
- }
34
- });
35
24
  pi.on("before_agent_start", (event, ctx) => {
36
25
  activation.refreshActivation(ctx);
37
26
  if (lifecycle.active) return lifecycle.beforeStart(ctx, event);
@@ -81,7 +81,7 @@ export async function registerNotesSharing(
81
81
  const { externalNotesResponses, ...details } = result.details;
82
82
  if (!remote.projectResult || remote.canProject?.(ctx) === false) {
83
83
  const warning =
84
- "Shared encrypted contents are unavailable on this model. Local results remain available. Use a compatible Codex model with PCC for remote contents.";
84
+ "Some notes cannot be read with this model. Available notes are shown; select a compatible Codex model to read the rest.";
85
85
  const data = result.structuredContent;
86
86
  if (!data || typeof data !== "object" || Array.isArray(data))
87
87
  throw new Error(
@@ -0,0 +1,234 @@
1
+ import { dirname, join } from "node:path";
2
+ import type {
3
+ ExtensionAPI,
4
+ ExtensionCommandContext,
5
+ ExtensionContext,
6
+ } from "@earendil-works/pi-coding-agent";
7
+ import type { CommsFeature } from "../guidance.js";
8
+ import type { OpenSettings } from "../settings-navigation.js";
9
+ import type { NotesRuntime } from "./index.js";
10
+ import {
11
+ admitSessionCleanup,
12
+ previousCleanupFailure,
13
+ } from "./session-pruning-exit.mjs";
14
+ import { cleanupStatus } from "./session-pruning-state.mjs";
15
+ import {
16
+ legacyNotesConfigured,
17
+ loadNotesSettings,
18
+ modelIsActive,
19
+ type NotesSettings,
20
+ notesSettingsPath,
21
+ writeSettings,
22
+ } from "./settings.js";
23
+ import type { NotesSettingsOwner } from "./settings-command.js";
24
+ import type { NotesStore } from "./store.js";
25
+
26
+ /** Dormant sessions need only settings; saved continuity still owns its history cuts. */
27
+ function hasSavedContinuity(ctx: ExtensionContext): boolean {
28
+ return ctx.sessionManager.getEntries().some((entry) => {
29
+ if (entry.type === "custom" || entry.type === "custom_message")
30
+ return (
31
+ entry.customType.startsWith("notes-compaction:") ||
32
+ [
33
+ "codex-context-agent",
34
+ "codex-context-backend",
35
+ "codex-context-note",
36
+ "codex-context-note-snapshot",
37
+ "codex-context-window",
38
+ "codex-context-tree-archive",
39
+ "codex-note-save-marker",
40
+ ].includes(entry.customType)
41
+ );
42
+ return (
43
+ entry.type === "compaction" &&
44
+ entry.details !== null &&
45
+ typeof entry.details === "object" &&
46
+ "pi-notes-compaction/window-checkpoint" in entry.details
47
+ );
48
+ });
49
+ }
50
+
51
+ export function createNotesFeature(
52
+ pi: ExtensionAPI,
53
+ openSettings?: OpenSettings,
54
+ ): CommsFeature {
55
+ let settings: NotesSettings | undefined;
56
+ let runtime: NotesRuntime | undefined;
57
+ let loading: Promise<NotesRuntime> | undefined;
58
+ let store: NotesStore | undefined;
59
+ let storage: Promise<NotesStore> | undefined;
60
+ let closed = false;
61
+ let staleOpening = false;
62
+ let releaseCleanupOwner: (() => void) | undefined;
63
+ const admit = async (ctx: ExtensionContext) => {
64
+ if (releaseCleanupOwner) return !staleOpening;
65
+ let reason: string | undefined;
66
+ try {
67
+ const claim = await admitSessionCleanup(
68
+ ctx.sessionManager.getSessionFile(),
69
+ ctx.sessionManager.getSessionId(),
70
+ ctx.sessionManager.getHeader(),
71
+ );
72
+ releaseCleanupOwner = claim?.release;
73
+ if (claim?.stale)
74
+ reason =
75
+ "This session was cleaned while Pi was opening it. Restart Pi to load its saved tree safely.";
76
+ } catch (error) {
77
+ reason = `Pi could not safely open this session: ${String(error)}. Resolve the problem and restart Pi.`;
78
+ }
79
+ if (!reason) return true;
80
+ staleOpening = true;
81
+ console.error(reason);
82
+ ctx.ui.notify(reason, "error");
83
+ if (ctx.mode === "rpc") {
84
+ // RPC honours startup shutdown after attaching input. Returning lets it
85
+ // reach that check before accepting normal commands.
86
+ ctx.shutdown();
87
+ return false;
88
+ }
89
+ if (ctx.mode === "tui") ctx.shutdown();
90
+ else {
91
+ // Released print/json mode installs graceful SIGTERM handling before
92
+ // session_start, but does not bind the documented ctx.shutdown action.
93
+ process.kill(process.pid, "SIGTERM");
94
+ }
95
+ // Do not finish binding or start an initial prompt/tree while exiting.
96
+ await new Promise<never>(() => {});
97
+ };
98
+ const ensureStore = () =>
99
+ (storage ??= (async () => {
100
+ const { NotesStore } = await import("./store.js");
101
+ if (closed) throw new Error("Session changed; reopen Notes settings");
102
+ return (store = new NotesStore(
103
+ join(dirname(notesSettingsPath()), "notes.sqlite"),
104
+ ));
105
+ })());
106
+ const ensure = (ctx: ExtensionContext): Promise<NotesRuntime> => {
107
+ if (loading) return loading;
108
+ loading = (async () => {
109
+ const { createNotesRuntime } = await import("./index.js");
110
+ if (closed) throw new Error("Session changed; reopen Notes settings");
111
+ runtime = await createNotesRuntime(pi, await ensureStore());
112
+ if (closed) throw new Error("Session changed; reopen Notes settings");
113
+ await runtime.start(ctx, settings);
114
+ return runtime;
115
+ })();
116
+ return loading;
117
+ };
118
+ const owner: NotesSettingsOwner = {
119
+ get settings() {
120
+ if (!settings)
121
+ throw new Error(
122
+ "Notes settings are unavailable; fix the settings file and reload Pi",
123
+ );
124
+ return settings;
125
+ },
126
+ get runtime() {
127
+ return runtime;
128
+ },
129
+ async save(value, ctx) {
130
+ if (runtime) runtime.activation.save(value, ctx);
131
+ else writeSettings(notesSettingsPath(), value);
132
+ settings = value;
133
+ if (!runtime && modelIsActive(value, ctx.model)) await ensure(ctx);
134
+ },
135
+ start: ensure,
136
+ };
137
+ const open = async (
138
+ ctx: ExtensionCommandContext,
139
+ initial: "notes" | "roles" = "notes",
140
+ ) => {
141
+ const { createNotesSettings } = await import("./settings-command.js");
142
+ return createNotesSettings(
143
+ owner,
144
+ await ensureStore(),
145
+ notesSettingsPath(),
146
+ Boolean(openSettings),
147
+ )(ctx, initial);
148
+ };
149
+ const routeSettings: OpenSettings =
150
+ openSettings ??
151
+ (async (ctx) => {
152
+ await open(ctx);
153
+ });
154
+ pi.on("session_start", async (_event, ctx) => {
155
+ if (!(await admit(ctx))) return;
156
+ const file = ctx.sessionManager.getSessionFile();
157
+ if (file) {
158
+ try {
159
+ const error = previousCleanupFailure(
160
+ cleanupStatus(file, dirname(notesSettingsPath())),
161
+ ctx.sessionManager.getSessionId(),
162
+ );
163
+ if (error) ctx.ui.notify(error, "warning");
164
+ } catch (error) {
165
+ ctx.ui.notify(
166
+ `Previous session cleanup status could not be read: ${String(error)}. Check Notes storage permissions and retry.`,
167
+ "warning",
168
+ );
169
+ }
170
+ }
171
+ try {
172
+ settings = loadNotesSettings(ctx);
173
+ writeSettings(notesSettingsPath(), settings);
174
+ if (
175
+ modelIsActive(settings, ctx.model) ||
176
+ hasSavedContinuity(ctx) ||
177
+ legacyNotesConfigured(ctx)
178
+ )
179
+ await ensure(ctx);
180
+ } catch (error) {
181
+ ctx.ui.notify(
182
+ `Notes continuity inactive: ${error instanceof Error ? error.message : String(error)}`,
183
+ "warning",
184
+ );
185
+ }
186
+ });
187
+ pi.on("agent_settled", async (_event, ctx) => {
188
+ // A brand-new session has no JSONL file until its first persisted turn.
189
+ await admit(ctx);
190
+ });
191
+ pi.on("model_select", async (_event, ctx) => {
192
+ if (!runtime && settings && modelIsActive(settings, ctx.model))
193
+ await ensure(ctx);
194
+ });
195
+ pi.on("session_shutdown", (event) => {
196
+ closed = true;
197
+ if (event.reason !== "quit") {
198
+ releaseCleanupOwner?.();
199
+ releaseCleanupOwner = undefined;
200
+ }
201
+ store?.close();
202
+ });
203
+ pi.registerCommand("notes", {
204
+ description: "Manage local notes and context rollover",
205
+ handler: async (args, ctx) => {
206
+ if (args.trim()) {
207
+ ctx.ui.notify("Use /notes to open settings.", "info");
208
+ return;
209
+ }
210
+ await routeSettings(ctx, "notes");
211
+ },
212
+ });
213
+ pi.registerCommand("roles", {
214
+ description: "View model roles and JSON configuration guidance",
215
+ handler: async (args, ctx) => {
216
+ if (args.trim()) {
217
+ ctx.ui.notify(
218
+ "Use /roles to view roles and JSON configuration guidance.",
219
+ "info",
220
+ );
221
+ return;
222
+ }
223
+ const next = await open(ctx, "roles");
224
+ if (next) await routeSettings(ctx, next);
225
+ },
226
+ });
227
+ return {
228
+ enabled: () => runtime?.lifecycle.active ?? false,
229
+ gptPromptMode: () => runtime?.lifecycle.gptPromptMode() ?? false,
230
+ async openSettings(ctx) {
231
+ return open(ctx);
232
+ },
233
+ };
234
+ }
@@ -0,0 +1,154 @@
1
+ import { open } from "node:fs/promises";
2
+ import { StringDecoder } from "node:string_decoder";
3
+
4
+ /** @typedef {{file:string; sessionId:string; generation:string; size:number; surviving:Set<string>; excluded:Set<string>}} PrunedHistory */
5
+
6
+ /** The live SessionManager still contains old text until restart. Read the committed
7
+ * source so later indexing, settings toggles and /reload cannot resurrect its terms.
8
+ */
9
+ export function createHistoryPruning() {
10
+ /** @type {PrunedHistory | undefined} */
11
+ let cached;
12
+ const empty = () => false;
13
+ /** @param {import("./sqlite.mjs").Database} db
14
+ * @param {string} sessionId
15
+ * @param {string | undefined} file
16
+ * @returns {Promise<(id:string) => boolean>}
17
+ */
18
+ return async (db, sessionId, file) => {
19
+ if (!file) return empty;
20
+ let input;
21
+ try {
22
+ input = await open(file, "r");
23
+ } catch (error) {
24
+ // A new Pi session may not have flushed its first message yet.
25
+ if (error instanceof Error && "code" in error && error.code === "ENOENT")
26
+ return empty;
27
+ throw error;
28
+ }
29
+ try {
30
+ const { size } = await input.stat();
31
+ if (!size) return empty;
32
+ let header;
33
+ for await (const entry of records(input, 0, size)) {
34
+ header = entry;
35
+ break;
36
+ }
37
+ if (header?.type !== "session" || header.id !== sessionId)
38
+ throw new Error("History source identity changed");
39
+ const marker = header.notesPruning;
40
+ if (marker === undefined) return empty;
41
+ if (
42
+ marker?.version !== 1 ||
43
+ marker.sessionId !== sessionId ||
44
+ typeof marker.generation !== "string"
45
+ )
46
+ throw new Error("Invalid history pruning marker");
47
+ const previous =
48
+ cached?.file === file &&
49
+ cached.sessionId === sessionId &&
50
+ cached.generation === marker.generation &&
51
+ cached.size <= size
52
+ ? cached
53
+ : undefined;
54
+ const incremental = previous !== undefined;
55
+ if (previous?.size === size) return exclusion(previous);
56
+ /** @type {PrunedHistory} */
57
+ const source = previous ?? {
58
+ file,
59
+ sessionId,
60
+ generation: marker.generation,
61
+ size: 0,
62
+ surviving: new Set(),
63
+ excluded: new Set(),
64
+ };
65
+ // A failed suffix scan invalidates mutated cache sets. Ordinary appends
66
+ // scan only their suffix, not the full session or every indexed document.
67
+ cached = undefined;
68
+ for await (const entry of records(input, source.size, size)) {
69
+ if (!incremental && entry.type === "session") continue;
70
+ if (typeof entry.id !== "string" || source.surviving.has(entry.id))
71
+ throw new Error("Invalid history source entry identity");
72
+ source.surviving.add(entry.id);
73
+ const message = entry.type === "message" ? entry.message : undefined;
74
+ if (
75
+ (entry.type === "custom" &&
76
+ entry.customType === "notes-compaction:pruned:v1") ||
77
+ (["user", "assistant"].includes(message?.role) &&
78
+ !visibleText(message.content))
79
+ )
80
+ source.excluded.add(entry.id);
81
+ }
82
+ source.size = size;
83
+ const excluded = exclusion(source);
84
+ if (!incremental) {
85
+ const removeTerms = db.prepare(
86
+ "DELETE FROM history_terms WHERE rowid = ?",
87
+ );
88
+ const removeDocument = db.prepare(
89
+ "DELETE FROM history_documents WHERE id = ?",
90
+ );
91
+ /** @type {import("./sqlite.mjs").Statement<[string], {id:number; item_id:string}>} */
92
+ const documents = db.prepare(
93
+ "SELECT id, item_id FROM history_documents WHERE session_id = ?",
94
+ );
95
+ db.transaction(() => {
96
+ for (const row of documents.all(sessionId)) {
97
+ if (!excluded(row.item_id)) continue;
98
+ removeTerms.run(row.id);
99
+ removeDocument.run(row.id);
100
+ }
101
+ }).immediate();
102
+ }
103
+ cached = source;
104
+ return excluded;
105
+ } finally {
106
+ await input.close();
107
+ }
108
+ };
109
+ }
110
+
111
+ /** @param {import("node:fs/promises").FileHandle} input @param {number} start @param {number} size */
112
+ async function* records(input, start, size) {
113
+ const buffer = Buffer.alloc(64 * 1024);
114
+ const decoder = new StringDecoder("utf8");
115
+ let pending = "";
116
+ for (let position = start; position < size; ) {
117
+ const { bytesRead } = await input.read(
118
+ buffer,
119
+ 0,
120
+ Math.min(buffer.length, size - position),
121
+ position,
122
+ );
123
+ if (!bytesRead)
124
+ throw new Error("History source changed during reconciliation");
125
+ position += bytesRead;
126
+ pending += decoder.write(buffer.subarray(0, bytesRead));
127
+ let offset = 0;
128
+ let end;
129
+ while ((end = pending.indexOf("\n", offset)) >= 0) {
130
+ yield JSON.parse(pending.slice(offset, end));
131
+ offset = end + 1;
132
+ }
133
+ pending = pending.slice(offset);
134
+ }
135
+ pending += decoder.end();
136
+ if (pending.length)
137
+ throw new Error(
138
+ "History source has an incomplete record; retry after the turn completes",
139
+ );
140
+ }
141
+
142
+ /** @param {PrunedHistory} source */
143
+ function exclusion(source) {
144
+ /** @param {string} id */
145
+ return (id) => !source.surviving.has(id) || source.excluded.has(id);
146
+ }
147
+
148
+ /** @param {unknown} content */
149
+ function visibleText(content) {
150
+ return typeof content === "string"
151
+ ? content.length > 0
152
+ : Array.isArray(content) &&
153
+ content.some((part) => part?.type === "text" && part.text?.length > 0);
154
+ }
@@ -97,7 +97,7 @@ async function index(request) {
97
97
  }
98
98
  }
99
99
  for (const item of request.items) {
100
- if (excluded.has(item.item_id)) continue;
100
+ if (excluded(item.item_id)) continue;
101
101
  const fingerprint = createHash("sha256")
102
102
  .update(item.content)
103
103
  .digest("hex");
@@ -12,23 +12,13 @@ interface HistoryItem {
12
12
  content: string;
13
13
  }
14
14
 
15
- // Pi entries are immutable once persisted. Rendering is lazy so indexing never
16
- // serializes the ongoing context, and repeated lookups reuse unchanged text.
17
- const rendered = new WeakMap<SessionEntry, string>();
15
+ // Cache lazy metadata, not rendered content: Pi retains persisted entries, so
16
+ // caching serialized tool arguments would duplicate them for the session lifetime.
18
17
  const cachedItems = new WeakMap<SessionEntry, HistoryItem>();
19
18
  const indexItems = new WeakMap<
20
19
  HistoryItem,
21
20
  Pick<HistoryItem, "item_id" | "content">
22
21
  >();
23
- function content(entry: SessionEntry, render: () => string): string {
24
- let value = rendered.get(entry);
25
- if (value === undefined) {
26
- value = render();
27
- rendered.set(entry, value);
28
- }
29
- return value;
30
- }
31
-
32
22
  function customHistoryRole(
33
23
  entry: Extract<SessionEntry, { type: "custom_message" }>,
34
24
  ): string {
@@ -180,7 +170,7 @@ export async function historyWindows(
180
170
  }
181
171
  : {}),
182
172
  get content() {
183
- return content(entry, render);
173
+ return render();
184
174
  },
185
175
  };
186
176
  if (message.role === "user" || message.role === "assistant") {
@@ -212,15 +202,13 @@ export async function historyWindows(
212
202
  item_id: entry.id,
213
203
  role: customHistoryRole(entry),
214
204
  get content() {
215
- return content(entry, () =>
216
- typeof entry.content === "string"
217
- ? entry.content
218
- : entry.content
219
- .map((block) =>
220
- block.type === "text" ? block.text : "[image]",
221
- )
222
- .join("\n"),
223
- );
205
+ return typeof entry.content === "string"
206
+ ? entry.content
207
+ : entry.content
208
+ .map((block) =>
209
+ block.type === "text" ? block.text : "[image]",
210
+ )
211
+ .join("\n");
224
212
  },
225
213
  };
226
214
  }
@@ -70,15 +70,16 @@ export async function useHistory(
70
70
  : undefined);
71
71
  const item = items?.find((item) => item.item_id === itemId);
72
72
  if (!item) return { item: null };
73
- const offset = integer(params["offset_chars"], 0, item.content.length);
73
+ const { content: fullContent, ...metadata } = item;
74
+ const offset = integer(params["offset_chars"], 0, fullContent.length);
74
75
  const limit = integer(params["limit_chars"], 8000, 8000);
75
- const content = item.content.slice(offset, offset + limit);
76
+ const content = fullContent.slice(offset, offset + limit);
76
77
  return {
77
78
  item: {
78
- ...item,
79
+ ...metadata,
79
80
  content,
80
- total_chars: item.content.length,
81
- ...(offset + content.length < item.content.length
81
+ total_chars: fullContent.length,
82
+ ...(offset + content.length < fullContent.length
82
83
  ? { next_offset_chars: offset + content.length }
83
84
  : {}),
84
85
  },