@howaboua/pi-notes-compaction 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -5
- package/PCC-INTEGRATION.md +7 -98
- package/README.md +21 -13
- package/package.json +7 -3
- package/src/activation.ts +152 -0
- package/src/agent-identity.ts +119 -0
- package/src/attachment-snapshot.ts +103 -0
- package/src/binding-startup.ts +30 -0
- package/src/bridge.ts +1 -43
- package/src/compaction-projection.ts +88 -0
- package/src/context-owner.ts +103 -0
- package/src/context-sharing.ts +144 -0
- package/src/extension-hooks.ts +198 -0
- package/src/extension-sharing.ts +118 -0
- package/src/fresh-checkpoint.ts +117 -0
- package/src/fresh-notes.ts +141 -26
- package/src/history-search-protocol.ts +39 -0
- package/src/history-search-pruning.mjs +120 -0
- package/src/history-search-worker.mjs +256 -0
- package/src/history-search.ts +269 -0
- package/src/history.ts +212 -40
- package/src/idle-admission.ts +201 -0
- package/src/index.ts +71 -401
- package/src/integrations.ts +159 -0
- package/src/lifecycle.ts +63 -448
- package/src/note-save-marker.ts +112 -0
- package/src/notes-lookup.ts +112 -0
- package/src/notes.ts +24 -28
- package/src/query-executor.ts +190 -0
- package/src/remote.ts +357 -0
- package/src/rollover.ts +212 -0
- package/src/saved-session.ts +44 -7
- package/src/session-pruning-content.ts +237 -0
- package/src/session-pruning-plan.ts +243 -0
- package/src/session-pruning-policy.ts +20 -0
- package/src/session-pruning-screen.ts +130 -0
- package/src/session-pruning-source.ts +44 -0
- package/src/session-pruning-transaction.ts +141 -0
- package/src/session-pruning.ts +142 -0
- package/src/settings-command.ts +77 -0
- package/src/settings-screen.ts +84 -19
- package/src/settings.ts +143 -11
- package/src/shared-notes.ts +37 -14
- package/src/sharing-service.ts +348 -0
- package/src/store.ts +31 -0
- package/src/task-boundary.ts +6 -1
- package/src/tool-contracts.ts +279 -0
- package/src/tools.ts +19 -560
- package/src/tree-archive.ts +149 -0
- package/src/tree-checkpoint.ts +170 -0
- package/src/tree-handoff-read.ts +192 -0
- package/src/window-checkpoint.ts +105 -0
- package/src/windows.ts +25 -9
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export interface SearchItem {
|
|
2
|
+
item_id: string;
|
|
3
|
+
content: string;
|
|
4
|
+
}
|
|
5
|
+
export type SearchRequest =
|
|
6
|
+
| {
|
|
7
|
+
action: "index";
|
|
8
|
+
sessionId: string;
|
|
9
|
+
file: string | undefined;
|
|
10
|
+
items: readonly SearchItem[];
|
|
11
|
+
removeIds: string[];
|
|
12
|
+
}
|
|
13
|
+
| {
|
|
14
|
+
action: "search";
|
|
15
|
+
sessionId: string;
|
|
16
|
+
ids: string[];
|
|
17
|
+
match: string;
|
|
18
|
+
limit: number;
|
|
19
|
+
recentFirst: boolean;
|
|
20
|
+
}
|
|
21
|
+
| {
|
|
22
|
+
action: "prune_session";
|
|
23
|
+
sessionId: string;
|
|
24
|
+
file: string;
|
|
25
|
+
}
|
|
26
|
+
| {
|
|
27
|
+
action: "prune";
|
|
28
|
+
activeSessionId: string;
|
|
29
|
+
sources: Record<string, string | null>;
|
|
30
|
+
};
|
|
31
|
+
export interface PruneResult {
|
|
32
|
+
removed: number;
|
|
33
|
+
unknownPaths: number;
|
|
34
|
+
inaccessible: number;
|
|
35
|
+
storageBusy: boolean;
|
|
36
|
+
}
|
|
37
|
+
export type SearchResponse =
|
|
38
|
+
| { id: number; value: string[] | PruneResult | null }
|
|
39
|
+
| { id: number; error: string };
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { open } from "node:fs/promises";
|
|
2
|
+
|
|
3
|
+
/** @typedef {{file:string; sessionId:string; generation:string; excluded:Set<string>}} PrunedHistory */
|
|
4
|
+
|
|
5
|
+
/** The live SessionManager still contains old text until restart. Read the committed
|
|
6
|
+
* source so later indexing, settings toggles and /reload cannot resurrect its terms.
|
|
7
|
+
*/
|
|
8
|
+
export function createHistoryPruning() {
|
|
9
|
+
/** @type {PrunedHistory | undefined} */
|
|
10
|
+
let cached;
|
|
11
|
+
const empty = new Set();
|
|
12
|
+
/** @param {import("better-sqlite3").Database} db
|
|
13
|
+
* @param {string} sessionId
|
|
14
|
+
* @param {string | undefined} file
|
|
15
|
+
* @returns {Promise<ReadonlySet<string>>}
|
|
16
|
+
*/
|
|
17
|
+
return async (db, sessionId, file) => {
|
|
18
|
+
if (!file) return empty;
|
|
19
|
+
let input;
|
|
20
|
+
try {
|
|
21
|
+
input = await open(file, "r");
|
|
22
|
+
} catch (error) {
|
|
23
|
+
// A new Pi session may not have flushed its first message yet.
|
|
24
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT")
|
|
25
|
+
return empty;
|
|
26
|
+
throw error;
|
|
27
|
+
}
|
|
28
|
+
let stream;
|
|
29
|
+
try {
|
|
30
|
+
const { size } = await input.stat();
|
|
31
|
+
if (!size) return empty;
|
|
32
|
+
stream = input.createReadStream({
|
|
33
|
+
start: 0,
|
|
34
|
+
end: size - 1,
|
|
35
|
+
autoClose: false,
|
|
36
|
+
encoding: "utf8",
|
|
37
|
+
});
|
|
38
|
+
/** @type {PrunedHistory | undefined} */
|
|
39
|
+
let source;
|
|
40
|
+
let pending = "";
|
|
41
|
+
for await (const chunk of stream) {
|
|
42
|
+
pending += chunk;
|
|
43
|
+
let start = 0;
|
|
44
|
+
let end;
|
|
45
|
+
while ((end = pending.indexOf("\n", start)) >= 0) {
|
|
46
|
+
const entry = JSON.parse(pending.slice(start, end));
|
|
47
|
+
start = end + 1;
|
|
48
|
+
if (!source) {
|
|
49
|
+
if (entry.type !== "session" || entry.id !== sessionId)
|
|
50
|
+
throw new Error("History source identity changed");
|
|
51
|
+
const marker = entry.notesPruning;
|
|
52
|
+
if (marker === undefined) return empty;
|
|
53
|
+
if (
|
|
54
|
+
marker?.version !== 1 ||
|
|
55
|
+
marker.sessionId !== sessionId ||
|
|
56
|
+
typeof marker.generation !== "string"
|
|
57
|
+
)
|
|
58
|
+
throw new Error("Invalid history pruning marker");
|
|
59
|
+
if (
|
|
60
|
+
cached?.file === file &&
|
|
61
|
+
cached.sessionId === sessionId &&
|
|
62
|
+
cached.generation === marker.generation
|
|
63
|
+
)
|
|
64
|
+
return cached.excluded;
|
|
65
|
+
source = {
|
|
66
|
+
file,
|
|
67
|
+
sessionId,
|
|
68
|
+
generation: marker.generation,
|
|
69
|
+
excluded: new Set(),
|
|
70
|
+
};
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const message = entry.type === "message" ? entry.message : undefined;
|
|
74
|
+
if (
|
|
75
|
+
(entry.type === "custom" &&
|
|
76
|
+
entry.customType === "notes-compaction:pruned:v1") ||
|
|
77
|
+
(["user", "assistant"].includes(message?.role) &&
|
|
78
|
+
!visibleText(message.content))
|
|
79
|
+
)
|
|
80
|
+
source.excluded.add(entry.id);
|
|
81
|
+
}
|
|
82
|
+
pending = pending.slice(start);
|
|
83
|
+
}
|
|
84
|
+
// Only complete lines from the opened inode count. A concurrent append's
|
|
85
|
+
// incomplete suffix is new history, never an old pruned conversation.
|
|
86
|
+
if (!source) throw new Error("History source header is missing");
|
|
87
|
+
const excluded = source.excluded;
|
|
88
|
+
const removeTerms = db.prepare(
|
|
89
|
+
"DELETE FROM history_terms WHERE rowid = ?",
|
|
90
|
+
);
|
|
91
|
+
const removeDocument = db.prepare(
|
|
92
|
+
"DELETE FROM history_documents WHERE id = ?",
|
|
93
|
+
);
|
|
94
|
+
/** @type {import("better-sqlite3").Statement<[string], {id:number; item_id:string}>} */
|
|
95
|
+
const documents = db.prepare(
|
|
96
|
+
"SELECT id, item_id FROM history_documents WHERE session_id = ?",
|
|
97
|
+
);
|
|
98
|
+
db.transaction(() => {
|
|
99
|
+
for (const row of documents.all(sessionId)) {
|
|
100
|
+
if (!excluded.has(row.item_id)) continue;
|
|
101
|
+
removeTerms.run(row.id);
|
|
102
|
+
removeDocument.run(row.id);
|
|
103
|
+
}
|
|
104
|
+
}).immediate();
|
|
105
|
+
cached = source;
|
|
106
|
+
return excluded;
|
|
107
|
+
} finally {
|
|
108
|
+
stream?.destroy();
|
|
109
|
+
await input.close();
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** @param {unknown} content */
|
|
115
|
+
function visibleText(content) {
|
|
116
|
+
return typeof content === "string"
|
|
117
|
+
? content.length > 0
|
|
118
|
+
: Array.isArray(content) &&
|
|
119
|
+
content.some((part) => part?.type === "text" && part.text?.length > 0);
|
|
120
|
+
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
5
|
+
import Database from "better-sqlite3";
|
|
6
|
+
import { createHistoryPruning } from "./history-search-pruning.mjs";
|
|
7
|
+
|
|
8
|
+
// Shipped as checked JavaScript: Node cannot strip TypeScript in node_modules.
|
|
9
|
+
// Pi's jiti resolver is not installed in worker threads.
|
|
10
|
+
/** @typedef {import("./history-search-protocol.js").SearchRequest} SearchRequest */
|
|
11
|
+
/** @typedef {import("./history-search-protocol.js").SearchResponse} SearchResponse */
|
|
12
|
+
/** @typedef {import("./history-search-protocol.js").PruneResult} PruneResult */
|
|
13
|
+
/** @type {import("better-sqlite3").Database | undefined} */
|
|
14
|
+
let database;
|
|
15
|
+
const pruneSession = createHistoryPruning();
|
|
16
|
+
/** @returns {import("better-sqlite3").Database} */
|
|
17
|
+
function open() {
|
|
18
|
+
if (database) return database;
|
|
19
|
+
mkdirSync(dirname(workerData.path), { recursive: true, mode: 0o700 });
|
|
20
|
+
const db = new Database(workerData.path);
|
|
21
|
+
try {
|
|
22
|
+
db.pragma("busy_timeout = 5000");
|
|
23
|
+
db.pragma("journal_mode = WAL");
|
|
24
|
+
db.transaction(() => {
|
|
25
|
+
const version = db.pragma("user_version", { simple: true });
|
|
26
|
+
if (version !== 0 && version !== 1 && version !== 2 && version !== 3)
|
|
27
|
+
throw new Error("Unsupported history index version");
|
|
28
|
+
// Older formats indexed full transcripts without source-kind metadata.
|
|
29
|
+
// They cannot be filtered safely: discard derived rows once, then
|
|
30
|
+
// rebuild only the opted-in session's visible conversation text.
|
|
31
|
+
if (version !== 3)
|
|
32
|
+
db.exec(`
|
|
33
|
+
DROP TABLE IF EXISTS history_terms;
|
|
34
|
+
DROP TABLE IF EXISTS history_documents;
|
|
35
|
+
DROP TABLE IF EXISTS history_sessions;
|
|
36
|
+
`);
|
|
37
|
+
db.exec(`
|
|
38
|
+
CREATE TABLE IF NOT EXISTS history_documents (
|
|
39
|
+
id INTEGER PRIMARY KEY, session_id TEXT NOT NULL,
|
|
40
|
+
item_id TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
|
41
|
+
UNIQUE(session_id, item_id)
|
|
42
|
+
);
|
|
43
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS history_terms USING fts5(
|
|
44
|
+
content, content='', contentless_delete=1,
|
|
45
|
+
tokenize='unicode61 remove_diacritics 2'
|
|
46
|
+
);
|
|
47
|
+
CREATE TABLE IF NOT EXISTS history_sessions (
|
|
48
|
+
session_id TEXT PRIMARY KEY, file_path TEXT
|
|
49
|
+
);
|
|
50
|
+
PRAGMA user_version = 3;
|
|
51
|
+
`);
|
|
52
|
+
}).immediate();
|
|
53
|
+
db.exec(
|
|
54
|
+
"CREATE TEMP TABLE search_scope (document_id INTEGER PRIMARY KEY, position INTEGER NOT NULL)",
|
|
55
|
+
);
|
|
56
|
+
database = db;
|
|
57
|
+
return db;
|
|
58
|
+
} catch (error) {
|
|
59
|
+
db.close();
|
|
60
|
+
throw error;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/** @param {Extract<SearchRequest, {action:"index"}>} request */
|
|
64
|
+
async function index(request) {
|
|
65
|
+
if (!database && !request.items.length && !existsSync(workerData.path))
|
|
66
|
+
return null;
|
|
67
|
+
const db = open();
|
|
68
|
+
const excluded = await pruneSession(db, request.sessionId, request.file);
|
|
69
|
+
/** @type {import("better-sqlite3").Statement<[string,string], {id:number; fingerprint:string}>} */
|
|
70
|
+
const find = db.prepare(
|
|
71
|
+
"SELECT id, fingerprint FROM history_documents WHERE session_id = ? AND item_id = ?",
|
|
72
|
+
);
|
|
73
|
+
const insert = db.prepare(
|
|
74
|
+
"INSERT INTO history_documents(session_id, item_id, fingerprint) VALUES (?, ?, ?)",
|
|
75
|
+
);
|
|
76
|
+
const update = db.prepare(
|
|
77
|
+
"UPDATE history_documents SET fingerprint = ? WHERE id = ?",
|
|
78
|
+
);
|
|
79
|
+
const removeTerms = db.prepare("DELETE FROM history_terms WHERE rowid = ?");
|
|
80
|
+
const addTerms = db.prepare(
|
|
81
|
+
"INSERT INTO history_terms(rowid, content) VALUES (?, ?)",
|
|
82
|
+
);
|
|
83
|
+
db.transaction(() => {
|
|
84
|
+
db.prepare(
|
|
85
|
+
"INSERT INTO history_sessions VALUES (?, ?) ON CONFLICT(session_id) DO UPDATE SET file_path = coalesce(excluded.file_path, file_path)",
|
|
86
|
+
).run(request.sessionId, request.file ?? null);
|
|
87
|
+
const removeDocument = db.prepare(
|
|
88
|
+
"DELETE FROM history_documents WHERE session_id = ? AND item_id = ?",
|
|
89
|
+
);
|
|
90
|
+
for (const id of request.removeIds) {
|
|
91
|
+
const existing = find.get(request.sessionId, id);
|
|
92
|
+
if (existing) {
|
|
93
|
+
removeTerms.run(existing.id);
|
|
94
|
+
removeDocument.run(request.sessionId, id);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
for (const item of request.items) {
|
|
98
|
+
if (excluded.has(item.item_id)) continue;
|
|
99
|
+
const fingerprint = createHash("sha256")
|
|
100
|
+
.update(item.content)
|
|
101
|
+
.digest("hex");
|
|
102
|
+
const existing = find.get(request.sessionId, item.item_id);
|
|
103
|
+
if (existing?.fingerprint === fingerprint) continue;
|
|
104
|
+
const id =
|
|
105
|
+
existing?.id ??
|
|
106
|
+
Number(
|
|
107
|
+
insert.run(request.sessionId, item.item_id, fingerprint)
|
|
108
|
+
.lastInsertRowid,
|
|
109
|
+
);
|
|
110
|
+
if (existing) {
|
|
111
|
+
removeTerms.run(id);
|
|
112
|
+
update.run(fingerprint, id);
|
|
113
|
+
}
|
|
114
|
+
addTerms.run(id, item.content);
|
|
115
|
+
}
|
|
116
|
+
}).immediate();
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
/** @param {Extract<SearchRequest, {action:"search"}>} request */
|
|
120
|
+
function search(request) {
|
|
121
|
+
const db = open();
|
|
122
|
+
return db
|
|
123
|
+
.transaction(() => {
|
|
124
|
+
db.exec("DELETE FROM search_scope");
|
|
125
|
+
const scope = db.prepare(
|
|
126
|
+
"INSERT INTO search_scope SELECT id, ? FROM history_documents WHERE session_id = ? AND item_id = ?",
|
|
127
|
+
);
|
|
128
|
+
for (const [position, id] of request.ids.entries())
|
|
129
|
+
scope.run(position, request.sessionId, id);
|
|
130
|
+
/** @type {import("better-sqlite3").Statement<[string,number], {item_id:string}>} */
|
|
131
|
+
const select = db.prepare(`
|
|
132
|
+
SELECT history_documents.item_id FROM history_terms
|
|
133
|
+
JOIN search_scope ON search_scope.document_id = history_terms.rowid
|
|
134
|
+
JOIN history_documents ON history_documents.id = history_terms.rowid
|
|
135
|
+
WHERE history_terms MATCH ?
|
|
136
|
+
ORDER BY ${request.recentFirst ? "search_scope.position DESC" : "rank, search_scope.position"}
|
|
137
|
+
LIMIT ?
|
|
138
|
+
`);
|
|
139
|
+
return select
|
|
140
|
+
.all(request.match, request.limit + 1)
|
|
141
|
+
.map((row) => row.item_id);
|
|
142
|
+
})
|
|
143
|
+
.immediate();
|
|
144
|
+
}
|
|
145
|
+
/** @param {string} activeSessionId
|
|
146
|
+
* @param {Record<string,string|null>} sources
|
|
147
|
+
* @returns {PruneResult}
|
|
148
|
+
*/
|
|
149
|
+
function prune(activeSessionId, sources) {
|
|
150
|
+
const result = {
|
|
151
|
+
removed: 0,
|
|
152
|
+
unknownPaths: 0,
|
|
153
|
+
inaccessible: 0,
|
|
154
|
+
storageBusy: false,
|
|
155
|
+
};
|
|
156
|
+
// Prune does not create an index for users who never enabled ranking.
|
|
157
|
+
if (!database && !existsSync(workerData.path)) return result;
|
|
158
|
+
const db = open();
|
|
159
|
+
// A source can acquire a saved path after indexing. Borrow proven ownership
|
|
160
|
+
// before Notes removes its missing-file records; never infer unknown paths.
|
|
161
|
+
db.transaction(() => {
|
|
162
|
+
const source = db.prepare(
|
|
163
|
+
"INSERT INTO history_sessions SELECT DISTINCT session_id, ? FROM history_documents WHERE session_id = ? ON CONFLICT(session_id) DO UPDATE SET file_path = coalesce(excluded.file_path, file_path)",
|
|
164
|
+
);
|
|
165
|
+
for (const [id, file] of Object.entries(sources)) source.run(file, id);
|
|
166
|
+
}).immediate();
|
|
167
|
+
/** @type {import("better-sqlite3").Statement<[], {session_id:string;file_path:string|null}>} */
|
|
168
|
+
const owners = db.prepare(`
|
|
169
|
+
SELECT session_id, file_path FROM history_sessions
|
|
170
|
+
UNION SELECT DISTINCT session_id, NULL FROM history_documents
|
|
171
|
+
WHERE session_id NOT IN (SELECT session_id FROM history_sessions)
|
|
172
|
+
`);
|
|
173
|
+
const sessions = owners.all();
|
|
174
|
+
const remove = db.transaction(
|
|
175
|
+
/** @param {string} id */ (id) => {
|
|
176
|
+
db.prepare(
|
|
177
|
+
"DELETE FROM history_terms WHERE rowid IN (SELECT id FROM history_documents WHERE session_id = ?)",
|
|
178
|
+
).run(id);
|
|
179
|
+
db.prepare("DELETE FROM history_documents WHERE session_id = ?").run(id);
|
|
180
|
+
db.prepare("DELETE FROM history_sessions WHERE session_id = ?").run(id);
|
|
181
|
+
},
|
|
182
|
+
);
|
|
183
|
+
for (const session of sessions) {
|
|
184
|
+
if (session.session_id === activeSessionId) continue;
|
|
185
|
+
if (!session.file_path) {
|
|
186
|
+
result.unknownPaths++;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
statSync(session.file_path);
|
|
191
|
+
} catch (error) {
|
|
192
|
+
if (
|
|
193
|
+
error instanceof Error &&
|
|
194
|
+
"code" in error &&
|
|
195
|
+
error.code === "ENOENT"
|
|
196
|
+
) {
|
|
197
|
+
remove.immediate(session.session_id);
|
|
198
|
+
result.removed++;
|
|
199
|
+
} else result.inaccessible++;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// Manual maintenance only: merge FTS deletion segments, reclaim free pages,
|
|
203
|
+
// then truncate the WAL. Other live agents may hold a reader/writer lock.
|
|
204
|
+
try {
|
|
205
|
+
db.exec("INSERT INTO history_terms(history_terms) VALUES ('optimize')");
|
|
206
|
+
db.exec("VACUUM");
|
|
207
|
+
/** @type {import("better-sqlite3").Statement<[], {busy:number}>} */
|
|
208
|
+
const checkpoint = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
209
|
+
result.storageBusy = checkpoint.all().some((row) => row.busy !== 0);
|
|
210
|
+
} catch (error) {
|
|
211
|
+
if (
|
|
212
|
+
error instanceof Error &&
|
|
213
|
+
"code" in error &&
|
|
214
|
+
typeof error.code === "string" &&
|
|
215
|
+
error.code.startsWith("SQLITE_BUSY")
|
|
216
|
+
)
|
|
217
|
+
result.storageBusy = true;
|
|
218
|
+
else throw error;
|
|
219
|
+
}
|
|
220
|
+
return result;
|
|
221
|
+
}
|
|
222
|
+
const port = parentPort;
|
|
223
|
+
if (!port) throw new Error("History search must run in its worker thread");
|
|
224
|
+
let queue = Promise.resolve();
|
|
225
|
+
port.on(
|
|
226
|
+
"message",
|
|
227
|
+
/** @param {{id:number;request:SearchRequest}} message */
|
|
228
|
+
({ id, request }) => {
|
|
229
|
+
// Source reconciliation is asynchronous; preserve the worker's former
|
|
230
|
+
// synchronous request order so a search cannot overtake index cleanup.
|
|
231
|
+
queue = queue.then(async () => {
|
|
232
|
+
try {
|
|
233
|
+
let value;
|
|
234
|
+
if (request.action === "prune_session") {
|
|
235
|
+
if (database || existsSync(workerData.path))
|
|
236
|
+
await pruneSession(open(), request.sessionId, request.file);
|
|
237
|
+
value = null;
|
|
238
|
+
} else
|
|
239
|
+
value =
|
|
240
|
+
request.action === "index"
|
|
241
|
+
? await index(request)
|
|
242
|
+
: request.action === "search"
|
|
243
|
+
? search(request)
|
|
244
|
+
: prune(request.activeSessionId, request.sources);
|
|
245
|
+
port.postMessage(/** @satisfies {SearchResponse} */ ({ id, value }));
|
|
246
|
+
} catch (error) {
|
|
247
|
+
port.postMessage(
|
|
248
|
+
/** @satisfies {SearchResponse} */ ({
|
|
249
|
+
id,
|
|
250
|
+
error: error instanceof Error ? error.message : String(error),
|
|
251
|
+
}),
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
});
|
|
255
|
+
},
|
|
256
|
+
);
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { setImmediate } from "node:timers/promises";
|
|
2
|
+
import { Worker } from "node:worker_threads";
|
|
3
|
+
import type {
|
|
4
|
+
PruneResult,
|
|
5
|
+
SearchItem,
|
|
6
|
+
SearchRequest,
|
|
7
|
+
SearchResponse,
|
|
8
|
+
} from "./history-search-protocol.js";
|
|
9
|
+
|
|
10
|
+
const UNAVAILABLE =
|
|
11
|
+
"Ranked history search is unavailable. list_items and read_item still work. Ask the user to check local storage or turn off Ranked history search in /notes.";
|
|
12
|
+
|
|
13
|
+
/** Worker owns derived storage; JSONL and the caller own content and scope. */
|
|
14
|
+
export class HistorySearch {
|
|
15
|
+
private worker: Worker | undefined;
|
|
16
|
+
private sessionId: string | undefined;
|
|
17
|
+
private file: string | undefined;
|
|
18
|
+
private sequence = 0;
|
|
19
|
+
private indexed = new Map<string, SearchItem>();
|
|
20
|
+
private excluded = new Set<string>();
|
|
21
|
+
private indexing: Promise<void> | undefined;
|
|
22
|
+
private generation = 0;
|
|
23
|
+
private pending = new Map<
|
|
24
|
+
number,
|
|
25
|
+
{
|
|
26
|
+
resolve: (value: string[] | PruneResult | null) => void;
|
|
27
|
+
reject: (error: Error) => void;
|
|
28
|
+
}
|
|
29
|
+
>();
|
|
30
|
+
private readonly path: string;
|
|
31
|
+
constructor(path: string) {
|
|
32
|
+
this.path = path;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
setSession(sessionId: string | undefined, file?: string): boolean {
|
|
36
|
+
if (sessionId === this.sessionId) {
|
|
37
|
+
this.file = file ?? this.file;
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
this.close();
|
|
41
|
+
this.sessionId = sessionId;
|
|
42
|
+
this.file = file;
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
isEnabled(sessionId: string): boolean {
|
|
47
|
+
return this.sessionId === sessionId;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private start(): Worker {
|
|
51
|
+
if (this.worker) return this.worker;
|
|
52
|
+
// Workers run outside jiti. Checked JavaScript also works when the
|
|
53
|
+
// published source package is installed beneath node_modules.
|
|
54
|
+
const worker = new Worker(
|
|
55
|
+
new URL("./history-search-worker.mjs", import.meta.url),
|
|
56
|
+
{
|
|
57
|
+
workerData: { path: this.path },
|
|
58
|
+
execArgv: [],
|
|
59
|
+
},
|
|
60
|
+
);
|
|
61
|
+
this.worker = worker;
|
|
62
|
+
worker.on("message", (response: SearchResponse) => {
|
|
63
|
+
if (worker !== this.worker) return;
|
|
64
|
+
const pending = this.pending.get(response.id);
|
|
65
|
+
if (!pending) return;
|
|
66
|
+
this.pending.delete(response.id);
|
|
67
|
+
if ("error" in response) {
|
|
68
|
+
const error = new Error(UNAVAILABLE, { cause: response.error });
|
|
69
|
+
pending.reject(error);
|
|
70
|
+
this.stop(error);
|
|
71
|
+
} else pending.resolve(response.value);
|
|
72
|
+
if (!this.pending.size) worker.unref();
|
|
73
|
+
});
|
|
74
|
+
const failed = (cause: unknown) => {
|
|
75
|
+
if (worker !== this.worker) return;
|
|
76
|
+
this.stop(new Error(UNAVAILABLE, { cause }));
|
|
77
|
+
};
|
|
78
|
+
worker.on("error", failed);
|
|
79
|
+
worker.on("exit", (code) =>
|
|
80
|
+
failed(new Error(`History worker exited (${code})`)),
|
|
81
|
+
);
|
|
82
|
+
return worker;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private request(
|
|
86
|
+
request: SearchRequest,
|
|
87
|
+
signal?: AbortSignal,
|
|
88
|
+
): Promise<string[] | PruneResult | null> {
|
|
89
|
+
signal?.throwIfAborted();
|
|
90
|
+
return new Promise((resolve, reject) => {
|
|
91
|
+
const id = ++this.sequence;
|
|
92
|
+
const abort = () => {
|
|
93
|
+
// SQLite is synchronous inside the worker. Termination is the only
|
|
94
|
+
// cancellation that also stops CPU/IO and rolls back its transaction.
|
|
95
|
+
this.stop(
|
|
96
|
+
new Error("History search cancelled; retry the query", {
|
|
97
|
+
cause: signal?.reason,
|
|
98
|
+
}),
|
|
99
|
+
);
|
|
100
|
+
};
|
|
101
|
+
const finish = () => signal?.removeEventListener("abort", abort);
|
|
102
|
+
try {
|
|
103
|
+
const worker = this.start();
|
|
104
|
+
this.pending.set(id, {
|
|
105
|
+
resolve: (value) => {
|
|
106
|
+
finish();
|
|
107
|
+
resolve(value);
|
|
108
|
+
},
|
|
109
|
+
reject: (error) => {
|
|
110
|
+
finish();
|
|
111
|
+
reject(error);
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
115
|
+
worker.ref();
|
|
116
|
+
worker.postMessage({ id, request });
|
|
117
|
+
} catch (cause) {
|
|
118
|
+
finish();
|
|
119
|
+
this.pending.delete(id);
|
|
120
|
+
reject(new Error(UNAVAILABLE, { cause }));
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async index(
|
|
126
|
+
sessionId: string,
|
|
127
|
+
items: readonly SearchItem[],
|
|
128
|
+
signal?: AbortSignal,
|
|
129
|
+
excludedIds?: readonly string[],
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
signal?.throwIfAborted();
|
|
132
|
+
if (!this.isEnabled(sessionId)) return;
|
|
133
|
+
const removeIds = excludedIds?.filter((id) => !this.excluded.has(id)) ?? [];
|
|
134
|
+
if (excludedIds) this.excluded = new Set(excludedIds);
|
|
135
|
+
for (const id of removeIds) this.indexed.delete(id);
|
|
136
|
+
const changed = items.filter((item) => {
|
|
137
|
+
const existing = this.indexed.get(item.item_id);
|
|
138
|
+
return (
|
|
139
|
+
!existing || (existing !== item && existing.content !== item.content)
|
|
140
|
+
);
|
|
141
|
+
});
|
|
142
|
+
if (!changed.length && !removeIds.length) {
|
|
143
|
+
// Cache hits still yield: consecutive tool turns must not monopolize
|
|
144
|
+
// the UI through an uninterrupted chain of resolved promises.
|
|
145
|
+
await (this.indexing ?? setImmediate());
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
for (const item of changed) this.indexed.set(item.item_id, item);
|
|
149
|
+
const generation = this.generation;
|
|
150
|
+
const previous = this.indexing;
|
|
151
|
+
const work = (async () => {
|
|
152
|
+
await previous;
|
|
153
|
+
let offset = 0;
|
|
154
|
+
do {
|
|
155
|
+
if (generation !== this.generation)
|
|
156
|
+
throw new Error("History session changed; retry the query");
|
|
157
|
+
// Bound structured-clone stalls as well as worker work. The next
|
|
158
|
+
// batch waits for completion; searches await the whole index fence.
|
|
159
|
+
const batch: SearchItem[] = [];
|
|
160
|
+
let chars = 0;
|
|
161
|
+
while (
|
|
162
|
+
offset < changed.length &&
|
|
163
|
+
batch.length < 128 &&
|
|
164
|
+
chars < 1_000_000
|
|
165
|
+
) {
|
|
166
|
+
const item = changed[offset++]!;
|
|
167
|
+
batch.push({ item_id: item.item_id, content: item.content });
|
|
168
|
+
chars += item.content.length;
|
|
169
|
+
}
|
|
170
|
+
await this.request(
|
|
171
|
+
{
|
|
172
|
+
action: "index",
|
|
173
|
+
sessionId,
|
|
174
|
+
file: this.file,
|
|
175
|
+
items: batch,
|
|
176
|
+
removeIds: offset <= batch.length ? removeIds : [],
|
|
177
|
+
},
|
|
178
|
+
signal,
|
|
179
|
+
);
|
|
180
|
+
} while (offset < changed.length);
|
|
181
|
+
})();
|
|
182
|
+
this.indexing = work;
|
|
183
|
+
try {
|
|
184
|
+
await work;
|
|
185
|
+
} catch (error) {
|
|
186
|
+
if (generation === this.generation)
|
|
187
|
+
this.stop(
|
|
188
|
+
error instanceof Error
|
|
189
|
+
? error
|
|
190
|
+
: new Error("History indexing failed", { cause: error }),
|
|
191
|
+
);
|
|
192
|
+
throw error;
|
|
193
|
+
} finally {
|
|
194
|
+
if (this.indexing === work) this.indexing = undefined;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async search<T extends SearchItem>(
|
|
199
|
+
sessionId: string,
|
|
200
|
+
items: readonly T[],
|
|
201
|
+
query: string,
|
|
202
|
+
limit: number,
|
|
203
|
+
recentFirst: boolean,
|
|
204
|
+
signal?: AbortSignal,
|
|
205
|
+
): Promise<T[]> {
|
|
206
|
+
if (!this.isEnabled(sessionId))
|
|
207
|
+
throw new Error("Ranked history search is not enabled for this session");
|
|
208
|
+
const words = [
|
|
209
|
+
...new Set(query.toLowerCase().match(/[\p{L}\p{N}\p{M}\p{Co}]+/gu)),
|
|
210
|
+
];
|
|
211
|
+
if (!words.length)
|
|
212
|
+
throw new Error("Supply at least one word or number to search history");
|
|
213
|
+
if (!items.length) return [];
|
|
214
|
+
const session = this.sessionId;
|
|
215
|
+
await this.index(sessionId, items, signal);
|
|
216
|
+
signal?.throwIfAborted();
|
|
217
|
+
if (session !== this.sessionId)
|
|
218
|
+
throw new Error("History session changed; retry the query");
|
|
219
|
+
const ids = await this.request(
|
|
220
|
+
{
|
|
221
|
+
action: "search",
|
|
222
|
+
sessionId,
|
|
223
|
+
ids: items.map((item) => item.item_id),
|
|
224
|
+
match: words.map((word) => `"${word}"`).join(" AND "),
|
|
225
|
+
limit,
|
|
226
|
+
recentFirst,
|
|
227
|
+
},
|
|
228
|
+
signal,
|
|
229
|
+
);
|
|
230
|
+
if (!Array.isArray(ids)) throw new Error(UNAVAILABLE);
|
|
231
|
+
const scope = new Map(items.map((item) => [item.item_id, item]));
|
|
232
|
+
return ids.map((id) => scope.get(id)!);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async pruneSession(sessionId: string, file: string): Promise<void> {
|
|
236
|
+
await this.request({ action: "prune_session", sessionId, file });
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async prune(
|
|
240
|
+
activeSessionId: string,
|
|
241
|
+
sources: Record<string, string | null> = {},
|
|
242
|
+
): Promise<PruneResult> {
|
|
243
|
+
const result = await this.request({
|
|
244
|
+
action: "prune",
|
|
245
|
+
activeSessionId,
|
|
246
|
+
sources,
|
|
247
|
+
});
|
|
248
|
+
if (!result || Array.isArray(result)) throw new Error(UNAVAILABLE);
|
|
249
|
+
return result;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
private stop(error: Error): void {
|
|
253
|
+
this.generation++;
|
|
254
|
+
const worker = this.worker;
|
|
255
|
+
this.worker = undefined;
|
|
256
|
+
for (const pending of this.pending.values()) pending.reject(error);
|
|
257
|
+
this.pending.clear();
|
|
258
|
+
this.indexed.clear();
|
|
259
|
+
this.excluded.clear();
|
|
260
|
+
this.indexing = undefined;
|
|
261
|
+
if (worker) void worker.terminate();
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
close(): void {
|
|
265
|
+
this.stop(new Error("History session changed; retry the query"));
|
|
266
|
+
this.sessionId = undefined;
|
|
267
|
+
this.file = undefined;
|
|
268
|
+
}
|
|
269
|
+
}
|