@henryqw/pi-session-recall 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +56 -0
- package/extensions/hydrate.ts +253 -0
- package/extensions/search-core.ts +1123 -0
- package/extensions/session-recall.ts +379 -0
- package/extensions/types.ts +45 -0
- package/package.json +48 -0
|
@@ -0,0 +1,1123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index engine: SQLite schema (WAL, external-content trigram FTS5), capped
|
|
3
|
+
* incremental sync from Pi session JSONL files, query sanitize ladder,
|
|
4
|
+
* discovery search with one-hop lineage suppression. No pi runtime imports —
|
|
5
|
+
* pure Node + node:sqlite so it is testable headless.
|
|
6
|
+
*/
|
|
7
|
+
import { DatabaseSync } from "node:sqlite";
|
|
8
|
+
import type { SQLInputValue, SQLOutputValue } from "node:sqlite";
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import type { SearchHit, SessionRow, SyncResult } from "./types.ts";
|
|
12
|
+
export const DEFAULT_SYNC_CAP = 50;
|
|
13
|
+
/** Hard byte ceiling per session file: larger files are skipped (and retried
|
|
14
|
+
* behind fresh work) instead of being read whole into memory. */
|
|
15
|
+
export const MAX_SESSION_FILE_BYTES = 32 * 1024 * 1024;
|
|
16
|
+
/** Hard ceiling for the internal/test `opts.cap` work bound of syncSessions. */
|
|
17
|
+
const MAX_SYNC_CAP = DEFAULT_SYNC_CAP * 10;
|
|
18
|
+
export const MAX_QUERY_CHARS = 512;
|
|
19
|
+
const MAX_TEXT_CHARS = 20000;
|
|
20
|
+
const SCAN_LIMIT = 300;
|
|
21
|
+
/** Best-ranked candidate retained per session after live-entry filtering. */
|
|
22
|
+
const ROWS_PER_FILE = 1;
|
|
23
|
+
|
|
24
|
+
const SCHEMA_SQL = `
|
|
25
|
+
-- ctime_ms is raw stat.ctimeMs stored as REAL (SQLite REALs are IEEE doubles,
|
|
26
|
+
-- so JS numbers round-trip losslessly): flooring would collapse sub-ms restores.
|
|
27
|
+
CREATE TABLE IF NOT EXISTS session_files (
|
|
28
|
+
path TEXT PRIMARY KEY,
|
|
29
|
+
size INTEGER NOT NULL,
|
|
30
|
+
mtime_ms INTEGER NOT NULL,
|
|
31
|
+
ctime_ms REAL NOT NULL
|
|
32
|
+
);
|
|
33
|
+
-- Retry markers affect ordering only: unchanged failures move behind fresh work,
|
|
34
|
+
-- but remain eligible on every pass once fresh work is drained.
|
|
35
|
+
CREATE TABLE IF NOT EXISTS session_failures (
|
|
36
|
+
path TEXT PRIMARY KEY,
|
|
37
|
+
size INTEGER NOT NULL,
|
|
38
|
+
mtime_ms INTEGER NOT NULL,
|
|
39
|
+
ctime_ms REAL NOT NULL,
|
|
40
|
+
attempts INTEGER NOT NULL DEFAULT 0
|
|
41
|
+
);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
43
|
+
path TEXT PRIMARY KEY,
|
|
44
|
+
cwd TEXT,
|
|
45
|
+
name TEXT,
|
|
46
|
+
started_at TEXT,
|
|
47
|
+
preview TEXT,
|
|
48
|
+
parent_session TEXT
|
|
49
|
+
);
|
|
50
|
+
-- Oversized messages split into head/tail columns: one row per source
|
|
51
|
+
-- message keeps AND matching across both retained regions (FTS5 implicit AND
|
|
52
|
+
-- spans columns) while phrases and NEAR cannot cross column boundaries, so
|
|
53
|
+
-- truncation never manufactures proximity matches over the elided middle.
|
|
54
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
55
|
+
rowid INTEGER PRIMARY KEY,
|
|
56
|
+
path TEXT NOT NULL,
|
|
57
|
+
entry_id TEXT NOT NULL,
|
|
58
|
+
role TEXT NOT NULL,
|
|
59
|
+
timestamp TEXT,
|
|
60
|
+
head TEXT NOT NULL,
|
|
61
|
+
tail TEXT NOT NULL,
|
|
62
|
+
UNIQUE(path, entry_id)
|
|
63
|
+
);
|
|
64
|
+
CREATE INDEX IF NOT EXISTS messages_path ON messages(path);
|
|
65
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
|
|
66
|
+
head,
|
|
67
|
+
tail,
|
|
68
|
+
content='messages',
|
|
69
|
+
content_rowid='rowid',
|
|
70
|
+
tokenize='trigram'
|
|
71
|
+
);
|
|
72
|
+
CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
|
|
73
|
+
INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
|
|
74
|
+
END;
|
|
75
|
+
CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
|
|
76
|
+
INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
|
|
77
|
+
END;
|
|
78
|
+
CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
|
|
79
|
+
INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
|
|
80
|
+
INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
|
|
81
|
+
END;
|
|
82
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
83
|
+
key TEXT PRIMARY KEY,
|
|
84
|
+
value TEXT NOT NULL
|
|
85
|
+
);
|
|
86
|
+
`;
|
|
87
|
+
|
|
88
|
+
/** Owner-only modes for the index directory and database files. The
|
|
89
|
+
* extension-owned parent dir is the only node tightened; intermediate config
|
|
90
|
+
* dirs are created restrictive but never chmod'ed if they predate this fix. */
|
|
91
|
+
const DB_FILE_MODE = 0o600;
|
|
92
|
+
const DB_DIR_MODE = 0o700;
|
|
93
|
+
|
|
94
|
+
/** Create or tighten the index's private directory and owner-only database
|
|
95
|
+
* file BEFORE SQLite writes any transcript content into it. Descriptor-based
|
|
96
|
+
* O_NOFOLLOW|O_NONBLOCK open + fstat rejects symlinks and other non-regular
|
|
97
|
+
* nodes instead of following them; fchmod tightens an existing permissive db.
|
|
98
|
+
* WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
|
|
99
|
+
* behind by older builds are tightened here too. Always closes the fd. */
|
|
100
|
+
function secureIndexNode(dbPath: string): void {
|
|
101
|
+
const posix = process.platform !== "win32";
|
|
102
|
+
const dir = path.dirname(dbPath);
|
|
103
|
+
fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
|
|
104
|
+
// O_NOFOLLOW on the db path does not protect parent components; reject a
|
|
105
|
+
// symlinked extension directory before any chmod or database open follows it.
|
|
106
|
+
if (!fs.lstatSync(dir).isDirectory()) throw new Error(`index directory is not a real directory: ${dir}`);
|
|
107
|
+
if (posix) {
|
|
108
|
+
// Tighten an existing permissive dir (e.g. created under umask 022).
|
|
109
|
+
if ((fs.statSync(dir).mode & 0o777) !== DB_DIR_MODE) {
|
|
110
|
+
fs.chmodSync(dir, DB_DIR_MODE);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
let fd: number | undefined;
|
|
114
|
+
try {
|
|
115
|
+
fd = fs.openSync(
|
|
116
|
+
dbPath,
|
|
117
|
+
(fs.constants.O_RDWR | fs.constants.O_CREAT | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0)),
|
|
118
|
+
DB_FILE_MODE,
|
|
119
|
+
);
|
|
120
|
+
const st = fs.fstatSync(fd);
|
|
121
|
+
if (!st.isFile()) throw new Error("index database path is not a regular file");
|
|
122
|
+
if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
|
|
123
|
+
} finally {
|
|
124
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
125
|
+
}
|
|
126
|
+
if (!posix) return;
|
|
127
|
+
for (const suffix of ["-wal", "-shm"]) {
|
|
128
|
+
// Same descriptor no-follow safety as the db path: chmodSync(path) would
|
|
129
|
+
// follow a swapped-in sidecar symlink and retarget its referent's mode.
|
|
130
|
+
let fd: number | undefined;
|
|
131
|
+
try {
|
|
132
|
+
fd = fs.openSync(
|
|
133
|
+
dbPath + suffix,
|
|
134
|
+
fs.constants.O_RDWR | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0),
|
|
135
|
+
);
|
|
136
|
+
const st = fs.fstatSync(fd);
|
|
137
|
+
if (!st.isFile()) throw new Error(`index ${suffix} sidecar is not a regular file`);
|
|
138
|
+
if ((st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
|
|
139
|
+
} catch (err) {
|
|
140
|
+
// Sidecars are created by SQLite; a missing legacy sidecar stays harmless.
|
|
141
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
|
|
142
|
+
} finally {
|
|
143
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function openDb(dbPath: string): DatabaseSync {
|
|
149
|
+
secureIndexNode(dbPath);
|
|
150
|
+
const db = new DatabaseSync(dbPath);
|
|
151
|
+
try {
|
|
152
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
153
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
154
|
+
// The index is disposable derived state with no migration path. Only the
|
|
155
|
+
// current schema is defined; an incompatible existing index fails on use
|
|
156
|
+
// and is never deleted or rewritten here — remove it manually to rebuild.
|
|
157
|
+
// SQLite's LIKE folds ASCII only, missing é/É, Greek, Cyrillic, …; every
|
|
158
|
+
// LIKE below wraps columns in ulower and binds pre-folded operands so
|
|
159
|
+
// both sides case-fold through the same foldCase helper.
|
|
160
|
+
db.function("ulower", (s: SQLOutputValue): string => typeof s === "string" ? foldCase(s) : "");
|
|
161
|
+
db.function("unear", nearLike);
|
|
162
|
+
db.exec(SCHEMA_SQL);
|
|
163
|
+
// CREATE TABLE IF NOT EXISTS cannot repair a predecessor index whose
|
|
164
|
+
// tables exist but lack current watermark columns (e.g. no ctime_ms): its
|
|
165
|
+
// search rows would silently serve stale hits while change detection
|
|
166
|
+
// breaks. Fail visibly; the index is disposable — remove it to rebuild.
|
|
167
|
+
for (const [table, required] of [
|
|
168
|
+
["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
|
|
169
|
+
["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
|
|
170
|
+
] as const) {
|
|
171
|
+
const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
|
|
172
|
+
for (const col of required) {
|
|
173
|
+
if (!have.has(col)) throw new Error(`index schema incompatible: ${table}.${col} missing — delete the index to rebuild`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
} catch (err) {
|
|
177
|
+
db.close();
|
|
178
|
+
throw err;
|
|
179
|
+
}
|
|
180
|
+
return db;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Replacement/edit-sensitive file fingerprint from one fs.Stats. Size plus a
|
|
184
|
+
* floored mtime_ms misses same-length rewrites whose millisecond mtime is
|
|
185
|
+
* restored (backup restores, redactions). Exact ctime closes that: POSIX
|
|
186
|
+
* utimensat can set mtime but never ctime, so every restore bumps it. */
|
|
187
|
+
function fileFingerprint(stat: fs.Stats): { size: number; mtimeMs: number; ctimeMs: number } {
|
|
188
|
+
return { size: stat.size, mtimeMs: Math.floor(stat.mtimeMs), ctimeMs: stat.ctimeMs };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// --- Parsing ---
|
|
192
|
+
|
|
193
|
+
interface ParsedMessage {
|
|
194
|
+
entryId: string;
|
|
195
|
+
role: string;
|
|
196
|
+
timestamp: string | null;
|
|
197
|
+
head: string;
|
|
198
|
+
tail: string;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
interface ParsedFile {
|
|
202
|
+
cwd: string | null;
|
|
203
|
+
name: string | null;
|
|
204
|
+
startedAt: string | null;
|
|
205
|
+
parentSession: string | null;
|
|
206
|
+
preview: string | null;
|
|
207
|
+
messages: ParsedMessage[];
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Middle-out truncation into SEPARATE head/tail index columns: concatenating
|
|
211
|
+
* them would make boundary terms adjacent and manufacture phrase/proximity
|
|
212
|
+
* matches across the elided middle. Raw text only — no synthetic notices:
|
|
213
|
+
* these strings are FTS-indexed, so injected terms would pollute search
|
|
214
|
+
* results and snippets. */
|
|
215
|
+
function truncateRegions(text: string, max = MAX_TEXT_CHARS): { head: string; tail: string } | null {
|
|
216
|
+
if (text.length <= max) return null;
|
|
217
|
+
const kept = max - 1;
|
|
218
|
+
return {
|
|
219
|
+
head: text.slice(0, Math.ceil(kept / 2)),
|
|
220
|
+
tail: text.slice(-Math.floor(kept / 2)),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Cap an untrusted string at a parse boundary so malformed JSONL cannot
|
|
225
|
+
* produce unbounded metadata downstream. */
|
|
226
|
+
function capStr(value: unknown, max: number): string | null {
|
|
227
|
+
return typeof value === "string" ? value.slice(0, max) : null;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Narrow ISO timestamp shape; captures wall-clock fields for validation. */
|
|
231
|
+
const ISO_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
232
|
+
|
|
233
|
+
/** Canonicalize a session-header timestamp to UTC ISO so its lexicographic
|
|
234
|
+
* order matches chronological order (browse sorts started_at DESC as TEXT).
|
|
235
|
+
* Anything else is stored null so it sorts behind every valid timestamp. */
|
|
236
|
+
function normalizeTimestamp(value: unknown): string | null {
|
|
237
|
+
if (typeof value !== "string") return null;
|
|
238
|
+
const m = ISO_TIMESTAMP_RE.exec(value);
|
|
239
|
+
if (!m) return null;
|
|
240
|
+
const [, y, mo, d, h] = m;
|
|
241
|
+
if (+h > 23) return null;
|
|
242
|
+
const probe = new Date(Date.UTC(+y, +mo - 1, +d));
|
|
243
|
+
// Date parsing rejects invalid time fields, but silently rolls impossible
|
|
244
|
+
// calendar dates such as February 30.
|
|
245
|
+
if (probe.getUTCMonth() !== +mo - 1 || probe.getUTCDate() !== +d) return null;
|
|
246
|
+
const date = new Date(value);
|
|
247
|
+
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function extractText(content: unknown): string {
|
|
251
|
+
if (typeof content === "string") return content;
|
|
252
|
+
if (!Array.isArray(content)) return "";
|
|
253
|
+
const parts: string[] = [];
|
|
254
|
+
for (const block of content) {
|
|
255
|
+
if (block && typeof block === "object" && (block as any).type === "text" && typeof (block as any).text === "string") {
|
|
256
|
+
parts.push((block as any).text);
|
|
257
|
+
}
|
|
258
|
+
// thinking / toolUse / tool_result blocks are skipped entirely
|
|
259
|
+
}
|
|
260
|
+
return parts.join("\n").trim();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Open the path once, validate that exact descriptor (type + size), and read
|
|
264
|
+
* only the validated snapshot from it. A concurrent append/replacement between
|
|
265
|
+
* the walk's stat and this open cannot grow the allocation or the read beyond
|
|
266
|
+
* maxBytes: the fd pins the inode, and fstat on that fd fixes both bounds.
|
|
267
|
+
* Deliberately not fs.readFileSync(fd) — that re-reads to EOF unbounded.
|
|
268
|
+
* Shared with hydration: callers pass their own byte ceiling (both use
|
|
269
|
+
* MAX_SESSION_FILE_BYTES in production). */
|
|
270
|
+
export function readBoundedSnapshot(filePath: string, maxBytes: number): string {
|
|
271
|
+
// O_NONBLOCK keeps a writerless FIFO (regular .jsonl swapped mid-walk) from
|
|
272
|
+
// blocking this open before fstat rejects it; O_NOFOLLOW (absent on Windows)
|
|
273
|
+
// rejects a symlink swapped in after the walk instead of following it.
|
|
274
|
+
const fd = fs.openSync(
|
|
275
|
+
filePath,
|
|
276
|
+
fs.constants.O_RDONLY | fs.constants.O_NONBLOCK | (fs.constants.O_NOFOLLOW ?? 0),
|
|
277
|
+
);
|
|
278
|
+
try {
|
|
279
|
+
const st = fs.fstatSync(fd);
|
|
280
|
+
if (!st.isFile()) throw new Error(`session path is not a regular file: ${filePath}`);
|
|
281
|
+
if (st.size > maxBytes) {
|
|
282
|
+
throw new Error(`session file exceeds ${Math.round(maxBytes / (1024 * 1024))} MiB snapshot limit: ${filePath}`);
|
|
283
|
+
}
|
|
284
|
+
const buf = Buffer.allocUnsafe(st.size);
|
|
285
|
+
let read = 0;
|
|
286
|
+
while (read < buf.length) {
|
|
287
|
+
const n = fs.readSync(fd, buf, read, buf.length - read, read);
|
|
288
|
+
if (n === 0) break; // truncated concurrently after fstat: index what was there
|
|
289
|
+
read += n;
|
|
290
|
+
}
|
|
291
|
+
return buf.toString("utf-8", 0, read);
|
|
292
|
+
} finally {
|
|
293
|
+
fs.closeSync(fd);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
|
|
298
|
+
const seenEntryIds = new Set<string>();
|
|
299
|
+
const parsed: ParsedFile = {
|
|
300
|
+
cwd: null,
|
|
301
|
+
name: null,
|
|
302
|
+
startedAt: null,
|
|
303
|
+
parentSession: null,
|
|
304
|
+
preview: null,
|
|
305
|
+
messages: [],
|
|
306
|
+
};
|
|
307
|
+
// Unreadable/oversized file must throw so the sync transaction rolls back
|
|
308
|
+
// instead of wiping previously indexed rows and advancing the watermark over a hole.
|
|
309
|
+
const content = readBoundedSnapshot(filePath, maxBytes);
|
|
310
|
+
for (const line of content.split("\n")) {
|
|
311
|
+
if (!line.trim()) continue;
|
|
312
|
+
let entry: any;
|
|
313
|
+
try {
|
|
314
|
+
entry = JSON.parse(line);
|
|
315
|
+
} catch {
|
|
316
|
+
continue; // skip malformed lines
|
|
317
|
+
}
|
|
318
|
+
const hasValidId = typeof entry?.id === "string" && entry.id.length > 0 && entry.id.length <= 256
|
|
319
|
+
&& (entry.parentId == null || (typeof entry.parentId === "string" && entry.parentId.length <= 256));
|
|
320
|
+
if (hasValidId) {
|
|
321
|
+
// Hydration is first-wins across every entry type; reserve IDs at the
|
|
322
|
+
// same boundary so discovery can never point at a different duplicate.
|
|
323
|
+
if (seenEntryIds.has(entry.id)) continue;
|
|
324
|
+
seenEntryIds.add(entry.id);
|
|
325
|
+
}
|
|
326
|
+
switch (entry?.type) {
|
|
327
|
+
case "session":
|
|
328
|
+
// Untrusted header strings are capped here so every consumer
|
|
329
|
+
// (browse rows, discovery meta) inherits the bound.
|
|
330
|
+
parsed.cwd = capStr(entry.cwd, 500) ?? parsed.cwd;
|
|
331
|
+
parsed.startedAt = normalizeTimestamp(entry.timestamp) ?? parsed.startedAt;
|
|
332
|
+
parsed.parentSession = capStr(entry.parentSession, 1024) ?? parsed.parentSession;
|
|
333
|
+
break;
|
|
334
|
+
case "session_info":
|
|
335
|
+
if (typeof entry.name === "string") parsed.name = entry.name.slice(0, 500);
|
|
336
|
+
break;
|
|
337
|
+
case "message": {
|
|
338
|
+
const role = entry.message?.role;
|
|
339
|
+
if (role !== "user" && role !== "assistant") break;
|
|
340
|
+
const full = extractText(entry.message.content);
|
|
341
|
+
if (!full) break;
|
|
342
|
+
if (!hasValidId) break;
|
|
343
|
+
if (role === "user" && parsed.preview === null) {
|
|
344
|
+
parsed.preview = full.slice(0, 200);
|
|
345
|
+
}
|
|
346
|
+
const regions = truncateRegions(full);
|
|
347
|
+
parsed.messages.push({
|
|
348
|
+
entryId: entry.id,
|
|
349
|
+
role,
|
|
350
|
+
timestamp: capStr(entry.timestamp, 128),
|
|
351
|
+
head: regions ? regions.head : full,
|
|
352
|
+
tail: regions ? regions.tail : "",
|
|
353
|
+
});
|
|
354
|
+
break;
|
|
355
|
+
}
|
|
356
|
+
// compaction, branch_summary, custom, custom_message, label,
|
|
357
|
+
// session_info (name handled above), model_change, … are skipped
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
return parsed;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function isJunkEncodedDir(relSegments: string[]): boolean {
|
|
364
|
+
return relSegments.some((seg) => seg.startsWith("--tmp-") || seg.startsWith("--private-tmp-"));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/** Walk result. `complete: false` means the tree could not be fully read
|
|
368
|
+
* (missing root, readdir/stat failure) — callers must not treat unseen
|
|
369
|
+
* indexed paths as deleted. */
|
|
370
|
+
interface WalkResult {
|
|
371
|
+
files: Map<string, fs.Stats>;
|
|
372
|
+
complete: boolean;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function walkJsonlFiles(sessionsDir: string): WalkResult {
|
|
376
|
+
const files = new Map<string, fs.Stats>();
|
|
377
|
+
if (!fs.existsSync(sessionsDir)) return { files, complete: false };
|
|
378
|
+
const stack: { dir: string; rel: string[] }[] = [{ dir: sessionsDir, rel: [] }];
|
|
379
|
+
let complete = true;
|
|
380
|
+
while (stack.length > 0) {
|
|
381
|
+
const { dir, rel } = stack.pop()!;;
|
|
382
|
+
let entries: fs.Dirent[];
|
|
383
|
+
try {
|
|
384
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
385
|
+
} catch {
|
|
386
|
+
complete = false;
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
for (const ent of entries) {
|
|
390
|
+
const full = path.join(dir, ent.name);
|
|
391
|
+
if (ent.isDirectory()) {
|
|
392
|
+
// Junk dirs are pruned before descent so a large ignored tree never
|
|
393
|
+
// costs a walk + per-file stats on every sync pass.
|
|
394
|
+
if (isJunkEncodedDir([...rel, ent.name])) continue;
|
|
395
|
+
stack.push({ dir: full, rel: [...rel, ent.name] });
|
|
396
|
+
} else if (ent.isFile() && ent.name.endsWith(".jsonl")) {
|
|
397
|
+
try {
|
|
398
|
+
files.set(full, fs.statSync(full));
|
|
399
|
+
} catch {
|
|
400
|
+
complete = false; // vanished or became unreadable mid-walk
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return { files, complete };
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// --- Sync ---
|
|
409
|
+
|
|
410
|
+
export function syncSessions(
|
|
411
|
+
sessionsDir: string,
|
|
412
|
+
dbPath: string,
|
|
413
|
+
opts?: { cap?: number; maxFileBytes?: number },
|
|
414
|
+
): SyncResult {
|
|
415
|
+
const requestedCap = opts?.cap ?? DEFAULT_SYNC_CAP;
|
|
416
|
+
const cap = Number.isFinite(requestedCap)
|
|
417
|
+
? Math.max(1, Math.min(MAX_SYNC_CAP, Math.floor(requestedCap)))
|
|
418
|
+
: DEFAULT_SYNC_CAP;
|
|
419
|
+
const db = openDb(dbPath);
|
|
420
|
+
try {
|
|
421
|
+
|
|
422
|
+
// Junk dirs never reach the walk result: pruned during descent.
|
|
423
|
+
const walk = walkJsonlFiles(sessionsDir);
|
|
424
|
+
const all = [...walk.files];
|
|
425
|
+
const watermarks = new Map(
|
|
426
|
+
(db.prepare("SELECT path, size, mtime_ms, ctime_ms FROM session_files").all() as any[]).map((r) => [r.path, r]),
|
|
427
|
+
);
|
|
428
|
+
const failures = new Map(
|
|
429
|
+
(db.prepare("SELECT path, size, mtime_ms, ctime_ms, attempts FROM session_failures").all() as any[]).map((r) => [r.path, r]),
|
|
430
|
+
);
|
|
431
|
+
// An incomplete traversal proves nothing about absence: purging here would
|
|
432
|
+
// wipe healthy sessions over a transient readdir/stat failure. Changed/new
|
|
433
|
+
// files already discovered still process normally.
|
|
434
|
+
const discovered = new Set(all.map(([fp]) => fp));
|
|
435
|
+
const deleted = walk.complete ? [...watermarks.keys()].filter((p) => !discovered.has(p)) : [];
|
|
436
|
+
const deletedFailures = walk.complete ? [...failures.keys()].filter((p) => !discovered.has(p)) : [];
|
|
437
|
+
|
|
438
|
+
const changed: { path: string; stat: fs.Stats }[] = [];
|
|
439
|
+
for (const [p, stat] of all) {
|
|
440
|
+
const fp = fileFingerprint(stat);
|
|
441
|
+
const wm = watermarks.get(p);
|
|
442
|
+
if (wm && wm.size === fp.size && wm.mtime_ms === fp.mtimeMs && wm.ctime_ms === fp.ctimeMs) continue;
|
|
443
|
+
changed.push({ path: p, stat });
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Newest-first, but an unchanged prior failure moves behind untouched work.
|
|
447
|
+
// Among retries, fewest attempts first so one persistent failure cannot
|
|
448
|
+
// monopolize a small cap forever.
|
|
449
|
+
const retryAttempts = (p: string, stat: fs.Stats): number | null => {
|
|
450
|
+
const f = failures.get(p);
|
|
451
|
+
if (!f) return null;
|
|
452
|
+
const fp = fileFingerprint(stat);
|
|
453
|
+
return f.size === fp.size && f.mtime_ms === fp.mtimeMs && f.ctime_ms === fp.ctimeMs ? (f.attempts ?? 0) : null;
|
|
454
|
+
};
|
|
455
|
+
changed.sort((a, b) => {
|
|
456
|
+
const aRetry = retryAttempts(a.path, a.stat);
|
|
457
|
+
const bRetry = retryAttempts(b.path, b.stat);
|
|
458
|
+
if (aRetry !== null && bRetry !== null) return aRetry - bRetry || b.stat.mtimeMs - a.stat.mtimeMs;
|
|
459
|
+
if (aRetry !== null) return 1;
|
|
460
|
+
if (bRetry !== null) return -1;
|
|
461
|
+
return b.stat.mtimeMs - a.stat.mtimeMs;
|
|
462
|
+
});
|
|
463
|
+
deleted.sort((a, b) => {
|
|
464
|
+
const ma = watermarks.get(a)!.mtime_ms;
|
|
465
|
+
const mb = watermarks.get(b)!.mtime_ms;
|
|
466
|
+
return mb - ma;
|
|
467
|
+
});
|
|
468
|
+
|
|
469
|
+
const delStmt = db.prepare("DELETE FROM messages WHERE path = ?");
|
|
470
|
+
const delSession = db.prepare("DELETE FROM sessions WHERE path = ?");
|
|
471
|
+
const delFile = db.prepare("DELETE FROM session_files WHERE path = ?");
|
|
472
|
+
const delFailure = db.prepare("DELETE FROM session_failures WHERE path = ?");
|
|
473
|
+
const upsertFailure = db.prepare(
|
|
474
|
+
"INSERT INTO session_failures(path, size, mtime_ms, ctime_ms, attempts) VALUES (?, ?, ?, ?, 1) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, attempts=CASE WHEN size=excluded.size AND mtime_ms=excluded.mtime_ms AND ctime_ms=excluded.ctime_ms THEN attempts+1 ELSE 1 END",
|
|
475
|
+
);
|
|
476
|
+
const upsertFile = db.prepare(
|
|
477
|
+
"INSERT INTO session_files(path, size, mtime_ms, ctime_ms) VALUES (?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms",
|
|
478
|
+
);
|
|
479
|
+
const upsertSession = db.prepare(
|
|
480
|
+
`INSERT INTO sessions(path, cwd, name, started_at, preview, parent_session) VALUES (?, ?, ?, ?, ?, ?)
|
|
481
|
+
ON CONFLICT(path) DO UPDATE SET cwd=excluded.cwd, name=excluded.name, started_at=excluded.started_at,
|
|
482
|
+
preview=excluded.preview, parent_session=excluded.parent_session`,
|
|
483
|
+
);
|
|
484
|
+
const insertMsg = db.prepare(
|
|
485
|
+
"INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
|
|
486
|
+
);
|
|
487
|
+
|
|
488
|
+
const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
|
|
489
|
+
|
|
490
|
+
// Returns messages indexed for this file.
|
|
491
|
+
const tx = (filePath: string, stat: fs.Stats): number => {
|
|
492
|
+
db.exec("BEGIN");
|
|
493
|
+
try {
|
|
494
|
+
const parsed = parseSessionFile(filePath, maxFileBytes);
|
|
495
|
+
delStmt.run(filePath);
|
|
496
|
+
upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
|
|
497
|
+
let count = 0;
|
|
498
|
+
for (const msg of parsed.messages) {
|
|
499
|
+
insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
|
|
500
|
+
count++;
|
|
501
|
+
}
|
|
502
|
+
{
|
|
503
|
+
const fp = fileFingerprint(stat);
|
|
504
|
+
upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
|
|
505
|
+
}
|
|
506
|
+
delFailure.run(filePath);
|
|
507
|
+
db.exec("COMMIT");
|
|
508
|
+
return count;
|
|
509
|
+
} catch (err) {
|
|
510
|
+
db.exec("ROLLBACK");
|
|
511
|
+
throw err;
|
|
512
|
+
}
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
let filesProcessed = 0;
|
|
516
|
+
let messagesIndexed = 0;
|
|
517
|
+
|
|
518
|
+
// The cap bounds total indexing attempts per pass, not successes.
|
|
519
|
+
const batch = changed.slice(0, cap);
|
|
520
|
+
for (const { path: p, stat } of batch) {
|
|
521
|
+
try {
|
|
522
|
+
// ponytail: byte-level work bound — skip instead of streaming files that
|
|
523
|
+
// are huge enough to block the event loop; raise the cap or stream the
|
|
524
|
+
// parser if real sessions ever hit it. Optimization only: readBoundedSnapshot
|
|
525
|
+
// re-validates size/type on its own descriptor at the parse boundary.
|
|
526
|
+
if (stat.size > maxFileBytes) throw new Error("session file exceeds indexing size cap");
|
|
527
|
+
messagesIndexed += tx(p, stat);
|
|
528
|
+
filesProcessed++;
|
|
529
|
+
} catch {
|
|
530
|
+
// Keep the file retryable, but put this unchanged fingerprint behind
|
|
531
|
+
// untouched work on the next pass.
|
|
532
|
+
{
|
|
533
|
+
const fp = fileFingerprint(stat);
|
|
534
|
+
upsertFailure.run(p, fp.size, fp.mtimeMs, fp.ctimeMs);
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
const deletedRemaining = Math.max(0, deleted.length - cap);
|
|
540
|
+
for (const p of deleted.slice(0, cap)) {
|
|
541
|
+
delStmt.run(p);
|
|
542
|
+
delSession.run(p);
|
|
543
|
+
delFile.run(p);
|
|
544
|
+
delFailure.run(p);
|
|
545
|
+
}
|
|
546
|
+
for (const p of deletedFailures) delFailure.run(p);
|
|
547
|
+
|
|
548
|
+
// Attempted failures remain unsynced and therefore remain in the backlog.
|
|
549
|
+
const backlog = changed.length - filesProcessed + deletedRemaining;
|
|
550
|
+
db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
|
|
551
|
+
|
|
552
|
+
return { filesProcessed, messagesIndexed, backlogRemaining: backlog };
|
|
553
|
+
} finally {
|
|
554
|
+
db.close();
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function getBacklog(db: DatabaseSync): number {
|
|
559
|
+
const row = db.prepare("SELECT value FROM meta WHERE key = 'backlog'").get() as { value: string } | undefined;
|
|
560
|
+
return row ? Number(row.value) : 0;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// --- Query sanitize ladder ---
|
|
564
|
+
|
|
565
|
+
const OPERATOR_RE = /\b(OR|AND|NOT|NEAR)\b/;
|
|
566
|
+
// FTS5 string syntax: `""` inside a quoted phrase is one literal quote.
|
|
567
|
+
const TOKEN_RE = /"((?:[^"]|"")*)"|(\S+)/g;
|
|
568
|
+
const unescapePhrase = (s: string): string => s.replaceAll('""', '"');
|
|
569
|
+
|
|
570
|
+
interface QueryTerm {
|
|
571
|
+
text: string;
|
|
572
|
+
operator: boolean;
|
|
573
|
+
nearDistance: boolean;
|
|
574
|
+
quoted: boolean;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
function collectQueryTerms(query: string): QueryTerm[] {
|
|
578
|
+
const terms: QueryTerm[] = [];
|
|
579
|
+
let depth = 0;
|
|
580
|
+
let pendingNear = false;
|
|
581
|
+
let nearDepth = 0;
|
|
582
|
+
let afterNearComma = false;
|
|
583
|
+
for (const match of spaceParensOutsideQuotes(query).matchAll(TOKEN_RE)) {
|
|
584
|
+
const phrase = match[1];
|
|
585
|
+
let raw = phrase === undefined ? match[2] ?? "" : unescapePhrase(phrase);
|
|
586
|
+
if (phrase === undefined && raw === "(") {
|
|
587
|
+
depth++;
|
|
588
|
+
if (pendingNear) {
|
|
589
|
+
nearDepth = depth;
|
|
590
|
+
pendingNear = false;
|
|
591
|
+
}
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
if (phrase === undefined && raw === ")") {
|
|
595
|
+
if (depth === nearDepth) {
|
|
596
|
+
nearDepth = 0;
|
|
597
|
+
afterNearComma = false;
|
|
598
|
+
}
|
|
599
|
+
depth = Math.max(0, depth - 1);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
const hasNearComma = phrase === undefined && nearDepth > 0 && raw.endsWith(",");
|
|
603
|
+
// An attached distance like `NEAR(Go Rust,10)` has no standalone comma
|
|
604
|
+
// token; split it so the numeric tail is recognized as the distance.
|
|
605
|
+
let attachedDistance: string | undefined;
|
|
606
|
+
if (!hasNearComma && phrase === undefined && nearDepth > 0) {
|
|
607
|
+
const attached = /^(.*),(\d+)$/.exec(raw);
|
|
608
|
+
if (attached) {
|
|
609
|
+
raw = attached[1];
|
|
610
|
+
attachedDistance = attached[2];
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
// Unmatched quote delimiters are malformed syntax, not searchable text.
|
|
614
|
+
// Phrases arrive already unescaped via raw.
|
|
615
|
+
const text = phrase === undefined ? raw.replace(/^[.,!?;:()]+|[.,!?;:()]+$/g, "").replace(/"/g, "") : raw;
|
|
616
|
+
if (text) {
|
|
617
|
+
const operator = phrase === undefined && /^(?:OR|AND|NOT|NEAR)$/.test(text);
|
|
618
|
+
const nearDistance = nearDepth > 0 && afterNearComma && /^\d+$/.test(text);
|
|
619
|
+
terms.push({ text, operator, nearDistance, quoted: phrase !== undefined });
|
|
620
|
+
if (operator && text === "NEAR") pendingNear = true;
|
|
621
|
+
}
|
|
622
|
+
if (attachedDistance !== undefined) terms.push({ text: attachedDistance, operator: false, nearDistance: true, quoted: false });
|
|
623
|
+
if (hasNearComma) afterNearComma = true;
|
|
624
|
+
}
|
|
625
|
+
return terms;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
function quoteTerm(term: QueryTerm): string {
|
|
629
|
+
// Prefix expansion belongs only to an unquoted trailing star. An explicitly
|
|
630
|
+
// quoted `"deploy*"` searches for the literal asterisk.
|
|
631
|
+
if (!term.quoted && term.text.endsWith("*")) return `"${term.text.slice(0, -1).replace(/"/g, '""')}"*`;
|
|
632
|
+
return `"${term.text.replace(/"/g, '""')}"`;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function quoteTerms(terms: QueryTerm[], sep: string): string {
|
|
636
|
+
return terms.map(quoteTerm).join(sep);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
export interface FtsQueryPlan {
|
|
640
|
+
/** Candidate FTS5 MATCH expressions in try order. */
|
|
641
|
+
ftsCandidates: string[];
|
|
642
|
+
/** When true, fall back to SQL LIKE (also forced when every term < 3 chars). */
|
|
643
|
+
forceLike: boolean;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/**
|
|
647
|
+
* Sanitize ladder: trim + 512 cap; implicit-AND quoting when no explicit
|
|
648
|
+
* FTS5 operator; raw pass-through otherwise; recovery candidates are the
|
|
649
|
+
* fully-quoted form, then OR-expansion; LIKE covers everything else.
|
|
650
|
+
*/
|
|
651
|
+
export function buildFtsQueryPlan(rawQuery: string): FtsQueryPlan {
|
|
652
|
+
let query = rawQuery.trim();
|
|
653
|
+
if (query.length > MAX_QUERY_CHARS) query = query.slice(0, MAX_QUERY_CHARS);
|
|
654
|
+
if (!query) return { ftsCandidates: [], forceLike: false };
|
|
655
|
+
|
|
656
|
+
const queryTerms = collectQueryTerms(query);
|
|
657
|
+
const hasOperator = OPERATOR_RE.test(query);
|
|
658
|
+
// Short terms vanish under trigram MATCH — for natural-language queries
|
|
659
|
+
// (AND semantics) that silently breaks the query. Explicit-operator queries
|
|
660
|
+
// route there too (`Go OR Rust` silently drops the Go side); the boolean
|
|
661
|
+
// LIKE fallback preserves their AND/OR/NOT semantics.
|
|
662
|
+
if (
|
|
663
|
+
queryTerms.some(
|
|
664
|
+
(term) => !term.operator && !term.nearDistance && [...term.text.replace(/["()*]/g, "")].length < 3,
|
|
665
|
+
)
|
|
666
|
+
) {
|
|
667
|
+
return { ftsCandidates: [], forceLike: true };
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const operands = queryTerms.filter((term) => !term.operator && !term.nearDistance);
|
|
671
|
+
// Recovery operands exclude syntax operators so a malformed query can never
|
|
672
|
+
// broaden into matches on AND/OR/NOT/NEAR themselves.
|
|
673
|
+
const natural = quoteTerms(operands, " ");
|
|
674
|
+
const orExpanded = operands.length > 1 ? quoteTerms(operands, " OR ") : null;
|
|
675
|
+
|
|
676
|
+
if (!hasOperator) {
|
|
677
|
+
// Quoted form cannot fail to parse; OR-expand only as breadth fallback.
|
|
678
|
+
return { ftsCandidates: orExpanded ? [natural, orExpanded] : [natural], forceLike: false };
|
|
679
|
+
}
|
|
680
|
+
// Explicit operators: raw first, then recovery paths.
|
|
681
|
+
const candidates = [query, natural];
|
|
682
|
+
if (orExpanded) candidates.push(orExpanded);
|
|
683
|
+
return { ftsCandidates: candidates, forceLike: false };
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** Unicode-aware case fold for all LIKE comparisons (column values via ulower,
|
|
687
|
+
* bound operand patterns, snippet matching). toLowerCase alone is not an
|
|
688
|
+
* equivalence for Greek: word-final Σ lowercases to ς while a typed query uses
|
|
689
|
+
* σ. ponytail: normalizes final sigma only, not full Unicode CaseFolding.txt
|
|
690
|
+
* (e.g. ß→ss stays unmatched); add a fold table if that ever matters. */
|
|
691
|
+
function foldCase(s: string): string {
|
|
692
|
+
return s.toLowerCase().replaceAll("ς", "σ");
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
function normalizeLikeTerm(term: string, quoted = false): string {
|
|
696
|
+
// Only an unquoted trailing star is prefix syntax. Quoted stars stay literal.
|
|
697
|
+
return !quoted && term.endsWith("*") ? term.slice(0, -1) : term;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
function likePattern(term: string): string {
|
|
701
|
+
// Fold here too: escaping is unaffected because % _ \ have no case variants.
|
|
702
|
+
return `%${foldCase(term).replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// --- Boolean LIKE fallback ---
|
|
706
|
+
// Parameterized translation of simple AND/OR/NOT/NEAR queries to SQL LIKE so
|
|
707
|
+
// short operands (trigram floor) keep boolean semantics. User input only ever
|
|
708
|
+
// reaches SQL as bound data.
|
|
709
|
+
|
|
710
|
+
function likeClause(term: string, quoted = false): { clause: string; params: string[] } | null {
|
|
711
|
+
term = normalizeLikeTerm(term, quoted);
|
|
712
|
+
if (!term) return null;
|
|
713
|
+
const pattern = likePattern(term);
|
|
714
|
+
return {
|
|
715
|
+
clause: "(ulower(m.head) LIKE ? ESCAPE '\\' OR ulower(m.tail) LIKE ? ESCAPE '\\')",
|
|
716
|
+
params: [pattern, pattern],
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
interface LikeSql {
|
|
721
|
+
where: string;
|
|
722
|
+
params: SQLInputValue[];
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/** Character-position analogue of trigram FTS5 NEAR for LIKE-only operands.
|
|
726
|
+
* Trigram positions make N allow at most N-2 characters between phrases. */
|
|
727
|
+
function nearLike(textValue: SQLOutputValue, termsValue: SQLOutputValue, distanceValue: SQLOutputValue): number {
|
|
728
|
+
if (typeof textValue !== "string" || typeof termsValue !== "string" || typeof distanceValue !== "number") return 0;
|
|
729
|
+
// Duplicate operands cannot affect the all-distinct-terms-present predicate,
|
|
730
|
+
// and the 512-char query cap otherwise admits hundreds of them.
|
|
731
|
+
const needles = [...new Set((JSON.parse(termsValue) as string[]).map(foldCase))];
|
|
732
|
+
const text = foldCase(textValue);
|
|
733
|
+
// Deduplication can leave one needle: presence satisfies any distance,
|
|
734
|
+
// while the multi-needle span math below demands an impossible negative gap.
|
|
735
|
+
if (needles.length === 1) return text.includes(needles[0]) ? 1 : 0;
|
|
736
|
+
const codePointAt = new Uint32Array(text.length + 1);
|
|
737
|
+
let point = 0;
|
|
738
|
+
for (let i = 0; i < text.length; point++) {
|
|
739
|
+
const width = (text.codePointAt(i) ?? 0) > 0xffff ? 2 : 1;
|
|
740
|
+
codePointAt[i] = point;
|
|
741
|
+
if (width === 2) codePointAt[i + 1] = point;
|
|
742
|
+
i += width;
|
|
743
|
+
}
|
|
744
|
+
codePointAt[text.length] = point;
|
|
745
|
+
|
|
746
|
+
const occurrences: { start: number; end: number; term: number }[] = [];
|
|
747
|
+
for (const [term, needle] of needles.entries()) {
|
|
748
|
+
for (let at = text.indexOf(needle); at >= 0; at = text.indexOf(needle, at + 1)) {
|
|
749
|
+
occurrences.push({ start: codePointAt[at], end: codePointAt[at + needle.length], term });
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
occurrences.sort((a, b) => a.start - b.start || a.end - b.end);
|
|
753
|
+
|
|
754
|
+
const counts = new Uint16Array(needles.length);
|
|
755
|
+
let present = 0;
|
|
756
|
+
let left = 0;
|
|
757
|
+
for (let right = 0; right < occurrences.length; right++) {
|
|
758
|
+
if (counts[occurrences[right].term]++ === 0) present++;
|
|
759
|
+
while (present === needles.length) {
|
|
760
|
+
if (occurrences[right].start - occurrences[left].end + 2 <= distanceValue) return 1;
|
|
761
|
+
if (--counts[occurrences[left++].term] === 0) present--;
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
return 0;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
interface LikeToken {
|
|
768
|
+
phrase?: string;
|
|
769
|
+
word?: string;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function parseNearLikeSql(tokens: LikeToken[], start: number): { sql: LikeSql; end: number } | null {
|
|
773
|
+
if (tokens[start + 1]?.word !== "(") return null;
|
|
774
|
+
const operands: { text: string; quoted: boolean }[] = [];
|
|
775
|
+
let sawComma = false;
|
|
776
|
+
let distanceText: string | undefined;
|
|
777
|
+
|
|
778
|
+
for (let i = start + 2; i < tokens.length; i++) {
|
|
779
|
+
const token = tokens[i];
|
|
780
|
+
if (token.word === ")") {
|
|
781
|
+
if (operands.length < 2 || (sawComma && distanceText === undefined)) return null;
|
|
782
|
+
const distance = distanceText === undefined ? 10 : Number(distanceText);
|
|
783
|
+
if (distanceText !== undefined && (!/^\d+$/.test(distanceText) || !Number.isSafeInteger(distance))) return null;
|
|
784
|
+
const terms = operands.map((term) => normalizeLikeTerm(term.text, term.quoted));
|
|
785
|
+
if (terms.some((term) => !term)) return null;
|
|
786
|
+
const clauses = terms.map((term) => likeClause(term, true)!);
|
|
787
|
+
const encoded = JSON.stringify(terms);
|
|
788
|
+
return {
|
|
789
|
+
sql: {
|
|
790
|
+
where: `(${clauses.map((clause) => clause.clause).join(" AND ")} AND (unear(m.head, ?, ?) OR unear(m.tail, ?, ?)))`,
|
|
791
|
+
params: [...clauses.flatMap((clause) => clause.params), encoded, distance, encoded, distance],
|
|
792
|
+
},
|
|
793
|
+
end: i,
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
if (token.phrase !== undefined) {
|
|
797
|
+
if (sawComma) return null;
|
|
798
|
+
operands.push({ text: token.phrase, quoted: true });
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
const word = token.word ?? "";
|
|
803
|
+
if (!word || word === "(" || /^(?:AND|OR|NOT|NEAR)$/.test(word) || word.includes('"')) return null;
|
|
804
|
+
const comma = word.indexOf(",");
|
|
805
|
+
if (comma >= 0) {
|
|
806
|
+
if (sawComma || word.indexOf(",", comma + 1) >= 0) return null;
|
|
807
|
+
const before = word.slice(0, comma).replace(/^[.!?;:]+|[.!?;:]+$/g, "");
|
|
808
|
+
if (before) operands.push({ text: before, quoted: false });
|
|
809
|
+
sawComma = true;
|
|
810
|
+
distanceText = word.slice(comma + 1) || undefined;
|
|
811
|
+
} else if (sawComma) {
|
|
812
|
+
if (distanceText !== undefined) return null;
|
|
813
|
+
distanceText = word;
|
|
814
|
+
} else {
|
|
815
|
+
const text = word.replace(/^[.!?;:]+|[.!?;:]+$/g, "");
|
|
816
|
+
if (!text) return null;
|
|
817
|
+
operands.push({ text, quoted: false });
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
return null;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
/** Space out parens that act as grouping syntax while leaving quoted phrases
|
|
824
|
+
* like "C(ABI)" intact. */
|
|
825
|
+
function spaceParensOutsideQuotes(q: string): string {
|
|
826
|
+
let out = "";
|
|
827
|
+
let inQuote = false;
|
|
828
|
+
for (const c of q) {
|
|
829
|
+
if (c === '"') inQuote = !inQuote;
|
|
830
|
+
out += !inQuote && (c === "(" || c === ")") ? ` ${c} ` : c;
|
|
831
|
+
}
|
|
832
|
+
return out;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
|
|
836
|
+
let query = rawQuery.trim();
|
|
837
|
+
if (!query) return null;
|
|
838
|
+
// Imbalance detection/recovery is quote-aware: parentheses inside quoted
|
|
839
|
+
// operands ("func(") are literals and must survive.
|
|
840
|
+
let depth = 0;
|
|
841
|
+
let inQuote = false;
|
|
842
|
+
for (const c of query) {
|
|
843
|
+
if (c === '"') inQuote = !inQuote;
|
|
844
|
+
else if (!inQuote && c === "(") depth++;
|
|
845
|
+
else if (!inQuote && c === ")" && --depth < 0) break;
|
|
846
|
+
}
|
|
847
|
+
if (depth !== 0) {
|
|
848
|
+
inQuote = false;
|
|
849
|
+
query = [...query].map((c) => {
|
|
850
|
+
if (c === '"') inQuote = !inQuote;
|
|
851
|
+
return !inQuote && (c === "(" || c === ")") ? " " : c;
|
|
852
|
+
}).join("");
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
const tokens: LikeToken[] = [...spaceParensOutsideQuotes(query).matchAll(TOKEN_RE)]
|
|
856
|
+
.map((m) => (m[1] !== undefined ? { phrase: unescapePhrase(m[1]) } : { word: m[2] ?? "" }))
|
|
857
|
+
.filter((t) => (t.phrase !== undefined ? t.phrase !== "" : t.word !== ""));
|
|
858
|
+
const fail = (): LikeSql | null => tokens.some((token) => token.word === "NEAR") ? { where: "0", params: [] } : null;
|
|
859
|
+
const sql: string[] = [];
|
|
860
|
+
const params: SQLInputValue[] = [];
|
|
861
|
+
let expectingOperand = true;
|
|
862
|
+
depth = 0;
|
|
863
|
+
|
|
864
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
865
|
+
const token = tokens[i];
|
|
866
|
+
const word = token.word;
|
|
867
|
+
if (word === "(") {
|
|
868
|
+
if (!expectingOperand) sql.push("AND");
|
|
869
|
+
sql.push("(");
|
|
870
|
+
depth++;
|
|
871
|
+
expectingOperand = true;
|
|
872
|
+
continue;
|
|
873
|
+
}
|
|
874
|
+
if (word === ")") {
|
|
875
|
+
if (expectingOperand || depth-- === 0) return fail();
|
|
876
|
+
sql.push(")");
|
|
877
|
+
expectingOperand = false;
|
|
878
|
+
continue;
|
|
879
|
+
}
|
|
880
|
+
if (word === "AND" || word === "OR") {
|
|
881
|
+
if (expectingOperand) return fail();
|
|
882
|
+
sql.push(word);
|
|
883
|
+
expectingOperand = true;
|
|
884
|
+
continue;
|
|
885
|
+
}
|
|
886
|
+
if (word === "NOT") {
|
|
887
|
+
if (!expectingOperand) sql.push("AND");
|
|
888
|
+
sql.push("NOT");
|
|
889
|
+
expectingOperand = true;
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
if (word === "NEAR") {
|
|
893
|
+
const near = parseNearLikeSql(tokens, i);
|
|
894
|
+
if (near === null) return fail();
|
|
895
|
+
if (!expectingOperand) sql.push("AND");
|
|
896
|
+
sql.push(near.sql.where);
|
|
897
|
+
params.push(...near.sql.params);
|
|
898
|
+
expectingOperand = false;
|
|
899
|
+
i = near.end;
|
|
900
|
+
continue;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
const term = token.phrase ?? word?.replace(/^[.,!?;:]+|[.,!?;:]+$/g, "").replace(/"/g, "") ?? "";
|
|
904
|
+
const clause = likeClause(term, token.phrase !== undefined);
|
|
905
|
+
if (clause === null) return fail();
|
|
906
|
+
if (!expectingOperand) sql.push("AND");
|
|
907
|
+
sql.push(clause.clause);
|
|
908
|
+
params.push(...clause.params);
|
|
909
|
+
expectingOperand = false;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
return expectingOperand || depth !== 0 ? fail() : { where: sql.join(" "), params };
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// --- Search ---
|
|
916
|
+
|
|
917
|
+
export interface SearchOptions {
|
|
918
|
+
limit?: number;
|
|
919
|
+
/** Entry ids on the current session's live branch — hits here are skipped. */
|
|
920
|
+
currentLiveEntryIds?: Set<string>;
|
|
921
|
+
/** Path of the current session file. */
|
|
922
|
+
currentSessionPath?: string;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
// Live-entry suppression happens INSIDE SQL (before ROW_NUMBER/caps) so
|
|
926
|
+
// inactive matches on the current file compete only against each other, and
|
|
927
|
+
// there is no pre-partition LIMIT: one verbose session cannot starve others.
|
|
928
|
+
// live_filter is a per-connection TEMP table populated from SearchOptions.
|
|
929
|
+
const LIVE_FILTER_SQL = `NOT EXISTS (
|
|
930
|
+
SELECT 1 FROM live_filter lf WHERE lf.path = m.path AND lf.entry_id = m.entry_id
|
|
931
|
+
)`;
|
|
932
|
+
const BASE_SELECT = `
|
|
933
|
+
WITH matches AS (
|
|
934
|
+
SELECT m.path, m.entry_id, m.role, m.timestamp, s.cwd, s.name, s.started_at,
|
|
935
|
+
s.parent_session, m.rowid AS rid,
|
|
936
|
+
snippet(session_fts, -1, '[', ']', '…', 16) AS snip,
|
|
937
|
+
bm25(session_fts) AS score
|
|
938
|
+
FROM session_fts JOIN messages m ON m.rowid = session_fts.rowid
|
|
939
|
+
LEFT JOIN sessions s ON s.path = m.path
|
|
940
|
+
WHERE session_fts MATCH ? AND ${LIVE_FILTER_SQL}
|
|
941
|
+
), ranked AS (
|
|
942
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY score, rid) AS rn
|
|
943
|
+
FROM matches
|
|
944
|
+
)
|
|
945
|
+
SELECT path, entry_id, role, timestamp, snip, cwd, name, started_at
|
|
946
|
+
FROM ranked r
|
|
947
|
+
WHERE rn <= ${ROWS_PER_FILE}
|
|
948
|
+
AND NOT EXISTS (
|
|
949
|
+
SELECT 1 FROM ranked parent
|
|
950
|
+
WHERE parent.path = r.parent_session
|
|
951
|
+
AND parent.path <> r.path
|
|
952
|
+
AND parent.rn <= ${ROWS_PER_FILE}
|
|
953
|
+
-- Direct mutual cycle (A<->B) would suppress both sides; keep the
|
|
954
|
+
-- lexicographically smaller path deterministically.
|
|
955
|
+
AND NOT (parent.parent_session IS r.path AND r.path < parent.path)
|
|
956
|
+
)
|
|
957
|
+
ORDER BY score, rid
|
|
958
|
+
LIMIT ${SCAN_LIMIT}`;
|
|
959
|
+
|
|
960
|
+
/** Bounded excerpt around the first matched term for the LIKE fallback,
|
|
961
|
+
* searching head then tail. */
|
|
962
|
+
function likeSnippet(head: string, tail: string, terms: string[]): string {
|
|
963
|
+
for (const text of [head, tail]) {
|
|
964
|
+
const lower = foldCase(text);
|
|
965
|
+
let at = -1;
|
|
966
|
+
for (const t of terms) {
|
|
967
|
+
const i = lower.indexOf(foldCase(t));
|
|
968
|
+
if (i >= 0 && (at < 0 || i < at)) at = i;
|
|
969
|
+
}
|
|
970
|
+
if (at >= 0) {
|
|
971
|
+
const start = Math.max(0, at - 60);
|
|
972
|
+
return `${start > 0 ? "…" : ""}${text.slice(start, start + 120)}…`;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
return head.slice(0, 120);
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
export function searchIndex(
|
|
979
|
+
dbPath: string,
|
|
980
|
+
query: string,
|
|
981
|
+
opts?: SearchOptions,
|
|
982
|
+
): { hits: SearchHit[]; backlogRemaining: number } {
|
|
983
|
+
const limit = opts?.limit ?? 3;
|
|
984
|
+
const db = openDb(dbPath);
|
|
985
|
+
try {
|
|
986
|
+
// Per-connection TEMP table of (path, entry_id) pairs to suppress —
|
|
987
|
+
// parameterized, immune to SQLite variable limits. Primary key indexes
|
|
988
|
+
// the correlated NOT EXISTS probe and dedupes inserts.
|
|
989
|
+
db.exec("CREATE TEMP TABLE IF NOT EXISTS live_filter (path TEXT NOT NULL, entry_id TEXT NOT NULL, PRIMARY KEY (path, entry_id))");
|
|
990
|
+
if (opts?.currentSessionPath && opts?.currentLiveEntryIds?.size) {
|
|
991
|
+
const ins = db.prepare("INSERT INTO live_filter(path, entry_id) VALUES (?, ?)");
|
|
992
|
+
db.exec("BEGIN");
|
|
993
|
+
try {
|
|
994
|
+
for (const id of opts.currentLiveEntryIds!) ins.run(opts.currentSessionPath!, id);
|
|
995
|
+
db.exec("COMMIT");
|
|
996
|
+
} catch (err) {
|
|
997
|
+
db.exec("ROLLBACK");
|
|
998
|
+
throw err;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
const plan = buildFtsQueryPlan(query);
|
|
1003
|
+
if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
|
|
1004
|
+
return { hits: [], backlogRemaining: getBacklog(db) };
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
interface RawRow {
|
|
1008
|
+
path: string;
|
|
1009
|
+
entry_id: string;
|
|
1010
|
+
role: string;
|
|
1011
|
+
timestamp: string | null;
|
|
1012
|
+
snip: string;
|
|
1013
|
+
cwd: string | null;
|
|
1014
|
+
name: string | null;
|
|
1015
|
+
started_at: string | null;
|
|
1016
|
+
}
|
|
1017
|
+
let rows: Omit<RawRow, "snip">[] = [];
|
|
1018
|
+
let usedLike = plan.forceLike;
|
|
1019
|
+
|
|
1020
|
+
if (!plan.forceLike) {
|
|
1021
|
+
// Recovery candidates run only after an FTS5 PARSE error. A candidate
|
|
1022
|
+
// that parses defines the result even with zero rows (`a AND b` with
|
|
1023
|
+
// no co-occurrence stays empty instead of degrading to OR/LIKE).
|
|
1024
|
+
let parseFailed = false;
|
|
1025
|
+
for (const cand of plan.ftsCandidates) {
|
|
1026
|
+
try {
|
|
1027
|
+
rows = db.prepare(BASE_SELECT).all(cand) as any;
|
|
1028
|
+
// First/raw success defines the result even with zero rows. After a
|
|
1029
|
+
// parse error, an empty recovery candidate keeps trying later ones.
|
|
1030
|
+
if (!parseFailed || rows.length > 0) {
|
|
1031
|
+
usedLike = false;
|
|
1032
|
+
break;
|
|
1033
|
+
}
|
|
1034
|
+
usedLike = true;
|
|
1035
|
+
} catch {
|
|
1036
|
+
parseFailed = true;
|
|
1037
|
+
usedLike = true; // malformed syntax → try next candidate
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
if (usedLike) {
|
|
1043
|
+
const trimmed = query.trim().slice(0, MAX_QUERY_CHARS);
|
|
1044
|
+
// Keep quoted operator words as operands and omit NEAR's optional numeric
|
|
1045
|
+
// distance; only unquoted syntax tokens are excluded.
|
|
1046
|
+
const operandTerms = collectQueryTerms(trimmed)
|
|
1047
|
+
.filter((term) => !term.operator && !term.nearDistance)
|
|
1048
|
+
.map((term) => normalizeLikeTerm(term.text, term.quoted))
|
|
1049
|
+
.filter(Boolean);
|
|
1050
|
+
if (operandTerms.length === 0) return { hits: [], backlogRemaining: getBacklog(db) };
|
|
1051
|
+
const terms = operandTerms;
|
|
1052
|
+
// Snippets anchor on operand terms only — operator words like OR would
|
|
1053
|
+
// otherwise match common substrings and hide the real match.
|
|
1054
|
+
const snippetTerms = terms;
|
|
1055
|
+
// Boolean LIKE preserves simple AND/OR/NOT; unsupported shapes degrade
|
|
1056
|
+
// to AND-of-terms. Both forms are fully parameterized.
|
|
1057
|
+
const bool = buildBooleanLikeSql(trimmed);
|
|
1058
|
+
const where = bool?.where ?? terms.map(() => "(ulower(m.head) LIKE ? ESCAPE '\\' OR ulower(m.tail) LIKE ? ESCAPE '\\')").join(" AND ");
|
|
1059
|
+
const params = bool?.params ?? terms.flatMap((t) => [likePattern(t), likePattern(t)]);
|
|
1060
|
+
rows = db.prepare(`WITH ranked AS (
|
|
1061
|
+
SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
|
|
1062
|
+
s.parent_session,
|
|
1063
|
+
ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
|
|
1064
|
+
COUNT(*) OVER (PARTITION BY m.path) AS matches
|
|
1065
|
+
FROM messages m LEFT JOIN sessions s ON s.path = m.path
|
|
1066
|
+
WHERE ${where} AND ${LIVE_FILTER_SQL}
|
|
1067
|
+
)
|
|
1068
|
+
SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
|
|
1069
|
+
FROM ranked r
|
|
1070
|
+
WHERE rn <= ${ROWS_PER_FILE}
|
|
1071
|
+
AND NOT EXISTS (
|
|
1072
|
+
SELECT 1 FROM ranked parent
|
|
1073
|
+
WHERE parent.path = r.parent_session
|
|
1074
|
+
AND parent.path <> r.path
|
|
1075
|
+
AND parent.rn <= ${ROWS_PER_FILE}
|
|
1076
|
+
-- mutual-cycle tie-break: keep the lexicographically smaller path
|
|
1077
|
+
AND NOT (parent.parent_session IS r.path AND r.path < parent.path)
|
|
1078
|
+
)
|
|
1079
|
+
ORDER BY matches DESC, started_at DESC, path
|
|
1080
|
+
LIMIT ${SCAN_LIMIT}`).all(...params) as any;
|
|
1081
|
+
for (const r of rows as any[]) r.snip = likeSnippet((r as any).head ?? "", (r as any).tail ?? "", snippetTerms.length > 0 ? snippetTerms : terms);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// Live-entry and one-hop lineage suppression already happened in SQL,
|
|
1085
|
+
// before the scan limit, so fork rows cannot starve unrelated matches.
|
|
1086
|
+
const seenFiles = new Set<string>();
|
|
1087
|
+
const hits: SearchHit[] = [];
|
|
1088
|
+
let rankCounter = 0;
|
|
1089
|
+
for (const r of rows as RawRow[]) {
|
|
1090
|
+
if (seenFiles.has(r.path)) continue;
|
|
1091
|
+
seenFiles.add(r.path);
|
|
1092
|
+
hits.push({
|
|
1093
|
+
path: r.path,
|
|
1094
|
+
entryId: r.entry_id,
|
|
1095
|
+
role: r.role,
|
|
1096
|
+
timestamp: r.timestamp ?? "",
|
|
1097
|
+
snippet: r.snip ?? "",
|
|
1098
|
+
rank: rankCounter++,
|
|
1099
|
+
cwd: r.cwd ?? undefined,
|
|
1100
|
+
name: r.name ?? undefined,
|
|
1101
|
+
startedAt: r.started_at ?? undefined,
|
|
1102
|
+
});
|
|
1103
|
+
if (hits.length >= limit) break;
|
|
1104
|
+
}
|
|
1105
|
+
return { hits, backlogRemaining: getBacklog(db) };
|
|
1106
|
+
} finally {
|
|
1107
|
+
db.close();
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// --- Browse ---
|
|
1112
|
+
|
|
1113
|
+
export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
|
|
1114
|
+
const db = openDb(dbPath);
|
|
1115
|
+
try {
|
|
1116
|
+
const rows = db
|
|
1117
|
+
.prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
|
|
1118
|
+
.all(limit) as any[];
|
|
1119
|
+
return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
|
|
1120
|
+
} finally {
|
|
1121
|
+
db.close();
|
|
1122
|
+
}
|
|
1123
|
+
}
|