@hcflabs/pi-session-recall 0.0.0-stage → 5.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,927 @@
1
+ /**
2
+ * Index engine: SQLite schema (WAL, external-content trigram FTS5), capped
3
+ * incremental sync, and discovery search with one-hop lineage suppression.
4
+ * No pi runtime imports — pure Node + node:sqlite so it is testable headless.
5
+ */
6
+ import { DatabaseSync } from "node:sqlite";
7
+ import type { SQLOutputValue } from "node:sqlite";
8
+ import { createHash } from "node:crypto";
9
+ import fs from "node:fs";
10
+ import path from "node:path";
11
+ import { buildFtsQueryPlan, buildLikeQueryPlan, foldCase, nearLike } from "./query.ts";
12
+ import { MAX_SESSION_FILE_BYTES, parseTranscriptLines, readBoundedSnapshotBytes } from "./transcript.ts";
13
+ import type { PreparationSessionRow, SearchHit, SessionRow, SyncResult } from "./types.ts";
14
+ export const DEFAULT_SYNC_CAP = 50;
15
+ const MAX_TEXT_CHARS = 20000;
16
+
17
+ const SCHEMA_SQL = `
18
+ -- ctime_ms is raw stat.ctimeMs stored as REAL (SQLite REALs are IEEE doubles,
19
+ -- so JS numbers round-trip losslessly): flooring would collapse sub-ms restores.
20
+ CREATE TABLE IF NOT EXISTS session_files (
21
+ path TEXT PRIMARY KEY,
22
+ size INTEGER NOT NULL,
23
+ mtime_ms INTEGER NOT NULL,
24
+ ctime_ms REAL NOT NULL
25
+ );
26
+ -- Retry markers affect ordering only: unchanged failures move behind fresh work,
27
+ -- but remain eligible on every pass once fresh work is drained.
28
+ CREATE TABLE IF NOT EXISTS session_failures (
29
+ path TEXT PRIMARY KEY,
30
+ size INTEGER NOT NULL,
31
+ mtime_ms INTEGER NOT NULL,
32
+ ctime_ms REAL NOT NULL,
33
+ attempts INTEGER NOT NULL DEFAULT 0
34
+ );
35
+ CREATE TABLE IF NOT EXISTS sessions (
36
+ path TEXT PRIMARY KEY,
37
+ cwd TEXT,
38
+ name TEXT,
39
+ started_at TEXT,
40
+ preview TEXT,
41
+ parent_session TEXT
42
+ );
43
+ -- Oversized messages split into head/tail columns: one row per source
44
+ -- message keeps AND matching across both retained regions (FTS5 implicit AND
45
+ -- spans columns) while phrases and NEAR cannot cross column boundaries, so
46
+ -- truncation never manufactures proximity matches over the elided middle.
47
+ CREATE TABLE IF NOT EXISTS messages (
48
+ rowid INTEGER PRIMARY KEY,
49
+ path TEXT NOT NULL,
50
+ entry_id TEXT NOT NULL,
51
+ role TEXT NOT NULL,
52
+ timestamp TEXT,
53
+ head TEXT NOT NULL,
54
+ tail TEXT NOT NULL,
55
+ UNIQUE(path, entry_id)
56
+ );
57
+ CREATE INDEX IF NOT EXISTS messages_path ON messages(path);
58
+ CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
59
+ head,
60
+ tail,
61
+ content='messages',
62
+ content_rowid='rowid',
63
+ tokenize='trigram'
64
+ );
65
+ CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
66
+ INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
67
+ END;
68
+ CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
69
+ INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
70
+ END;
71
+ CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
72
+ INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
73
+ INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
74
+ END;
75
+ CREATE TABLE IF NOT EXISTS meta (
76
+ key TEXT PRIMARY KEY,
77
+ value TEXT NOT NULL
78
+ );
79
+ -- Append state lets sync parse only the complete lines after byte_offset.
80
+ -- A row is valid only while its fingerprint equals the session_files row.
81
+ -- prefix_sha256 covers bytes [0, byte_offset), and byte_offset ends a line.
82
+ -- entry_ids is the JSON array of valid ids that the prefix reserved.
83
+ -- An index from an older version has no rows here. Sync then re-indexes a
84
+ -- changed file in full and writes its first row.
85
+ CREATE TABLE IF NOT EXISTS session_append_state (
86
+ path TEXT PRIMARY KEY,
87
+ size INTEGER NOT NULL,
88
+ mtime_ms INTEGER NOT NULL,
89
+ ctime_ms REAL NOT NULL,
90
+ byte_offset INTEGER NOT NULL,
91
+ prefix_sha256 TEXT NOT NULL,
92
+ entry_ids TEXT NOT NULL
93
+ );
94
+ `;
95
+
96
+ /** Owner-only modes for the index directory and database files. The
97
+ * extension-owned parent dir is the only node tightened; intermediate config
98
+ * dirs are created restrictive but never chmod'ed if they predate this fix. */
99
+ const DB_FILE_MODE = 0o600;
100
+ const DB_DIR_MODE = 0o700;
101
+
102
+ /** Create or tighten the index's private directory and owner-only database
103
+ * file BEFORE SQLite writes any transcript content into it. Descriptor-based
104
+ * O_NOFOLLOW|O_NONBLOCK open + fstat rejects symlinks and other non-regular
105
+ * nodes instead of following them; fchmod tightens an existing permissive db.
106
+ * WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
107
+ * behind by older builds are tightened here too. Always closes the fd. */
108
+ function secureIndexNode(dbPath: string): { dev: number; ino: number } {
109
+ const posix = process.platform !== "win32";
110
+ const dir = path.dirname(dbPath);
111
+ fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
112
+ // O_NOFOLLOW on the db path does not protect parent components; reject a
113
+ // symlinked extension directory before any chmod or database open follows it.
114
+ if (!fs.lstatSync(dir).isDirectory()) throw new Error(`index directory is not a real directory: ${dir}`);
115
+ if (posix) {
116
+ // Tighten an existing permissive dir (e.g. created under umask 022).
117
+ if ((fs.statSync(dir).mode & 0o777) !== DB_DIR_MODE) {
118
+ fs.chmodSync(dir, DB_DIR_MODE);
119
+ }
120
+ }
121
+ let fd: number | undefined;
122
+ let identity: { dev: number; ino: number };
123
+ try {
124
+ fd = fs.openSync(
125
+ dbPath,
126
+ (fs.constants.O_RDWR | fs.constants.O_CREAT | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0)),
127
+ DB_FILE_MODE,
128
+ );
129
+ const st = fs.fstatSync(fd);
130
+ if (!st.isFile()) throw new Error("index database path is not a regular file");
131
+ if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
132
+ identity = { dev: st.dev, ino: st.ino };
133
+ } finally {
134
+ if (fd !== undefined) fs.closeSync(fd);
135
+ }
136
+ if (!posix) return identity;
137
+ for (const suffix of ["-wal", "-shm"]) {
138
+ // Same descriptor no-follow safety as the db path: chmodSync(path) would
139
+ // follow a swapped-in sidecar symlink and retarget its referent's mode.
140
+ let fd: number | undefined;
141
+ try {
142
+ fd = fs.openSync(
143
+ dbPath + suffix,
144
+ fs.constants.O_RDWR | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0),
145
+ );
146
+ const st = fs.fstatSync(fd);
147
+ if (!st.isFile()) throw new Error(`index ${suffix} sidecar is not a regular file`);
148
+ if ((st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
149
+ } catch (err) {
150
+ // Sidecars are created by SQLite; a missing legacy sidecar stays harmless.
151
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
152
+ } finally {
153
+ if (fd !== undefined) fs.closeSync(fd);
154
+ }
155
+ }
156
+ return identity;
157
+ }
158
+
159
+ /** One open connection per index path. The connection keeps the file
160
+ * identity and the schema version that its setup checked. */
161
+ interface CachedDb {
162
+ db: DatabaseSync;
163
+ dev: number;
164
+ ino: number;
165
+ schemaVersion: number;
166
+ }
167
+
168
+ const openDbs = new Map<string, CachedDb>();
169
+
170
+ function schemaVersion(db: DatabaseSync): number {
171
+ return Number((db.prepare("PRAGMA schema_version").get() as { schema_version: number }).schema_version);
172
+ }
173
+
174
+ function evictDb(dbPath: string, db?: DatabaseSync): void {
175
+ const cached = openDbs.get(dbPath);
176
+ if (!cached || (db !== undefined && cached.db !== db)) return;
177
+ openDbs.delete(dbPath);
178
+ try {
179
+ cached.db.close();
180
+ } catch {
181
+ // The connection is unusable. A later call opens a new connection.
182
+ }
183
+ }
184
+
185
+ /** Close the cached index connection for `dbPath`, or all cached connections
186
+ * when `dbPath` is absent. A later index call opens a new connection. */
187
+ export function closeIndexDatabases(dbPath?: string): void {
188
+ for (const key of dbPath === undefined ? [...openDbs.keys()] : [dbPath]) evictDb(key);
189
+ }
190
+
191
+ /** Return the cached connection for `dbPath`, or open and check a new one.
192
+ * secureIndexNode runs on every call, so mode tightening and node checks
193
+ * do not change. A deleted or replaced index file, or a schema change by
194
+ * another connection, discards the cached connection and repeats setup. */
195
+ function openDb(dbPath: string): DatabaseSync {
196
+ const cached = openDbs.get(dbPath);
197
+ if (cached) {
198
+ // Close a connection to a deleted or replaced file before
199
+ // secureIndexNode creates the new file at this path.
200
+ let st: fs.Stats | undefined;
201
+ try {
202
+ st = fs.lstatSync(dbPath);
203
+ } catch {
204
+ st = undefined;
205
+ }
206
+ if (!st || st.dev !== cached.dev || st.ino !== cached.ino) evictDb(dbPath);
207
+ }
208
+ let identity: { dev: number; ino: number };
209
+ try {
210
+ identity = secureIndexNode(dbPath);
211
+ } catch (err) {
212
+ evictDb(dbPath);
213
+ throw err;
214
+ }
215
+ const current = openDbs.get(dbPath);
216
+ if (current) {
217
+ let reusable = false;
218
+ if (current.dev === identity.dev && current.ino === identity.ino) {
219
+ try {
220
+ reusable = schemaVersion(current.db) === current.schemaVersion;
221
+ } catch {
222
+ reusable = false;
223
+ }
224
+ }
225
+ if (reusable) return current.db;
226
+ evictDb(dbPath);
227
+ }
228
+ const db = setUpDb(dbPath);
229
+ try {
230
+ openDbs.set(dbPath, { db, ...identity, schemaVersion: schemaVersion(db) });
231
+ } catch (err) {
232
+ db.close();
233
+ throw err;
234
+ }
235
+ return db;
236
+ }
237
+
238
+ /** Run `fn` with the cached connection. An error discards the connection, so
239
+ * the next call starts from a new connection as before the cache existed. */
240
+ function withDb<T>(dbPath: string, fn: (db: DatabaseSync) => T): T {
241
+ const db = openDb(dbPath);
242
+ try {
243
+ return fn(db);
244
+ } catch (err) {
245
+ evictDb(dbPath, db);
246
+ throw err;
247
+ }
248
+ }
249
+
250
+ function setUpDb(dbPath: string): DatabaseSync {
251
+ const db = new DatabaseSync(dbPath);
252
+ try {
253
+ db.exec("PRAGMA journal_mode = WAL");
254
+ db.exec("PRAGMA busy_timeout = 5000");
255
+ // The index is disposable derived state with no migration path. Only the
256
+ // current schema is defined; an incompatible existing index fails on use
257
+ // and is never deleted or rewritten here — remove it manually to rebuild.
258
+ // SQLite's LIKE folds ASCII only, missing é/É, Greek, Cyrillic, …; every
259
+ // LIKE below wraps columns in ulower and binds pre-folded operands so
260
+ // both sides case-fold through the same foldCase helper.
261
+ db.function("ulower", (s: SQLOutputValue): string => typeof s === "string" ? foldCase(s) : "");
262
+ db.function("unear", nearLike);
263
+ // Creating a missing external-content FTS table after messages and their
264
+ // watermarks persist leaves it blank: unchanged files never replay inserts.
265
+ // The disposable index must be deleted and rebuilt instead of repaired here.
266
+ const hasFts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'session_fts'").get() !== undefined;
267
+ const hasPersistedSchema = db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' LIMIT 1").get() !== undefined;
268
+ if (!hasFts && hasPersistedSchema) {
269
+ throw new Error("index schema incompatible: session_fts missing — delete the index to rebuild");
270
+ }
271
+ db.exec(SCHEMA_SQL);
272
+ // CREATE TABLE IF NOT EXISTS cannot repair a predecessor index whose
273
+ // tables exist but lack current watermark columns (e.g. no ctime_ms): its
274
+ // search rows would silently serve stale hits while change detection
275
+ // breaks. Fail visibly; the index is disposable — remove it to rebuild.
276
+ for (const [table, required] of [
277
+ ["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
278
+ ["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
279
+ ["session_append_state", ["path", "size", "mtime_ms", "ctime_ms", "byte_offset", "prefix_sha256", "entry_ids"]],
280
+ ] as const) {
281
+ const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
282
+ for (const col of required) {
283
+ if (!have.has(col)) throw new Error(`index schema incompatible: ${table}.${col} missing — delete the index to rebuild`);
284
+ }
285
+ }
286
+ } catch (err) {
287
+ db.close();
288
+ throw err;
289
+ }
290
+ return db;
291
+ }
292
+
293
+ /** Replacement/edit-sensitive file fingerprint from one fs.Stats. Size plus a
294
+ * floored mtime_ms misses same-length rewrites whose millisecond mtime is
295
+ * restored (backup restores, redactions). Exact ctime closes that: POSIX
296
+ * utimensat can set mtime but never ctime, so every restore bumps it. */
297
+ function fileFingerprint(stat: fs.Stats): { size: number; mtimeMs: number; ctimeMs: number } {
298
+ return { size: stat.size, mtimeMs: Math.floor(stat.mtimeMs), ctimeMs: stat.ctimeMs };
299
+ }
300
+
301
+ // --- Parsing ---
302
+
303
+ interface ParsedMessage {
304
+ entryId: string;
305
+ role: string;
306
+ timestamp: string | null;
307
+ head: string;
308
+ tail: string;
309
+ }
310
+
311
+ interface ParsedFile {
312
+ cwd: string | null;
313
+ name: string | null;
314
+ startedAt: string | null;
315
+ parentSession: string | null;
316
+ preview: string | null;
317
+ messages: ParsedMessage[];
318
+ }
319
+
320
+ /** Middle-out truncation into SEPARATE head/tail index columns: concatenating
321
+ * them would make boundary terms adjacent and manufacture phrase/proximity
322
+ * matches across the elided middle. Raw text only — no synthetic notices:
323
+ * these strings are FTS-indexed, so injected terms would pollute search
324
+ * results and snippets. */
325
+ function truncateRegions(text: string, max = MAX_TEXT_CHARS): { head: string; tail: string } | null {
326
+ if (text.length <= max) return null;
327
+ const kept = max - 1;
328
+ return {
329
+ head: text.slice(0, Math.ceil(kept / 2)),
330
+ tail: text.slice(-Math.floor(kept / 2)),
331
+ };
332
+ }
333
+
334
+ /** Cap an untrusted string at a parse boundary so malformed JSONL cannot
335
+ * produce unbounded metadata downstream. */
336
+ function capStr(value: unknown, max: number): string | null {
337
+ return typeof value === "string" ? value.slice(0, max) : null;
338
+ }
339
+
340
+ /** Narrow ISO timestamp shape; captures wall-clock fields for validation. */
341
+ const ISO_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
342
+
343
+ /** Canonicalize a session-header timestamp to UTC ISO so its lexicographic
344
+ * order matches chronological order (browse sorts started_at DESC as TEXT).
345
+ * Anything else is stored null so it sorts behind every valid timestamp. */
346
+ function normalizeTimestamp(value: unknown): string | null {
347
+ if (typeof value !== "string") return null;
348
+ const m = ISO_TIMESTAMP_RE.exec(value);
349
+ if (!m) return null;
350
+ const [, y, mo, d, h] = m;
351
+ if (+h > 23) return null;
352
+ const probe = new Date(Date.UTC(+y, +mo - 1, +d));
353
+ // Date parsing rejects invalid time fields, but silently rolls impossible
354
+ // calendar dates such as February 30.
355
+ if (probe.getUTCMonth() !== +mo - 1 || probe.getUTCDate() !== +d) return null;
356
+ const date = new Date(value);
357
+ return Number.isNaN(date.getTime()) ? null : date.toISOString();
358
+ }
359
+
360
+ function extractText(content: unknown): string {
361
+ if (typeof content === "string") return content;
362
+ if (!Array.isArray(content)) return "";
363
+ const parts: string[] = [];
364
+ for (const block of content) {
365
+ if (block && typeof block === "object" && (block as any).type === "text" && typeof (block as any).text === "string") {
366
+ parts.push((block as any).text);
367
+ }
368
+ // thinking / toolUse / tool_result blocks are skipped entirely
369
+ }
370
+ return parts.join("\n").trim();
371
+ }
372
+
373
+ /** Apply the entries of decoded snapshot text to `parsed`. Metadata in
374
+ * `parsed` and ids in `seenIds` hold the state after the earlier lines, so
375
+ * a parse that continues after a line boundary gives the full-parse result. */
376
+ function parseSessionText(parsed: ParsedFile, content: string, seenIds: Set<string>): ParsedFile {
377
+ for (const { data: entryRaw, id } of parseTranscriptLines(content, seenIds)) {
378
+ const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
379
+ switch (entry?.type) {
380
+ case "session":
381
+ // Untrusted header strings are capped here so every consumer
382
+ // (browse rows, discovery meta) inherits the bound.
383
+ parsed.cwd = capStr(entry.cwd, 500) ?? parsed.cwd;
384
+ parsed.startedAt = normalizeTimestamp(entry.timestamp) ?? parsed.startedAt;
385
+ parsed.parentSession = capStr(entry.parentSession, 1024) ?? parsed.parentSession;
386
+ break;
387
+ case "session_info":
388
+ if (typeof entry.name === "string") parsed.name = entry.name.slice(0, 500);
389
+ break;
390
+ case "message": {
391
+ const msg = entry.message as { role?: unknown; content?: unknown } | undefined;
392
+ const role = msg?.role;
393
+ if (!msg || (role !== "user" && role !== "assistant")) break;
394
+ const full = extractText(msg.content);
395
+ if (!full || id === undefined) break;
396
+ if (role === "user" && parsed.preview === null) {
397
+ parsed.preview = full.slice(0, 200);
398
+ }
399
+ const regions = truncateRegions(full);
400
+ parsed.messages.push({
401
+ entryId: id,
402
+ role,
403
+ timestamp: capStr(entry.timestamp, 128),
404
+ head: regions ? regions.head : full,
405
+ tail: regions ? regions.tail : "",
406
+ });
407
+ break;
408
+ }
409
+ // compaction, branch_summary, custom, custom_message, label,
410
+ // session_info (name handled above), model_change, … are skipped
411
+ }
412
+ }
413
+ return parsed;
414
+ }
415
+
416
+ function emptyParsedFile(): ParsedFile {
417
+ return { cwd: null, name: null, startedAt: null, parentSession: null, preview: null, messages: [] };
418
+ }
419
+
420
+ /** Append state above this JSON size is not stored. The next change of
421
+ * that file then uses a full re-index. */
422
+ const MAX_APPEND_ID_CHARS = 1024 * 1024;
423
+ const LF = 0x0a;
424
+
425
+ function sha256(bytes: Uint8Array): string {
426
+ return createHash("sha256").update(bytes).digest("base64");
427
+ }
428
+
429
+ interface AppendStateRow {
430
+ byte_offset: number;
431
+ prefix_sha256: string;
432
+ entry_ids: string;
433
+ cwd: string | null;
434
+ name: string | null;
435
+ started_at: string | null;
436
+ preview: string | null;
437
+ parent_session: string | null;
438
+ }
439
+
440
+ /** Parse only the complete lines after the stored offset. Return null when
441
+ * the result can differ from a full parse:
442
+ * - no append state matches the current watermark,
443
+ * - the file is shorter than the offset (truncation),
444
+ * - the bytes before the offset changed (rewrite),
445
+ * - the snapshot ends in a partial line.
446
+ * A UTF-8 decoder restarts cleanly after a LF byte, so decoding only the
447
+ * appended bytes gives the same lines as decoding the whole snapshot. */
448
+ function parseAppendedLines(
449
+ state: AppendStateRow | undefined,
450
+ snapshot: Buffer,
451
+ ): { parsed: ParsedFile; seenIds: Set<string>; snapshotSha256: string } | null {
452
+ if (!state) return null;
453
+ const offset = Number(state.byte_offset);
454
+ if (!Number.isSafeInteger(offset) || offset < 0 || snapshot.length < offset) return null;
455
+ if (snapshot.length > offset && snapshot[snapshot.length - 1] !== LF) return null;
456
+ const hash = createHash("sha256").update(snapshot.subarray(0, offset));
457
+ if (hash.copy().digest("base64") !== state.prefix_sha256) return null;
458
+ let ids: unknown;
459
+ try {
460
+ ids = JSON.parse(state.entry_ids);
461
+ } catch {
462
+ return null;
463
+ }
464
+ if (!Array.isArray(ids) || !ids.every((id) => typeof id === "string")) return null;
465
+ const seenIds = new Set<string>(ids);
466
+ const parsed: ParsedFile = {
467
+ cwd: state.cwd,
468
+ name: state.name,
469
+ startedAt: state.started_at,
470
+ parentSession: state.parent_session,
471
+ preview: state.preview,
472
+ messages: [],
473
+ };
474
+ return {
475
+ parsed: parseSessionText(parsed, snapshot.toString("utf-8", offset), seenIds),
476
+ seenIds,
477
+ snapshotSha256: hash.update(snapshot.subarray(offset)).digest("base64"),
478
+ };
479
+ }
480
+
481
+ /** Walk result. `complete: false` means the tree could not be fully read
482
+ * (missing root, readdir/stat failure) — callers must not treat unseen
483
+ * indexed paths as deleted. */
484
+ interface WalkResult {
485
+ files: Map<string, fs.Stats>;
486
+ complete: boolean;
487
+ }
488
+
489
+ function walkJsonlFiles(sessionsDir: string): WalkResult {
490
+ const files = new Map<string, fs.Stats>();
491
+ const stack = [sessionsDir];
492
+ let complete = true;
493
+ while (stack.length > 0) {
494
+ const dir = stack.pop()!;
495
+ let entries: fs.Dirent[];
496
+ try {
497
+ entries = fs.readdirSync(dir, { withFileTypes: true });
498
+ } catch {
499
+ complete = false;
500
+ continue;
501
+ }
502
+ for (const ent of entries) {
503
+ const full = path.join(dir, ent.name);
504
+ if (ent.isDirectory()) {
505
+ // Junk dirs are pruned before descent so a large ignored tree never
506
+ // costs a walk + per-file stats on every sync pass.
507
+ if (ent.name.startsWith("--tmp-") || ent.name.startsWith("--private-tmp-")) continue;
508
+ stack.push(full);
509
+ } else if (ent.isFile() && ent.name.endsWith(".jsonl")) {
510
+ try {
511
+ files.set(full, fs.statSync(full));
512
+ } catch {
513
+ complete = false; // vanished or became unreadable mid-walk
514
+ }
515
+ }
516
+ }
517
+ }
518
+ return { files, complete };
519
+ }
520
+
521
+ // --- Sync ---
522
+
523
+ export function syncSessions(
524
+ sessionsDir: string,
525
+ dbPath: string,
526
+ opts?: { cap?: number; maxFileBytes?: number },
527
+ ): SyncResult {
528
+ const cap = opts?.cap ?? DEFAULT_SYNC_CAP;
529
+ return withDb(dbPath, (db) => {
530
+
531
+ // Junk dirs never reach the walk result: pruned during descent.
532
+ const walk = walkJsonlFiles(sessionsDir);
533
+ const all = [...walk.files];
534
+ const watermarks = new Map(
535
+ (db.prepare("SELECT path, size, mtime_ms, ctime_ms FROM session_files").all() as any[]).map((r) => [r.path, r]),
536
+ );
537
+ const failures = new Map(
538
+ (db.prepare("SELECT path, size, mtime_ms, ctime_ms, attempts FROM session_failures").all() as any[]).map((r) => [r.path, r]),
539
+ );
540
+ // An incomplete traversal proves nothing about absence: purging here would
541
+ // wipe healthy sessions over a transient readdir/stat failure. Changed/new
542
+ // files already discovered still process normally.
543
+ const discovered = new Set(all.map(([fp]) => fp));
544
+ const deleted = walk.complete ? [...watermarks.keys()].filter((p) => !discovered.has(p)) : [];
545
+ const deletedFailures = walk.complete ? [...failures.keys()].filter((p) => !discovered.has(p)) : [];
546
+
547
+ const changed: { path: string; stat: fs.Stats }[] = [];
548
+ for (const [p, stat] of all) {
549
+ const fp = fileFingerprint(stat);
550
+ const wm = watermarks.get(p);
551
+ if (wm && wm.size === fp.size && wm.mtime_ms === fp.mtimeMs && wm.ctime_ms === fp.ctimeMs) continue;
552
+ changed.push({ path: p, stat });
553
+ }
554
+
555
+ // Newest-first, but an unchanged prior failure moves behind untouched work.
556
+ // Among retries, fewest attempts first so one persistent failure cannot
557
+ // monopolize a small cap forever.
558
+ const retryAttempts = (p: string, stat: fs.Stats): number | null => {
559
+ const f = failures.get(p);
560
+ if (!f) return null;
561
+ const fp = fileFingerprint(stat);
562
+ return f.size === fp.size && f.mtime_ms === fp.mtimeMs && f.ctime_ms === fp.ctimeMs ? f.attempts : null;
563
+ };
564
+ changed.sort((a, b) => {
565
+ const aRetry = retryAttempts(a.path, a.stat);
566
+ const bRetry = retryAttempts(b.path, b.stat);
567
+ if (aRetry !== null && bRetry !== null) return aRetry - bRetry || b.stat.mtimeMs - a.stat.mtimeMs;
568
+ if (aRetry !== null) return 1;
569
+ if (bRetry !== null) return -1;
570
+ return b.stat.mtimeMs - a.stat.mtimeMs;
571
+ });
572
+ deleted.sort((a, b) => {
573
+ const ma = watermarks.get(a)!.mtime_ms;
574
+ const mb = watermarks.get(b)!.mtime_ms;
575
+ return mb - ma;
576
+ });
577
+
578
+ const delStmt = db.prepare("DELETE FROM messages WHERE path = ?");
579
+ const delSession = db.prepare("DELETE FROM sessions WHERE path = ?");
580
+ const delFile = db.prepare("DELETE FROM session_files WHERE path = ?");
581
+ const delFailure = db.prepare("DELETE FROM session_failures WHERE path = ?");
582
+ const upsertFailure = db.prepare(
583
+ "INSERT INTO session_failures(path, size, mtime_ms, ctime_ms, attempts) VALUES (?, ?, ?, ?, 1) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, attempts=CASE WHEN size=excluded.size AND mtime_ms=excluded.mtime_ms AND ctime_ms=excluded.ctime_ms THEN attempts+1 ELSE 1 END",
584
+ );
585
+ const upsertFile = db.prepare(
586
+ "INSERT INTO session_files(path, size, mtime_ms, ctime_ms) VALUES (?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms",
587
+ );
588
+ const upsertSession = db.prepare(
589
+ `INSERT INTO sessions(path, cwd, name, started_at, preview, parent_session) VALUES (?, ?, ?, ?, ?, ?)
590
+ ON CONFLICT(path) DO UPDATE SET cwd=excluded.cwd, name=excluded.name, started_at=excluded.started_at,
591
+ preview=excluded.preview, parent_session=excluded.parent_session`,
592
+ );
593
+ const insertMsg = db.prepare(
594
+ "INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
595
+ );
596
+ // The join accepts append state only when it belongs to the current
597
+ // watermark. A writer that does not know this table (an older version)
598
+ // changes the watermark, so sync then re-indexes the file in full.
599
+ const getAppendState = db.prepare(
600
+ `SELECT a.byte_offset, a.prefix_sha256, a.entry_ids, s.cwd, s.name, s.started_at, s.preview, s.parent_session
601
+ FROM session_append_state a
602
+ JOIN session_files f ON f.path = a.path AND f.size = a.size AND f.mtime_ms = a.mtime_ms AND f.ctime_ms = a.ctime_ms
603
+ JOIN sessions s ON s.path = a.path
604
+ WHERE a.path = ?`,
605
+ );
606
+ const upsertAppendState = db.prepare(
607
+ `INSERT INTO session_append_state(path, size, mtime_ms, ctime_ms, byte_offset, prefix_sha256, entry_ids) VALUES (?, ?, ?, ?, ?, ?, ?)
608
+ ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms,
609
+ byte_offset=excluded.byte_offset, prefix_sha256=excluded.prefix_sha256, entry_ids=excluded.entry_ids`,
610
+ );
611
+ const delAppendState = db.prepare("DELETE FROM session_append_state WHERE path = ?");
612
+
613
+ const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
614
+
615
+ const tx = (filePath: string, stat: fs.Stats): void => {
616
+ db.exec("BEGIN");
617
+ try {
618
+ // Unreadable/oversized file must throw so the sync transaction rolls back
619
+ // instead of wiping previously indexed rows and advancing the watermark over a hole.
620
+ const snapshot = readBoundedSnapshotBytes(filePath, maxFileBytes);
621
+ const appended = parseAppendedLines(getAppendState.get(filePath) as AppendStateRow | undefined, snapshot);
622
+ const seenIds = appended?.seenIds ?? new Set<string>();
623
+ const parsed = appended?.parsed ?? parseSessionText(emptyParsedFile(), snapshot.toString("utf-8"), seenIds);
624
+ // Indexed rows of an appended file are the full-parse rows of its
625
+ // unchanged prefix, so only the new messages are inserted.
626
+ if (!appended) delStmt.run(filePath);
627
+ upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
628
+ for (const msg of parsed.messages) {
629
+ insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
630
+ }
631
+ const fp = fileFingerprint(stat);
632
+ upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
633
+ // A partial last line can parse differently after more bytes arrive,
634
+ // so only a snapshot that ends a line gets append state.
635
+ const entryIds = JSON.stringify([...seenIds]);
636
+ if ((snapshot.length === 0 || snapshot[snapshot.length - 1] === LF) && entryIds.length <= MAX_APPEND_ID_CHARS) {
637
+ upsertAppendState.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs, snapshot.length, appended?.snapshotSha256 ?? sha256(snapshot), entryIds);
638
+ } else {
639
+ delAppendState.run(filePath);
640
+ }
641
+ delFailure.run(filePath);
642
+ db.exec("COMMIT");
643
+ } catch (err) {
644
+ db.exec("ROLLBACK");
645
+ throw err;
646
+ }
647
+ };
648
+
649
+ let filesProcessed = 0;
650
+
651
+ // The cap bounds total indexing attempts per pass, not successes.
652
+ const batch = changed.slice(0, cap);
653
+ for (const { path: p, stat } of batch) {
654
+ try {
655
+ tx(p, stat);
656
+ filesProcessed++;
657
+ } catch {
658
+ // Keep the file retryable, but put this unchanged fingerprint behind
659
+ // untouched work on the next pass.
660
+ {
661
+ const fp = fileFingerprint(stat);
662
+ upsertFailure.run(p, fp.size, fp.mtimeMs, fp.ctimeMs);
663
+ }
664
+ }
665
+ }
666
+
667
+ const deletedRemaining = Math.max(0, deleted.length - cap);
668
+ for (const p of deleted.slice(0, cap)) {
669
+ delStmt.run(p);
670
+ delSession.run(p);
671
+ delFile.run(p);
672
+ delFailure.run(p);
673
+ delAppendState.run(p);
674
+ }
675
+ for (const p of deletedFailures) delFailure.run(p);
676
+
677
+ // Attempted failures remain unsynced and therefore remain in the backlog.
678
+ const backlog = changed.length - filesProcessed + deletedRemaining;
679
+ db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
680
+
681
+ return { filesProcessed, backlogRemaining: backlog, walkComplete: walk.complete };
682
+ });
683
+ }
684
+
685
+ function getBacklog(db: DatabaseSync): number {
686
+ const row = db.prepare("SELECT value FROM meta WHERE key = 'backlog'").get() as { value: string } | undefined;
687
+ return row ? Number(row.value) : 0;
688
+ }
689
+
690
+ // --- Search ---
691
+
692
+ export interface SearchOptions {
693
+ limit?: number;
694
+ /** Path of the current session file — the whole file is skipped. */
695
+ currentSessionPath?: string;
696
+ }
697
+
698
+ // Current-session exclusion happens INSIDE SQL (before ROW_NUMBER/caps), so
699
+ // the excluded file cannot consume the per-file rank, lineage, or scan limit.
700
+ const CURRENT_SESSION_FILTER_SQL = "(? IS NULL OR m.path <> ?)";
701
+ const LINEAGE_FILTER_SQL = `NOT EXISTS (
702
+ SELECT 1 FROM ranked parent
703
+ WHERE parent.path = r.parent_session
704
+ AND parent.path <> r.path
705
+ AND parent.rn <= 1
706
+ -- Direct mutual cycle (A<->B) would suppress both sides; keep the
707
+ -- lexicographically smaller path deterministically.
708
+ AND NOT (parent.parent_session IS r.path AND r.path < parent.path)
709
+ )`;
710
+ const BASE_SELECT = `
711
+ WITH matches AS (
712
+ SELECT m.path, m.entry_id, m.role, m.timestamp, s.cwd, s.name, s.started_at,
713
+ s.parent_session, m.rowid AS rid,
714
+ snippet(session_fts, -1, '[', ']', '…', 16) AS snip,
715
+ bm25(session_fts) AS score
716
+ FROM session_fts JOIN messages m ON m.rowid = session_fts.rowid
717
+ LEFT JOIN sessions s ON s.path = m.path
718
+ WHERE session_fts MATCH ? AND ${CURRENT_SESSION_FILTER_SQL}
719
+ ), ranked AS (
720
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY score, rid) AS rn
721
+ FROM matches
722
+ )
723
+ SELECT path, entry_id, role, timestamp, snip, cwd, name, started_at
724
+ FROM ranked r
725
+ WHERE rn <= 1
726
+ AND ${LINEAGE_FILTER_SQL}
727
+ ORDER BY score, rid
728
+ LIMIT ?`;
729
+
730
+ /** Bounded excerpt around the first matched term for the LIKE fallback,
731
+ * searching head then tail. */
732
+ function likeSnippet(head: string, tail: string, terms: string[]): string {
733
+ for (const text of [head, tail]) {
734
+ const lower = foldCase(text);
735
+ let at = -1;
736
+ for (const t of terms) {
737
+ const i = lower.indexOf(foldCase(t));
738
+ if (i >= 0 && (at < 0 || i < at)) at = i;
739
+ }
740
+ if (at >= 0) {
741
+ const start = Math.max(0, at - 60);
742
+ return `${start > 0 ? "…" : ""}${text.slice(start, start + 120)}…`;
743
+ }
744
+ }
745
+ return head.slice(0, 120);
746
+ }
747
+
748
+ export function searchIndex(
749
+ dbPath: string,
750
+ query: string,
751
+ opts?: SearchOptions,
752
+ ): { hits: SearchHit[]; backlogRemaining: number } {
753
+ const limit = opts?.limit ?? 3;
754
+ return withDb(dbPath, (db) => {
755
+ const currentSessionPath = opts?.currentSessionPath ?? null;
756
+ const plan = buildFtsQueryPlan(query);
757
+ if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
758
+ return { hits: [], backlogRemaining: getBacklog(db) };
759
+ }
760
+
761
+ interface RawRow {
762
+ path: string;
763
+ entry_id: string;
764
+ role: string;
765
+ timestamp: string | null;
766
+ snip: string;
767
+ cwd: string | null;
768
+ name: string | null;
769
+ started_at: string | null;
770
+ }
771
+ let rows: Omit<RawRow, "snip">[] = [];
772
+ let usedLike = plan.forceLike;
773
+
774
+ if (!plan.forceLike) {
775
+ // Recovery candidates run only after an FTS5 PARSE error. A candidate
776
+ // that parses defines the result even with zero rows (`a AND b` with
777
+ // no co-occurrence stays empty instead of degrading to OR/LIKE).
778
+ let parseFailed = false;
779
+ for (const cand of plan.ftsCandidates) {
780
+ try {
781
+ rows = db.prepare(BASE_SELECT).all(cand, currentSessionPath, currentSessionPath, limit) as any;
782
+ // First/raw success defines the result even with zero rows. After a
783
+ // parse error, an empty recovery candidate keeps trying later ones.
784
+ if (!parseFailed || rows.length > 0) {
785
+ usedLike = false;
786
+ break;
787
+ }
788
+ usedLike = true;
789
+ } catch {
790
+ parseFailed = true;
791
+ usedLike = true; // malformed syntax → try next candidate
792
+ }
793
+ }
794
+ }
795
+
796
+ if (usedLike) {
797
+ const likePlan = buildLikeQueryPlan(query);
798
+ if (likePlan === null) return { hits: [], backlogRemaining: getBacklog(db) };
799
+ // The FTS candidate filter only removes rows that the LIKE plan cannot
800
+ // match, so it leaves the matched set, the ranking, and the order the same.
801
+ const likeRows = (prefilter: string | null) => db.prepare(`WITH ranked AS (
802
+ SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
803
+ s.parent_session,
804
+ ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
805
+ COUNT(*) OVER (PARTITION BY m.path) AS matches
806
+ FROM messages m LEFT JOIN sessions s ON s.path = m.path
807
+ WHERE ${likePlan.where}
808
+ ${prefilter === null ? "" : "AND m.rowid IN (SELECT rowid FROM session_fts WHERE session_fts MATCH ?)"}
809
+ AND ${CURRENT_SESSION_FILTER_SQL}
810
+ )
811
+ SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
812
+ FROM ranked r
813
+ WHERE rn <= 1
814
+ AND ${LINEAGE_FILTER_SQL}
815
+ ORDER BY matches DESC, started_at DESC, path
816
+ LIMIT ?`).all(
817
+ ...likePlan.params,
818
+ ...(prefilter === null ? [] : [prefilter]),
819
+ currentSessionPath,
820
+ currentSessionPath,
821
+ limit,
822
+ ) as any;
823
+ try {
824
+ rows = likeRows(likePlan.ftsPrefilter);
825
+ } catch (err) {
826
+ // The full LIKE scan stays the result of record if the filter fails.
827
+ if (likePlan.ftsPrefilter === null) throw err;
828
+ rows = likeRows(null);
829
+ }
830
+ // Snippets anchor on operand terms only — operator words like OR would
831
+ // otherwise match common substrings and hide the real match.
832
+ for (const r of rows as any[]) r.snip = likeSnippet(r.head, r.tail, likePlan.terms);
833
+ }
834
+
835
+ // Current-file and one-hop lineage suppression already happened in SQL,
836
+ // before the LIMIT, so excluded or fork rows cannot starve matches.
837
+ const hits: SearchHit[] = (rows as RawRow[]).map((r, rank) => ({
838
+ path: r.path,
839
+ snippet: r.snip ?? "",
840
+ rank,
841
+ matchMessageId: r.entry_id,
842
+ role: r.role,
843
+ timestamp: r.timestamp ?? "",
844
+ cwd: r.cwd ?? undefined,
845
+ name: r.name ?? undefined,
846
+ startedAt: r.started_at ?? undefined,
847
+ }));
848
+ return { hits, backlogRemaining: getBacklog(db) };
849
+ });
850
+ }
851
+
852
+ // --- Preparation / browse ---
853
+
854
+ export interface PreparationRowOptions {
855
+ limit: number;
856
+ /** Canonical repository root. Omit to sample all indexed sessions. */
857
+ repositoryRoot?: string;
858
+ currentSessionPath?: string;
859
+ }
860
+
861
+ /** Select deterministic, repository-scoped pattern-miner candidates from the
862
+ * existing index. Scope, current-session exclusion, and one-hop lineage
863
+ * collapse all happen before the sample limit. */
864
+ export function getPreparationRows(
865
+ dbPath: string,
866
+ opts: PreparationRowOptions,
867
+ ): PreparationSessionRow[] {
868
+ const clauses: string[] = [];
869
+ const params: string[] = [];
870
+ if (opts.repositoryRoot !== undefined) {
871
+ const prefix = opts.repositoryRoot.endsWith(path.sep)
872
+ ? opts.repositoryRoot
873
+ : opts.repositoryRoot + path.sep;
874
+ clauses.push("(s.cwd = ? OR substr(s.cwd, 1, length(?)) = ?)");
875
+ params.push(opts.repositoryRoot, prefix, prefix);
876
+ }
877
+ if (opts.currentSessionPath !== undefined) {
878
+ clauses.push("s.path <> ?");
879
+ params.push(opts.currentSessionPath);
880
+ }
881
+ const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
882
+ return withDb(dbPath, (db) => {
883
+ const rows = db.prepare(`
884
+ WITH eligible AS (
885
+ SELECT s.path, s.cwd, s.name, s.started_at, s.preview,
886
+ CASE
887
+ WHEN s.parent_session IS NULL THEN s.path
888
+ WHEN parent.path <> s.path AND parent.parent_session = s.path
889
+ THEN min(s.path, parent.path)
890
+ ELSE s.parent_session
891
+ END AS lineage_id
892
+ FROM sessions s
893
+ LEFT JOIN sessions parent ON parent.path = s.parent_session
894
+ ${where}
895
+ ), ranked AS (
896
+ SELECT *, ROW_NUMBER() OVER (
897
+ PARTITION BY lineage_id
898
+ ORDER BY CASE WHEN path = lineage_id THEN 0 ELSE 1 END,
899
+ started_at DESC, path
900
+ ) AS rn
901
+ FROM eligible
902
+ )
903
+ SELECT path, cwd, name, started_at, preview, lineage_id
904
+ FROM ranked
905
+ WHERE rn = 1
906
+ ORDER BY started_at DESC, path
907
+ LIMIT ?
908
+ `).all(...params, opts.limit) as any[];
909
+ return rows.map((r) => ({
910
+ path: r.path,
911
+ cwd: r.cwd ?? "",
912
+ name: r.name ?? undefined,
913
+ startedAt: r.started_at ?? undefined,
914
+ preview: r.preview ?? undefined,
915
+ lineageId: r.lineage_id,
916
+ }));
917
+ });
918
+ }
919
+
920
+ export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
921
+ return withDb(dbPath, (db) => {
922
+ const rows = db
923
+ .prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
924
+ .all(limit) as any[];
925
+ return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
926
+ });
927
+ }