@hcflabs/pi-session-recall 0.0.0-stage → 5.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,724 @@
1
+ /**
2
+ * Index engine: SQLite schema (WAL, external-content trigram FTS5), capped
3
+ * incremental sync, and discovery search with one-hop lineage suppression.
4
+ * No pi runtime imports — pure Node + node:sqlite so it is testable headless.
5
+ */
6
+ import { DatabaseSync } from "node:sqlite";
7
+ import type { SQLOutputValue } from "node:sqlite";
8
+ import fs from "node:fs";
9
+ import path from "node:path";
10
+ import { buildFtsQueryPlan, buildLikeQueryPlan, foldCase, nearLike } from "./query.ts";
11
+ import { MAX_SESSION_FILE_BYTES, readTranscriptEntries } from "./transcript.ts";
12
+ import type { PreparationSessionRow, SearchHit, SessionRow, SyncResult } from "./types.ts";
13
+ export const DEFAULT_SYNC_CAP = 50;
14
+ const MAX_TEXT_CHARS = 20000;
15
+
16
+ const SCHEMA_SQL = `
17
+ -- ctime_ms is raw stat.ctimeMs stored as REAL (SQLite REALs are IEEE doubles,
18
+ -- so JS numbers round-trip losslessly): flooring would collapse sub-ms restores.
19
+ CREATE TABLE IF NOT EXISTS session_files (
20
+ path TEXT PRIMARY KEY,
21
+ size INTEGER NOT NULL,
22
+ mtime_ms INTEGER NOT NULL,
23
+ ctime_ms REAL NOT NULL
24
+ );
25
+ -- Retry markers affect ordering only: unchanged failures move behind fresh work,
26
+ -- but remain eligible on every pass once fresh work is drained.
27
+ CREATE TABLE IF NOT EXISTS session_failures (
28
+ path TEXT PRIMARY KEY,
29
+ size INTEGER NOT NULL,
30
+ mtime_ms INTEGER NOT NULL,
31
+ ctime_ms REAL NOT NULL,
32
+ attempts INTEGER NOT NULL DEFAULT 0
33
+ );
34
+ CREATE TABLE IF NOT EXISTS sessions (
35
+ path TEXT PRIMARY KEY,
36
+ cwd TEXT,
37
+ name TEXT,
38
+ started_at TEXT,
39
+ preview TEXT,
40
+ parent_session TEXT
41
+ );
42
+ -- Oversized messages split into head/tail columns: one row per source
43
+ -- message keeps AND matching across both retained regions (FTS5 implicit AND
44
+ -- spans columns) while phrases and NEAR cannot cross column boundaries, so
45
+ -- truncation never manufactures proximity matches over the elided middle.
46
+ CREATE TABLE IF NOT EXISTS messages (
47
+ rowid INTEGER PRIMARY KEY,
48
+ path TEXT NOT NULL,
49
+ entry_id TEXT NOT NULL,
50
+ role TEXT NOT NULL,
51
+ timestamp TEXT,
52
+ head TEXT NOT NULL,
53
+ tail TEXT NOT NULL,
54
+ UNIQUE(path, entry_id)
55
+ );
56
+ CREATE INDEX IF NOT EXISTS messages_path ON messages(path);
57
+ CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
58
+ head,
59
+ tail,
60
+ content='messages',
61
+ content_rowid='rowid',
62
+ tokenize='trigram'
63
+ );
64
+ CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
65
+ INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
66
+ END;
67
+ CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
68
+ INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
69
+ END;
70
+ CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
71
+ INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
72
+ INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
73
+ END;
74
+ CREATE TABLE IF NOT EXISTS meta (
75
+ key TEXT PRIMARY KEY,
76
+ value TEXT NOT NULL
77
+ );
78
+ `;
79
+
80
+ /** Owner-only modes for the index directory and database files. The
81
+ * extension-owned parent dir is the only node tightened; intermediate config
82
+ * dirs are created restrictive but never chmod'ed if they predate this fix. */
83
+ const DB_FILE_MODE = 0o600;
84
+ const DB_DIR_MODE = 0o700;
85
+
86
+ /** Create or tighten the index's private directory and owner-only database
87
+ * file BEFORE SQLite writes any transcript content into it. Descriptor-based
88
+ * O_NOFOLLOW|O_NONBLOCK open + fstat rejects symlinks and other non-regular
89
+ * nodes instead of following them; fchmod tightens an existing permissive db.
90
+ * WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
91
+ * behind by older builds are tightened here too. Always closes the fd. */
92
+ function secureIndexNode(dbPath: string): void {
93
+ const posix = process.platform !== "win32";
94
+ const dir = path.dirname(dbPath);
95
+ fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
96
+ // O_NOFOLLOW on the db path does not protect parent components; reject a
97
+ // symlinked extension directory before any chmod or database open follows it.
98
+ if (!fs.lstatSync(dir).isDirectory()) throw new Error(`index directory is not a real directory: ${dir}`);
99
+ if (posix) {
100
+ // Tighten an existing permissive dir (e.g. created under umask 022).
101
+ if ((fs.statSync(dir).mode & 0o777) !== DB_DIR_MODE) {
102
+ fs.chmodSync(dir, DB_DIR_MODE);
103
+ }
104
+ }
105
+ let fd: number | undefined;
106
+ try {
107
+ fd = fs.openSync(
108
+ dbPath,
109
+ (fs.constants.O_RDWR | fs.constants.O_CREAT | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0)),
110
+ DB_FILE_MODE,
111
+ );
112
+ const st = fs.fstatSync(fd);
113
+ if (!st.isFile()) throw new Error("index database path is not a regular file");
114
+ if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
115
+ } finally {
116
+ if (fd !== undefined) fs.closeSync(fd);
117
+ }
118
+ if (!posix) return;
119
+ for (const suffix of ["-wal", "-shm"]) {
120
+ // Same descriptor no-follow safety as the db path: chmodSync(path) would
121
+ // follow a swapped-in sidecar symlink and retarget its referent's mode.
122
+ let fd: number | undefined;
123
+ try {
124
+ fd = fs.openSync(
125
+ dbPath + suffix,
126
+ fs.constants.O_RDWR | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0),
127
+ );
128
+ const st = fs.fstatSync(fd);
129
+ if (!st.isFile()) throw new Error(`index ${suffix} sidecar is not a regular file`);
130
+ if ((st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
131
+ } catch (err) {
132
+ // Sidecars are created by SQLite; a missing legacy sidecar stays harmless.
133
+ if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
134
+ } finally {
135
+ if (fd !== undefined) fs.closeSync(fd);
136
+ }
137
+ }
138
+ }
139
+
140
+ function openDb(dbPath: string): DatabaseSync {
141
+ secureIndexNode(dbPath);
142
+ const db = new DatabaseSync(dbPath);
143
+ try {
144
+ db.exec("PRAGMA journal_mode = WAL");
145
+ db.exec("PRAGMA busy_timeout = 5000");
146
+ // The index is disposable derived state with no migration path. Only the
147
+ // current schema is defined; an incompatible existing index fails on use
148
+ // and is never deleted or rewritten here — remove it manually to rebuild.
149
+ // SQLite's LIKE folds ASCII only, missing é/É, Greek, Cyrillic, …; every
150
+ // LIKE below wraps columns in ulower and binds pre-folded operands so
151
+ // both sides case-fold through the same foldCase helper.
152
+ db.function("ulower", (s: SQLOutputValue): string => typeof s === "string" ? foldCase(s) : "");
153
+ db.function("unear", nearLike);
154
+ // Creating a missing external-content FTS table after messages and their
155
+ // watermarks persist leaves it blank: unchanged files never replay inserts.
156
+ // The disposable index must be deleted and rebuilt instead of repaired here.
157
+ const hasFts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'session_fts'").get() !== undefined;
158
+ const hasPersistedSchema = db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' LIMIT 1").get() !== undefined;
159
+ if (!hasFts && hasPersistedSchema) {
160
+ throw new Error("index schema incompatible: session_fts missing — delete the index to rebuild");
161
+ }
162
+ db.exec(SCHEMA_SQL);
163
+ // CREATE TABLE IF NOT EXISTS cannot repair a predecessor index whose
164
+ // tables exist but lack current watermark columns (e.g. no ctime_ms): its
165
+ // search rows would silently serve stale hits while change detection
166
+ // breaks. Fail visibly; the index is disposable — remove it to rebuild.
167
+ for (const [table, required] of [
168
+ ["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
169
+ ["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
170
+ ] as const) {
171
+ const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
172
+ for (const col of required) {
173
+ if (!have.has(col)) throw new Error(`index schema incompatible: ${table}.${col} missing — delete the index to rebuild`);
174
+ }
175
+ }
176
+ } catch (err) {
177
+ db.close();
178
+ throw err;
179
+ }
180
+ return db;
181
+ }
182
+
183
+ /** Replacement/edit-sensitive file fingerprint from one fs.Stats. Size plus a
184
+ * floored mtime_ms misses same-length rewrites whose millisecond mtime is
185
+ * restored (backup restores, redactions). Exact ctime closes that: POSIX
186
+ * utimensat can set mtime but never ctime, so every restore bumps it. */
187
+ function fileFingerprint(stat: fs.Stats): { size: number; mtimeMs: number; ctimeMs: number } {
188
+ return { size: stat.size, mtimeMs: Math.floor(stat.mtimeMs), ctimeMs: stat.ctimeMs };
189
+ }
190
+
191
+ // --- Parsing ---
192
+
193
+ interface ParsedMessage {
194
+ entryId: string;
195
+ role: string;
196
+ timestamp: string | null;
197
+ head: string;
198
+ tail: string;
199
+ }
200
+
201
+ interface ParsedFile {
202
+ cwd: string | null;
203
+ name: string | null;
204
+ startedAt: string | null;
205
+ parentSession: string | null;
206
+ preview: string | null;
207
+ messages: ParsedMessage[];
208
+ }
209
+
210
+ /** Middle-out truncation into SEPARATE head/tail index columns: concatenating
211
+ * them would make boundary terms adjacent and manufacture phrase/proximity
212
+ * matches across the elided middle. Raw text only — no synthetic notices:
213
+ * these strings are FTS-indexed, so injected terms would pollute search
214
+ * results and snippets. */
215
+ function truncateRegions(text: string, max = MAX_TEXT_CHARS): { head: string; tail: string } | null {
216
+ if (text.length <= max) return null;
217
+ const kept = max - 1;
218
+ return {
219
+ head: text.slice(0, Math.ceil(kept / 2)),
220
+ tail: text.slice(-Math.floor(kept / 2)),
221
+ };
222
+ }
223
+
224
+ /** Cap an untrusted string at a parse boundary so malformed JSONL cannot
225
+ * produce unbounded metadata downstream. */
226
+ function capStr(value: unknown, max: number): string | null {
227
+ return typeof value === "string" ? value.slice(0, max) : null;
228
+ }
229
+
230
+ /** Narrow ISO timestamp shape; captures wall-clock fields for validation. */
231
+ const ISO_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
232
+
233
+ /** Canonicalize a session-header timestamp to UTC ISO so its lexicographic
234
+ * order matches chronological order (browse sorts started_at DESC as TEXT).
235
+ * Anything else is stored null so it sorts behind every valid timestamp. */
236
+ function normalizeTimestamp(value: unknown): string | null {
237
+ if (typeof value !== "string") return null;
238
+ const m = ISO_TIMESTAMP_RE.exec(value);
239
+ if (!m) return null;
240
+ const [, y, mo, d, h] = m;
241
+ if (+h > 23) return null;
242
+ const probe = new Date(Date.UTC(+y, +mo - 1, +d));
243
+ // Date parsing rejects invalid time fields, but silently rolls impossible
244
+ // calendar dates such as February 30.
245
+ if (probe.getUTCMonth() !== +mo - 1 || probe.getUTCDate() !== +d) return null;
246
+ const date = new Date(value);
247
+ return Number.isNaN(date.getTime()) ? null : date.toISOString();
248
+ }
249
+
250
+ function extractText(content: unknown): string {
251
+ if (typeof content === "string") return content;
252
+ if (!Array.isArray(content)) return "";
253
+ const parts: string[] = [];
254
+ for (const block of content) {
255
+ if (block && typeof block === "object" && (block as any).type === "text" && typeof (block as any).text === "string") {
256
+ parts.push((block as any).text);
257
+ }
258
+ // thinking / toolUse / tool_result blocks are skipped entirely
259
+ }
260
+ return parts.join("\n").trim();
261
+ }
262
+
263
+ function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
264
+ const parsed: ParsedFile = {
265
+ cwd: null,
266
+ name: null,
267
+ startedAt: null,
268
+ parentSession: null,
269
+ preview: null,
270
+ messages: [],
271
+ };
272
+ // Unreadable/oversized file must throw so the sync transaction rolls back
273
+ // instead of wiping previously indexed rows and advancing the watermark over a hole.
274
+ for (const { data: entryRaw, id } of readTranscriptEntries(filePath, maxBytes)) {
275
+ const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
276
+ switch (entry?.type) {
277
+ case "session":
278
+ // Untrusted header strings are capped here so every consumer
279
+ // (browse rows, discovery meta) inherits the bound.
280
+ parsed.cwd = capStr(entry.cwd, 500) ?? parsed.cwd;
281
+ parsed.startedAt = normalizeTimestamp(entry.timestamp) ?? parsed.startedAt;
282
+ parsed.parentSession = capStr(entry.parentSession, 1024) ?? parsed.parentSession;
283
+ break;
284
+ case "session_info":
285
+ if (typeof entry.name === "string") parsed.name = entry.name.slice(0, 500);
286
+ break;
287
+ case "message": {
288
+ const msg = entry.message as { role?: unknown; content?: unknown } | undefined;
289
+ const role = msg?.role;
290
+ if (!msg || (role !== "user" && role !== "assistant")) break;
291
+ const full = extractText(msg.content);
292
+ if (!full || id === undefined) break;
293
+ if (role === "user" && parsed.preview === null) {
294
+ parsed.preview = full.slice(0, 200);
295
+ }
296
+ const regions = truncateRegions(full);
297
+ parsed.messages.push({
298
+ entryId: id,
299
+ role,
300
+ timestamp: capStr(entry.timestamp, 128),
301
+ head: regions ? regions.head : full,
302
+ tail: regions ? regions.tail : "",
303
+ });
304
+ break;
305
+ }
306
+ // compaction, branch_summary, custom, custom_message, label,
307
+ // session_info (name handled above), model_change, … are skipped
308
+ }
309
+ }
310
+ return parsed;
311
+ }
312
+
313
+ /** Walk result. `complete: false` means the tree could not be fully read
314
+ * (missing root, readdir/stat failure) — callers must not treat unseen
315
+ * indexed paths as deleted. */
316
+ interface WalkResult {
317
+ files: Map<string, fs.Stats>;
318
+ complete: boolean;
319
+ }
320
+
321
+ function walkJsonlFiles(sessionsDir: string): WalkResult {
322
+ const files = new Map<string, fs.Stats>();
323
+ const stack = [sessionsDir];
324
+ let complete = true;
325
+ while (stack.length > 0) {
326
+ const dir = stack.pop()!;
327
+ let entries: fs.Dirent[];
328
+ try {
329
+ entries = fs.readdirSync(dir, { withFileTypes: true });
330
+ } catch {
331
+ complete = false;
332
+ continue;
333
+ }
334
+ for (const ent of entries) {
335
+ const full = path.join(dir, ent.name);
336
+ if (ent.isDirectory()) {
337
+ // Junk dirs are pruned before descent so a large ignored tree never
338
+ // costs a walk + per-file stats on every sync pass.
339
+ if (ent.name.startsWith("--tmp-") || ent.name.startsWith("--private-tmp-")) continue;
340
+ stack.push(full);
341
+ } else if (ent.isFile() && ent.name.endsWith(".jsonl")) {
342
+ try {
343
+ files.set(full, fs.statSync(full));
344
+ } catch {
345
+ complete = false; // vanished or became unreadable mid-walk
346
+ }
347
+ }
348
+ }
349
+ }
350
+ return { files, complete };
351
+ }
352
+
353
+ // --- Sync ---
354
+
355
+ export function syncSessions(
356
+ sessionsDir: string,
357
+ dbPath: string,
358
+ opts?: { cap?: number; maxFileBytes?: number },
359
+ ): SyncResult {
360
+ const cap = opts?.cap ?? DEFAULT_SYNC_CAP;
361
+ const db = openDb(dbPath);
362
+ try {
363
+
364
+ // Junk dirs never reach the walk result: pruned during descent.
365
+ const walk = walkJsonlFiles(sessionsDir);
366
+ const all = [...walk.files];
367
+ const watermarks = new Map(
368
+ (db.prepare("SELECT path, size, mtime_ms, ctime_ms FROM session_files").all() as any[]).map((r) => [r.path, r]),
369
+ );
370
+ const failures = new Map(
371
+ (db.prepare("SELECT path, size, mtime_ms, ctime_ms, attempts FROM session_failures").all() as any[]).map((r) => [r.path, r]),
372
+ );
373
+ // An incomplete traversal proves nothing about absence: purging here would
374
+ // wipe healthy sessions over a transient readdir/stat failure. Changed/new
375
+ // files already discovered still process normally.
376
+ const discovered = new Set(all.map(([fp]) => fp));
377
+ const deleted = walk.complete ? [...watermarks.keys()].filter((p) => !discovered.has(p)) : [];
378
+ const deletedFailures = walk.complete ? [...failures.keys()].filter((p) => !discovered.has(p)) : [];
379
+
380
+ const changed: { path: string; stat: fs.Stats }[] = [];
381
+ for (const [p, stat] of all) {
382
+ const fp = fileFingerprint(stat);
383
+ const wm = watermarks.get(p);
384
+ if (wm && wm.size === fp.size && wm.mtime_ms === fp.mtimeMs && wm.ctime_ms === fp.ctimeMs) continue;
385
+ changed.push({ path: p, stat });
386
+ }
387
+
388
+ // Newest-first, but an unchanged prior failure moves behind untouched work.
389
+ // Among retries, fewest attempts first so one persistent failure cannot
390
+ // monopolize a small cap forever.
391
+ const retryAttempts = (p: string, stat: fs.Stats): number | null => {
392
+ const f = failures.get(p);
393
+ if (!f) return null;
394
+ const fp = fileFingerprint(stat);
395
+ return f.size === fp.size && f.mtime_ms === fp.mtimeMs && f.ctime_ms === fp.ctimeMs ? f.attempts : null;
396
+ };
397
+ changed.sort((a, b) => {
398
+ const aRetry = retryAttempts(a.path, a.stat);
399
+ const bRetry = retryAttempts(b.path, b.stat);
400
+ if (aRetry !== null && bRetry !== null) return aRetry - bRetry || b.stat.mtimeMs - a.stat.mtimeMs;
401
+ if (aRetry !== null) return 1;
402
+ if (bRetry !== null) return -1;
403
+ return b.stat.mtimeMs - a.stat.mtimeMs;
404
+ });
405
+ deleted.sort((a, b) => {
406
+ const ma = watermarks.get(a)!.mtime_ms;
407
+ const mb = watermarks.get(b)!.mtime_ms;
408
+ return mb - ma;
409
+ });
410
+
411
+ const delStmt = db.prepare("DELETE FROM messages WHERE path = ?");
412
+ const delSession = db.prepare("DELETE FROM sessions WHERE path = ?");
413
+ const delFile = db.prepare("DELETE FROM session_files WHERE path = ?");
414
+ const delFailure = db.prepare("DELETE FROM session_failures WHERE path = ?");
415
+ const upsertFailure = db.prepare(
416
+ "INSERT INTO session_failures(path, size, mtime_ms, ctime_ms, attempts) VALUES (?, ?, ?, ?, 1) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, attempts=CASE WHEN size=excluded.size AND mtime_ms=excluded.mtime_ms AND ctime_ms=excluded.ctime_ms THEN attempts+1 ELSE 1 END",
417
+ );
418
+ const upsertFile = db.prepare(
419
+ "INSERT INTO session_files(path, size, mtime_ms, ctime_ms) VALUES (?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms",
420
+ );
421
+ const upsertSession = db.prepare(
422
+ `INSERT INTO sessions(path, cwd, name, started_at, preview, parent_session) VALUES (?, ?, ?, ?, ?, ?)
423
+ ON CONFLICT(path) DO UPDATE SET cwd=excluded.cwd, name=excluded.name, started_at=excluded.started_at,
424
+ preview=excluded.preview, parent_session=excluded.parent_session`,
425
+ );
426
+ const insertMsg = db.prepare(
427
+ "INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
428
+ );
429
+
430
+ const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
431
+
432
+ const tx = (filePath: string, stat: fs.Stats): void => {
433
+ db.exec("BEGIN");
434
+ try {
435
+ const parsed = parseSessionFile(filePath, maxFileBytes);
436
+ delStmt.run(filePath);
437
+ upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
438
+ for (const msg of parsed.messages) {
439
+ insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
440
+ }
441
+ {
442
+ const fp = fileFingerprint(stat);
443
+ upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
444
+ }
445
+ delFailure.run(filePath);
446
+ db.exec("COMMIT");
447
+ } catch (err) {
448
+ db.exec("ROLLBACK");
449
+ throw err;
450
+ }
451
+ };
452
+
453
+ let filesProcessed = 0;
454
+
455
+ // The cap bounds total indexing attempts per pass, not successes.
456
+ const batch = changed.slice(0, cap);
457
+ for (const { path: p, stat } of batch) {
458
+ try {
459
+ tx(p, stat);
460
+ filesProcessed++;
461
+ } catch {
462
+ // Keep the file retryable, but put this unchanged fingerprint behind
463
+ // untouched work on the next pass.
464
+ {
465
+ const fp = fileFingerprint(stat);
466
+ upsertFailure.run(p, fp.size, fp.mtimeMs, fp.ctimeMs);
467
+ }
468
+ }
469
+ }
470
+
471
+ const deletedRemaining = Math.max(0, deleted.length - cap);
472
+ for (const p of deleted.slice(0, cap)) {
473
+ delStmt.run(p);
474
+ delSession.run(p);
475
+ delFile.run(p);
476
+ delFailure.run(p);
477
+ }
478
+ for (const p of deletedFailures) delFailure.run(p);
479
+
480
+ // Attempted failures remain unsynced and therefore remain in the backlog.
481
+ const backlog = changed.length - filesProcessed + deletedRemaining;
482
+ db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
483
+
484
+ return { filesProcessed, backlogRemaining: backlog, walkComplete: walk.complete };
485
+ } finally {
486
+ db.close();
487
+ }
488
+ }
489
+
490
+ function getBacklog(db: DatabaseSync): number {
491
+ const row = db.prepare("SELECT value FROM meta WHERE key = 'backlog'").get() as { value: string } | undefined;
492
+ return row ? Number(row.value) : 0;
493
+ }
494
+
495
+ // --- Search ---
496
+
497
+ export interface SearchOptions {
498
+ limit?: number;
499
+ /** Path of the current session file — the whole file is skipped. */
500
+ currentSessionPath?: string;
501
+ }
502
+
503
+ // Current-session exclusion happens INSIDE SQL (before ROW_NUMBER/caps), so
504
+ // the excluded file cannot consume the per-file rank, lineage, or scan limit.
505
+ const CURRENT_SESSION_FILTER_SQL = "(? IS NULL OR m.path <> ?)";
506
+ const LINEAGE_FILTER_SQL = `NOT EXISTS (
507
+ SELECT 1 FROM ranked parent
508
+ WHERE parent.path = r.parent_session
509
+ AND parent.path <> r.path
510
+ AND parent.rn <= 1
511
+ -- Direct mutual cycle (A<->B) would suppress both sides; keep the
512
+ -- lexicographically smaller path deterministically.
513
+ AND NOT (parent.parent_session IS r.path AND r.path < parent.path)
514
+ )`;
515
+ const BASE_SELECT = `
516
+ WITH matches AS (
517
+ SELECT m.path, m.entry_id, m.role, m.timestamp, s.cwd, s.name, s.started_at,
518
+ s.parent_session, m.rowid AS rid,
519
+ snippet(session_fts, -1, '[', ']', '…', 16) AS snip,
520
+ bm25(session_fts) AS score
521
+ FROM session_fts JOIN messages m ON m.rowid = session_fts.rowid
522
+ LEFT JOIN sessions s ON s.path = m.path
523
+ WHERE session_fts MATCH ? AND ${CURRENT_SESSION_FILTER_SQL}
524
+ ), ranked AS (
525
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY score, rid) AS rn
526
+ FROM matches
527
+ )
528
+ SELECT path, entry_id, role, timestamp, snip, cwd, name, started_at
529
+ FROM ranked r
530
+ WHERE rn <= 1
531
+ AND ${LINEAGE_FILTER_SQL}
532
+ ORDER BY score, rid
533
+ LIMIT ?`;
534
+
535
+ /** Bounded excerpt around the first matched term for the LIKE fallback,
536
+ * searching head then tail. */
537
+ function likeSnippet(head: string, tail: string, terms: string[]): string {
538
+ for (const text of [head, tail]) {
539
+ const lower = foldCase(text);
540
+ let at = -1;
541
+ for (const t of terms) {
542
+ const i = lower.indexOf(foldCase(t));
543
+ if (i >= 0 && (at < 0 || i < at)) at = i;
544
+ }
545
+ if (at >= 0) {
546
+ const start = Math.max(0, at - 60);
547
+ return `${start > 0 ? "…" : ""}${text.slice(start, start + 120)}…`;
548
+ }
549
+ }
550
+ return head.slice(0, 120);
551
+ }
552
+
553
+ export function searchIndex(
554
+ dbPath: string,
555
+ query: string,
556
+ opts?: SearchOptions,
557
+ ): { hits: SearchHit[]; backlogRemaining: number } {
558
+ const limit = opts?.limit ?? 3;
559
+ const db = openDb(dbPath);
560
+ try {
561
+ const currentSessionPath = opts?.currentSessionPath ?? null;
562
+ const plan = buildFtsQueryPlan(query);
563
+ if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
564
+ return { hits: [], backlogRemaining: getBacklog(db) };
565
+ }
566
+
567
+ interface RawRow {
568
+ path: string;
569
+ entry_id: string;
570
+ role: string;
571
+ timestamp: string | null;
572
+ snip: string;
573
+ cwd: string | null;
574
+ name: string | null;
575
+ started_at: string | null;
576
+ }
577
+ let rows: Omit<RawRow, "snip">[] = [];
578
+ let usedLike = plan.forceLike;
579
+
580
+ if (!plan.forceLike) {
581
+ // Recovery candidates run only after an FTS5 PARSE error. A candidate
582
+ // that parses defines the result even with zero rows (`a AND b` with
583
+ // no co-occurrence stays empty instead of degrading to OR/LIKE).
584
+ let parseFailed = false;
585
+ for (const cand of plan.ftsCandidates) {
586
+ try {
587
+ rows = db.prepare(BASE_SELECT).all(cand, currentSessionPath, currentSessionPath, limit) as any;
588
+ // First/raw success defines the result even with zero rows. After a
589
+ // parse error, an empty recovery candidate keeps trying later ones.
590
+ if (!parseFailed || rows.length > 0) {
591
+ usedLike = false;
592
+ break;
593
+ }
594
+ usedLike = true;
595
+ } catch {
596
+ parseFailed = true;
597
+ usedLike = true; // malformed syntax → try next candidate
598
+ }
599
+ }
600
+ }
601
+
602
+ if (usedLike) {
603
+ const likePlan = buildLikeQueryPlan(query);
604
+ if (likePlan === null) return { hits: [], backlogRemaining: getBacklog(db) };
605
+ rows = db.prepare(`WITH ranked AS (
606
+ SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
607
+ s.parent_session,
608
+ ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
609
+ COUNT(*) OVER (PARTITION BY m.path) AS matches
610
+ FROM messages m LEFT JOIN sessions s ON s.path = m.path
611
+ WHERE ${likePlan.where} AND ${CURRENT_SESSION_FILTER_SQL}
612
+ )
613
+ SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
614
+ FROM ranked r
615
+ WHERE rn <= 1
616
+ AND ${LINEAGE_FILTER_SQL}
617
+ ORDER BY matches DESC, started_at DESC, path
618
+ LIMIT ?`).all(...likePlan.params, currentSessionPath, currentSessionPath, limit) as any;
619
+ // Snippets anchor on operand terms only — operator words like OR would
620
+ // otherwise match common substrings and hide the real match.
621
+ for (const r of rows as any[]) r.snip = likeSnippet(r.head, r.tail, likePlan.terms);
622
+ }
623
+
624
+ // Current-file and one-hop lineage suppression already happened in SQL,
625
+ // before the LIMIT, so excluded or fork rows cannot starve matches.
626
+ const hits: SearchHit[] = (rows as RawRow[]).map((r, rank) => ({
627
+ path: r.path,
628
+ snippet: r.snip ?? "",
629
+ rank,
630
+ matchMessageId: r.entry_id,
631
+ role: r.role,
632
+ timestamp: r.timestamp ?? "",
633
+ cwd: r.cwd ?? undefined,
634
+ name: r.name ?? undefined,
635
+ startedAt: r.started_at ?? undefined,
636
+ }));
637
+ return { hits, backlogRemaining: getBacklog(db) };
638
+ } finally {
639
+ db.close();
640
+ }
641
+ }
642
+
643
+ // --- Preparation / browse ---
644
+
645
+ export interface PreparationRowOptions {
646
+ limit: number;
647
+ /** Canonical repository root. Omit to sample all indexed sessions. */
648
+ repositoryRoot?: string;
649
+ currentSessionPath?: string;
650
+ }
651
+
652
+ /** Select deterministic, repository-scoped pattern-miner candidates from the
653
+ * existing index. Scope, current-session exclusion, and one-hop lineage
654
+ * collapse all happen before the sample limit. */
655
+ export function getPreparationRows(
656
+ dbPath: string,
657
+ opts: PreparationRowOptions,
658
+ ): PreparationSessionRow[] {
659
+ const clauses: string[] = [];
660
+ const params: string[] = [];
661
+ if (opts.repositoryRoot !== undefined) {
662
+ const prefix = opts.repositoryRoot.endsWith(path.sep)
663
+ ? opts.repositoryRoot
664
+ : opts.repositoryRoot + path.sep;
665
+ clauses.push("(s.cwd = ? OR substr(s.cwd, 1, length(?)) = ?)");
666
+ params.push(opts.repositoryRoot, prefix, prefix);
667
+ }
668
+ if (opts.currentSessionPath !== undefined) {
669
+ clauses.push("s.path <> ?");
670
+ params.push(opts.currentSessionPath);
671
+ }
672
+ const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
673
+ const db = openDb(dbPath);
674
+ try {
675
+ const rows = db.prepare(`
676
+ WITH eligible AS (
677
+ SELECT s.path, s.cwd, s.name, s.started_at, s.preview,
678
+ CASE
679
+ WHEN s.parent_session IS NULL THEN s.path
680
+ WHEN parent.path <> s.path AND parent.parent_session = s.path
681
+ THEN min(s.path, parent.path)
682
+ ELSE s.parent_session
683
+ END AS lineage_id
684
+ FROM sessions s
685
+ LEFT JOIN sessions parent ON parent.path = s.parent_session
686
+ ${where}
687
+ ), ranked AS (
688
+ SELECT *, ROW_NUMBER() OVER (
689
+ PARTITION BY lineage_id
690
+ ORDER BY CASE WHEN path = lineage_id THEN 0 ELSE 1 END,
691
+ started_at DESC, path
692
+ ) AS rn
693
+ FROM eligible
694
+ )
695
+ SELECT path, cwd, name, started_at, preview, lineage_id
696
+ FROM ranked
697
+ WHERE rn = 1
698
+ ORDER BY started_at DESC, path
699
+ LIMIT ?
700
+ `).all(...params, opts.limit) as any[];
701
+ return rows.map((r) => ({
702
+ path: r.path,
703
+ cwd: r.cwd ?? "",
704
+ name: r.name ?? undefined,
705
+ startedAt: r.started_at ?? undefined,
706
+ preview: r.preview ?? undefined,
707
+ lineageId: r.lineage_id,
708
+ }));
709
+ } finally {
710
+ db.close();
711
+ }
712
+ }
713
+
714
+ export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
715
+ const db = openDb(dbPath);
716
+ try {
717
+ const rows = db
718
+ .prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
719
+ .all(limit) as any[];
720
+ return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
721
+ } finally {
722
+ db.close();
723
+ }
724
+ }