@hcflabs/pi-session-recall 0.0.0-stage → 5.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +146 -2
- package/docs/session-search-routing.html +138 -0
- package/docs/session-search-routing.svg +121 -0
- package/extensions/hydrate.ts +258 -0
- package/extensions/query.ts +379 -0
- package/extensions/repository-inventory.ts +499 -0
- package/extensions/search-core.ts +724 -0
- package/extensions/session-recall.ts +556 -0
- package/extensions/transcript.ts +112 -0
- package/extensions/types.ts +53 -0
- package/package.json +60 -4
- package/skills/pi-session-pattern-miner/SKILL.md +97 -0
|
@@ -0,0 +1,724 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index engine: SQLite schema (WAL, external-content trigram FTS5), capped
|
|
3
|
+
* incremental sync, and discovery search with one-hop lineage suppression.
|
|
4
|
+
* No pi runtime imports — pure Node + node:sqlite so it is testable headless.
|
|
5
|
+
*/
|
|
6
|
+
import { DatabaseSync } from "node:sqlite";
|
|
7
|
+
import type { SQLOutputValue } from "node:sqlite";
|
|
8
|
+
import fs from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { buildFtsQueryPlan, buildLikeQueryPlan, foldCase, nearLike } from "./query.ts";
|
|
11
|
+
import { MAX_SESSION_FILE_BYTES, readTranscriptEntries } from "./transcript.ts";
|
|
12
|
+
import type { PreparationSessionRow, SearchHit, SessionRow, SyncResult } from "./types.ts";
|
|
13
|
+
export const DEFAULT_SYNC_CAP = 50;
|
|
14
|
+
const MAX_TEXT_CHARS = 20000;
|
|
15
|
+
|
|
16
|
+
const SCHEMA_SQL = `
|
|
17
|
+
-- ctime_ms is raw stat.ctimeMs stored as REAL (SQLite REALs are IEEE doubles,
|
|
18
|
+
-- so JS numbers round-trip losslessly): flooring would collapse sub-ms restores.
|
|
19
|
+
CREATE TABLE IF NOT EXISTS session_files (
|
|
20
|
+
path TEXT PRIMARY KEY,
|
|
21
|
+
size INTEGER NOT NULL,
|
|
22
|
+
mtime_ms INTEGER NOT NULL,
|
|
23
|
+
ctime_ms REAL NOT NULL
|
|
24
|
+
);
|
|
25
|
+
-- Retry markers affect ordering only: unchanged failures move behind fresh work,
|
|
26
|
+
-- but remain eligible on every pass once fresh work is drained.
|
|
27
|
+
CREATE TABLE IF NOT EXISTS session_failures (
|
|
28
|
+
path TEXT PRIMARY KEY,
|
|
29
|
+
size INTEGER NOT NULL,
|
|
30
|
+
mtime_ms INTEGER NOT NULL,
|
|
31
|
+
ctime_ms REAL NOT NULL,
|
|
32
|
+
attempts INTEGER NOT NULL DEFAULT 0
|
|
33
|
+
);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
35
|
+
path TEXT PRIMARY KEY,
|
|
36
|
+
cwd TEXT,
|
|
37
|
+
name TEXT,
|
|
38
|
+
started_at TEXT,
|
|
39
|
+
preview TEXT,
|
|
40
|
+
parent_session TEXT
|
|
41
|
+
);
|
|
42
|
+
-- Oversized messages split into head/tail columns: one row per source
|
|
43
|
+
-- message keeps AND matching across both retained regions (FTS5 implicit AND
|
|
44
|
+
-- spans columns) while phrases and NEAR cannot cross column boundaries, so
|
|
45
|
+
-- truncation never manufactures proximity matches over the elided middle.
|
|
46
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
47
|
+
rowid INTEGER PRIMARY KEY,
|
|
48
|
+
path TEXT NOT NULL,
|
|
49
|
+
entry_id TEXT NOT NULL,
|
|
50
|
+
role TEXT NOT NULL,
|
|
51
|
+
timestamp TEXT,
|
|
52
|
+
head TEXT NOT NULL,
|
|
53
|
+
tail TEXT NOT NULL,
|
|
54
|
+
UNIQUE(path, entry_id)
|
|
55
|
+
);
|
|
56
|
+
CREATE INDEX IF NOT EXISTS messages_path ON messages(path);
|
|
57
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS session_fts USING fts5(
|
|
58
|
+
head,
|
|
59
|
+
tail,
|
|
60
|
+
content='messages',
|
|
61
|
+
content_rowid='rowid',
|
|
62
|
+
tokenize='trigram'
|
|
63
|
+
);
|
|
64
|
+
CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
|
|
65
|
+
INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
|
|
66
|
+
END;
|
|
67
|
+
CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
|
|
68
|
+
INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
|
|
69
|
+
END;
|
|
70
|
+
CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
|
|
71
|
+
INSERT INTO session_fts(session_fts, rowid, head, tail) VALUES ('delete', old.rowid, old.head, old.tail);
|
|
72
|
+
INSERT INTO session_fts(rowid, head, tail) VALUES (new.rowid, new.head, new.tail);
|
|
73
|
+
END;
|
|
74
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
75
|
+
key TEXT PRIMARY KEY,
|
|
76
|
+
value TEXT NOT NULL
|
|
77
|
+
);
|
|
78
|
+
`;
|
|
79
|
+
|
|
80
|
+
/** Owner-only modes for the index directory and database files. The
|
|
81
|
+
* extension-owned parent dir is the only node tightened; intermediate config
|
|
82
|
+
* dirs are created restrictive but never chmod'ed if they predate this fix. */
|
|
83
|
+
const DB_FILE_MODE = 0o600;
|
|
84
|
+
const DB_DIR_MODE = 0o700;
|
|
85
|
+
|
|
86
|
+
/** Create or tighten the index's private directory and owner-only database
|
|
87
|
+
* file BEFORE SQLite writes any transcript content into it. Descriptor-based
|
|
88
|
+
* O_NOFOLLOW|O_NONBLOCK open + fstat rejects symlinks and other non-regular
|
|
89
|
+
* nodes instead of following them; fchmod tightens an existing permissive db.
|
|
90
|
+
* WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
|
|
91
|
+
* behind by older builds are tightened here too. Always closes the fd. */
|
|
92
|
+
function secureIndexNode(dbPath: string): void {
|
|
93
|
+
const posix = process.platform !== "win32";
|
|
94
|
+
const dir = path.dirname(dbPath);
|
|
95
|
+
fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
|
|
96
|
+
// O_NOFOLLOW on the db path does not protect parent components; reject a
|
|
97
|
+
// symlinked extension directory before any chmod or database open follows it.
|
|
98
|
+
if (!fs.lstatSync(dir).isDirectory()) throw new Error(`index directory is not a real directory: ${dir}`);
|
|
99
|
+
if (posix) {
|
|
100
|
+
// Tighten an existing permissive dir (e.g. created under umask 022).
|
|
101
|
+
if ((fs.statSync(dir).mode & 0o777) !== DB_DIR_MODE) {
|
|
102
|
+
fs.chmodSync(dir, DB_DIR_MODE);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
let fd: number | undefined;
|
|
106
|
+
try {
|
|
107
|
+
fd = fs.openSync(
|
|
108
|
+
dbPath,
|
|
109
|
+
(fs.constants.O_RDWR | fs.constants.O_CREAT | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0)),
|
|
110
|
+
DB_FILE_MODE,
|
|
111
|
+
);
|
|
112
|
+
const st = fs.fstatSync(fd);
|
|
113
|
+
if (!st.isFile()) throw new Error("index database path is not a regular file");
|
|
114
|
+
if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
|
|
115
|
+
} finally {
|
|
116
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
117
|
+
}
|
|
118
|
+
if (!posix) return;
|
|
119
|
+
for (const suffix of ["-wal", "-shm"]) {
|
|
120
|
+
// Same descriptor no-follow safety as the db path: chmodSync(path) would
|
|
121
|
+
// follow a swapped-in sidecar symlink and retarget its referent's mode.
|
|
122
|
+
let fd: number | undefined;
|
|
123
|
+
try {
|
|
124
|
+
fd = fs.openSync(
|
|
125
|
+
dbPath + suffix,
|
|
126
|
+
fs.constants.O_RDWR | (fs.constants.O_NONBLOCK ?? 0) | (fs.constants.O_NOFOLLOW ?? 0),
|
|
127
|
+
);
|
|
128
|
+
const st = fs.fstatSync(fd);
|
|
129
|
+
if (!st.isFile()) throw new Error(`index ${suffix} sidecar is not a regular file`);
|
|
130
|
+
if ((st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
|
|
131
|
+
} catch (err) {
|
|
132
|
+
// Sidecars are created by SQLite; a missing legacy sidecar stays harmless.
|
|
133
|
+
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
|
|
134
|
+
} finally {
|
|
135
|
+
if (fd !== undefined) fs.closeSync(fd);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function openDb(dbPath: string): DatabaseSync {
|
|
141
|
+
secureIndexNode(dbPath);
|
|
142
|
+
const db = new DatabaseSync(dbPath);
|
|
143
|
+
try {
|
|
144
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
145
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
146
|
+
// The index is disposable derived state with no migration path. Only the
|
|
147
|
+
// current schema is defined; an incompatible existing index fails on use
|
|
148
|
+
// and is never deleted or rewritten here — remove it manually to rebuild.
|
|
149
|
+
// SQLite's LIKE folds ASCII only, missing é/É, Greek, Cyrillic, …; every
|
|
150
|
+
// LIKE below wraps columns in ulower and binds pre-folded operands so
|
|
151
|
+
// both sides case-fold through the same foldCase helper.
|
|
152
|
+
db.function("ulower", (s: SQLOutputValue): string => typeof s === "string" ? foldCase(s) : "");
|
|
153
|
+
db.function("unear", nearLike);
|
|
154
|
+
// Creating a missing external-content FTS table after messages and their
|
|
155
|
+
// watermarks persist leaves it blank: unchanged files never replay inserts.
|
|
156
|
+
// The disposable index must be deleted and rebuilt instead of repaired here.
|
|
157
|
+
const hasFts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'session_fts'").get() !== undefined;
|
|
158
|
+
const hasPersistedSchema = db.prepare("SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' LIMIT 1").get() !== undefined;
|
|
159
|
+
if (!hasFts && hasPersistedSchema) {
|
|
160
|
+
throw new Error("index schema incompatible: session_fts missing — delete the index to rebuild");
|
|
161
|
+
}
|
|
162
|
+
db.exec(SCHEMA_SQL);
|
|
163
|
+
// CREATE TABLE IF NOT EXISTS cannot repair a predecessor index whose
|
|
164
|
+
// tables exist but lack current watermark columns (e.g. no ctime_ms): its
|
|
165
|
+
// search rows would silently serve stale hits while change detection
|
|
166
|
+
// breaks. Fail visibly; the index is disposable — remove it to rebuild.
|
|
167
|
+
for (const [table, required] of [
|
|
168
|
+
["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
|
|
169
|
+
["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
|
|
170
|
+
] as const) {
|
|
171
|
+
const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
|
|
172
|
+
for (const col of required) {
|
|
173
|
+
if (!have.has(col)) throw new Error(`index schema incompatible: ${table}.${col} missing — delete the index to rebuild`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
} catch (err) {
|
|
177
|
+
db.close();
|
|
178
|
+
throw err;
|
|
179
|
+
}
|
|
180
|
+
return db;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Replacement/edit-sensitive file fingerprint from one fs.Stats. Size plus a
|
|
184
|
+
* floored mtime_ms misses same-length rewrites whose millisecond mtime is
|
|
185
|
+
* restored (backup restores, redactions). Exact ctime closes that: POSIX
|
|
186
|
+
* utimensat can set mtime but never ctime, so every restore bumps it. */
|
|
187
|
+
function fileFingerprint(stat: fs.Stats): { size: number; mtimeMs: number; ctimeMs: number } {
|
|
188
|
+
return { size: stat.size, mtimeMs: Math.floor(stat.mtimeMs), ctimeMs: stat.ctimeMs };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// --- Parsing ---
|
|
192
|
+
|
|
193
|
+
interface ParsedMessage {
|
|
194
|
+
entryId: string;
|
|
195
|
+
role: string;
|
|
196
|
+
timestamp: string | null;
|
|
197
|
+
head: string;
|
|
198
|
+
tail: string;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
interface ParsedFile {
|
|
202
|
+
cwd: string | null;
|
|
203
|
+
name: string | null;
|
|
204
|
+
startedAt: string | null;
|
|
205
|
+
parentSession: string | null;
|
|
206
|
+
preview: string | null;
|
|
207
|
+
messages: ParsedMessage[];
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Middle-out truncation into SEPARATE head/tail index columns: concatenating
|
|
211
|
+
* them would make boundary terms adjacent and manufacture phrase/proximity
|
|
212
|
+
* matches across the elided middle. Raw text only — no synthetic notices:
|
|
213
|
+
* these strings are FTS-indexed, so injected terms would pollute search
|
|
214
|
+
* results and snippets. */
|
|
215
|
+
function truncateRegions(text: string, max = MAX_TEXT_CHARS): { head: string; tail: string } | null {
|
|
216
|
+
if (text.length <= max) return null;
|
|
217
|
+
const kept = max - 1;
|
|
218
|
+
return {
|
|
219
|
+
head: text.slice(0, Math.ceil(kept / 2)),
|
|
220
|
+
tail: text.slice(-Math.floor(kept / 2)),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Cap an untrusted string at a parse boundary so malformed JSONL cannot
|
|
225
|
+
* produce unbounded metadata downstream. */
|
|
226
|
+
function capStr(value: unknown, max: number): string | null {
|
|
227
|
+
return typeof value === "string" ? value.slice(0, max) : null;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Narrow ISO timestamp shape; captures wall-clock fields for validation. */
|
|
231
|
+
const ISO_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
232
|
+
|
|
233
|
+
/** Canonicalize a session-header timestamp to UTC ISO so its lexicographic
|
|
234
|
+
* order matches chronological order (browse sorts started_at DESC as TEXT).
|
|
235
|
+
* Anything else is stored null so it sorts behind every valid timestamp. */
|
|
236
|
+
function normalizeTimestamp(value: unknown): string | null {
|
|
237
|
+
if (typeof value !== "string") return null;
|
|
238
|
+
const m = ISO_TIMESTAMP_RE.exec(value);
|
|
239
|
+
if (!m) return null;
|
|
240
|
+
const [, y, mo, d, h] = m;
|
|
241
|
+
if (+h > 23) return null;
|
|
242
|
+
const probe = new Date(Date.UTC(+y, +mo - 1, +d));
|
|
243
|
+
// Date parsing rejects invalid time fields, but silently rolls impossible
|
|
244
|
+
// calendar dates such as February 30.
|
|
245
|
+
if (probe.getUTCMonth() !== +mo - 1 || probe.getUTCDate() !== +d) return null;
|
|
246
|
+
const date = new Date(value);
|
|
247
|
+
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function extractText(content: unknown): string {
|
|
251
|
+
if (typeof content === "string") return content;
|
|
252
|
+
if (!Array.isArray(content)) return "";
|
|
253
|
+
const parts: string[] = [];
|
|
254
|
+
for (const block of content) {
|
|
255
|
+
if (block && typeof block === "object" && (block as any).type === "text" && typeof (block as any).text === "string") {
|
|
256
|
+
parts.push((block as any).text);
|
|
257
|
+
}
|
|
258
|
+
// thinking / toolUse / tool_result blocks are skipped entirely
|
|
259
|
+
}
|
|
260
|
+
return parts.join("\n").trim();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
|
|
264
|
+
const parsed: ParsedFile = {
|
|
265
|
+
cwd: null,
|
|
266
|
+
name: null,
|
|
267
|
+
startedAt: null,
|
|
268
|
+
parentSession: null,
|
|
269
|
+
preview: null,
|
|
270
|
+
messages: [],
|
|
271
|
+
};
|
|
272
|
+
// Unreadable/oversized file must throw so the sync transaction rolls back
|
|
273
|
+
// instead of wiping previously indexed rows and advancing the watermark over a hole.
|
|
274
|
+
for (const { data: entryRaw, id } of readTranscriptEntries(filePath, maxBytes)) {
|
|
275
|
+
const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
|
|
276
|
+
switch (entry?.type) {
|
|
277
|
+
case "session":
|
|
278
|
+
// Untrusted header strings are capped here so every consumer
|
|
279
|
+
// (browse rows, discovery meta) inherits the bound.
|
|
280
|
+
parsed.cwd = capStr(entry.cwd, 500) ?? parsed.cwd;
|
|
281
|
+
parsed.startedAt = normalizeTimestamp(entry.timestamp) ?? parsed.startedAt;
|
|
282
|
+
parsed.parentSession = capStr(entry.parentSession, 1024) ?? parsed.parentSession;
|
|
283
|
+
break;
|
|
284
|
+
case "session_info":
|
|
285
|
+
if (typeof entry.name === "string") parsed.name = entry.name.slice(0, 500);
|
|
286
|
+
break;
|
|
287
|
+
case "message": {
|
|
288
|
+
const msg = entry.message as { role?: unknown; content?: unknown } | undefined;
|
|
289
|
+
const role = msg?.role;
|
|
290
|
+
if (!msg || (role !== "user" && role !== "assistant")) break;
|
|
291
|
+
const full = extractText(msg.content);
|
|
292
|
+
if (!full || id === undefined) break;
|
|
293
|
+
if (role === "user" && parsed.preview === null) {
|
|
294
|
+
parsed.preview = full.slice(0, 200);
|
|
295
|
+
}
|
|
296
|
+
const regions = truncateRegions(full);
|
|
297
|
+
parsed.messages.push({
|
|
298
|
+
entryId: id,
|
|
299
|
+
role,
|
|
300
|
+
timestamp: capStr(entry.timestamp, 128),
|
|
301
|
+
head: regions ? regions.head : full,
|
|
302
|
+
tail: regions ? regions.tail : "",
|
|
303
|
+
});
|
|
304
|
+
break;
|
|
305
|
+
}
|
|
306
|
+
// compaction, branch_summary, custom, custom_message, label,
|
|
307
|
+
// session_info (name handled above), model_change, … are skipped
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return parsed;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Walk result. `complete: false` means the tree could not be fully read
|
|
314
|
+
* (missing root, readdir/stat failure) — callers must not treat unseen
|
|
315
|
+
* indexed paths as deleted. */
|
|
316
|
+
interface WalkResult {
|
|
317
|
+
files: Map<string, fs.Stats>;
|
|
318
|
+
complete: boolean;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function walkJsonlFiles(sessionsDir: string): WalkResult {
|
|
322
|
+
const files = new Map<string, fs.Stats>();
|
|
323
|
+
const stack = [sessionsDir];
|
|
324
|
+
let complete = true;
|
|
325
|
+
while (stack.length > 0) {
|
|
326
|
+
const dir = stack.pop()!;
|
|
327
|
+
let entries: fs.Dirent[];
|
|
328
|
+
try {
|
|
329
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
330
|
+
} catch {
|
|
331
|
+
complete = false;
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
for (const ent of entries) {
|
|
335
|
+
const full = path.join(dir, ent.name);
|
|
336
|
+
if (ent.isDirectory()) {
|
|
337
|
+
// Junk dirs are pruned before descent so a large ignored tree never
|
|
338
|
+
// costs a walk + per-file stats on every sync pass.
|
|
339
|
+
if (ent.name.startsWith("--tmp-") || ent.name.startsWith("--private-tmp-")) continue;
|
|
340
|
+
stack.push(full);
|
|
341
|
+
} else if (ent.isFile() && ent.name.endsWith(".jsonl")) {
|
|
342
|
+
try {
|
|
343
|
+
files.set(full, fs.statSync(full));
|
|
344
|
+
} catch {
|
|
345
|
+
complete = false; // vanished or became unreadable mid-walk
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
return { files, complete };
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// --- Sync ---
|
|
354
|
+
|
|
355
|
+
export function syncSessions(
|
|
356
|
+
sessionsDir: string,
|
|
357
|
+
dbPath: string,
|
|
358
|
+
opts?: { cap?: number; maxFileBytes?: number },
|
|
359
|
+
): SyncResult {
|
|
360
|
+
const cap = opts?.cap ?? DEFAULT_SYNC_CAP;
|
|
361
|
+
const db = openDb(dbPath);
|
|
362
|
+
try {
|
|
363
|
+
|
|
364
|
+
// Junk dirs never reach the walk result: pruned during descent.
|
|
365
|
+
const walk = walkJsonlFiles(sessionsDir);
|
|
366
|
+
const all = [...walk.files];
|
|
367
|
+
const watermarks = new Map(
|
|
368
|
+
(db.prepare("SELECT path, size, mtime_ms, ctime_ms FROM session_files").all() as any[]).map((r) => [r.path, r]),
|
|
369
|
+
);
|
|
370
|
+
const failures = new Map(
|
|
371
|
+
(db.prepare("SELECT path, size, mtime_ms, ctime_ms, attempts FROM session_failures").all() as any[]).map((r) => [r.path, r]),
|
|
372
|
+
);
|
|
373
|
+
// An incomplete traversal proves nothing about absence: purging here would
|
|
374
|
+
// wipe healthy sessions over a transient readdir/stat failure. Changed/new
|
|
375
|
+
// files already discovered still process normally.
|
|
376
|
+
const discovered = new Set(all.map(([fp]) => fp));
|
|
377
|
+
const deleted = walk.complete ? [...watermarks.keys()].filter((p) => !discovered.has(p)) : [];
|
|
378
|
+
const deletedFailures = walk.complete ? [...failures.keys()].filter((p) => !discovered.has(p)) : [];
|
|
379
|
+
|
|
380
|
+
const changed: { path: string; stat: fs.Stats }[] = [];
|
|
381
|
+
for (const [p, stat] of all) {
|
|
382
|
+
const fp = fileFingerprint(stat);
|
|
383
|
+
const wm = watermarks.get(p);
|
|
384
|
+
if (wm && wm.size === fp.size && wm.mtime_ms === fp.mtimeMs && wm.ctime_ms === fp.ctimeMs) continue;
|
|
385
|
+
changed.push({ path: p, stat });
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Newest-first, but an unchanged prior failure moves behind untouched work.
|
|
389
|
+
// Among retries, fewest attempts first so one persistent failure cannot
|
|
390
|
+
// monopolize a small cap forever.
|
|
391
|
+
const retryAttempts = (p: string, stat: fs.Stats): number | null => {
|
|
392
|
+
const f = failures.get(p);
|
|
393
|
+
if (!f) return null;
|
|
394
|
+
const fp = fileFingerprint(stat);
|
|
395
|
+
return f.size === fp.size && f.mtime_ms === fp.mtimeMs && f.ctime_ms === fp.ctimeMs ? f.attempts : null;
|
|
396
|
+
};
|
|
397
|
+
changed.sort((a, b) => {
|
|
398
|
+
const aRetry = retryAttempts(a.path, a.stat);
|
|
399
|
+
const bRetry = retryAttempts(b.path, b.stat);
|
|
400
|
+
if (aRetry !== null && bRetry !== null) return aRetry - bRetry || b.stat.mtimeMs - a.stat.mtimeMs;
|
|
401
|
+
if (aRetry !== null) return 1;
|
|
402
|
+
if (bRetry !== null) return -1;
|
|
403
|
+
return b.stat.mtimeMs - a.stat.mtimeMs;
|
|
404
|
+
});
|
|
405
|
+
deleted.sort((a, b) => {
|
|
406
|
+
const ma = watermarks.get(a)!.mtime_ms;
|
|
407
|
+
const mb = watermarks.get(b)!.mtime_ms;
|
|
408
|
+
return mb - ma;
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
const delStmt = db.prepare("DELETE FROM messages WHERE path = ?");
|
|
412
|
+
const delSession = db.prepare("DELETE FROM sessions WHERE path = ?");
|
|
413
|
+
const delFile = db.prepare("DELETE FROM session_files WHERE path = ?");
|
|
414
|
+
const delFailure = db.prepare("DELETE FROM session_failures WHERE path = ?");
|
|
415
|
+
const upsertFailure = db.prepare(
|
|
416
|
+
"INSERT INTO session_failures(path, size, mtime_ms, ctime_ms, attempts) VALUES (?, ?, ?, ?, 1) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, attempts=CASE WHEN size=excluded.size AND mtime_ms=excluded.mtime_ms AND ctime_ms=excluded.ctime_ms THEN attempts+1 ELSE 1 END",
|
|
417
|
+
);
|
|
418
|
+
const upsertFile = db.prepare(
|
|
419
|
+
"INSERT INTO session_files(path, size, mtime_ms, ctime_ms) VALUES (?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms",
|
|
420
|
+
);
|
|
421
|
+
const upsertSession = db.prepare(
|
|
422
|
+
`INSERT INTO sessions(path, cwd, name, started_at, preview, parent_session) VALUES (?, ?, ?, ?, ?, ?)
|
|
423
|
+
ON CONFLICT(path) DO UPDATE SET cwd=excluded.cwd, name=excluded.name, started_at=excluded.started_at,
|
|
424
|
+
preview=excluded.preview, parent_session=excluded.parent_session`,
|
|
425
|
+
);
|
|
426
|
+
const insertMsg = db.prepare(
|
|
427
|
+
"INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
|
|
428
|
+
);
|
|
429
|
+
|
|
430
|
+
const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
|
|
431
|
+
|
|
432
|
+
const tx = (filePath: string, stat: fs.Stats): void => {
|
|
433
|
+
db.exec("BEGIN");
|
|
434
|
+
try {
|
|
435
|
+
const parsed = parseSessionFile(filePath, maxFileBytes);
|
|
436
|
+
delStmt.run(filePath);
|
|
437
|
+
upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
|
|
438
|
+
for (const msg of parsed.messages) {
|
|
439
|
+
insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
|
|
440
|
+
}
|
|
441
|
+
{
|
|
442
|
+
const fp = fileFingerprint(stat);
|
|
443
|
+
upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
|
|
444
|
+
}
|
|
445
|
+
delFailure.run(filePath);
|
|
446
|
+
db.exec("COMMIT");
|
|
447
|
+
} catch (err) {
|
|
448
|
+
db.exec("ROLLBACK");
|
|
449
|
+
throw err;
|
|
450
|
+
}
|
|
451
|
+
};
|
|
452
|
+
|
|
453
|
+
let filesProcessed = 0;
|
|
454
|
+
|
|
455
|
+
// The cap bounds total indexing attempts per pass, not successes.
|
|
456
|
+
const batch = changed.slice(0, cap);
|
|
457
|
+
for (const { path: p, stat } of batch) {
|
|
458
|
+
try {
|
|
459
|
+
tx(p, stat);
|
|
460
|
+
filesProcessed++;
|
|
461
|
+
} catch {
|
|
462
|
+
// Keep the file retryable, but put this unchanged fingerprint behind
|
|
463
|
+
// untouched work on the next pass.
|
|
464
|
+
{
|
|
465
|
+
const fp = fileFingerprint(stat);
|
|
466
|
+
upsertFailure.run(p, fp.size, fp.mtimeMs, fp.ctimeMs);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
const deletedRemaining = Math.max(0, deleted.length - cap);
|
|
472
|
+
for (const p of deleted.slice(0, cap)) {
|
|
473
|
+
delStmt.run(p);
|
|
474
|
+
delSession.run(p);
|
|
475
|
+
delFile.run(p);
|
|
476
|
+
delFailure.run(p);
|
|
477
|
+
}
|
|
478
|
+
for (const p of deletedFailures) delFailure.run(p);
|
|
479
|
+
|
|
480
|
+
// Attempted failures remain unsynced and therefore remain in the backlog.
|
|
481
|
+
const backlog = changed.length - filesProcessed + deletedRemaining;
|
|
482
|
+
db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
|
|
483
|
+
|
|
484
|
+
return { filesProcessed, backlogRemaining: backlog, walkComplete: walk.complete };
|
|
485
|
+
} finally {
|
|
486
|
+
db.close();
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function getBacklog(db: DatabaseSync): number {
|
|
491
|
+
const row = db.prepare("SELECT value FROM meta WHERE key = 'backlog'").get() as { value: string } | undefined;
|
|
492
|
+
return row ? Number(row.value) : 0;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// --- Search ---
|
|
496
|
+
|
|
497
|
+
export interface SearchOptions {
|
|
498
|
+
limit?: number;
|
|
499
|
+
/** Path of the current session file — the whole file is skipped. */
|
|
500
|
+
currentSessionPath?: string;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// Current-session exclusion happens INSIDE SQL (before ROW_NUMBER/caps), so
|
|
504
|
+
// the excluded file cannot consume the per-file rank, lineage, or scan limit.
|
|
505
|
+
const CURRENT_SESSION_FILTER_SQL = "(? IS NULL OR m.path <> ?)";
|
|
506
|
+
const LINEAGE_FILTER_SQL = `NOT EXISTS (
|
|
507
|
+
SELECT 1 FROM ranked parent
|
|
508
|
+
WHERE parent.path = r.parent_session
|
|
509
|
+
AND parent.path <> r.path
|
|
510
|
+
AND parent.rn <= 1
|
|
511
|
+
-- Direct mutual cycle (A<->B) would suppress both sides; keep the
|
|
512
|
+
-- lexicographically smaller path deterministically.
|
|
513
|
+
AND NOT (parent.parent_session IS r.path AND r.path < parent.path)
|
|
514
|
+
)`;
|
|
515
|
+
const BASE_SELECT = `
|
|
516
|
+
WITH matches AS (
|
|
517
|
+
SELECT m.path, m.entry_id, m.role, m.timestamp, s.cwd, s.name, s.started_at,
|
|
518
|
+
s.parent_session, m.rowid AS rid,
|
|
519
|
+
snippet(session_fts, -1, '[', ']', '…', 16) AS snip,
|
|
520
|
+
bm25(session_fts) AS score
|
|
521
|
+
FROM session_fts JOIN messages m ON m.rowid = session_fts.rowid
|
|
522
|
+
LEFT JOIN sessions s ON s.path = m.path
|
|
523
|
+
WHERE session_fts MATCH ? AND ${CURRENT_SESSION_FILTER_SQL}
|
|
524
|
+
), ranked AS (
|
|
525
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY path ORDER BY score, rid) AS rn
|
|
526
|
+
FROM matches
|
|
527
|
+
)
|
|
528
|
+
SELECT path, entry_id, role, timestamp, snip, cwd, name, started_at
|
|
529
|
+
FROM ranked r
|
|
530
|
+
WHERE rn <= 1
|
|
531
|
+
AND ${LINEAGE_FILTER_SQL}
|
|
532
|
+
ORDER BY score, rid
|
|
533
|
+
LIMIT ?`;
|
|
534
|
+
|
|
535
|
+
/** Bounded excerpt around the first matched term for the LIKE fallback,
|
|
536
|
+
* searching head then tail. */
|
|
537
|
+
function likeSnippet(head: string, tail: string, terms: string[]): string {
|
|
538
|
+
for (const text of [head, tail]) {
|
|
539
|
+
const lower = foldCase(text);
|
|
540
|
+
let at = -1;
|
|
541
|
+
for (const t of terms) {
|
|
542
|
+
const i = lower.indexOf(foldCase(t));
|
|
543
|
+
if (i >= 0 && (at < 0 || i < at)) at = i;
|
|
544
|
+
}
|
|
545
|
+
if (at >= 0) {
|
|
546
|
+
const start = Math.max(0, at - 60);
|
|
547
|
+
return `${start > 0 ? "…" : ""}${text.slice(start, start + 120)}…`;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return head.slice(0, 120);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
export function searchIndex(
|
|
554
|
+
dbPath: string,
|
|
555
|
+
query: string,
|
|
556
|
+
opts?: SearchOptions,
|
|
557
|
+
): { hits: SearchHit[]; backlogRemaining: number } {
|
|
558
|
+
const limit = opts?.limit ?? 3;
|
|
559
|
+
const db = openDb(dbPath);
|
|
560
|
+
try {
|
|
561
|
+
const currentSessionPath = opts?.currentSessionPath ?? null;
|
|
562
|
+
const plan = buildFtsQueryPlan(query);
|
|
563
|
+
if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
|
|
564
|
+
return { hits: [], backlogRemaining: getBacklog(db) };
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
interface RawRow {
|
|
568
|
+
path: string;
|
|
569
|
+
entry_id: string;
|
|
570
|
+
role: string;
|
|
571
|
+
timestamp: string | null;
|
|
572
|
+
snip: string;
|
|
573
|
+
cwd: string | null;
|
|
574
|
+
name: string | null;
|
|
575
|
+
started_at: string | null;
|
|
576
|
+
}
|
|
577
|
+
let rows: Omit<RawRow, "snip">[] = [];
|
|
578
|
+
let usedLike = plan.forceLike;
|
|
579
|
+
|
|
580
|
+
if (!plan.forceLike) {
|
|
581
|
+
// Recovery candidates run only after an FTS5 PARSE error. A candidate
|
|
582
|
+
// that parses defines the result even with zero rows (`a AND b` with
|
|
583
|
+
// no co-occurrence stays empty instead of degrading to OR/LIKE).
|
|
584
|
+
let parseFailed = false;
|
|
585
|
+
for (const cand of plan.ftsCandidates) {
|
|
586
|
+
try {
|
|
587
|
+
rows = db.prepare(BASE_SELECT).all(cand, currentSessionPath, currentSessionPath, limit) as any;
|
|
588
|
+
// First/raw success defines the result even with zero rows. After a
|
|
589
|
+
// parse error, an empty recovery candidate keeps trying later ones.
|
|
590
|
+
if (!parseFailed || rows.length > 0) {
|
|
591
|
+
usedLike = false;
|
|
592
|
+
break;
|
|
593
|
+
}
|
|
594
|
+
usedLike = true;
|
|
595
|
+
} catch {
|
|
596
|
+
parseFailed = true;
|
|
597
|
+
usedLike = true; // malformed syntax → try next candidate
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
if (usedLike) {
|
|
603
|
+
const likePlan = buildLikeQueryPlan(query);
|
|
604
|
+
if (likePlan === null) return { hits: [], backlogRemaining: getBacklog(db) };
|
|
605
|
+
rows = db.prepare(`WITH ranked AS (
|
|
606
|
+
SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
|
|
607
|
+
s.parent_session,
|
|
608
|
+
ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
|
|
609
|
+
COUNT(*) OVER (PARTITION BY m.path) AS matches
|
|
610
|
+
FROM messages m LEFT JOIN sessions s ON s.path = m.path
|
|
611
|
+
WHERE ${likePlan.where} AND ${CURRENT_SESSION_FILTER_SQL}
|
|
612
|
+
)
|
|
613
|
+
SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
|
|
614
|
+
FROM ranked r
|
|
615
|
+
WHERE rn <= 1
|
|
616
|
+
AND ${LINEAGE_FILTER_SQL}
|
|
617
|
+
ORDER BY matches DESC, started_at DESC, path
|
|
618
|
+
LIMIT ?`).all(...likePlan.params, currentSessionPath, currentSessionPath, limit) as any;
|
|
619
|
+
// Snippets anchor on operand terms only — operator words like OR would
|
|
620
|
+
// otherwise match common substrings and hide the real match.
|
|
621
|
+
for (const r of rows as any[]) r.snip = likeSnippet(r.head, r.tail, likePlan.terms);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// Current-file and one-hop lineage suppression already happened in SQL,
|
|
625
|
+
// before the LIMIT, so excluded or fork rows cannot starve matches.
|
|
626
|
+
const hits: SearchHit[] = (rows as RawRow[]).map((r, rank) => ({
|
|
627
|
+
path: r.path,
|
|
628
|
+
snippet: r.snip ?? "",
|
|
629
|
+
rank,
|
|
630
|
+
matchMessageId: r.entry_id,
|
|
631
|
+
role: r.role,
|
|
632
|
+
timestamp: r.timestamp ?? "",
|
|
633
|
+
cwd: r.cwd ?? undefined,
|
|
634
|
+
name: r.name ?? undefined,
|
|
635
|
+
startedAt: r.started_at ?? undefined,
|
|
636
|
+
}));
|
|
637
|
+
return { hits, backlogRemaining: getBacklog(db) };
|
|
638
|
+
} finally {
|
|
639
|
+
db.close();
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// --- Preparation / browse ---
|
|
644
|
+
|
|
645
|
+
export interface PreparationRowOptions {
|
|
646
|
+
limit: number;
|
|
647
|
+
/** Canonical repository root. Omit to sample all indexed sessions. */
|
|
648
|
+
repositoryRoot?: string;
|
|
649
|
+
currentSessionPath?: string;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/** Select deterministic, repository-scoped pattern-miner candidates from the
|
|
653
|
+
* existing index. Scope, current-session exclusion, and one-hop lineage
|
|
654
|
+
* collapse all happen before the sample limit. */
|
|
655
|
+
export function getPreparationRows(
|
|
656
|
+
dbPath: string,
|
|
657
|
+
opts: PreparationRowOptions,
|
|
658
|
+
): PreparationSessionRow[] {
|
|
659
|
+
const clauses: string[] = [];
|
|
660
|
+
const params: string[] = [];
|
|
661
|
+
if (opts.repositoryRoot !== undefined) {
|
|
662
|
+
const prefix = opts.repositoryRoot.endsWith(path.sep)
|
|
663
|
+
? opts.repositoryRoot
|
|
664
|
+
: opts.repositoryRoot + path.sep;
|
|
665
|
+
clauses.push("(s.cwd = ? OR substr(s.cwd, 1, length(?)) = ?)");
|
|
666
|
+
params.push(opts.repositoryRoot, prefix, prefix);
|
|
667
|
+
}
|
|
668
|
+
if (opts.currentSessionPath !== undefined) {
|
|
669
|
+
clauses.push("s.path <> ?");
|
|
670
|
+
params.push(opts.currentSessionPath);
|
|
671
|
+
}
|
|
672
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
673
|
+
const db = openDb(dbPath);
|
|
674
|
+
try {
|
|
675
|
+
const rows = db.prepare(`
|
|
676
|
+
WITH eligible AS (
|
|
677
|
+
SELECT s.path, s.cwd, s.name, s.started_at, s.preview,
|
|
678
|
+
CASE
|
|
679
|
+
WHEN s.parent_session IS NULL THEN s.path
|
|
680
|
+
WHEN parent.path <> s.path AND parent.parent_session = s.path
|
|
681
|
+
THEN min(s.path, parent.path)
|
|
682
|
+
ELSE s.parent_session
|
|
683
|
+
END AS lineage_id
|
|
684
|
+
FROM sessions s
|
|
685
|
+
LEFT JOIN sessions parent ON parent.path = s.parent_session
|
|
686
|
+
${where}
|
|
687
|
+
), ranked AS (
|
|
688
|
+
SELECT *, ROW_NUMBER() OVER (
|
|
689
|
+
PARTITION BY lineage_id
|
|
690
|
+
ORDER BY CASE WHEN path = lineage_id THEN 0 ELSE 1 END,
|
|
691
|
+
started_at DESC, path
|
|
692
|
+
) AS rn
|
|
693
|
+
FROM eligible
|
|
694
|
+
)
|
|
695
|
+
SELECT path, cwd, name, started_at, preview, lineage_id
|
|
696
|
+
FROM ranked
|
|
697
|
+
WHERE rn = 1
|
|
698
|
+
ORDER BY started_at DESC, path
|
|
699
|
+
LIMIT ?
|
|
700
|
+
`).all(...params, opts.limit) as any[];
|
|
701
|
+
return rows.map((r) => ({
|
|
702
|
+
path: r.path,
|
|
703
|
+
cwd: r.cwd ?? "",
|
|
704
|
+
name: r.name ?? undefined,
|
|
705
|
+
startedAt: r.started_at ?? undefined,
|
|
706
|
+
preview: r.preview ?? undefined,
|
|
707
|
+
lineageId: r.lineage_id,
|
|
708
|
+
}));
|
|
709
|
+
} finally {
|
|
710
|
+
db.close();
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
|
|
715
|
+
const db = openDb(dbPath);
|
|
716
|
+
try {
|
|
717
|
+
const rows = db
|
|
718
|
+
.prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
|
|
719
|
+
.all(limit) as any[];
|
|
720
|
+
return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
|
|
721
|
+
} finally {
|
|
722
|
+
db.close();
|
|
723
|
+
}
|
|
724
|
+
}
|