@hcflabs/pi-session-recall 5.2.8 → 5.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/query.ts +33 -3
- package/extensions/search-core.ts +247 -44
- package/extensions/session-recall.ts +6 -1
- package/extensions/transcript.ts +18 -3
- package/package.json +1 -1
package/extensions/query.ts
CHANGED
|
@@ -273,7 +273,7 @@ function spaceParensOutsideQuotes(q: string): string {
|
|
|
273
273
|
return out;
|
|
274
274
|
}
|
|
275
275
|
|
|
276
|
-
function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
|
|
276
|
+
function buildBooleanLikeSql(rawQuery: string): (LikeSql & { operands: string[] }) | null {
|
|
277
277
|
let query = rawQuery.trim();
|
|
278
278
|
if (!query) return null;
|
|
279
279
|
// Imbalance detection/recovery is quote-aware: parentheses inside quoted
|
|
@@ -296,9 +296,11 @@ function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
|
|
|
296
296
|
const tokens: LikeToken[] = [...spaceParensOutsideQuotes(query).matchAll(TOKEN_RE)]
|
|
297
297
|
.map((m) => (m[1] !== undefined ? { phrase: unescapePhrase(m[1]) } : { word: m[2] ?? "" }))
|
|
298
298
|
.filter((t) => (t.phrase !== undefined ? t.phrase !== "" : t.word !== ""));
|
|
299
|
-
const fail = (): LikeSql | null => tokens.some((token) => token.word === "NEAR") ? { where: "0", params: [] } : null;
|
|
299
|
+
const fail = (): (LikeSql & { operands: string[] }) | null => tokens.some((token) => token.word === "NEAR") ? { where: "0", params: [], operands: [] } : null;
|
|
300
300
|
const sql: string[] = [];
|
|
301
301
|
const params: SQLInputValue[] = [];
|
|
302
|
+
// Plain operand terms in LIKE form. NEAR operands are not included.
|
|
303
|
+
const operands: string[] = [];
|
|
302
304
|
let expectingOperand = true;
|
|
303
305
|
depth = 0;
|
|
304
306
|
|
|
@@ -347,14 +349,41 @@ function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
|
|
|
347
349
|
if (!expectingOperand) sql.push("AND");
|
|
348
350
|
sql.push(clause.clause);
|
|
349
351
|
params.push(...clause.params);
|
|
352
|
+
operands.push(normalizeLikeTerm(term, token.phrase !== undefined));
|
|
350
353
|
expectingOperand = false;
|
|
351
354
|
}
|
|
352
355
|
|
|
353
|
-
return expectingOperand || depth !== 0 ? fail() : { where: sql.join(" "), params };
|
|
356
|
+
return expectingOperand || depth !== 0 ? fail() : { where: sql.join(" "), params, operands };
|
|
354
357
|
}
|
|
355
358
|
|
|
356
359
|
interface LikeQueryPlan extends LikeSql {
|
|
357
360
|
terms: string[];
|
|
361
|
+
/** FTS5 MATCH expression whose rows include every row that `where`
|
|
362
|
+
* matches, or null. A caller can add it to `where` as a candidate filter. */
|
|
363
|
+
ftsPrefilter: string | null;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** Build a candidate filter for a pure-AND LIKE plan. Each operand of such a
|
|
367
|
+
* plan is required, so an AND of FTS5 phrases for some operands is a valid
|
|
368
|
+
* filter when each phrase matches every row that its LIKE clause matches.
|
|
369
|
+
* Trigram MATCH of a phrase is a substring test on FTS5-folded text. LIKE is
|
|
370
|
+
* a substring test on foldCase text. The two folds agree for printable ASCII
|
|
371
|
+
* operands with one exception:
|
|
372
|
+
* - foldCase changes İ (U+0130) to "i" + U+0307, but FTS5 keeps İ.
|
|
373
|
+
* That "i" can only be the last character of an ASCII match, so the filter
|
|
374
|
+
* removes a final "i". KELVIN SIGN (U+212A) folds to "k" in both. Other
|
|
375
|
+
* operands, and operands below 3 characters, stay out of the filter. */
|
|
376
|
+
function likeFtsPrefilter(query: string, operands: string[]): string | null {
|
|
377
|
+
if (OPERATOR_RE.test(query)) return null;
|
|
378
|
+
const phrases: string[] = [];
|
|
379
|
+
for (const operand of operands) {
|
|
380
|
+
let folded = foldCase(operand);
|
|
381
|
+
if (!/^[\x20-\x7e]*$/.test(folded)) continue;
|
|
382
|
+
if (folded.endsWith("i")) folded = folded.slice(0, -1);
|
|
383
|
+
if (folded.length < 3) continue;
|
|
384
|
+
phrases.push(`"${folded.replace(/"/g, '""')}"`);
|
|
385
|
+
}
|
|
386
|
+
return phrases.length > 0 ? phrases.join(" ") : null;
|
|
358
387
|
}
|
|
359
388
|
|
|
360
389
|
export function buildLikeQueryPlan(rawQuery: string): LikeQueryPlan | null {
|
|
@@ -375,5 +404,6 @@ export function buildLikeQueryPlan(rawQuery: string): LikeQueryPlan | null {
|
|
|
375
404
|
terms,
|
|
376
405
|
where: bool?.where ?? clauses.map((clause) => clause.clause).join(" AND "),
|
|
377
406
|
params: bool?.params ?? clauses.flatMap((clause) => clause.params),
|
|
407
|
+
ftsPrefilter: likeFtsPrefilter(query, bool?.operands ?? terms),
|
|
378
408
|
};
|
|
379
409
|
}
|
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { DatabaseSync } from "node:sqlite";
|
|
7
7
|
import type { SQLOutputValue } from "node:sqlite";
|
|
8
|
+
import { createHash } from "node:crypto";
|
|
8
9
|
import fs from "node:fs";
|
|
9
10
|
import path from "node:path";
|
|
10
11
|
import { buildFtsQueryPlan, buildLikeQueryPlan, foldCase, nearLike } from "./query.ts";
|
|
11
|
-
import { MAX_SESSION_FILE_BYTES,
|
|
12
|
+
import { MAX_SESSION_FILE_BYTES, parseTranscriptLines, readBoundedSnapshotBytes } from "./transcript.ts";
|
|
12
13
|
import type { PreparationSessionRow, SearchHit, SessionRow, SyncResult } from "./types.ts";
|
|
13
14
|
export const DEFAULT_SYNC_CAP = 50;
|
|
14
15
|
const MAX_TEXT_CHARS = 20000;
|
|
@@ -75,6 +76,21 @@ CREATE TABLE IF NOT EXISTS meta (
|
|
|
75
76
|
key TEXT PRIMARY KEY,
|
|
76
77
|
value TEXT NOT NULL
|
|
77
78
|
);
|
|
79
|
+
-- Append state lets sync parse only the complete lines after byte_offset.
|
|
80
|
+
-- A row is valid only while its fingerprint equals the session_files row.
|
|
81
|
+
-- prefix_sha256 covers bytes [0, byte_offset), and byte_offset ends a line.
|
|
82
|
+
-- entry_ids is the JSON array of valid ids that the prefix reserved.
|
|
83
|
+
-- An index from an older version has no rows here. Sync then re-indexes a
|
|
84
|
+
-- changed file in full and writes its first row.
|
|
85
|
+
CREATE TABLE IF NOT EXISTS session_append_state (
|
|
86
|
+
path TEXT PRIMARY KEY,
|
|
87
|
+
size INTEGER NOT NULL,
|
|
88
|
+
mtime_ms INTEGER NOT NULL,
|
|
89
|
+
ctime_ms REAL NOT NULL,
|
|
90
|
+
byte_offset INTEGER NOT NULL,
|
|
91
|
+
prefix_sha256 TEXT NOT NULL,
|
|
92
|
+
entry_ids TEXT NOT NULL
|
|
93
|
+
);
|
|
78
94
|
`;
|
|
79
95
|
|
|
80
96
|
/** Owner-only modes for the index directory and database files. The
|
|
@@ -89,7 +105,7 @@ const DB_DIR_MODE = 0o700;
|
|
|
89
105
|
* nodes instead of following them; fchmod tightens an existing permissive db.
|
|
90
106
|
* WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
|
|
91
107
|
* behind by older builds are tightened here too. Always closes the fd. */
|
|
92
|
-
function secureIndexNode(dbPath: string):
|
|
108
|
+
function secureIndexNode(dbPath: string): { dev: number; ino: number } {
|
|
93
109
|
const posix = process.platform !== "win32";
|
|
94
110
|
const dir = path.dirname(dbPath);
|
|
95
111
|
fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
|
|
@@ -103,6 +119,7 @@ function secureIndexNode(dbPath: string): void {
|
|
|
103
119
|
}
|
|
104
120
|
}
|
|
105
121
|
let fd: number | undefined;
|
|
122
|
+
let identity: { dev: number; ino: number };
|
|
106
123
|
try {
|
|
107
124
|
fd = fs.openSync(
|
|
108
125
|
dbPath,
|
|
@@ -112,10 +129,11 @@ function secureIndexNode(dbPath: string): void {
|
|
|
112
129
|
const st = fs.fstatSync(fd);
|
|
113
130
|
if (!st.isFile()) throw new Error("index database path is not a regular file");
|
|
114
131
|
if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
|
|
132
|
+
identity = { dev: st.dev, ino: st.ino };
|
|
115
133
|
} finally {
|
|
116
134
|
if (fd !== undefined) fs.closeSync(fd);
|
|
117
135
|
}
|
|
118
|
-
if (!posix) return;
|
|
136
|
+
if (!posix) return identity;
|
|
119
137
|
for (const suffix of ["-wal", "-shm"]) {
|
|
120
138
|
// Same descriptor no-follow safety as the db path: chmodSync(path) would
|
|
121
139
|
// follow a swapped-in sidecar symlink and retarget its referent's mode.
|
|
@@ -135,10 +153,101 @@ function secureIndexNode(dbPath: string): void {
|
|
|
135
153
|
if (fd !== undefined) fs.closeSync(fd);
|
|
136
154
|
}
|
|
137
155
|
}
|
|
156
|
+
return identity;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** One open connection per index path. The connection keeps the file
|
|
160
|
+
* identity and the schema version that its setup checked. */
|
|
161
|
+
interface CachedDb {
|
|
162
|
+
db: DatabaseSync;
|
|
163
|
+
dev: number;
|
|
164
|
+
ino: number;
|
|
165
|
+
schemaVersion: number;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const openDbs = new Map<string, CachedDb>();
|
|
169
|
+
|
|
170
|
+
function schemaVersion(db: DatabaseSync): number {
|
|
171
|
+
return Number((db.prepare("PRAGMA schema_version").get() as { schema_version: number }).schema_version);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function evictDb(dbPath: string, db?: DatabaseSync): void {
|
|
175
|
+
const cached = openDbs.get(dbPath);
|
|
176
|
+
if (!cached || (db !== undefined && cached.db !== db)) return;
|
|
177
|
+
openDbs.delete(dbPath);
|
|
178
|
+
try {
|
|
179
|
+
cached.db.close();
|
|
180
|
+
} catch {
|
|
181
|
+
// The connection is unusable. A later call opens a new connection.
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Close the cached index connection for `dbPath`, or all cached connections
|
|
186
|
+
* when `dbPath` is absent. A later index call opens a new connection. */
|
|
187
|
+
export function closeIndexDatabases(dbPath?: string): void {
|
|
188
|
+
for (const key of dbPath === undefined ? [...openDbs.keys()] : [dbPath]) evictDb(key);
|
|
138
189
|
}
|
|
139
190
|
|
|
191
|
+
/** Return the cached connection for `dbPath`, or open and check a new one.
|
|
192
|
+
* secureIndexNode runs on every call, so mode tightening and node checks
|
|
193
|
+
* do not change. A deleted or replaced index file, or a schema change by
|
|
194
|
+
* another connection, discards the cached connection and repeats setup. */
|
|
140
195
|
function openDb(dbPath: string): DatabaseSync {
|
|
141
|
-
|
|
196
|
+
const cached = openDbs.get(dbPath);
|
|
197
|
+
if (cached) {
|
|
198
|
+
// Close a connection to a deleted or replaced file before
|
|
199
|
+
// secureIndexNode creates the new file at this path.
|
|
200
|
+
let st: fs.Stats | undefined;
|
|
201
|
+
try {
|
|
202
|
+
st = fs.lstatSync(dbPath);
|
|
203
|
+
} catch {
|
|
204
|
+
st = undefined;
|
|
205
|
+
}
|
|
206
|
+
if (!st || st.dev !== cached.dev || st.ino !== cached.ino) evictDb(dbPath);
|
|
207
|
+
}
|
|
208
|
+
let identity: { dev: number; ino: number };
|
|
209
|
+
try {
|
|
210
|
+
identity = secureIndexNode(dbPath);
|
|
211
|
+
} catch (err) {
|
|
212
|
+
evictDb(dbPath);
|
|
213
|
+
throw err;
|
|
214
|
+
}
|
|
215
|
+
const current = openDbs.get(dbPath);
|
|
216
|
+
if (current) {
|
|
217
|
+
let reusable = false;
|
|
218
|
+
if (current.dev === identity.dev && current.ino === identity.ino) {
|
|
219
|
+
try {
|
|
220
|
+
reusable = schemaVersion(current.db) === current.schemaVersion;
|
|
221
|
+
} catch {
|
|
222
|
+
reusable = false;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (reusable) return current.db;
|
|
226
|
+
evictDb(dbPath);
|
|
227
|
+
}
|
|
228
|
+
const db = setUpDb(dbPath);
|
|
229
|
+
try {
|
|
230
|
+
openDbs.set(dbPath, { db, ...identity, schemaVersion: schemaVersion(db) });
|
|
231
|
+
} catch (err) {
|
|
232
|
+
db.close();
|
|
233
|
+
throw err;
|
|
234
|
+
}
|
|
235
|
+
return db;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Run `fn` with the cached connection. An error discards the connection, so
|
|
239
|
+
* the next call starts from a new connection as before the cache existed. */
|
|
240
|
+
function withDb<T>(dbPath: string, fn: (db: DatabaseSync) => T): T {
|
|
241
|
+
const db = openDb(dbPath);
|
|
242
|
+
try {
|
|
243
|
+
return fn(db);
|
|
244
|
+
} catch (err) {
|
|
245
|
+
evictDb(dbPath, db);
|
|
246
|
+
throw err;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function setUpDb(dbPath: string): DatabaseSync {
|
|
142
251
|
const db = new DatabaseSync(dbPath);
|
|
143
252
|
try {
|
|
144
253
|
db.exec("PRAGMA journal_mode = WAL");
|
|
@@ -167,6 +276,7 @@ function openDb(dbPath: string): DatabaseSync {
|
|
|
167
276
|
for (const [table, required] of [
|
|
168
277
|
["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
|
|
169
278
|
["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
|
|
279
|
+
["session_append_state", ["path", "size", "mtime_ms", "ctime_ms", "byte_offset", "prefix_sha256", "entry_ids"]],
|
|
170
280
|
] as const) {
|
|
171
281
|
const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
|
|
172
282
|
for (const col of required) {
|
|
@@ -260,18 +370,11 @@ function extractText(content: unknown): string {
|
|
|
260
370
|
return parts.join("\n").trim();
|
|
261
371
|
}
|
|
262
372
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
parentSession: null,
|
|
269
|
-
preview: null,
|
|
270
|
-
messages: [],
|
|
271
|
-
};
|
|
272
|
-
// Unreadable/oversized file must throw so the sync transaction rolls back
|
|
273
|
-
// instead of wiping previously indexed rows and advancing the watermark over a hole.
|
|
274
|
-
for (const { data: entryRaw, id } of readTranscriptEntries(filePath, maxBytes)) {
|
|
373
|
+
/** Apply the entries of decoded snapshot text to `parsed`. Metadata in
|
|
374
|
+
* `parsed` and ids in `seenIds` hold the state after the earlier lines, so
|
|
375
|
+
* a parse that continues after a line boundary gives the full-parse result. */
|
|
376
|
+
function parseSessionText(parsed: ParsedFile, content: string, seenIds: Set<string>): ParsedFile {
|
|
377
|
+
for (const { data: entryRaw, id } of parseTranscriptLines(content, seenIds)) {
|
|
275
378
|
const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
|
|
276
379
|
switch (entry?.type) {
|
|
277
380
|
case "session":
|
|
@@ -310,6 +413,71 @@ function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
|
|
|
310
413
|
return parsed;
|
|
311
414
|
}
|
|
312
415
|
|
|
416
|
+
function emptyParsedFile(): ParsedFile {
|
|
417
|
+
return { cwd: null, name: null, startedAt: null, parentSession: null, preview: null, messages: [] };
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** Append state above this JSON size is not stored. The next change of
|
|
421
|
+
* that file then uses a full re-index. */
|
|
422
|
+
const MAX_APPEND_ID_CHARS = 1024 * 1024;
|
|
423
|
+
const LF = 0x0a;
|
|
424
|
+
|
|
425
|
+
function sha256(bytes: Uint8Array): string {
|
|
426
|
+
return createHash("sha256").update(bytes).digest("base64");
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
interface AppendStateRow {
|
|
430
|
+
byte_offset: number;
|
|
431
|
+
prefix_sha256: string;
|
|
432
|
+
entry_ids: string;
|
|
433
|
+
cwd: string | null;
|
|
434
|
+
name: string | null;
|
|
435
|
+
started_at: string | null;
|
|
436
|
+
preview: string | null;
|
|
437
|
+
parent_session: string | null;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/** Parse only the complete lines after the stored offset. Return null when
|
|
441
|
+
* the result can differ from a full parse:
|
|
442
|
+
* - no append state matches the current watermark,
|
|
443
|
+
* - the file is shorter than the offset (truncation),
|
|
444
|
+
* - the bytes before the offset changed (rewrite),
|
|
445
|
+
* - the snapshot ends in a partial line.
|
|
446
|
+
* A UTF-8 decoder restarts cleanly after a LF byte, so decoding only the
|
|
447
|
+
* appended bytes gives the same lines as decoding the whole snapshot. */
|
|
448
|
+
function parseAppendedLines(
|
|
449
|
+
state: AppendStateRow | undefined,
|
|
450
|
+
snapshot: Buffer,
|
|
451
|
+
): { parsed: ParsedFile; seenIds: Set<string>; snapshotSha256: string } | null {
|
|
452
|
+
if (!state) return null;
|
|
453
|
+
const offset = Number(state.byte_offset);
|
|
454
|
+
if (!Number.isSafeInteger(offset) || offset < 0 || snapshot.length < offset) return null;
|
|
455
|
+
if (snapshot.length > offset && snapshot[snapshot.length - 1] !== LF) return null;
|
|
456
|
+
const hash = createHash("sha256").update(snapshot.subarray(0, offset));
|
|
457
|
+
if (hash.copy().digest("base64") !== state.prefix_sha256) return null;
|
|
458
|
+
let ids: unknown;
|
|
459
|
+
try {
|
|
460
|
+
ids = JSON.parse(state.entry_ids);
|
|
461
|
+
} catch {
|
|
462
|
+
return null;
|
|
463
|
+
}
|
|
464
|
+
if (!Array.isArray(ids) || !ids.every((id) => typeof id === "string")) return null;
|
|
465
|
+
const seenIds = new Set<string>(ids);
|
|
466
|
+
const parsed: ParsedFile = {
|
|
467
|
+
cwd: state.cwd,
|
|
468
|
+
name: state.name,
|
|
469
|
+
startedAt: state.started_at,
|
|
470
|
+
parentSession: state.parent_session,
|
|
471
|
+
preview: state.preview,
|
|
472
|
+
messages: [],
|
|
473
|
+
};
|
|
474
|
+
return {
|
|
475
|
+
parsed: parseSessionText(parsed, snapshot.toString("utf-8", offset), seenIds),
|
|
476
|
+
seenIds,
|
|
477
|
+
snapshotSha256: hash.update(snapshot.subarray(offset)).digest("base64"),
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
|
|
313
481
|
/** Walk result. `complete: false` means the tree could not be fully read
|
|
314
482
|
* (missing root, readdir/stat failure) — callers must not treat unseen
|
|
315
483
|
* indexed paths as deleted. */
|
|
@@ -358,8 +526,7 @@ export function syncSessions(
|
|
|
358
526
|
opts?: { cap?: number; maxFileBytes?: number },
|
|
359
527
|
): SyncResult {
|
|
360
528
|
const cap = opts?.cap ?? DEFAULT_SYNC_CAP;
|
|
361
|
-
|
|
362
|
-
try {
|
|
529
|
+
return withDb(dbPath, (db) => {
|
|
363
530
|
|
|
364
531
|
// Junk dirs never reach the walk result: pruned during descent.
|
|
365
532
|
const walk = walkJsonlFiles(sessionsDir);
|
|
@@ -426,21 +593,50 @@ export function syncSessions(
|
|
|
426
593
|
const insertMsg = db.prepare(
|
|
427
594
|
"INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
|
|
428
595
|
);
|
|
596
|
+
// The join accepts append state only when it belongs to the current
|
|
597
|
+
// watermark. A writer that does not know this table (an older version)
|
|
598
|
+
// changes the watermark, so sync then re-indexes the file in full.
|
|
599
|
+
const getAppendState = db.prepare(
|
|
600
|
+
`SELECT a.byte_offset, a.prefix_sha256, a.entry_ids, s.cwd, s.name, s.started_at, s.preview, s.parent_session
|
|
601
|
+
FROM session_append_state a
|
|
602
|
+
JOIN session_files f ON f.path = a.path AND f.size = a.size AND f.mtime_ms = a.mtime_ms AND f.ctime_ms = a.ctime_ms
|
|
603
|
+
JOIN sessions s ON s.path = a.path
|
|
604
|
+
WHERE a.path = ?`,
|
|
605
|
+
);
|
|
606
|
+
const upsertAppendState = db.prepare(
|
|
607
|
+
`INSERT INTO session_append_state(path, size, mtime_ms, ctime_ms, byte_offset, prefix_sha256, entry_ids) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
608
|
+
ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms,
|
|
609
|
+
byte_offset=excluded.byte_offset, prefix_sha256=excluded.prefix_sha256, entry_ids=excluded.entry_ids`,
|
|
610
|
+
);
|
|
611
|
+
const delAppendState = db.prepare("DELETE FROM session_append_state WHERE path = ?");
|
|
429
612
|
|
|
430
613
|
const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
|
|
431
614
|
|
|
432
615
|
const tx = (filePath: string, stat: fs.Stats): void => {
|
|
433
616
|
db.exec("BEGIN");
|
|
434
617
|
try {
|
|
435
|
-
|
|
436
|
-
|
|
618
|
+
// Unreadable/oversized file must throw so the sync transaction rolls back
|
|
619
|
+
// instead of wiping previously indexed rows and advancing the watermark over a hole.
|
|
620
|
+
const snapshot = readBoundedSnapshotBytes(filePath, maxFileBytes);
|
|
621
|
+
const appended = parseAppendedLines(getAppendState.get(filePath) as AppendStateRow | undefined, snapshot);
|
|
622
|
+
const seenIds = appended?.seenIds ?? new Set<string>();
|
|
623
|
+
const parsed = appended?.parsed ?? parseSessionText(emptyParsedFile(), snapshot.toString("utf-8"), seenIds);
|
|
624
|
+
// Indexed rows of an appended file are the full-parse rows of its
|
|
625
|
+
// unchanged prefix, so only the new messages are inserted.
|
|
626
|
+
if (!appended) delStmt.run(filePath);
|
|
437
627
|
upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
|
|
438
628
|
for (const msg of parsed.messages) {
|
|
439
629
|
insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
|
|
440
630
|
}
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
631
|
+
const fp = fileFingerprint(stat);
|
|
632
|
+
upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
|
|
633
|
+
// A partial last line can parse differently after more bytes arrive,
|
|
634
|
+
// so only a snapshot that ends a line gets append state.
|
|
635
|
+
const entryIds = JSON.stringify([...seenIds]);
|
|
636
|
+
if ((snapshot.length === 0 || snapshot[snapshot.length - 1] === LF) && entryIds.length <= MAX_APPEND_ID_CHARS) {
|
|
637
|
+
upsertAppendState.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs, snapshot.length, appended?.snapshotSha256 ?? sha256(snapshot), entryIds);
|
|
638
|
+
} else {
|
|
639
|
+
delAppendState.run(filePath);
|
|
444
640
|
}
|
|
445
641
|
delFailure.run(filePath);
|
|
446
642
|
db.exec("COMMIT");
|
|
@@ -474,6 +670,7 @@ export function syncSessions(
|
|
|
474
670
|
delSession.run(p);
|
|
475
671
|
delFile.run(p);
|
|
476
672
|
delFailure.run(p);
|
|
673
|
+
delAppendState.run(p);
|
|
477
674
|
}
|
|
478
675
|
for (const p of deletedFailures) delFailure.run(p);
|
|
479
676
|
|
|
@@ -482,9 +679,7 @@ export function syncSessions(
|
|
|
482
679
|
db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
|
|
483
680
|
|
|
484
681
|
return { filesProcessed, backlogRemaining: backlog, walkComplete: walk.complete };
|
|
485
|
-
}
|
|
486
|
-
db.close();
|
|
487
|
-
}
|
|
682
|
+
});
|
|
488
683
|
}
|
|
489
684
|
|
|
490
685
|
function getBacklog(db: DatabaseSync): number {
|
|
@@ -556,8 +751,7 @@ export function searchIndex(
|
|
|
556
751
|
opts?: SearchOptions,
|
|
557
752
|
): { hits: SearchHit[]; backlogRemaining: number } {
|
|
558
753
|
const limit = opts?.limit ?? 3;
|
|
559
|
-
|
|
560
|
-
try {
|
|
754
|
+
return withDb(dbPath, (db) => {
|
|
561
755
|
const currentSessionPath = opts?.currentSessionPath ?? null;
|
|
562
756
|
const plan = buildFtsQueryPlan(query);
|
|
563
757
|
if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
|
|
@@ -602,20 +796,37 @@ export function searchIndex(
|
|
|
602
796
|
if (usedLike) {
|
|
603
797
|
const likePlan = buildLikeQueryPlan(query);
|
|
604
798
|
if (likePlan === null) return { hits: [], backlogRemaining: getBacklog(db) };
|
|
605
|
-
rows
|
|
799
|
+
// The FTS candidate filter only removes rows that the LIKE plan cannot
|
|
800
|
+
// match, so it leaves the matched set, the ranking, and the order the same.
|
|
801
|
+
const likeRows = (prefilter: string | null) => db.prepare(`WITH ranked AS (
|
|
606
802
|
SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
|
|
607
803
|
s.parent_session,
|
|
608
804
|
ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
|
|
609
805
|
COUNT(*) OVER (PARTITION BY m.path) AS matches
|
|
610
806
|
FROM messages m LEFT JOIN sessions s ON s.path = m.path
|
|
611
|
-
WHERE ${likePlan.where}
|
|
807
|
+
WHERE ${likePlan.where}
|
|
808
|
+
${prefilter === null ? "" : "AND m.rowid IN (SELECT rowid FROM session_fts WHERE session_fts MATCH ?)"}
|
|
809
|
+
AND ${CURRENT_SESSION_FILTER_SQL}
|
|
612
810
|
)
|
|
613
811
|
SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
|
|
614
812
|
FROM ranked r
|
|
615
813
|
WHERE rn <= 1
|
|
616
814
|
AND ${LINEAGE_FILTER_SQL}
|
|
617
815
|
ORDER BY matches DESC, started_at DESC, path
|
|
618
|
-
LIMIT ?`).all(
|
|
816
|
+
LIMIT ?`).all(
|
|
817
|
+
...likePlan.params,
|
|
818
|
+
...(prefilter === null ? [] : [prefilter]),
|
|
819
|
+
currentSessionPath,
|
|
820
|
+
currentSessionPath,
|
|
821
|
+
limit,
|
|
822
|
+
) as any;
|
|
823
|
+
try {
|
|
824
|
+
rows = likeRows(likePlan.ftsPrefilter);
|
|
825
|
+
} catch (err) {
|
|
826
|
+
// The full LIKE scan stays the result of record if the filter fails.
|
|
827
|
+
if (likePlan.ftsPrefilter === null) throw err;
|
|
828
|
+
rows = likeRows(null);
|
|
829
|
+
}
|
|
619
830
|
// Snippets anchor on operand terms only — operator words like OR would
|
|
620
831
|
// otherwise match common substrings and hide the real match.
|
|
621
832
|
for (const r of rows as any[]) r.snip = likeSnippet(r.head, r.tail, likePlan.terms);
|
|
@@ -635,9 +846,7 @@ export function searchIndex(
|
|
|
635
846
|
startedAt: r.started_at ?? undefined,
|
|
636
847
|
}));
|
|
637
848
|
return { hits, backlogRemaining: getBacklog(db) };
|
|
638
|
-
}
|
|
639
|
-
db.close();
|
|
640
|
-
}
|
|
849
|
+
});
|
|
641
850
|
}
|
|
642
851
|
|
|
643
852
|
// --- Preparation / browse ---
|
|
@@ -670,8 +879,7 @@ export function getPreparationRows(
|
|
|
670
879
|
params.push(opts.currentSessionPath);
|
|
671
880
|
}
|
|
672
881
|
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
673
|
-
|
|
674
|
-
try {
|
|
882
|
+
return withDb(dbPath, (db) => {
|
|
675
883
|
const rows = db.prepare(`
|
|
676
884
|
WITH eligible AS (
|
|
677
885
|
SELECT s.path, s.cwd, s.name, s.started_at, s.preview,
|
|
@@ -706,19 +914,14 @@ export function getPreparationRows(
|
|
|
706
914
|
preview: r.preview ?? undefined,
|
|
707
915
|
lineageId: r.lineage_id,
|
|
708
916
|
}));
|
|
709
|
-
}
|
|
710
|
-
db.close();
|
|
711
|
-
}
|
|
917
|
+
});
|
|
712
918
|
}
|
|
713
919
|
|
|
714
920
|
export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
|
|
715
|
-
|
|
716
|
-
try {
|
|
921
|
+
return withDb(dbPath, (db) => {
|
|
717
922
|
const rows = db
|
|
718
923
|
.prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
|
|
719
924
|
.all(limit) as any[];
|
|
720
925
|
return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
|
|
721
|
-
}
|
|
722
|
-
db.close();
|
|
723
|
-
}
|
|
926
|
+
});
|
|
724
927
|
}
|
|
@@ -9,7 +9,7 @@ import { Text, truncateToWidth } from "@earendil-works/pi-tui";
|
|
|
9
9
|
import { Type, type TSchema } from "typebox";
|
|
10
10
|
import { realpathSync } from "node:fs";
|
|
11
11
|
import { join, sep } from "node:path";
|
|
12
|
-
import { getPreparationRows, getSessionRows, searchIndex, syncSessions } from "./search-core.ts";
|
|
12
|
+
import { closeIndexDatabases, getPreparationRows, getSessionRows, searchIndex, syncSessions } from "./search-core.ts";
|
|
13
13
|
import { getWindow, readSession } from "./hydrate.ts";
|
|
14
14
|
import { MAX_QUERY_CHARS } from "./query.ts";
|
|
15
15
|
import { inventoryRepository, type RepositoryInventory } from "./repository-inventory.ts";
|
|
@@ -369,6 +369,11 @@ export default function (pi: ExtensionAPI): void {
|
|
|
369
369
|
}
|
|
370
370
|
}, 0);
|
|
371
371
|
});
|
|
372
|
+
// search-core keeps one index connection open across tool calls. Close it
|
|
373
|
+
// when the runtime stops. A later call opens a new connection.
|
|
374
|
+
pi.on("session_shutdown", () => {
|
|
375
|
+
closeIndexDatabases();
|
|
376
|
+
});
|
|
372
377
|
|
|
373
378
|
pi.registerTool({
|
|
374
379
|
name: "session_search",
|
package/extensions/transcript.ts
CHANGED
|
@@ -18,6 +18,12 @@ export const MAX_SESSION_FILE_BYTES = 32 * 1024 * 1024;
|
|
|
18
18
|
* Shared with hydration: callers pass their own byte ceiling (both use
|
|
19
19
|
* MAX_SESSION_FILE_BYTES in production). */
|
|
20
20
|
export function readBoundedSnapshot(filePath: string, maxBytes: number): string {
|
|
21
|
+
return readBoundedSnapshotBytes(filePath, maxBytes).toString("utf-8");
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Byte form of readBoundedSnapshot. The index engine uses the bytes to
|
|
25
|
+
* verify an unchanged prefix before it parses only appended lines. */
|
|
26
|
+
export function readBoundedSnapshotBytes(filePath: string, maxBytes: number): Buffer {
|
|
21
27
|
// O_NONBLOCK keeps a writerless FIFO (regular .jsonl swapped mid-walk) from
|
|
22
28
|
// blocking this open before fstat rejects it; O_NOFOLLOW (absent on Windows)
|
|
23
29
|
// rejects a symlink swapped in after the walk instead of following it.
|
|
@@ -38,7 +44,7 @@ export function readBoundedSnapshot(filePath: string, maxBytes: number): string
|
|
|
38
44
|
if (n === 0) break; // truncated concurrently after fstat: index what was there
|
|
39
45
|
read += n;
|
|
40
46
|
}
|
|
41
|
-
return buf.
|
|
47
|
+
return buf.subarray(0, read);
|
|
42
48
|
} finally {
|
|
43
49
|
fs.closeSync(fd);
|
|
44
50
|
}
|
|
@@ -83,8 +89,17 @@ export function* readTranscriptEntries(
|
|
|
83
89
|
filePath: string,
|
|
84
90
|
maxBytes: number = MAX_SESSION_FILE_BYTES,
|
|
85
91
|
): Generator<TranscriptEntry> {
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
yield* parseTranscriptLines(readBoundedSnapshot(filePath, maxBytes));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Yield the JSONL entries of decoded snapshot text with the rules of
|
|
96
|
+
* readTranscriptEntries. `seenIds` holds the ids that earlier lines reserved.
|
|
97
|
+
* The parser adds each new valid id to it. Pass the set from an earlier
|
|
98
|
+
* parse to continue that parse after a line boundary. */
|
|
99
|
+
export function* parseTranscriptLines(
|
|
100
|
+
content: string,
|
|
101
|
+
seenIds: Set<string> = new Set<string>(),
|
|
102
|
+
): Generator<TranscriptEntry> {
|
|
88
103
|
let start = 0;
|
|
89
104
|
while (start < content.length) {
|
|
90
105
|
const nl = content.indexOf("\n", start);
|
package/package.json
CHANGED