@hcflabs/pi-session-recall 5.2.8 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -57,7 +57,11 @@ Start pattern mining with:
57
57
  }
58
58
  ```
59
59
 
60
- Repository scope selects exact and descendant `cwd` values before applying the limit. It excludes the current session file. Use `scope:"all"` across repositories, including when inventory is unavailable.
60
+ Repository scope selects sessions whose stored `cwd` is the scope root or below it, before it applies the limit. It excludes the current session file. The tool compares the real paths of both, so a symlink does not hide a session.
61
+
62
+ Inside a Git repository, the scope root is the Git root. Outside Git, for example in a parent directory that holds many repositories, the scope root is the real path of the current directory. The result then has `scope.gitRoot:null` and `inventory.available:false` with `reason:"not-a-git-repository"`. `scope.root` always names the directory that the tool used. It is `null` for `scope:"all"`.
63
+
64
+ Use `scope:"all"` to sample sessions from every directory.
61
65
 
62
66
  The TUI shows the last five visual lines and earlier-line count. Press the tool expand key (`Ctrl+O` by default) for the full bounded response; the model always receives it.
63
67
 
@@ -135,7 +139,7 @@ Files over 32 MiB are neither indexed nor retrieved. Discovery cannot find new o
135
139
 
136
140
  Preparation runs one sync pass. Backlog or incomplete walks limit the sample and set `sync.complete:false`. Total sync failure is a tool error. Preparation rejects `query`, session cursors, or `window`; `scope` requires the preparation operation.
137
141
 
138
- Repository scope fails outside Git or when inventory fails. All scope returns its corpus with `inventory.available:false` for nonzero Git-root lookup (`reason:"not-a-git-repository"`) or later inventory failure (`reason:"inventory-failed"`).
142
+ Both scopes return their corpus with `inventory.available:false` when the current directory is not in a Git repository (`reason:"not-a-git-repository"`). Repository scope fails when inventory fails inside a Git repository. All scope returns its corpus with `inventory.available:false` and `reason:"inventory-failed"` in that case.
139
143
 
140
144
  | Limit | Maximum |
141
145
  | --- | --- |
@@ -273,7 +273,7 @@ function spaceParensOutsideQuotes(q: string): string {
273
273
  return out;
274
274
  }
275
275
 
276
- function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
276
+ function buildBooleanLikeSql(rawQuery: string): (LikeSql & { operands: string[] }) | null {
277
277
  let query = rawQuery.trim();
278
278
  if (!query) return null;
279
279
  // Imbalance detection/recovery is quote-aware: parentheses inside quoted
@@ -296,9 +296,11 @@ function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
296
296
  const tokens: LikeToken[] = [...spaceParensOutsideQuotes(query).matchAll(TOKEN_RE)]
297
297
  .map((m) => (m[1] !== undefined ? { phrase: unescapePhrase(m[1]) } : { word: m[2] ?? "" }))
298
298
  .filter((t) => (t.phrase !== undefined ? t.phrase !== "" : t.word !== ""));
299
- const fail = (): LikeSql | null => tokens.some((token) => token.word === "NEAR") ? { where: "0", params: [] } : null;
299
+ const fail = (): (LikeSql & { operands: string[] }) | null => tokens.some((token) => token.word === "NEAR") ? { where: "0", params: [], operands: [] } : null;
300
300
  const sql: string[] = [];
301
301
  const params: SQLInputValue[] = [];
302
+ // Plain operand terms in LIKE form. NEAR operands are not included.
303
+ const operands: string[] = [];
302
304
  let expectingOperand = true;
303
305
  depth = 0;
304
306
 
@@ -347,14 +349,41 @@ function buildBooleanLikeSql(rawQuery: string): LikeSql | null {
347
349
  if (!expectingOperand) sql.push("AND");
348
350
  sql.push(clause.clause);
349
351
  params.push(...clause.params);
352
+ operands.push(normalizeLikeTerm(term, token.phrase !== undefined));
350
353
  expectingOperand = false;
351
354
  }
352
355
 
353
- return expectingOperand || depth !== 0 ? fail() : { where: sql.join(" "), params };
356
+ return expectingOperand || depth !== 0 ? fail() : { where: sql.join(" "), params, operands };
354
357
  }
355
358
 
356
359
  interface LikeQueryPlan extends LikeSql {
357
360
  terms: string[];
361
+ /** FTS5 MATCH expression whose rows include every row that `where`
362
+ * matches, or null. A caller can add it to `where` as a candidate filter. */
363
+ ftsPrefilter: string | null;
364
+ }
365
+
366
+ /** Build a candidate filter for a pure-AND LIKE plan. Each operand of such a
367
+ * plan is required, so an AND of FTS5 phrases for some operands is a valid
368
+ * filter when each phrase matches every row that its LIKE clause matches.
369
+ * Trigram MATCH of a phrase is a substring test on FTS5-folded text. LIKE is
370
+ * a substring test on foldCase text. The two folds agree for printable ASCII
371
+ * operands with one exception:
372
+ * - foldCase changes İ (U+0130) to "i" + U+0307, but FTS5 keeps İ.
373
+ * That "i" can only be the last character of an ASCII match, so the filter
374
+ * removes a final "i". KELVIN SIGN (U+212A) folds to "k" in both. Other
375
+ * operands, and operands below 3 characters, stay out of the filter. */
376
+ function likeFtsPrefilter(query: string, operands: string[]): string | null {
377
+ if (OPERATOR_RE.test(query)) return null;
378
+ const phrases: string[] = [];
379
+ for (const operand of operands) {
380
+ let folded = foldCase(operand);
381
+ if (!/^[\x20-\x7e]*$/.test(folded)) continue;
382
+ if (folded.endsWith("i")) folded = folded.slice(0, -1);
383
+ if (folded.length < 3) continue;
384
+ phrases.push(`"${folded.replace(/"/g, '""')}"`);
385
+ }
386
+ return phrases.length > 0 ? phrases.join(" ") : null;
358
387
  }
359
388
 
360
389
  export function buildLikeQueryPlan(rawQuery: string): LikeQueryPlan | null {
@@ -375,5 +404,6 @@ export function buildLikeQueryPlan(rawQuery: string): LikeQueryPlan | null {
375
404
  terms,
376
405
  where: bool?.where ?? clauses.map((clause) => clause.clause).join(" AND "),
377
406
  params: bool?.params ?? clauses.flatMap((clause) => clause.params),
407
+ ftsPrefilter: likeFtsPrefilter(query, bool?.operands ?? terms),
378
408
  };
379
409
  }
@@ -460,7 +460,9 @@ function unavailableInventory(
460
460
  };
461
461
  }
462
462
 
463
- /** Return bounded discovery hints from the Git index and Pi's effective skill registry. */
463
+ /** Return bounded discovery hints from the Git index and Pi's effective skill registry.
464
+ * Outside a Git repository, both modes return the unavailable inventory.
465
+ * After the Git root is known, "required" mode throws on a failure and "optional" mode returns the unavailable inventory. */
464
466
  export async function inventoryRepository(
465
467
  pi: InventoryPi,
466
468
  ctx: InventoryContext,
@@ -469,10 +471,7 @@ export async function inventoryRepository(
469
471
  throwIfAborted(ctx.signal);
470
472
  const gitRoot = await resolveGitRoot(ctx);
471
473
  throwIfAborted(ctx.signal);
472
- if (!gitRoot) {
473
- if (mode === "optional") return unavailableInventory(REPOSITORY_UNAVAILABLE_REASON);
474
- throw new Error("Repository inventory requires a Git repository.");
475
- }
474
+ if (!gitRoot) return unavailableInventory(REPOSITORY_UNAVAILABLE_REASON);
476
475
 
477
476
  try {
478
477
  const index = parseIndex((await runGit(gitRoot, ["--no-replace-objects", "ls-files", "--stage", "-z"], ctx.signal, INDEX_STDOUT_BYTES)).stdout);
@@ -5,10 +5,11 @@
5
5
  */
6
6
  import { DatabaseSync } from "node:sqlite";
7
7
  import type { SQLOutputValue } from "node:sqlite";
8
+ import { createHash } from "node:crypto";
8
9
  import fs from "node:fs";
9
10
  import path from "node:path";
10
11
  import { buildFtsQueryPlan, buildLikeQueryPlan, foldCase, nearLike } from "./query.ts";
11
- import { MAX_SESSION_FILE_BYTES, readTranscriptEntries } from "./transcript.ts";
12
+ import { MAX_SESSION_FILE_BYTES, parseTranscriptLines, readBoundedSnapshotBytes } from "./transcript.ts";
12
13
  import type { PreparationSessionRow, SearchHit, SessionRow, SyncResult } from "./types.ts";
13
14
  export const DEFAULT_SYNC_CAP = 50;
14
15
  const MAX_TEXT_CHARS = 20000;
@@ -75,6 +76,21 @@ CREATE TABLE IF NOT EXISTS meta (
75
76
  key TEXT PRIMARY KEY,
76
77
  value TEXT NOT NULL
77
78
  );
79
+ -- Append state lets sync parse only the complete lines after byte_offset.
80
+ -- A row is valid only while its fingerprint equals the session_files row.
81
+ -- prefix_sha256 covers bytes [0, byte_offset), and byte_offset ends a line.
82
+ -- entry_ids is the JSON array of valid ids that the prefix reserved.
83
+ -- An index from an older version has no rows here. Sync then re-indexes a
84
+ -- changed file in full and writes its first row.
85
+ CREATE TABLE IF NOT EXISTS session_append_state (
86
+ path TEXT PRIMARY KEY,
87
+ size INTEGER NOT NULL,
88
+ mtime_ms INTEGER NOT NULL,
89
+ ctime_ms REAL NOT NULL,
90
+ byte_offset INTEGER NOT NULL,
91
+ prefix_sha256 TEXT NOT NULL,
92
+ entry_ids TEXT NOT NULL
93
+ );
78
94
  `;
79
95
 
80
96
  /** Owner-only modes for the index directory and database files. The
@@ -89,7 +105,7 @@ const DB_DIR_MODE = 0o700;
89
105
  * nodes instead of following them; fchmod tightens an existing permissive db.
90
106
  * WAL/SHM sidecars inherit the main db mode from SQLite; legacy sidecars left
91
107
  * behind by older builds are tightened here too. Always closes the fd. */
92
- function secureIndexNode(dbPath: string): void {
108
+ function secureIndexNode(dbPath: string): { dev: number; ino: number } {
93
109
  const posix = process.platform !== "win32";
94
110
  const dir = path.dirname(dbPath);
95
111
  fs.mkdirSync(dir, { recursive: true, mode: DB_DIR_MODE });
@@ -103,6 +119,7 @@ function secureIndexNode(dbPath: string): void {
103
119
  }
104
120
  }
105
121
  let fd: number | undefined;
122
+ let identity: { dev: number; ino: number };
106
123
  try {
107
124
  fd = fs.openSync(
108
125
  dbPath,
@@ -112,10 +129,11 @@ function secureIndexNode(dbPath: string): void {
112
129
  const st = fs.fstatSync(fd);
113
130
  if (!st.isFile()) throw new Error("index database path is not a regular file");
114
131
  if (posix && (st.mode & 0o777) !== DB_FILE_MODE) fs.fchmodSync(fd, DB_FILE_MODE);
132
+ identity = { dev: st.dev, ino: st.ino };
115
133
  } finally {
116
134
  if (fd !== undefined) fs.closeSync(fd);
117
135
  }
118
- if (!posix) return;
136
+ if (!posix) return identity;
119
137
  for (const suffix of ["-wal", "-shm"]) {
120
138
  // Same descriptor no-follow safety as the db path: chmodSync(path) would
121
139
  // follow a swapped-in sidecar symlink and retarget its referent's mode.
@@ -135,10 +153,101 @@ function secureIndexNode(dbPath: string): void {
135
153
  if (fd !== undefined) fs.closeSync(fd);
136
154
  }
137
155
  }
156
+ return identity;
157
+ }
158
+
159
+ /** One open connection per index path. The connection keeps the file
160
+ * identity and the schema version that its setup checked. */
161
+ interface CachedDb {
162
+ db: DatabaseSync;
163
+ dev: number;
164
+ ino: number;
165
+ schemaVersion: number;
166
+ }
167
+
168
+ const openDbs = new Map<string, CachedDb>();
169
+
170
+ function schemaVersion(db: DatabaseSync): number {
171
+ return Number((db.prepare("PRAGMA schema_version").get() as { schema_version: number }).schema_version);
172
+ }
173
+
174
+ function evictDb(dbPath: string, db?: DatabaseSync): void {
175
+ const cached = openDbs.get(dbPath);
176
+ if (!cached || (db !== undefined && cached.db !== db)) return;
177
+ openDbs.delete(dbPath);
178
+ try {
179
+ cached.db.close();
180
+ } catch {
181
+ // The connection is unusable. A later call opens a new connection.
182
+ }
138
183
  }
139
184
 
185
+ /** Close the cached index connection for `dbPath`, or all cached connections
186
+ * when `dbPath` is absent. A later index call opens a new connection. */
187
+ export function closeIndexDatabases(dbPath?: string): void {
188
+ for (const key of dbPath === undefined ? [...openDbs.keys()] : [dbPath]) evictDb(key);
189
+ }
190
+
191
+ /** Return the cached connection for `dbPath`, or open and check a new one.
192
+ * secureIndexNode runs on every call, so mode tightening and node checks
193
+ * do not change. A deleted or replaced index file, or a schema change by
194
+ * another connection, discards the cached connection and repeats setup. */
140
195
  function openDb(dbPath: string): DatabaseSync {
141
- secureIndexNode(dbPath);
196
+ const cached = openDbs.get(dbPath);
197
+ if (cached) {
198
+ // Close a connection to a deleted or replaced file before
199
+ // secureIndexNode creates the new file at this path.
200
+ let st: fs.Stats | undefined;
201
+ try {
202
+ st = fs.lstatSync(dbPath);
203
+ } catch {
204
+ st = undefined;
205
+ }
206
+ if (!st || st.dev !== cached.dev || st.ino !== cached.ino) evictDb(dbPath);
207
+ }
208
+ let identity: { dev: number; ino: number };
209
+ try {
210
+ identity = secureIndexNode(dbPath);
211
+ } catch (err) {
212
+ evictDb(dbPath);
213
+ throw err;
214
+ }
215
+ const current = openDbs.get(dbPath);
216
+ if (current) {
217
+ let reusable = false;
218
+ if (current.dev === identity.dev && current.ino === identity.ino) {
219
+ try {
220
+ reusable = schemaVersion(current.db) === current.schemaVersion;
221
+ } catch {
222
+ reusable = false;
223
+ }
224
+ }
225
+ if (reusable) return current.db;
226
+ evictDb(dbPath);
227
+ }
228
+ const db = setUpDb(dbPath);
229
+ try {
230
+ openDbs.set(dbPath, { db, ...identity, schemaVersion: schemaVersion(db) });
231
+ } catch (err) {
232
+ db.close();
233
+ throw err;
234
+ }
235
+ return db;
236
+ }
237
+
238
+ /** Run `fn` with the cached connection. An error discards the connection, so
239
+ * the next call starts from a new connection as before the cache existed. */
240
+ function withDb<T>(dbPath: string, fn: (db: DatabaseSync) => T): T {
241
+ const db = openDb(dbPath);
242
+ try {
243
+ return fn(db);
244
+ } catch (err) {
245
+ evictDb(dbPath, db);
246
+ throw err;
247
+ }
248
+ }
249
+
250
+ function setUpDb(dbPath: string): DatabaseSync {
142
251
  const db = new DatabaseSync(dbPath);
143
252
  try {
144
253
  db.exec("PRAGMA journal_mode = WAL");
@@ -167,6 +276,7 @@ function openDb(dbPath: string): DatabaseSync {
167
276
  for (const [table, required] of [
168
277
  ["session_files", ["path", "size", "mtime_ms", "ctime_ms"]],
169
278
  ["session_failures", ["path", "size", "mtime_ms", "ctime_ms", "attempts"]],
279
+ ["session_append_state", ["path", "size", "mtime_ms", "ctime_ms", "byte_offset", "prefix_sha256", "entry_ids"]],
170
280
  ] as const) {
171
281
  const have = new Set((db.prepare(`PRAGMA table_info(${table})`).all() as any[]).map((c) => c.name as string));
172
282
  for (const col of required) {
@@ -260,18 +370,11 @@ function extractText(content: unknown): string {
260
370
  return parts.join("\n").trim();
261
371
  }
262
372
 
263
- function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
264
- const parsed: ParsedFile = {
265
- cwd: null,
266
- name: null,
267
- startedAt: null,
268
- parentSession: null,
269
- preview: null,
270
- messages: [],
271
- };
272
- // Unreadable/oversized file must throw so the sync transaction rolls back
273
- // instead of wiping previously indexed rows and advancing the watermark over a hole.
274
- for (const { data: entryRaw, id } of readTranscriptEntries(filePath, maxBytes)) {
373
+ /** Apply the entries of decoded snapshot text to `parsed`. Metadata in
374
+ * `parsed` and ids in `seenIds` hold the state after the earlier lines, so
375
+ * a parse that continues after a line boundary gives the full-parse result. */
376
+ function parseSessionText(parsed: ParsedFile, content: string, seenIds: Set<string>): ParsedFile {
377
+ for (const { data: entryRaw, id } of parseTranscriptLines(content, seenIds)) {
275
378
  const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
276
379
  switch (entry?.type) {
277
380
  case "session":
@@ -310,6 +413,71 @@ function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
310
413
  return parsed;
311
414
  }
312
415
 
416
+ function emptyParsedFile(): ParsedFile {
417
+ return { cwd: null, name: null, startedAt: null, parentSession: null, preview: null, messages: [] };
418
+ }
419
+
420
+ /** Append state above this JSON size is not stored. The next change of
421
+ * that file then uses a full re-index. */
422
+ const MAX_APPEND_ID_CHARS = 1024 * 1024;
423
+ const LF = 0x0a;
424
+
425
+ function sha256(bytes: Uint8Array): string {
426
+ return createHash("sha256").update(bytes).digest("base64");
427
+ }
428
+
429
+ interface AppendStateRow {
430
+ byte_offset: number;
431
+ prefix_sha256: string;
432
+ entry_ids: string;
433
+ cwd: string | null;
434
+ name: string | null;
435
+ started_at: string | null;
436
+ preview: string | null;
437
+ parent_session: string | null;
438
+ }
439
+
440
+ /** Parse only the complete lines after the stored offset. Return null when
441
+ * the result can differ from a full parse:
442
+ * - no append state matches the current watermark,
443
+ * - the file is shorter than the offset (truncation),
444
+ * - the bytes before the offset changed (rewrite),
445
+ * - the snapshot ends in a partial line.
446
+ * A UTF-8 decoder restarts cleanly after a LF byte, so decoding only the
447
+ * appended bytes gives the same lines as decoding the whole snapshot. */
448
+ function parseAppendedLines(
449
+ state: AppendStateRow | undefined,
450
+ snapshot: Buffer,
451
+ ): { parsed: ParsedFile; seenIds: Set<string>; snapshotSha256: string } | null {
452
+ if (!state) return null;
453
+ const offset = Number(state.byte_offset);
454
+ if (!Number.isSafeInteger(offset) || offset < 0 || snapshot.length < offset) return null;
455
+ if (snapshot.length > offset && snapshot[snapshot.length - 1] !== LF) return null;
456
+ const hash = createHash("sha256").update(snapshot.subarray(0, offset));
457
+ if (hash.copy().digest("base64") !== state.prefix_sha256) return null;
458
+ let ids: unknown;
459
+ try {
460
+ ids = JSON.parse(state.entry_ids);
461
+ } catch {
462
+ return null;
463
+ }
464
+ if (!Array.isArray(ids) || !ids.every((id) => typeof id === "string")) return null;
465
+ const seenIds = new Set<string>(ids);
466
+ const parsed: ParsedFile = {
467
+ cwd: state.cwd,
468
+ name: state.name,
469
+ startedAt: state.started_at,
470
+ parentSession: state.parent_session,
471
+ preview: state.preview,
472
+ messages: [],
473
+ };
474
+ return {
475
+ parsed: parseSessionText(parsed, snapshot.toString("utf-8", offset), seenIds),
476
+ seenIds,
477
+ snapshotSha256: hash.update(snapshot.subarray(offset)).digest("base64"),
478
+ };
479
+ }
480
+
313
481
  /** Walk result. `complete: false` means the tree could not be fully read
314
482
  * (missing root, readdir/stat failure) — callers must not treat unseen
315
483
  * indexed paths as deleted. */
@@ -358,8 +526,7 @@ export function syncSessions(
358
526
  opts?: { cap?: number; maxFileBytes?: number },
359
527
  ): SyncResult {
360
528
  const cap = opts?.cap ?? DEFAULT_SYNC_CAP;
361
- const db = openDb(dbPath);
362
- try {
529
+ return withDb(dbPath, (db) => {
363
530
 
364
531
  // Junk dirs never reach the walk result: pruned during descent.
365
532
  const walk = walkJsonlFiles(sessionsDir);
@@ -426,21 +593,50 @@ export function syncSessions(
426
593
  const insertMsg = db.prepare(
427
594
  "INSERT INTO messages(path, entry_id, role, timestamp, head, tail) VALUES (?, ?, ?, ?, ?, ?)",
428
595
  );
596
+ // The join accepts append state only when it belongs to the current
597
+ // watermark. A writer that does not know this table (an older version)
598
+ // changes the watermark, so sync then re-indexes the file in full.
599
+ const getAppendState = db.prepare(
600
+ `SELECT a.byte_offset, a.prefix_sha256, a.entry_ids, s.cwd, s.name, s.started_at, s.preview, s.parent_session
601
+ FROM session_append_state a
602
+ JOIN session_files f ON f.path = a.path AND f.size = a.size AND f.mtime_ms = a.mtime_ms AND f.ctime_ms = a.ctime_ms
603
+ JOIN sessions s ON s.path = a.path
604
+ WHERE a.path = ?`,
605
+ );
606
+ const upsertAppendState = db.prepare(
607
+ `INSERT INTO session_append_state(path, size, mtime_ms, ctime_ms, byte_offset, prefix_sha256, entry_ids) VALUES (?, ?, ?, ?, ?, ?, ?)
608
+ ON CONFLICT(path) DO UPDATE SET size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms,
609
+ byte_offset=excluded.byte_offset, prefix_sha256=excluded.prefix_sha256, entry_ids=excluded.entry_ids`,
610
+ );
611
+ const delAppendState = db.prepare("DELETE FROM session_append_state WHERE path = ?");
429
612
 
430
613
  const maxFileBytes = opts?.maxFileBytes ?? MAX_SESSION_FILE_BYTES;
431
614
 
432
615
  const tx = (filePath: string, stat: fs.Stats): void => {
433
616
  db.exec("BEGIN");
434
617
  try {
435
- const parsed = parseSessionFile(filePath, maxFileBytes);
436
- delStmt.run(filePath);
618
+ // Unreadable/oversized file must throw so the sync transaction rolls back
619
+ // instead of wiping previously indexed rows and advancing the watermark over a hole.
620
+ const snapshot = readBoundedSnapshotBytes(filePath, maxFileBytes);
621
+ const appended = parseAppendedLines(getAppendState.get(filePath) as AppendStateRow | undefined, snapshot);
622
+ const seenIds = appended?.seenIds ?? new Set<string>();
623
+ const parsed = appended?.parsed ?? parseSessionText(emptyParsedFile(), snapshot.toString("utf-8"), seenIds);
624
+ // Indexed rows of an appended file are the full-parse rows of its
625
+ // unchanged prefix, so only the new messages are inserted.
626
+ if (!appended) delStmt.run(filePath);
437
627
  upsertSession.run(filePath, parsed.cwd, parsed.name, parsed.startedAt, parsed.preview, parsed.parentSession);
438
628
  for (const msg of parsed.messages) {
439
629
  insertMsg.run(filePath, msg.entryId, msg.role, msg.timestamp, msg.head, msg.tail);
440
630
  }
441
- {
442
- const fp = fileFingerprint(stat);
443
- upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
631
+ const fp = fileFingerprint(stat);
632
+ upsertFile.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs);
633
+ // A partial last line can parse differently after more bytes arrive,
634
+ // so only a snapshot that ends a line gets append state.
635
+ const entryIds = JSON.stringify([...seenIds]);
636
+ if ((snapshot.length === 0 || snapshot[snapshot.length - 1] === LF) && entryIds.length <= MAX_APPEND_ID_CHARS) {
637
+ upsertAppendState.run(filePath, fp.size, fp.mtimeMs, fp.ctimeMs, snapshot.length, appended?.snapshotSha256 ?? sha256(snapshot), entryIds);
638
+ } else {
639
+ delAppendState.run(filePath);
444
640
  }
445
641
  delFailure.run(filePath);
446
642
  db.exec("COMMIT");
@@ -474,6 +670,7 @@ export function syncSessions(
474
670
  delSession.run(p);
475
671
  delFile.run(p);
476
672
  delFailure.run(p);
673
+ delAppendState.run(p);
477
674
  }
478
675
  for (const p of deletedFailures) delFailure.run(p);
479
676
 
@@ -482,9 +679,7 @@ export function syncSessions(
482
679
  db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
483
680
 
484
681
  return { filesProcessed, backlogRemaining: backlog, walkComplete: walk.complete };
485
- } finally {
486
- db.close();
487
- }
682
+ });
488
683
  }
489
684
 
490
685
  function getBacklog(db: DatabaseSync): number {
@@ -556,8 +751,7 @@ export function searchIndex(
556
751
  opts?: SearchOptions,
557
752
  ): { hits: SearchHit[]; backlogRemaining: number } {
558
753
  const limit = opts?.limit ?? 3;
559
- const db = openDb(dbPath);
560
- try {
754
+ return withDb(dbPath, (db) => {
561
755
  const currentSessionPath = opts?.currentSessionPath ?? null;
562
756
  const plan = buildFtsQueryPlan(query);
563
757
  if (plan.ftsCandidates.length === 0 && !plan.forceLike) {
@@ -602,20 +796,37 @@ export function searchIndex(
602
796
  if (usedLike) {
603
797
  const likePlan = buildLikeQueryPlan(query);
604
798
  if (likePlan === null) return { hits: [], backlogRemaining: getBacklog(db) };
605
- rows = db.prepare(`WITH ranked AS (
799
+ // The FTS candidate filter only removes rows that the LIKE plan cannot
800
+ // match, so it leaves the matched set, the ranking, and the order the same.
801
+ const likeRows = (prefilter: string | null) => db.prepare(`WITH ranked AS (
606
802
  SELECT m.path, m.entry_id, m.role, m.timestamp, m.head, m.tail, s.cwd, s.name, s.started_at,
607
803
  s.parent_session,
608
804
  ROW_NUMBER() OVER (PARTITION BY m.path ORDER BY m.rowid DESC) AS rn,
609
805
  COUNT(*) OVER (PARTITION BY m.path) AS matches
610
806
  FROM messages m LEFT JOIN sessions s ON s.path = m.path
611
- WHERE ${likePlan.where} AND ${CURRENT_SESSION_FILTER_SQL}
807
+ WHERE ${likePlan.where}
808
+ ${prefilter === null ? "" : "AND m.rowid IN (SELECT rowid FROM session_fts WHERE session_fts MATCH ?)"}
809
+ AND ${CURRENT_SESSION_FILTER_SQL}
612
810
  )
613
811
  SELECT path, entry_id, role, timestamp, head, tail, cwd, name, started_at
614
812
  FROM ranked r
615
813
  WHERE rn <= 1
616
814
  AND ${LINEAGE_FILTER_SQL}
617
815
  ORDER BY matches DESC, started_at DESC, path
618
- LIMIT ?`).all(...likePlan.params, currentSessionPath, currentSessionPath, limit) as any;
816
+ LIMIT ?`).all(
817
+ ...likePlan.params,
818
+ ...(prefilter === null ? [] : [prefilter]),
819
+ currentSessionPath,
820
+ currentSessionPath,
821
+ limit,
822
+ ) as any;
823
+ try {
824
+ rows = likeRows(likePlan.ftsPrefilter);
825
+ } catch (err) {
826
+ // The full LIKE scan stays the result of record if the filter fails.
827
+ if (likePlan.ftsPrefilter === null) throw err;
828
+ rows = likeRows(null);
829
+ }
619
830
  // Snippets anchor on operand terms only — operator words like OR would
620
831
  // otherwise match common substrings and hide the real match.
621
832
  for (const r of rows as any[]) r.snip = likeSnippet(r.head, r.tail, likePlan.terms);
@@ -635,20 +846,58 @@ export function searchIndex(
635
846
  startedAt: r.started_at ?? undefined,
636
847
  }));
637
848
  return { hits, backlogRemaining: getBacklog(db) };
638
- } finally {
639
- db.close();
640
- }
849
+ });
641
850
  }
642
851
 
643
852
  // --- Preparation / browse ---
644
853
 
645
854
  export interface PreparationRowOptions {
646
855
  limit: number;
647
- /** Canonical repository root. Omit to sample all indexed sessions. */
856
+ /** Repository root, or any directory. A session matches when its stored
857
+ * cwd is this directory or below it. Both sides are compared after symlink
858
+ * resolution. Omit to sample all indexed sessions. */
648
859
  repositoryRoot?: string;
649
860
  currentSessionPath?: string;
650
861
  }
651
862
 
863
+ /** Resolve symlinks in the deepest existing ancestor of an absolute path and
864
+ * keep the rest as written. A stored cwd can name a directory that no longer
865
+ * exists. Return `undefined` for a relative path. */
866
+ function canonicalizePath(value: string): string | undefined {
867
+ if (!path.isAbsolute(value)) return undefined;
868
+ let existing = path.resolve(value);
869
+ const rest: string[] = [];
870
+ for (;;) {
871
+ try {
872
+ return path.join(fs.realpathSync(existing), ...rest);
873
+ } catch {
874
+ const parent = path.dirname(existing);
875
+ if (parent === existing) return undefined;
876
+ rest.unshift(path.basename(existing));
877
+ existing = parent;
878
+ }
879
+ }
880
+ }
881
+
882
+ function isSameOrBelow(candidate: string, root: string): boolean {
883
+ const prefix = root.endsWith(path.sep) ? root : root + path.sep;
884
+ return candidate === root || candidate.startsWith(prefix);
885
+ }
886
+
887
+ /** List the stored session cwds that are the root or below it. SQL cannot
888
+ * resolve symlinks, so this check runs on each distinct cwd. */
889
+ function cwdsInScope(db: DatabaseSync, root: string): string[] {
890
+ const canonicalRoot = canonicalizePath(root);
891
+ const stored = db.prepare("SELECT DISTINCT cwd FROM sessions WHERE cwd IS NOT NULL").all() as { cwd: string }[];
892
+ return stored
893
+ .map((row) => row.cwd)
894
+ .filter((cwd) => {
895
+ if (isSameOrBelow(cwd, root)) return true;
896
+ const canonicalCwd = canonicalRoot === undefined ? undefined : canonicalizePath(cwd);
897
+ return canonicalCwd !== undefined && isSameOrBelow(canonicalCwd, canonicalRoot!);
898
+ });
899
+ }
900
+
652
901
  /** Select deterministic, repository-scoped pattern-miner candidates from the
653
902
  * existing index. Scope, current-session exclusion, and one-hop lineage
654
903
  * collapse all happen before the sample limit. */
@@ -656,22 +905,18 @@ export function getPreparationRows(
656
905
  dbPath: string,
657
906
  opts: PreparationRowOptions,
658
907
  ): PreparationSessionRow[] {
659
- const clauses: string[] = [];
660
- const params: string[] = [];
661
- if (opts.repositoryRoot !== undefined) {
662
- const prefix = opts.repositoryRoot.endsWith(path.sep)
663
- ? opts.repositoryRoot
664
- : opts.repositoryRoot + path.sep;
665
- clauses.push("(s.cwd = ? OR substr(s.cwd, 1, length(?)) = ?)");
666
- params.push(opts.repositoryRoot, prefix, prefix);
667
- }
668
- if (opts.currentSessionPath !== undefined) {
669
- clauses.push("s.path <> ?");
670
- params.push(opts.currentSessionPath);
671
- }
672
- const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
673
- const db = openDb(dbPath);
674
- try {
908
+ return withDb(dbPath, (db) => {
909
+ const clauses: string[] = [];
910
+ const params: string[] = [];
911
+ if (opts.repositoryRoot !== undefined) {
912
+ clauses.push("s.cwd IN (SELECT value FROM json_each(?))");
913
+ params.push(JSON.stringify(cwdsInScope(db, opts.repositoryRoot)));
914
+ }
915
+ if (opts.currentSessionPath !== undefined) {
916
+ clauses.push("s.path <> ?");
917
+ params.push(opts.currentSessionPath);
918
+ }
919
+ const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
675
920
  const rows = db.prepare(`
676
921
  WITH eligible AS (
677
922
  SELECT s.path, s.cwd, s.name, s.started_at, s.preview,
@@ -706,19 +951,14 @@ export function getPreparationRows(
706
951
  preview: r.preview ?? undefined,
707
952
  lineageId: r.lineage_id,
708
953
  }));
709
- } finally {
710
- db.close();
711
- }
954
+ });
712
955
  }
713
956
 
714
957
  export function getSessionRows(dbPath: string, limit: number): SessionRow[] {
715
- const db = openDb(dbPath);
716
- try {
958
+ return withDb(dbPath, (db) => {
717
959
  const rows = db
718
960
  .prepare("SELECT path, cwd, name, started_at, preview FROM sessions ORDER BY started_at DESC LIMIT ?")
719
961
  .all(limit) as any[];
720
962
  return rows.map((r) => ({ path: r.path, cwd: r.cwd ?? "", name: r.name ?? undefined, startedAt: r.started_at ?? undefined, preview: r.preview ?? undefined }));
721
- } finally {
722
- db.close();
723
- }
963
+ });
724
964
  }
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * pi-session-recall entry point: tool registration and mode dispatch.
3
3
  */
4
- import { getAgentDir, keyHint, truncateToVisualLines } from "@earendil-works/pi-coding-agent";
4
+ import { getAgentDir, truncateToVisualLines } from "@earendil-works/pi-coding-agent";
5
5
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
6
6
  import { StringEnum } from "@earendil-works/pi-ai";
7
7
  import { extensionConfigDir } from "@hcflabs/pi-config-store";
@@ -9,7 +9,7 @@ import { Text, truncateToWidth } from "@earendil-works/pi-tui";
9
9
  import { Type, type TSchema } from "typebox";
10
10
  import { realpathSync } from "node:fs";
11
11
  import { join, sep } from "node:path";
12
- import { getPreparationRows, getSessionRows, searchIndex, syncSessions } from "./search-core.ts";
12
+ import { closeIndexDatabases, getPreparationRows, getSessionRows, searchIndex, syncSessions } from "./search-core.ts";
13
13
  import { getWindow, readSession } from "./hydrate.ts";
14
14
  import { MAX_QUERY_CHARS } from "./query.ts";
15
15
  import { inventoryRepository, type RepositoryInventory } from "./repository-inventory.ts";
@@ -194,6 +194,7 @@ function buildPreparationResult(
194
194
  kind: "repository" | "all",
195
195
  requestedLimit: number,
196
196
  gitRoot: string | undefined,
197
+ root: string | undefined,
197
198
  syncResult: ReturnType<typeof syncSessions>,
198
199
  rows: PreparationSessionRow[],
199
200
  repositoryInventory: RepositoryInventory,
@@ -211,7 +212,7 @@ function buildPreparationResult(
211
212
  contentTruncated: boolean,
212
213
  ) => ({
213
214
  mode: "prepare-pattern-miner",
214
- scope: { kind, gitRoot: gitRoot ?? null, requestedLimit, sampledCount: preparedSessions.length },
215
+ scope: { kind, gitRoot: gitRoot ?? null, root: root ?? null, requestedLimit, sampledCount: preparedSessions.length },
215
216
  sync,
216
217
  sessions: preparedSessions,
217
218
  inventory,
@@ -302,6 +303,7 @@ const OUTPUT_SCHEMA = Type.Union([
302
303
  scope: Type.Object({
303
304
  kind: StringEnum(["repository", "all"] as const),
304
305
  gitRoot: Nullable(Type.String()),
306
+ root: Nullable(Type.String()),
305
307
  requestedLimit: Type.Number(),
306
308
  sampledCount: Type.Number(),
307
309
  }),
@@ -351,6 +353,7 @@ const OUTPUT_SCHEMA = Type.Union([
351
353
  const DESCRIPTION = `Search past Pi sessions locally with FTS5; returns indexed metadata and snippets.
352
354
 
353
355
  - \`operation: "prepare-pattern-miner"\` + \`scope\`: prepare one bounded corpus and repository inventory.
356
+ \`repository\` samples sessions in the Git root. Outside Git, it samples sessions in the current directory and below, and the inventory is unavailable. \`scope.root\` names the directory used. \`all\` samples every session.
354
357
  - \`query\`: discover matches. Prefer distinctive identifiers or uncommon terms; multi-word queries are AND. Use \`OR\`/\`NOT\` for Boolean queries and quotes only when exact wording is known.
355
358
  - \`sessionId\` + \`aroundMessageId\`: scroll ±\`window\`; retain \`branchTip\` across forks.
356
359
  - \`sessionId\` alone: read; no args: browse recent sessions.
@@ -369,6 +372,11 @@ export default function (pi: ExtensionAPI): void {
369
372
  }
370
373
  }, 0);
371
374
  });
375
+ // search-core keeps one index connection open across tool calls. Close it
376
+ // when the runtime stops. A later call opens a new connection.
377
+ pi.on("session_shutdown", () => {
378
+ closeIndexDatabases();
379
+ });
372
380
 
373
381
  pi.registerTool({
374
382
  name: "session_search",
@@ -390,7 +398,7 @@ export default function (pi: ExtensionAPI): void {
390
398
  }, { additionalProperties: false }),
391
399
  outputSchema: OUTPUT_SCHEMA,
392
400
  annotations: { readOnlyHint: true, openWorldHint: false },
393
- renderResult(result, { expanded }, theme) {
401
+ renderResult(result, { expanded }, theme, context) {
394
402
  const output = result.content.find((part) => part.type === "text")?.text ?? "";
395
403
  const styledOutput = theme.fg("toolOutput", output);
396
404
  if (expanded) return new Text(`\n${styledOutput}`, 0, 0);
@@ -405,7 +413,7 @@ export default function (pi: ExtensionAPI): void {
405
413
  cachedLines = ["", ...preview.visualLines];
406
414
  } else {
407
415
  const hint = theme.fg("muted", `... (${preview.skippedCount} earlier lines,`) +
408
- ` ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`;
416
+ ` ${context.keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`;
409
417
  cachedLines = ["", truncateToWidth(hint, width, "..."), ...preview.visualLines];
410
418
  }
411
419
  cachedWidth = width;
@@ -438,14 +446,16 @@ export default function (pi: ExtensionAPI): void {
438
446
  { cwd: ctx.cwd, signal },
439
447
  params.scope === "repository" ? "required" : "optional",
440
448
  );
449
+ // Repository scope uses the Git root. Outside Git it uses the real path of cwd.
450
+ const root = params.scope === "repository" ? inventory.gitRoot ?? realpathSync(ctx.cwd) : undefined;
441
451
  const sync = syncSessions(sessionsDir(), dbPath());
442
452
  const currentSessionPath = ctx.sessionManager.getSessionFile() ?? undefined;
443
453
  const rows = getPreparationRows(dbPath(), {
444
454
  limit,
445
- ...(params.scope === "repository" ? { repositoryRoot: inventory.gitRoot! } : {}),
455
+ ...(root === undefined ? {} : { repositoryRoot: root }),
446
456
  currentSessionPath,
447
457
  });
448
- return textResult(buildPreparationResult(params.scope, limit, inventory.gitRoot, sync, rows, inventory));
458
+ return textResult(buildPreparationResult(params.scope, limit, inventory.gitRoot, root, sync, rows, inventory));
449
459
  }
450
460
 
451
461
  let sessionId = params.sessionId?.trim() || undefined;
@@ -18,6 +18,12 @@ export const MAX_SESSION_FILE_BYTES = 32 * 1024 * 1024;
18
18
  * Shared with hydration: callers pass their own byte ceiling (both use
19
19
  * MAX_SESSION_FILE_BYTES in production). */
20
20
  export function readBoundedSnapshot(filePath: string, maxBytes: number): string {
21
+ return readBoundedSnapshotBytes(filePath, maxBytes).toString("utf-8");
22
+ }
23
+
24
+ /** Byte form of readBoundedSnapshot. The index engine uses the bytes to
25
+ * verify an unchanged prefix before it parses only appended lines. */
26
+ export function readBoundedSnapshotBytes(filePath: string, maxBytes: number): Buffer {
21
27
  // O_NONBLOCK keeps a writerless FIFO (regular .jsonl swapped mid-walk) from
22
28
  // blocking this open before fstat rejects it; O_NOFOLLOW (absent on Windows)
23
29
  // rejects a symlink swapped in after the walk instead of following it.
@@ -38,7 +44,7 @@ export function readBoundedSnapshot(filePath: string, maxBytes: number): string
38
44
  if (n === 0) break; // truncated concurrently after fstat: index what was there
39
45
  read += n;
40
46
  }
41
- return buf.toString("utf-8", 0, read);
47
+ return buf.subarray(0, read);
42
48
  } finally {
43
49
  fs.closeSync(fd);
44
50
  }
@@ -83,8 +89,17 @@ export function* readTranscriptEntries(
83
89
  filePath: string,
84
90
  maxBytes: number = MAX_SESSION_FILE_BYTES,
85
91
  ): Generator<TranscriptEntry> {
86
- const content = readBoundedSnapshot(filePath, maxBytes);
87
- const seenIds = new Set<string>();
92
+ yield* parseTranscriptLines(readBoundedSnapshot(filePath, maxBytes));
93
+ }
94
+
95
+ /** Yield the JSONL entries of decoded snapshot text with the rules of
96
+ * readTranscriptEntries. `seenIds` holds the ids that earlier lines reserved.
97
+ * The parser adds each new valid id to it. Pass the set from an earlier
98
+ * parse to continue that parse after a line boundary. */
99
+ export function* parseTranscriptLines(
100
+ content: string,
101
+ seenIds: Set<string> = new Set<string>(),
102
+ ): Generator<TranscriptEntry> {
88
103
  let start = 0;
89
104
  while (start < content.length) {
90
105
  const nl = content.indexOf("\n", start);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hcflabs/pi-session-recall",
3
- "version": "5.2.8",
3
+ "version": "5.4.0",
4
4
  "description": "Local FTS5 search over past Pi sessions plus a skill for turning recurring work into deterministic automation.",
5
5
  "keywords": [
6
6
  "pi-package",
@@ -27,9 +27,9 @@
27
27
  "pack:check": "npm pack --dry-run"
28
28
  },
29
29
  "peerDependencies": {
30
- "@earendil-works/pi-ai": "^1.3.0",
31
- "@earendil-works/pi-coding-agent": "^1.3.0",
32
- "@earendil-works/pi-tui": "^1.3.0",
30
+ "@earendil-works/pi-ai": "^1.6.0",
31
+ "@earendil-works/pi-coding-agent": "^1.6.0",
32
+ "@earendil-works/pi-tui": "^1.6.0",
33
33
  "typebox": "^1.3.15"
34
34
  },
35
35
  "repository": {
@@ -55,8 +55,8 @@
55
55
  "@hcflabs/pi-config-store": "^4.0.0"
56
56
  },
57
57
  "devDependencies": {
58
- "@earendil-works/pi-ai": "1.3.0",
59
- "@earendil-works/pi-coding-agent": "1.3.0",
60
- "@earendil-works/pi-tui": "1.3.0"
58
+ "@earendil-works/pi-ai": "1.6.0",
59
+ "@earendil-works/pi-coding-agent": "1.6.0",
60
+ "@earendil-works/pi-tui": "1.6.0"
61
61
  }
62
62
  }