@henryqw/pi-session-recall 0.1.4 → 0.1.5

package/README.md

@@ -31,6 +31,8 @@ Query syntax: FTS5 over a trigram index — multi-word = AND by default, `OR` fo
 
 Hits inside the current session's live context are suppressed; compacted-away or inactive-branch history stays discoverable. Forked sessions collapse into their parent when both match.
 
+If the lazy index sync before browse/discovery cannot fully enumerate the session tree, or throws entirely, results are still served from the current index — potentially partially updated and stale: files discovered before the failure may already reflect their new content, while rows for files the walk never reached remain stale — and carry a top-level `syncWarning`: `{kind:"incomplete-walk"}` for a partial walk (indexed-but-unseen paths are never purged in that case), or `{kind:"sync-failed", error}` with the capped failure message. The warning is omitted once a sync completes.
+
 ## Deliberate exclusions
 
 Session directories whose encoded path starts with `--tmp-` or `--private-tmp-` (sessions run from `/tmp` or `/private/tmp`) are never indexed. Session files over 32 MiB are excluded from indexing and hydration: discovery cannot newly find them; READ/SCROLL return an explicit size error, while a stale discovery hit retained from before the file grew is returned as metadata with empty messages and that error.

package/extensions/hydrate.ts

@@ -4,7 +4,7 @@
  * Pure functions over file paths; no SQLite access.
  */
 /// <reference types="node" />
-import { MAX_SESSION_FILE_BYTES, readBoundedSnapshot } from "./search-core.ts";
+import { readTranscriptEntries, type TranscriptEntry } from "./transcript.ts";
 import type { WindowMessage } from "./types.ts";
 
 export interface WindowResult {
@@ -36,34 +36,37 @@ interface Entry {
 	};
 }
 
-/** Parse JSONL lines; malformed lines are skipped. Header line (no id) skipped. */
+/** Project a boundary entry onto the shape hydration consumes. Entries without
+ *  a validated id pair are dropped (they carry no hydratable position); the
+ *  raw data must be a JSON object for any projection beyond id/parentId to
+ *  exist at all. */
+function toEntry(t: TranscriptEntry): Entry | null {
+	if (t.id === undefined) return null;
+	if (t.data === null || typeof t.data !== "object") return null;
+	const rec = t.data as Record<string, unknown>;
+	const message = rec.message !== null && typeof rec.message === "object"
+		? (rec.message as Entry["message"])
+		: undefined;
+	return {
+		id: t.id,
+		parentId: t.parentId ?? null,
+		type: typeof rec.type === "string" ? rec.type : "",
+		timestamp: typeof rec.timestamp === "string" ? rec.timestamp : undefined,
+		message,
+	};
+}
+
+/** Parse JSONL lines; malformed lines are skipped and duplicate/invalid ids are
+ *  handled by the shared transcript boundary. Entries stream out of the
+ *  boundary's generator and only successfully projected ones are pushed. */
 function parseSessionEntries(sessionPath: string): Entry[] {
-	// O_NONBLOCK + descriptor validation + fixed-size snapshot read are shared
-	// with the index engine; the fd pins the inode so a concurrent append after
-	// fstat waits until the next hydration call.
-	const raw = readBoundedSnapshot(sessionPath, MAX_SESSION_FILE_BYTES);
+	// The bounded snapshot read (O_NONBLOCK + descriptor validation + fixed-size
+	// read) is shared with the index engine; the fd pins the inode so a
+	// concurrent append after fstat waits until the next hydration call.
 	const entries: Entry[] = [];
-	const seenIds = new Set<string>();
-	for (const line of raw.split("\n")) {
-		if (!line.trim()) continue;
-		let obj: unknown;
-		try {
-			obj = JSON.parse(line);
-		} catch {
-			continue; // skip malformed line
-		}
-		const e = obj as Entry;
-		if (
-			e &&
-			typeof e.id === "string" &&
-			e.id.length > 0 &&
-			e.id.length <= 256 &&
-			!seenIds.has(e.id) &&
-			(e.parentId == null || (typeof e.parentId === "string" && e.parentId.length <= 256))
-		) {
-			seenIds.add(e.id);
-			entries.push(e);
-		}
+	for (const t of readTranscriptEntries(sessionPath)) {
+		const projected = toEntry(t);
+		if (projected !== null) entries.push(projected);
 	}
 	return entries;
 }

package/extensions/search-core.ts

@@ -8,11 +8,9 @@ import { DatabaseSync } from "node:sqlite";
 import type { SQLInputValue, SQLOutputValue } from "node:sqlite";
 import fs from "node:fs";
 import path from "node:path";
+import { MAX_SESSION_FILE_BYTES, readTranscriptEntries } from "./transcript.ts";
 import type { SearchHit, SessionRow, SyncResult } from "./types.ts";
 export const DEFAULT_SYNC_CAP = 50;
-/** Hard byte ceiling per session file: larger files are skipped (and retried
- *  behind fresh work) instead of being read whole into memory. */
-export const MAX_SESSION_FILE_BYTES = 32 * 1024 * 1024;
 /** Hard ceiling for the internal/test `opts.cap` work bound of syncSessions. */
 const MAX_SYNC_CAP = DEFAULT_SYNC_CAP * 10;
 export const MAX_QUERY_CHARS = 512;
@@ -260,42 +258,7 @@ function extractText(content: unknown): string {
 	return parts.join("\n").trim();
 }
 
-/** Open the path once, validate that exact descriptor (type + size), and read
- *  only the validated snapshot from it. A concurrent append/replacement between
- *  the walk's stat and this open cannot grow the allocation or the read beyond
- *  maxBytes: the fd pins the inode, and fstat on that fd fixes both bounds.
- *  Deliberately not fs.readFileSync(fd) — that re-reads to EOF unbounded.
- *  Shared with hydration: callers pass their own byte ceiling (both use
- *  MAX_SESSION_FILE_BYTES in production). */
-export function readBoundedSnapshot(filePath: string, maxBytes: number): string {
-	// O_NONBLOCK keeps a writerless FIFO (regular .jsonl swapped mid-walk) from
-	// blocking this open before fstat rejects it; O_NOFOLLOW (absent on Windows)
-	// rejects a symlink swapped in after the walk instead of following it.
-	const fd = fs.openSync(
-		filePath,
-		fs.constants.O_RDONLY | fs.constants.O_NONBLOCK | (fs.constants.O_NOFOLLOW ?? 0),
-	);
-	try {
-		const st = fs.fstatSync(fd);
-		if (!st.isFile()) throw new Error(`session path is not a regular file: ${filePath}`);
-		if (st.size > maxBytes) {
-			throw new Error(`session file exceeds ${Math.round(maxBytes / (1024 * 1024))} MiB snapshot limit: ${filePath}`);
-		}
-		const buf = Buffer.allocUnsafe(st.size);
-		let read = 0;
-		while (read < buf.length) {
-			const n = fs.readSync(fd, buf, read, buf.length - read, read);
-			if (n === 0) break; // truncated concurrently after fstat: index what was there
-			read += n;
-		}
-		return buf.toString("utf-8", 0, read);
-	} finally {
-		fs.closeSync(fd);
-	}
-}
-
 function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
-	const seenEntryIds = new Set<string>();
 	const parsed: ParsedFile = {
 		cwd: null,
 		name: null,
@@ -306,23 +269,8 @@ function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
 	};
 	// Unreadable/oversized file must throw so the sync transaction rolls back
 	// instead of wiping previously indexed rows and advancing the watermark over a hole.
-	const content = readBoundedSnapshot(filePath, maxBytes);
-	for (const line of content.split("\n")) {
-		if (!line.trim()) continue;
-		let entry: any;
-		try {
-			entry = JSON.parse(line);
-		} catch {
-			continue; // skip malformed lines
-		}
-		const hasValidId = typeof entry?.id === "string" && entry.id.length > 0 && entry.id.length <= 256
-			&& (entry.parentId == null || (typeof entry.parentId === "string" && entry.parentId.length <= 256));
-		if (hasValidId) {
-			// Hydration is first-wins across every entry type; reserve IDs at the
-			// same boundary so discovery can never point at a different duplicate.
-			if (seenEntryIds.has(entry.id)) continue;
-			seenEntryIds.add(entry.id);
-		}
+	for (const { data: entryRaw, id } of readTranscriptEntries(filePath, maxBytes)) {
+		const entry = entryRaw !== null && typeof entryRaw === "object" ? (entryRaw as Record<string, unknown>) : undefined;
 		switch (entry?.type) {
 			case "session":
 				// Untrusted header strings are capped here so every consumer
@@ -335,17 +283,17 @@ function parseSessionFile(filePath: string, maxBytes: number): ParsedFile {
 				if (typeof entry.name === "string") parsed.name = entry.name.slice(0, 500);
 				break;
 			case "message": {
-				const role = entry.message?.role;
-				if (role !== "user" && role !== "assistant") break;
-				const full = extractText(entry.message.content);
-				if (!full) break;
-				if (!hasValidId) break;
+				const msg = entry.message as { role?: unknown; content?: unknown } | undefined;
+				const role = msg?.role;
+				if (!msg || (role !== "user" && role !== "assistant")) break;
+				const full = extractText(msg.content);
+				if (!full || id === undefined) break;
 				if (role === "user" && parsed.preview === null) {
 					parsed.preview = full.slice(0, 200);
 				}
 				const regions = truncateRegions(full);
 				parsed.messages.push({
-					entryId: entry.id,
+					entryId: id,
 					role,
 					timestamp: capStr(entry.timestamp, 128),
 					head: regions ? regions.head : full,
@@ -549,7 +497,7 @@ export function syncSessions(
 	const backlog = changed.length - filesProcessed + deletedRemaining;
 	db.prepare("INSERT INTO meta(key, value) VALUES ('backlog', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(backlog));
 
-	return { filesProcessed, messagesIndexed, backlogRemaining: backlog };
+	return { filesProcessed, messagesIndexed, backlogRemaining: backlog, walkComplete: walk.complete };
 	} finally {
 		db.close();
 	}

package/extensions/session-recall.ts

@@ -208,17 +208,20 @@ export default function (pi: ExtensionAPI): void {
 					return textResult(result);
 				}
 
-				// Lazy sync: drains any backlog the capped startup pass left.
+				// Lazy sync: drains any backlog the capped startup pass left. A partial
+				// or failed sync degrades to a warning; the stale index stays usable.
+				let syncWarning: { kind: "incomplete-walk" } | { kind: "sync-failed"; error: string } | undefined;
 				try {
-					syncSessions(sessionsDir(), dbPath());
-				} catch {
-					// Serve from the possibly stale index rather than failing.
+					const sync = syncSessions(sessionsDir(), dbPath());
+					if (!sync.walkComplete) syncWarning = { kind: "incomplete-walk" };
+				} catch (error) {
+					syncWarning = { kind: "sync-failed", error: (error instanceof Error ? error.message : String(error)).slice(0, 512) };
 				}
 
 				// --- BROWSE ---
 				if (!params.query?.trim()) {
 					const rows = getSessionRows(dbPath(), clamp(params.limit, 1, 10, 3));
-					return textResult({ mode: "browse", sessions: rows });
+					return textResult({ mode: "browse", sessions: rows, ...(syncWarning ? { syncWarning } : {}) });
 				}
 
 				// --- DISCOVERY ---
@@ -249,7 +252,17 @@ export default function (pi: ExtensionAPI): void {
 				const resultQuery = params.query!.trim().slice(0, MAX_QUERY_CHARS);
 				// Reserve the complete response envelope and divide remaining space
 				// across hits so the first hydrated result cannot starve later metadata.
-				let used = JSON.stringify({ mode: "discovery", query: resultQuery, results: [], backlogRemaining }).length + Math.max(0, hits.length - 1);
+				// The same warning-bearing envelope is reused for the final result so the
+				// reservation matches what is returned (and textResult's trimming keeps
+				// top-level non-array keys like syncWarning).
+				const envelope: Record<string, unknown> = {
+					mode: "discovery",
+					query: resultQuery,
+					results: [],
+					backlogRemaining,
+					...(syncWarning ? { syncWarning } : {}),
+				};
+				let used = JSON.stringify(envelope).length + Math.max(0, hits.length - 1);
 				const results = hits.map((hit, index) => {
 					const remaining = Math.floor((OUTPUT_CHAR_BUDGET - used) / (hits.length - index));
 					const meta = {
@@ -339,7 +352,7 @@ export default function (pi: ExtensionAPI): void {
 					}
 				});
 
-				const result: Record<string, unknown> = { mode: "discovery", query: resultQuery, results, backlogRemaining };
+				const result: Record<string, unknown> = { ...envelope, results };
 				return textResult(result);
 			} catch (error) {
 				const message = error instanceof Error ? error.message : String(error);

package/extensions/transcript.ts

@@ -0,0 +1,112 @@
+/**
+ * Neutral JSONL trust boundary for Pi session transcripts: one bounded,
+ * descriptor-validated snapshot per read, parsed into entries whose only
+ * trusted fields are the validated id/parentId pair. No pi runtime imports —
+ * shared by the index engine (search-core) and hydration (hydrate).
+ */
+import fs from "node:fs";
+
+/** Hard byte ceiling per session file: larger files are skipped (and retried
+ *  behind fresh work) instead of being read whole into memory. */
+export const MAX_SESSION_FILE_BYTES = 32 * 1024 * 1024;
+
+/** Open the path once, validate that exact descriptor (type + size), and read
+ *  only the validated snapshot from it. A concurrent append/replacement between
+ *  the walk's stat and this open cannot grow the allocation or the read beyond
+ *  maxBytes: the fd pins the inode, and fstat on that fd fixes both bounds.
+ *  Deliberately not fs.readFileSync(fd) — that re-reads to EOF unbounded.
+ *  Shared with hydration: callers pass their own byte ceiling (both use
+ *  MAX_SESSION_FILE_BYTES in production). */
+export function readBoundedSnapshot(filePath: string, maxBytes: number): string {
+	// O_NONBLOCK keeps a writerless FIFO (regular .jsonl swapped mid-walk) from
+	// blocking this open before fstat rejects it; O_NOFOLLOW (absent on Windows)
+	// rejects a symlink swapped in after the walk instead of following it.
+	const fd = fs.openSync(
+		filePath,
+		fs.constants.O_RDONLY | fs.constants.O_NONBLOCK | (fs.constants.O_NOFOLLOW ?? 0),
+	);
+	try {
+		const st = fs.fstatSync(fd);
+		if (!st.isFile()) throw new Error(`session path is not a regular file: ${filePath}`);
+		if (st.size > maxBytes) {
+			throw new Error(`session file exceeds ${Math.round(maxBytes / (1024 * 1024))} MiB snapshot limit: ${filePath}`);
+		}
+		const buf = Buffer.allocUnsafe(st.size);
+		let read = 0;
+		while (read < buf.length) {
+			const n = fs.readSync(fd, buf, read, buf.length - read, read);
+			if (n === 0) break; // truncated concurrently after fstat: index what was there
+			read += n;
+		}
+		return buf.toString("utf-8", 0, read);
+	} finally {
+		fs.closeSync(fd);
+	}
+}
+
+/** One parseable JSONL line. `data` is the raw parsed JSON value — any JSON
+ *  type, never assumed to be an object. `id`/`parentId` exist only when both
+ *  passed validation at this boundary; consumers may rely on them without
+ *  re-validating. */
+export interface TranscriptEntry {
+	data: unknown;
+	/** Nonempty string ≤256 chars; absent when missing or invalid. */
+	id?: string;
+	/** null or string ≤256 chars; defined only when `id` is present. */
+	parentId?: string | null;
+}
+
+/** Validate the id pair exactly as both previous parsers did: a valid entry
+ *  needs a usable id AND a nullish-or-valid parentId; anything else is treated
+ *  as id-less data (still projected for headers, never hydrated). */
+function validateIds(data: unknown): { id: string; parentId: string | null } | undefined {
+	const rec = data !== null && typeof data === "object" ? (data as Record<string, unknown>) : undefined;
+	const id = rec?.id;
+	if (typeof id !== "string" || id.length === 0 || id.length > 256) return undefined;
+	const parentId = rec?.parentId;
+	if (parentId == null) return { id, parentId: null };
+	if (typeof parentId === "string" && parentId.length <= 256) return { id, parentId };
+	return undefined;
+}
+
+/** Lazily yield one bounded snapshot of `filePath` as JSONL entries in file
+ *  order — no materialized entry array, and lines are located by newline
+ *  scan rather than splitting the whole snapshot into a string array.
+ *  Blank and malformed JSON lines are skipped. The first occurrence of
+ *  every valid id is reserved across all entry types; later duplicates are
+ *  dropped entirely. Entries without a valid id pair are retained so index
+ *  header/session_info projection can still use them — hydration projects to
+ *  entries carrying a validated `id`. The snapshot is read once when iteration
+ *  starts (fd-pinned descriptor validation still applies); consumers iterate
+ *  immediately via for-of. */
+export function* readTranscriptEntries(
+	filePath: string,
+	maxBytes: number = MAX_SESSION_FILE_BYTES,
+): Generator<TranscriptEntry> {
+	const content = readBoundedSnapshot(filePath, maxBytes);
+	const seenIds = new Set<string>();
+	let start = 0;
+	while (start < content.length) {
+		const nl = content.indexOf("\n", start);
+		const end = nl === -1 ? content.length : nl;
+		const line = content.slice(start, end);
+		start = end + 1;
+		if (!line.trim()) continue;
+		let data: unknown;
+		try {
+			data = JSON.parse(line);
+		} catch {
+			continue; // skip malformed lines
+		}
+		const ids = validateIds(data);
+		if (!ids) {
+			yield { data };
+			continue;
+		}
+		// Hydration is first-wins across every entry type; reserve IDs here so
+		// no consumer can ever point at a different duplicate.
+		if (seenIds.has(ids.id)) continue;
+		seenIds.add(ids.id);
+		yield { data, id: ids.id, parentId: ids.parentId };
+	}
+}

package/extensions/types.ts

@@ -42,4 +42,7 @@ export interface SyncResult {
 	messagesIndexed: number;
 	/** Changed files still unindexed after this pass, including failures. */
 	backlogRemaining: number;
+	/** False when the filesystem walk could not fully enumerate the tree —
+	 *  indexed-but-unseen paths were NOT purged and results may be stale. */
+	walkComplete: boolean;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@henryqw/pi-session-recall",
-	"version": "0.1.4",
+	"version": "0.1.5",
 	"description": "FTS5 search over past Pi sessions: single tool, four arg-inferred modes (discovery/scroll/read/browse), zero LLM calls.",
 	"keywords": [
 		"pi-package",