memonaut 0.0.0 → 0.2.0

package/src/regex.ts

@@ -0,0 +1,378 @@
+import type {DB} from './db.js';
+import type {ChunkKind, SearchHit} from './model.js';
+import {extractEntry} from './pi-source.js';
+import {ripgrep, type RipgrepMatch} from './ripgrep.js';
+import {collectThreads, loadFiles, visibleSets} from './search.js';
+
+/**
+ * Exact regex search, delegated to ripgrep, resolved back through the index.
+ *
+ * The two search paths are complementary and neither pretends to be the other.
+ * FTS5 tokenises, so it cannot match inside an identifier and knows nothing
+ * about text the tier excluded; ripgrep reads the ORIGINAL transcripts, so it
+ * matches anything that was ever written down, notably tool output (42% of the
+ * bytes, excluded by default). What it does NOT have is ranking.
+ *
+ * Three things make this more than `rg ~/.pi/agent/sessions`:
+ *
+ * 1. The candidate file list comes from the INDEX, never from a directory walk,
+ *    so transcripts the user told the tool to ignore are still never read, and
+ *    `private` still holds.
+ * 2. A matched line is resolved to an entry by `(lineage_id, entry_key)`, read
+ *    out of the line's own JSON. Never by byte offset: `entry.byte_offset` is
+ *    the offset in the file that FIRST recorded the entry, and a fork's copy of
+ *    that entry sits somewhere else entirely in the child file.
+ * 3. Forks are deduplicated. A fork copies the shared prefix, so rg reports a
+ *    shared entry once per fork file; the result is ONE hit carrying N threads,
+ *    exactly as the FTS path would report it.
+ */
+
+export interface RegexQuery {
+	/** A ripgrep (Rust regex) pattern, or a literal string when `fixed`. */
+	pattern: string;
+	cwd?: string[];
+	project?: string[];
+	role?: string[];
+	tool?: string[];
+	kind?: ChunkKind[];
+	since?: string;
+	until?: string;
+	includePrivate?: boolean;
+	limit?: number;
+	/** Threads listed per group; the total is always reported. */
+	threadLimit?: number;
+	/** `rg -i`. */
+	ignoreCase?: boolean;
+	/** `rg -F`: the pattern is a literal string. */
+	fixed?: boolean;
+	/** `rg -m`: per-file cap. 0 disables it. */
+	maxPerFile?: number;
+	/** Stop the scan after this many distinct matching entries. */
+	maxMatches?: number;
+	/** Override the ripgrep binary (tests, unusual installs). */
+	binary?: string;
+}
+
+export interface RegexOutcome {
+	hits: SearchHit[];
+	/** Transcripts actually handed to rg, after index-side filtering. */
+	filesSearched: number;
+	/** Matching lines rg reported, before fork deduplication. */
+	rawMatches: number;
+	/** Distinct entries left after fork deduplication. */
+	entriesMatched: number;
+	/** Groups found before `limit` was applied. */
+	scanned: number;
+	/** True when the scan stopped at `maxMatches` rather than at the end. */
+	truncated: boolean;
+	/**
+	 * Matched lines that no longer resolve to an indexed entry: a transcript
+	 * that grew since the last sync, or a line the extractor skips.
+	 */
+	unresolved: number;
+}
+
+export const DEFAULT_MAX_PER_FILE = 200;
+export const DEFAULT_MAX_MATCHES = 2000;
+
+/** Context kept either side of the match in a snippet, in characters. */
+const SNIPPET_RADIUS = 70;
+
+interface EntryRow {
+	id: number;
+	entry_key: string;
+	lineage_id: number;
+	role: string;
+	tool: string | null;
+	ts: string | null;
+}
+
+interface Candidate {
+	entry: EntryRow;
+	kind?: ChunkKind;
+	snippet: string;
+	matchedIn: NonNullable<SearchHit['matchedIn']>;
+}
+
+/** Everything the extractor can see, so the snippet is not limited by the tier. */
+const FULL_EXTRACT = {
+	tier: 'full' as const,
+	toolResultHeadBytes: Number.MAX_SAFE_INTEGER,
+	toolArgsHeadBytes: Number.MAX_SAFE_INTEGER,
+};
+
+function windowAround(
+	text: string,
+	start: number,
+	length: number,
+	radius = SNIPPET_RADIUS,
+): string {
+	const from = Math.max(0, start - radius);
+	const to = Math.min(text.length, start + length + radius);
+	// \u0001/\u0002 are the same markers FTS5 `snippet()` emits, so the renderer
+	// paints a regex hit exactly like a normal one.
+	return (
+		(from > 0 ? '…' : '') +
+		text.slice(from, start) +
+		'\u0001' +
+		text.slice(start, start + length) +
+		'\u0002' +
+		text.slice(start + length, to) +
+		(to < text.length ? '…' : '')
+	);
+}
+
+function indexOfNeedle(
+	haystack: string,
+	needle: string,
+	ignoreCase: boolean,
+): number {
+	if (!needle) return -1;
+	if (!ignoreCase) return haystack.indexOf(needle);
+	return haystack.toLowerCase().indexOf(needle.toLowerCase());
+}
+
+/**
+ * Turn a matched raw line into something a human can read, and say where the
+ * matched bytes actually live.
+ *
+ * The needle is rg's own `submatches[0]`, which is a LITERAL whatever the
+ * pattern was, so nothing here has to re-implement Rust regex semantics in
+ * JavaScript.
+ *
+ * Three outcomes, and keeping them apart is the honesty of the whole feature:
+ * the needle is in extracted text that is also indexed (`index`); it is in
+ * extracted text that is NOT indexed, because of the tier or because the chunk
+ * was truncated before it (`transcript`); or it is in none of the extracted
+ * text at all, so it matched a JSON key, an id or an escape sequence
+ * (`structure`). Only the middle one means "the index could never have found
+ * this". Labelling a match on `"timestamp":` that way would be a lie about an
+ * entry whose text is indexed perfectly well.
+ */
+function describeMatch(
+	raw: Record<string, unknown>,
+	match: RipgrepMatch,
+	ignoreCase: boolean,
+	indexedTexts: string[],
+): {
+	kind?: ChunkKind;
+	snippet: string;
+	matchedIn: NonNullable<SearchHit['matchedIn']>;
+} {
+	const extracted = extractEntry(raw, 0, 0, FULL_EXTRACT)?.chunks ?? [];
+	for (const chunk of extracted) {
+		const at = indexOfNeedle(chunk.text, match.needle, ignoreCase);
+		if (at < 0) continue;
+		const indexed = indexedTexts.some(
+			(text) => indexOfNeedle(text, match.needle, ignoreCase) >= 0,
+		);
+		return {
+			kind: chunk.kind,
+			snippet: windowAround(chunk.text, at, match.needle.length),
+			matchedIn: indexed ? 'index' : 'transcript',
+		};
+	}
+	// Nothing anyone wrote: no kind fits, so none is invented.
+	const at = indexOfNeedle(match.line, match.needle, ignoreCase);
+	return {
+		snippet:
+			at >= 0
+				? windowAround(match.line.trimEnd(), at, match.needle.length)
+				: match.line.slice(0, SNIPPET_RADIUS * 2),
+		matchedIn: 'structure',
+	};
+}
+
+/**
+ * Search the transcripts themselves, and report the result exactly as the
+ * index-backed search would.
+ */
+export async function regexSearch(
+	db: DB,
+	query: RegexQuery,
+): Promise<RegexOutcome> {
+	const limit = query.limit ?? 20;
+	const threadLimit = query.threadLimit ?? 3;
+	// A cap that is NaN or zero would mean "collect every match in the corpus",
+	// which is exactly the unbounded read this is here to prevent.
+	const maxMatches =
+		query.maxMatches !== undefined &&
+		Number.isFinite(query.maxMatches) &&
+		query.maxMatches > 0
+			? query.maxMatches
+			: DEFAULT_MAX_MATCHES;
+	const files = loadFiles(db);
+	const visible = visibleSets(files, query);
+
+	// The earliest moment a lineage could hold: a fork's own `started` is later
+	// than the entries it inherited, so an `until` bound must be judged against
+	// the whole family or a fork whose parent is gone from disk would be skipped.
+	const lineageStart = new Map<number, string>();
+	for (const file of files.values()) {
+		if (file.lineage_id === null || !file.started) continue;
+		const id = Number(file.lineage_id);
+		const known = lineageStart.get(id);
+		if (!known || file.started < known) lineageStart.set(id, file.started);
+	}
+
+	// Path to lineage: the lineage is all a matched line needs, because entry
+	// identity is (lineage_id, entry_key) and a fork copy resolves to the entry
+	// its ancestor owns.
+	const byPath = new Map<string, number>();
+	const paths: string[] = [];
+	for (const file of files.values()) {
+		const fileId = Number(file.id);
+		if (!visible.files.has(fileId)) continue;
+		if (file.lineage_id === null) continue;
+		const lineageId = Number(file.lineage_id);
+		// Time prefilters, kept conservative: `last_activity` is the newest entry
+		// in this file INCLUDING inherited ones, so it is a sound `since` bound.
+		if (query.since && file.last_activity && file.last_activity < query.since)
+			continue;
+		const start = lineageStart.get(lineageId);
+		if (query.until && start && start > query.until) continue;
+		byPath.set(file.path, lineageId);
+		paths.push(file.path);
+	}
+
+	const outcome: RegexOutcome = {
+		hits: [],
+		filesSearched: paths.length,
+		rawMatches: 0,
+		entriesMatched: 0,
+		scanned: 0,
+		truncated: false,
+		unresolved: 0,
+	};
+	if (paths.length === 0) return outcome;
+
+	const entryStmt = db.prepare(
+		`SELECT id, entry_key, lineage_id, role, tool, ts
+       FROM entry WHERE lineage_id = ? AND entry_key = ?`,
+	);
+	const chunkStmt = db.prepare('SELECT text FROM chunk WHERE entry_id = ?');
+	const roleFilter = query.role?.length ? new Set(query.role) : null;
+	const toolFilter = query.tool?.length ? new Set(query.tool) : null;
+	const kindFilter = query.kind?.length ? new Set<string>(query.kind) : null;
+
+	const seen = new Set<number>();
+	const candidates: Candidate[] = [];
+
+	await ripgrep(
+		paths,
+		{
+			pattern: query.pattern,
+			ignoreCase: query.ignoreCase,
+			fixed: query.fixed,
+			maxPerFile: query.maxPerFile ?? DEFAULT_MAX_PER_FILE,
+			binary: query.binary,
+		},
+		(match) => {
+			outcome.rawMatches++;
+			const lineage = byPath.get(match.path);
+			if (lineage === undefined) return;
+			let raw: Record<string, unknown>;
+			try {
+				raw = JSON.parse(match.line) as Record<string, unknown>;
+			} catch {
+				outcome.unresolved++;
+				return;
+			}
+			const entryKey = typeof raw.id === 'string' ? raw.id : null;
+			if (!entryKey) return; // the session header line carries no identity
+			const entry = entryStmt.get(lineage, entryKey) as unknown as
+				EntryRow | undefined;
+			if (!entry) {
+				outcome.unresolved++;
+				return;
+			}
+			// THE fork rule: a shared entry is reported by rg once per fork file
+			// that copied it, and must collapse to one hit here. `(lineage, key)`
+			// resolved to one entry row, so its id is the deduplication key.
+			const entryId = Number(entry.id);
+			if (seen.has(entryId)) return;
+			seen.add(entryId);
+
+			if (roleFilter && !roleFilter.has(entry.role)) return;
+			if (toolFilter && !(entry.tool && toolFilter.has(entry.tool))) return;
+			if (query.since && (!entry.ts || entry.ts < query.since)) return;
+			if (query.until && (!entry.ts || entry.ts > query.until)) return;
+
+			const indexedTexts = (
+				chunkStmt.all(entryId) as unknown as Array<{text: string}>
+			).map((r) => r.text);
+			const described = describeMatch(
+				raw,
+				match,
+				Boolean(query.ignoreCase),
+				indexedTexts,
+			);
+			// A structural match has no kind, so a --kind filter excludes it: it is
+			// not text of that kind, and pretending otherwise would be a guess.
+			if (kindFilter && !(described.kind && kindFilter.has(described.kind)))
+				return;
+
+			candidates.push({entry, ...described});
+			if (candidates.length >= maxMatches) {
+				outcome.truncated = true;
+				return false;
+			}
+		},
+	);
+
+	outcome.entriesMatched = candidates.length;
+
+	// No bm25 here, and inventing a score would be a lie. Recency of the matched
+	// entry is the ordering, newest first.
+	candidates.sort((a, b) => (b.entry.ts ?? '').localeCompare(a.entry.ts ?? ''));
+
+	const memberStmt = db.prepare(
+		'SELECT file_id, seq FROM membership WHERE entry_id = ?',
+	);
+	const groups = new Map<number, SearchHit>();
+	const order: number[] = [];
+	for (const candidate of candidates) {
+		const lineageId = Number(candidate.entry.lineage_id);
+		const existing = groups.get(lineageId);
+		if (existing) {
+			// Same grouping as the FTS path: one chatty lineage cannot own the page,
+			// and the matches it displaced are counted rather than dropped silently.
+			existing.otherHits++;
+			continue;
+		}
+		const threads = collectThreads(
+			Number(candidate.entry.id),
+			memberStmt,
+			files,
+			visible.files,
+		);
+		if (threads.length === 0) continue;
+		const hit: SearchHit = {
+			entryId: Number(candidate.entry.id),
+			entryKey: candidate.entry.entry_key,
+			lineageId,
+			role: candidate.entry.role,
+			tool: candidate.entry.tool,
+			ts: candidate.entry.ts,
+			kind: candidate.kind,
+			snippet: candidate.snippet,
+			// No bm25 to rank with, so no score is claimed; the array order is the
+			// answer, newest match first.
+			score: 0,
+			threads:
+				threadLimit < 0 ? threads : threads.slice(0, Math.max(threadLimit, 1)),
+			threadTotal: threads.length,
+			otherHits: 0,
+			matchedIn: candidate.matchedIn,
+		};
+		groups.set(lineageId, hit);
+		order.push(lineageId);
+	}
+
+	outcome.scanned = order.length;
+	outcome.hits = order
+		.map((id) => groups.get(id))
+		.filter((h): h is SearchHit => Boolean(h))
+		.slice(0, limit);
+	return outcome;
+}

package/src/ripgrep.ts

@@ -0,0 +1,263 @@
+import {spawn} from 'node:child_process';
+
+/**
+ * Thin wrapper around `rg --json`.
+ *
+ * Regex over the raw transcripts is a genuinely different capability from the
+ * FTS index (see `docs/DESIGN.md`): FTS5 tokenises, so it can never do
+ * substring-inside-identifier or true regex, and the raw files hold text the
+ * index deliberately does not (tool output is 42% of the bytes). Rather than
+ * reimplement a scanner over 2 GB of JSONL in JavaScript, this shells out to
+ * the tool that already does it well, and refuses to pretend when it is absent.
+ */
+
+export const DEFAULT_RG = 'rg';
+
+export function ripgrepBinary(env = process.env): string {
+	return env.MEMONAUT_RG?.trim() || DEFAULT_RG;
+}
+
+export class RipgrepMissingError extends Error {
+	readonly binary: string;
+	constructor(binary: string) {
+		super(
+			[
+				`regex search needs ripgrep, and \`${binary}\` was not found.`,
+				'  install it:  apt install ripgrep · brew install ripgrep · cargo install ripgrep',
+				'  or point MEMONAUT_RG at the binary.',
+			].join('\n'),
+		);
+		this.name = 'RipgrepMissingError';
+		this.binary = binary;
+	}
+}
+
+export class RipgrepError extends Error {
+	constructor(message: string) {
+		super(`ripgrep: ${message}`);
+		this.name = 'RipgrepError';
+	}
+}
+
+/** One matching line, as rg reported it. */
+export interface RipgrepMatch {
+	/** Path exactly as it was handed to rg, so it can be looked up in the index. */
+	path: string;
+	/** The whole matching line, which for a transcript is one complete JSON entry. */
+	line: string;
+	/** Byte offset of the line within the file. */
+	offset: number;
+	/** The literal text the pattern matched. A literal, whatever the pattern was. */
+	needle: string;
+}
+
+export interface RipgrepOptions {
+	pattern: string;
+	ignoreCase?: boolean;
+	/** `-F`: the pattern is a literal string, not a regex. */
+	fixed?: boolean;
+	/** `-m`: per-file cap. 0 disables it. */
+	maxPerFile?: number;
+	binary?: string;
+}
+
+/**
+ * argv budget per invocation.
+ *
+ * `ARG_MAX` is ~2 MB on Linux and 256 KB on macOS, and a transcript path is
+ * ~110 bytes, so batching by both count and byte length keeps every platform
+ * far away from E2BIG.
+ */
+const BATCH_FILES = 400;
+const BATCH_BYTES = 96_000;
+
+function decode(field: {text?: string; bytes?: string} | undefined): string {
+	if (!field) return '';
+	if (typeof field.text === 'string') return field.text;
+	if (typeof field.bytes === 'string')
+		return Buffer.from(field.bytes, 'base64').toString('utf8');
+	return '';
+}
+
+function batches(paths: string[]): string[][] {
+	const out: string[][] = [];
+	let current: string[] = [];
+	let bytes = 0;
+	for (const p of paths) {
+		if (
+			current.length > 0 &&
+			(current.length >= BATCH_FILES || bytes + p.length + 1 > BATCH_BYTES)
+		) {
+			out.push(current);
+			current = [];
+			bytes = 0;
+		}
+		current.push(p);
+		bytes += p.length + 1;
+	}
+	if (current.length) out.push(current);
+	return out;
+}
+
+function baseArgs(opts: RipgrepOptions): string[] {
+	const args = [
+		'--json',
+		// Errors about files that vanished between the index scan and now are not
+		// the user's problem; a real pattern error still reaches stderr.
+		'--no-messages',
+		// The user's ripgrep config must not change what a search means here.
+		'--no-config',
+		// Explicit paths already come from the index, which is the only ignore
+		// list that matters. --text stops a transcript being silently skipped as
+		// binary; they are JSON, so there is nothing to blow up on.
+		'--no-ignore',
+		'--text',
+	];
+	if (opts.ignoreCase) args.push('-i');
+	if (opts.fixed) args.push('-F');
+	// Explicitly finite: a NaN cap must not take the same branch as the
+	// documented "0 means all", or a mistyped flag silently removes the bound.
+	if (Number.isFinite(opts.maxPerFile) && (opts.maxPerFile as number) > 0)
+		args.push('-m', String(opts.maxPerFile));
+	// -e, so a pattern starting with `-` is a pattern and not a flag.
+	args.push('-e', opts.pattern);
+	return args;
+}
+
+/**
+ * Run rg over `paths`, calling `onMatch` for each matching line.
+ *
+ * Output is consumed as it arrives, so at most ONE record is ever held: a
+ * single transcript line can be megabytes (a base64 image), and `--json` echoes
+ * the whole line back, so accumulating rg's output is not an option. What
+ * `onMatch` then does with that one line is its own business, and runs inline
+ * on this handler. Returning `false` stops the scan and kills rg.
+ */
+export async function ripgrep(
+	paths: string[],
+	opts: RipgrepOptions,
+	onMatch: (match: RipgrepMatch) => boolean | void,
+): Promise<void> {
+	if (paths.length === 0) return;
+	const binary = opts.binary ?? ripgrepBinary();
+	const args = baseArgs(opts);
+	let stop = false;
+	for (const batch of batches(paths)) {
+		if (stop) return;
+		await runOne(binary, [...args, '--', ...batch], (match) => {
+			const keepGoing = onMatch(match);
+			if (keepGoing === false) {
+				stop = true;
+				return false;
+			}
+			return true;
+		});
+	}
+}
+
+function runOne(
+	binary: string,
+	args: string[],
+	onMatch: (match: RipgrepMatch) => boolean,
+): Promise<void> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(binary, args, {stdio: ['ignore', 'pipe', 'pipe']});
+		let stderr = '';
+		let buffer = '';
+		let killed = false;
+		let settled = false;
+
+		const finish = (err?: Error) => {
+			if (settled) return;
+			settled = true;
+			if (err) reject(err);
+			else resolve();
+		};
+
+		child.on('error', (err) => {
+			const code = (err as NodeJS.ErrnoException).code;
+			if (code === 'ENOENT') finish(new RipgrepMissingError(binary));
+			else finish(err);
+		});
+		child.stderr.setEncoding('utf8');
+		child.stderr.on('data', (chunk: string) => {
+			if (stderr.length < 4096) stderr += chunk;
+		});
+		child.stdout.setEncoding('utf8');
+		child.stdout.on('error', () => {
+			/* EPIPE after an early kill */
+		});
+		child.stdout.on('data', (chunk: string) => {
+			if (killed) return;
+			buffer += chunk;
+			let nl: number;
+			while ((nl = buffer.indexOf('\n')) >= 0) {
+				const line = buffer.slice(0, nl);
+				buffer = buffer.slice(nl + 1);
+				if (!line) continue;
+				let record: RgRecord;
+				try {
+					record = JSON.parse(line) as RgRecord;
+				} catch {
+					continue;
+				}
+				if (record.type !== 'match') continue;
+				const data = record.data;
+				const sub = data.submatches?.[0];
+				const match: RipgrepMatch = {
+					path: decode(data.path),
+					line: decode(data.lines),
+					offset: Number(data.absolute_offset ?? 0),
+					needle: decode(sub?.match),
+				};
+				if (!onMatch(match)) {
+					killed = true;
+					buffer = '';
+					child.kill();
+					return;
+				}
+			}
+		});
+		child.on('close', (code) => {
+			if (killed) return finish();
+			// 0 = matches, 1 = no matches. 2 is "something went wrong", which with
+			// --no-messages is usually just a transcript that was deleted since the
+			// index last saw it; a real complaint (a bad pattern) reaches stderr.
+			if (code !== null && code >= 2 && stderr.trim())
+				return finish(new RipgrepError(stderr.trim()));
+			finish();
+		});
+	});
+}
+
+interface RgRecord {
+	type: string;
+	data: {
+		path?: {text?: string; bytes?: string};
+		lines?: {text?: string; bytes?: string};
+		absolute_offset?: number;
+		submatches?: Array<{
+			match?: {text?: string; bytes?: string};
+			start: number;
+			end: number;
+		}>;
+	};
+}
+
+/** True when a usable ripgrep is on PATH. For tests and for `--help` honesty. */
+export async function ripgrepAvailable(
+	binary = ripgrepBinary(),
+): Promise<boolean> {
+	try {
+		await new Promise<void>((resolve, reject) => {
+			const child = spawn(binary, ['--version'], {stdio: 'ignore'});
+			child.on('error', reject);
+			child.on('close', (code) =>
+				code === 0 ? resolve() : reject(new Error(String(code))),
+			);
+		});
+		return true;
+	} catch {
+		return false;
+	}
+}