memonaut 0.0.0 → 0.2.0

package/src/cli-main.ts

@@ -0,0 +1,618 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {parseArgs} from 'node:util';
+import {loadConfig, writeStarterConfig, type Config} from './config.js';
+import {getMeta, openDb} from './db.js';
+import {
+	colorsEnabled,
+	makeStyle,
+	renderHit,
+	renderTable,
+	relativeTime,
+	tildify,
+	type Style,
+} from './format.js';
+import {index} from './indexer.js';
+import {TIERS, type ChunkKind, type SearchHit, type Tier} from './model.js';
+import {readRawEntry} from './pi-source.js';
+import {
+	DEFAULT_MAX_MATCHES,
+	DEFAULT_MAX_PER_FILE,
+	regexSearch,
+} from './regex.js';
+import {indexStats, readThread, resolveThread, search} from './search.js';
+
+const USAGE = `memonaut — search your agent conversation transcripts
+
+USAGE
+  recall <command> [options]
+
+COMMANDS
+  index                  Bring the index up to date (incremental by default)
+  search <query...>      Search transcripts, grouped by fork lineage
+  search --regex <pat>   Exact regex search over the original transcripts
+  show <ref>             Print a thread (file id, session uuid prefix, name or path)
+  sql <query>            Run a read-only SQL query against the index
+  stats                  Summarise what is indexed
+  config                 Show config paths, or write a starter config
+
+SEARCH OPTIONS
+  --cwd <glob>           Filter by working directory (repeatable, globs, ~ ok)
+  --project <name>       Filter by project (last segment of the cwd, repeatable)
+  --role <role>          user | assistant | toolResult | bashExecution | ... (repeatable)
+  --tool <name>          Filter by tool name (repeatable)
+  --kind <kind>          user | assistant | thinking | toolCall | toolResult | bash | summary | name
+  --since <when>         30d, 6h, 2w, or an ISO date
+  --until <when>         Same forms as --since
+  --limit <n>            Result groups (default 20)
+  --threads <n|all>      Threads shown per group (default 3)
+  --private              Include transcripts matched by the config's private globs
+  --raw                  Pass the query to FTS5 verbatim (no quoting fallback)
+  --no-recency           Rank on bm25 x kind weight only
+  --path                 Show full transcript paths
+  --json                 Machine-readable output
+  --no-sync              Skip the incremental catch-up before querying
+
+REGEX OPTIONS
+  --regex <pattern>      Exact regex search, run by ripgrep over the ORIGINAL
+                         transcripts instead of the index. It can therefore
+                         match text the index does not hold at all, notably
+                         tool output (42% of the bytes, excluded by default),
+                         and it can match inside identifiers, which FTS5
+                         cannot. There is no relevance ranking: results are
+                         ordered by recency. Needs ripgrep installed.
+                         Every filter above applies, and the candidate files
+                         come from the index, so ignored and (unless --private)
+                         private transcripts are still never read.
+  -i, --ignore-case      Case-insensitive matching (put it AFTER the pattern)
+  -F, --fixed            Treat the pattern as a literal string, not a regex
+  --max-per-file <n>     Stop after n matches in one transcript (default ${DEFAULT_MAX_PER_FILE}, 0 = all)
+  --max-matches <n>      Stop the scan after n matching entries (default ${DEFAULT_MAX_MATCHES})
+
+INDEX OPTIONS
+  --full                 Rebuild from scratch
+  --tier <t>             slim | default | full (overrides config for this run)
+  --quiet                No progress output
+
+QUERY SYNTAX
+  The query is an FTS5 MATCH expression: bare words are ANDed, "quoted phrases"
+  match in order, OR / NOT / NEAR(a b, 5) work, and trailing * is a prefix.
+  If that fails to parse, the query is retried as quoted literal tokens.
+
+EXAMPLES
+  recall search steering queue --project wherever --since 30d
+  recall search '"fork point" OR parentSession' --threads all
+  recall search --tool bash --kind bash 'fuser -k'
+  recall search --regex 'ERR_[A-Z_]+' --since 30d
+  recall search --regex 'handleToolCall(' --fixed --project wherever
+  recall sql "select project, count(*) n from file group by 1 order by n desc limit 10"
+`;
+
+function fail(message: string): never {
+	process.stderr.write(`recall: ${message}\n`);
+	process.exit(1);
+}
+
+function version(): string {
+	try {
+		const here = path.dirname(fileURLToPath(import.meta.url));
+		for (const candidate of [
+			path.join(here, '../package.json'),
+			path.join(here, '../../package.json'),
+		]) {
+			if (fs.existsSync(candidate)) {
+				return (
+					(JSON.parse(fs.readFileSync(candidate, 'utf8')) as {version?: string})
+						.version ?? '0.0.0'
+				);
+			}
+		}
+	} catch {
+		/* ignore */
+	}
+	return '0.0.0';
+}
+
+/** Accept `30d`, `6h`, `2w`, `3mo`, or anything Date can parse. */
+function parseWhen(value: string | undefined): string | undefined {
+	if (!value) return undefined;
+	const m = /^(\d+)(mo|[smhdwy])$/.exec(value.trim());
+	if (m) {
+		const n = Number(m[1]);
+		const unit = m[2];
+		const ms =
+			unit === 's'
+				? 1000
+				: unit === 'm'
+					? 60_000
+					: unit === 'h'
+						? 3_600_000
+						: unit === 'd'
+							? 86_400_000
+							: unit === 'w'
+								? 604_800_000
+								: unit === 'mo'
+									? 2_592_000_000
+									: 31_536_000_000;
+		return new Date(Date.now() - n * ms).toISOString();
+	}
+	const parsed = Date.parse(value);
+	if (Number.isNaN(parsed)) fail(`cannot understand time "${value}"`);
+	return new Date(parsed).toISOString();
+}
+
+function withTier(config: Config, tier: string | undefined): Config {
+	if (!tier) return config;
+	if (!TIERS.includes(tier as Tier))
+		fail(`unknown tier "${tier}" (expected ${TIERS.join(', ')})`);
+	return {...config, tier: tier as Tier};
+}
+
+/** Cheap incremental catch-up so a query is never answered from a stale index. */
+function sync(config: Config, quiet: boolean): void {
+	if (!fs.existsSync(config.dbPath)) {
+		fail(`no index at ${tildify(config.dbPath)}. Run \`recall index\` first.`);
+	}
+	const stats = index({config, onProgress: () => {}});
+	if (!quiet && stats.filesIndexed > 0) {
+		process.stderr.write(
+			`recall: synced ${stats.filesIndexed} transcript(s) in ${stats.durationMs}ms\n`,
+		);
+	}
+}
+
+function cmdIndex(argv: string[]): void {
+	const {values} = parseArgs({
+		args: argv,
+		options: {
+			full: {type: 'boolean', default: false},
+			tier: {type: 'string'},
+			quiet: {type: 'boolean', default: false},
+			json: {type: 'boolean', default: false},
+		},
+		allowPositionals: false,
+	});
+	const config = withTier(loadConfig(), values.tier as string | undefined);
+	const quiet = Boolean(values.quiet) || Boolean(values.json);
+	let lastLine = 0;
+	const stats = index({
+		config,
+		full: Boolean(values.full),
+		onProgress: (p) => {
+			if (quiet || !process.stderr.isTTY) return;
+			const now = Date.now();
+			if (now - lastLine < 100) return;
+			lastLine = now;
+			process.stderr.write(`\r${p.phase} ${p.done}/${p.total}\u001b[K`);
+		},
+	});
+	if (!quiet && process.stderr.isTTY) process.stderr.write('\r\u001b[K');
+	if (values.json) {
+		process.stdout.write(JSON.stringify(stats, null, 2) + '\n');
+		return;
+	}
+	const seconds = (stats.durationMs / 1000).toFixed(1);
+	process.stdout.write(
+		[
+			`indexed ${stats.filesIndexed} transcript(s) in ${seconds}s${stats.fullRebuild ? ' (full rebuild)' : ''}`,
+			`  seen ${stats.filesSeen} · skipped ${stats.filesSkipped} · appended ${stats.filesAppended} · removed ${stats.filesRemoved}`,
+			`  ignored ${stats.filesIgnored} · private ${stats.filesPrivate} · lineages rebuilt ${stats.lineagesRebuilt}`,
+			`  entries ${stats.entriesInserted} new, ${stats.entriesShared} inherited by forks · chunks ${stats.chunksInserted}`,
+			`  index: ${tildify(config.dbPath)}`,
+		].join('\n') + '\n',
+	);
+}
+
+/** Flags that only mean something on the regex path. */
+const REGEX_ONLY_FLAGS = [
+	'ignore-case',
+	'fixed',
+	'max-per-file',
+	'max-matches',
+];
+/** Flags that only mean something to the full-text ranker. */
+const FTS_ONLY_FLAGS = ['raw', 'no-recency'];
+
+/** Numeric flags and the smallest value that means anything for each. */
+const NUMERIC_FLAGS: Array<[string, number]> = [
+	['max-per-file', 0], // 0 is documented as "no per-file cap"
+	['max-matches', 1],
+];
+
+/**
+ * Everything about a search that can be judged from the arguments alone.
+ *
+ * Pure, and exported, so the rules are testable without a process: the whole
+ * point of them is to refuse silently-wrong input, and "silently" is exactly
+ * what a hand-check misses. `Number('nonsense')` is NaN, and NaN quietly fails
+ * every `> 0` guard downstream, so a mistyped cap would REMOVE the bound it was
+ * meant to tighten. Those caps are the only thing between a pathological
+ * pattern and an unbounded read.
+ */
+export function searchFlagError(
+	given: Record<string, unknown>,
+	text: string,
+	pattern: string | undefined,
+): string | null {
+	if (pattern === undefined) {
+		if (!text) return 'nothing to search for. Try `recall search <words>`';
+		const stray = REGEX_ONLY_FLAGS.find((f) => given[f] !== undefined);
+		if (stray) return `--${stray} only applies to --regex`;
+	} else {
+		if (text)
+			return `--regex takes the pattern itself, so "${text}" has nowhere to go. Try \`recall search --regex '${pattern}'\``;
+		if (!pattern.trim()) return '--regex needs a pattern';
+		const stray = FTS_ONLY_FLAGS.find((f) => given[f] === true);
+		if (stray)
+			return `--${stray} applies to full-text ranking, and a regex search has no bm25 to rank with`;
+	}
+	for (const [flag, min] of NUMERIC_FLAGS) {
+		const raw = given[flag];
+		if (raw === undefined) continue;
+		const n = Number(raw);
+		if (!Number.isInteger(n) || n < min)
+			return `--${flag} needs a whole number ${min} or more, not "${String(raw)}"`;
+	}
+	return null;
+}
+
+/** One rendered block per hit, or a plain reason there are none. */
+function writeHits(
+	hits: SearchHit[],
+	opts: {style: Style; showPath: boolean},
+	nothing: string,
+): void {
+	if (hits.length === 0) {
+		process.stdout.write(nothing + '\n');
+		return;
+	}
+	process.stdout.write(
+		hits
+			.map((hit) =>
+				renderHit(hit, {style: opts.style, showPath: opts.showPath}),
+			)
+			.join('\n\n') + '\n',
+	);
+}
+
+async function cmdSearch(argv: string[]): Promise<void> {
+	const {values, positionals} = parseArgs({
+		args: argv,
+		options: {
+			// No defaults on the mode-specific flags: `undefined` is what makes
+			// "you passed this to the wrong search" detectable.
+			regex: {type: 'string'},
+			'ignore-case': {type: 'boolean', short: 'i'},
+			fixed: {type: 'boolean', short: 'F'},
+			'max-per-file': {type: 'string'},
+			'max-matches': {type: 'string'},
+			cwd: {type: 'string', multiple: true},
+			project: {type: 'string', multiple: true},
+			role: {type: 'string', multiple: true},
+			tool: {type: 'string', multiple: true},
+			kind: {type: 'string', multiple: true},
+			since: {type: 'string'},
+			until: {type: 'string'},
+			limit: {type: 'string', default: '20'},
+			threads: {type: 'string', default: '3'},
+			private: {type: 'boolean', default: false},
+			raw: {type: 'boolean', default: false},
+			'no-recency': {type: 'boolean', default: false},
+			path: {type: 'boolean', default: false},
+			json: {type: 'boolean', default: false},
+			'no-sync': {type: 'boolean', default: false},
+		},
+		allowPositionals: true,
+	});
+	const text = positionals.join(' ').trim();
+	const pattern = values.regex as string | undefined;
+
+	// Everything that can be judged from the arguments alone is judged here,
+	// before a config is loaded, an index is synced or a file is opened.
+	const given = values as Record<string, unknown>;
+	const problem = searchFlagError(given, text, pattern);
+	if (problem) fail(problem);
+	const maxPerFile =
+		given['max-per-file'] === undefined
+			? DEFAULT_MAX_PER_FILE
+			: Number(given['max-per-file']);
+	const maxMatches =
+		given['max-matches'] === undefined
+			? DEFAULT_MAX_MATCHES
+			: Number(given['max-matches']);
+
+	const config = loadConfig();
+	if (!values['no-sync']) sync(config, Boolean(values.json));
+
+	const db = openDb(config.dbPath, {readOnly: true});
+	const threadsRaw = String(values.threads);
+	const common = {
+		cwd: values.cwd as string[] | undefined,
+		project: values.project as string[] | undefined,
+		role: values.role as string[] | undefined,
+		tool: values.tool as string[] | undefined,
+		kind: values.kind as ChunkKind[] | undefined,
+		since: parseWhen(values.since as string | undefined),
+		until: parseWhen(values.until as string | undefined),
+		includePrivate: Boolean(values.private),
+		limit: Number(values.limit),
+		threadLimit: threadsRaw === 'all' ? -1 : Number(threadsRaw),
+	};
+	const style = makeStyle(colorsEnabled());
+	const render = {style, showPath: Boolean(values.path)};
+	const json = (payload: unknown): void => {
+		process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+	};
+	const note = (line: string): void => {
+		process.stderr.write(style.dim(line + '\n'));
+	};
+
+	try {
+		if (pattern !== undefined) {
+			const outcome = await regexSearch(db, {
+				...common,
+				pattern,
+				ignoreCase: Boolean(values['ignore-case']),
+				fixed: Boolean(values.fixed),
+				maxPerFile,
+				maxMatches,
+			});
+			if (values.json) return json(outcome);
+			writeHits(
+				outcome.hits,
+				render,
+				`no matches in ${outcome.filesSearched} transcript(s)`,
+			);
+			if (outcome.truncated)
+				note(
+					`(stopped at ${maxMatches} matching entries; narrow the pattern or raise --max-matches)`,
+				);
+			// The index is caught up before the query, so this means the transcripts
+			// moved underneath it: worth saying, since those lines were skipped.
+			if (outcome.unresolved > 0)
+				note(
+					`(${outcome.unresolved} matching line(s) are not in the index yet; \`recall index\` will pick them up)`,
+				);
+			return;
+		}
+
+		const outcome = search(db, {
+			...common,
+			text,
+			noRecency: Boolean(values['no-recency']),
+			raw: Boolean(values.raw),
+		});
+		if (values.json) return json(outcome);
+		if (outcome.quotedFallback)
+			note(`(query retried as literal tokens: ${outcome.usedQuery})`);
+		writeHits(outcome.hits, render, 'no matches');
+	} finally {
+		db.close();
+	}
+}
+
+function cmdShow(argv: string[]): void {
+	const {values, positionals} = parseArgs({
+		args: argv,
+		options: {
+			from: {type: 'string', default: '0'},
+			limit: {type: 'string', default: '40'},
+			full: {type: 'boolean', default: false},
+			json: {type: 'boolean', default: false},
+		},
+		allowPositionals: true,
+	});
+	const ref = positionals[0];
+	if (!ref)
+		fail(
+			'which thread? Pass a file id, a session uuid prefix, a name, or a path',
+		);
+
+	const config = loadConfig();
+	const db = openDb(config.dbPath, {readOnly: true});
+	const thread = resolveThread(db, ref);
+	if (!thread) fail(`no thread matching "${ref}"`);
+
+	const entries = readThread(
+		db,
+		Number(thread.id),
+		Number(values.from),
+		Number(values.limit),
+	);
+	if (values.json) {
+		process.stdout.write(JSON.stringify({thread, entries}, null, 2) + '\n');
+		db.close();
+		return;
+	}
+
+	const style = makeStyle(colorsEnabled());
+	const header = [
+		style.bold(thread.name ?? thread.project ?? String(thread.id)),
+		style.dim(tildify(thread.cwd ?? '')),
+		style.dim(`${thread.entry_count} entries`),
+		style.dim(`last ${relativeTime(thread.last_activity)}`),
+	].join(' · ');
+	process.stdout.write(
+		header + '\n' + style.dim(tildify(thread.path)) + '\n\n',
+	);
+
+	const indent = (text: string) =>
+		text
+			.split('\n')
+			.map((l) => '     ' + l)
+			.join('\n');
+
+	for (const entry of entries) {
+		const marks = [
+			style.label(entry.role + (entry.tool ? `:${entry.tool}` : '')),
+		];
+		if (entry.shared) marks.push(style.dim('(inherited)'));
+		process.stdout.write(
+			`${style.dim(String(entry.seq).padStart(4))} ${marks.join(' ')}\n`,
+		);
+		if (values.full && entry.ownerPath) {
+			// Full fidelity means going back to the transcript: the index keeps tool
+			// output truncated (or absent), by design, and the byte range is the pointer.
+			const raw = readRawEntry(
+				entry.ownerPath,
+				entry.byteOffset,
+				entry.byteLength,
+			);
+			if (raw !== null) {
+				process.stdout.write(indent(JSON.stringify(raw, null, 2)) + '\n');
+				continue;
+			}
+		}
+		for (const t of entry.texts) {
+			const body = t.text.length > 2000 ? t.text.slice(0, 2000) + ' …' : t.text;
+			process.stdout.write(indent(body) + '\n');
+		}
+	}
+	db.close();
+}
+
+function cmdSql(argv: string[]): void {
+	const {values, positionals} = parseArgs({
+		args: argv,
+		options: {
+			json: {type: 'boolean', default: false},
+			limit: {type: 'string', default: '200'},
+		},
+		allowPositionals: true,
+	});
+	const sql = positionals.join(' ').trim();
+	if (!sql) fail('no SQL given');
+	const config = loadConfig();
+	const db = openDb(config.dbPath, {readOnly: true});
+	let rows: Array<Record<string, unknown>>;
+	try {
+		rows = db.prepare(sql).all() as unknown as Array<Record<string, unknown>>;
+	} catch (err) {
+		db.close();
+		fail((err as Error).message);
+	}
+	const capped = rows.slice(0, Number(values.limit));
+	if (values.json) process.stdout.write(JSON.stringify(capped, null, 2) + '\n');
+	else {
+		process.stdout.write(renderTable(capped) + '\n');
+		if (rows.length > capped.length) {
+			process.stdout.write(
+				`(${rows.length - capped.length} more rows, raise --limit)\n`,
+			);
+		}
+	}
+	db.close();
+}
+
+function cmdStats(argv: string[]): void {
+	const {values} = parseArgs({
+		args: argv,
+		options: {json: {type: 'boolean', default: false}},
+		allowPositionals: false,
+	});
+	const config = loadConfig();
+	const db = openDb(config.dbPath, {readOnly: true});
+	const stats = indexStats(db);
+	const indexedAt = getMeta(db, 'indexed_at');
+	const tier = getMeta(db, 'tier');
+	const size = fs.existsSync(config.dbPath)
+		? fs.statSync(config.dbPath).size
+		: 0;
+	const payload = {
+		...stats,
+		tier,
+		indexedAt: indexedAt ? new Date(Number(indexedAt)).toISOString() : null,
+		dbBytes: size,
+	};
+	if (values.json) {
+		process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+		db.close();
+		return;
+	}
+	const style = makeStyle(colorsEnabled());
+	process.stdout.write(
+		[
+			`${style.bold(String(stats.files))} transcripts · ${stats.lineages} lineages · ${stats.forks} forks`,
+			`${stats.entries} entries · ${stats.memberships} memberships · ${stats.chunks} chunks`,
+			`${stats.projects} projects · ${stats.privateFiles} private · ${stats.orphans} orphaned`,
+			`span ${String(stats.oldest).slice(0, 10)} → ${String(stats.newest).slice(0, 10)}`,
+			`tier ${tier} · db ${(size / 1e6).toFixed(1)} MB · updated ${relativeTime(payload.indexedAt)}`,
+			style.dim(tildify(config.dbPath)),
+		].join('\n') + '\n',
+	);
+	db.close();
+}
+
+function cmdConfig(argv: string[]): void {
+	const {values} = parseArgs({
+		args: argv,
+		options: {
+			init: {type: 'boolean', default: false},
+			json: {type: 'boolean', default: false},
+		},
+		allowPositionals: false,
+	});
+	const config = loadConfig();
+	if (values.init) {
+		const written = writeStarterConfig(config);
+		process.stdout.write(`config: ${tildify(written)}\n`);
+		return;
+	}
+	if (values.json) {
+		process.stdout.write(JSON.stringify(config, null, 2) + '\n');
+		return;
+	}
+	process.stdout.write(
+		[
+			`config   ${tildify(config.configPath)}${config.loaded ? '' : ' (not created yet, using defaults)'}`,
+			`index    ${tildify(config.dbPath)}`,
+			`tier     ${config.tier}`,
+			`sources  ${config.sources.map((s) => `${s.id}:${tildify(s.root)}`).join(', ')}`,
+			`ignore   ${config.ignore.join(', ') || '(none)'}`,
+			`private  ${config.private.join(', ') || '(none)'}`,
+		].join('\n') + '\n',
+	);
+}
+
+async function main(): Promise<void> {
+	const argv = process.argv.slice(2);
+	const command = argv[0];
+	const rest = argv.slice(1);
+	switch (command) {
+		case 'index':
+			return cmdIndex(rest);
+		case 'search':
+		case 's':
+			return cmdSearch(rest);
+		case 'show':
+			return cmdShow(rest);
+		case 'sql':
+			return cmdSql(rest);
+		case 'stats':
+			return cmdStats(rest);
+		case 'config':
+			return cmdConfig(rest);
+		case '--version':
+		case '-v':
+			process.stdout.write(version() + '\n');
+			return;
+		case undefined:
+		case 'help':
+		case '--help':
+		case '-h':
+			process.stdout.write(USAGE);
+			return;
+		default:
+			fail(`unknown command "${command}". Try \`recall help\``);
+	}
+}
+
+/** Entry point. Loaded dynamically by `cli.ts`, see the note there. */
+export async function run(): Promise<void> {
+	try {
+		await main();
+	} catch (err) {
+		fail((err as Error).message);
+	}
+}

package/src/cli.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+import {silenceSqliteWarning} from './quiet.js';
+
+/**
+ * Launcher, deliberately thin.
+ *
+ * `node:sqlite` prints an ExperimentalWarning the moment it is LOADED, and ES
+ * modules load the whole graph before any module body runs. So a static import
+ * of anything that touches sqlite would emit the warning before the filter
+ * could be installed, no matter where the call sits. Installing the filter here
+ * and pulling the real CLI in dynamically is what keeps two lines of noise off
+ * every single invocation.
+ */
+silenceSqliteWarning();
+
+const {run} = await import('./cli-main.js');
+await run();