memonaut 0.0.0 → 0.2.0

package/src/config.ts

@@ -0,0 +1,130 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {expandTilde} from './glob.js';
+import type {Tier} from './model.js';
+import {TIERS} from './model.js';
+
+export interface SourceConfig {
+	/** Short label stored on every file row, so multi-source indexes stay legible. */
+	id: string;
+	/** Only `pi` for now. The adapter seam is here so other agents can be added. */
+	kind: 'pi';
+	root: string;
+}
+
+export interface Config {
+	sources: SourceConfig[];
+	/** cwd globs that are never read, never stored. */
+	ignore: string[];
+	/** cwd globs that are indexed but hidden from agents unless asked for. */
+	private: string[];
+	tier: Tier;
+	/** Bytes kept per tool result (tier `full` only). */
+	toolResultHeadBytes: number;
+	/** Bytes kept per tool call argument blob. */
+	toolArgsHeadBytes: number;
+	configDir: string;
+	dataDir: string;
+	dbPath: string;
+	configPath: string;
+	/** True when a config file was actually found on disk. */
+	loaded: boolean;
+}
+
+export const DEFAULT_TIER: Tier = 'default';
+
+function xdg(envVar: string, fallback: string): string {
+	const v = process.env[envVar];
+	if (v && v.trim()) return v;
+	return path.join(os.homedir(), fallback);
+}
+
+export function defaultConfigDir(env = process.env): string {
+	if (env.MEMONAUT_CONFIG_DIR) return env.MEMONAUT_CONFIG_DIR;
+	return path.join(xdg('XDG_CONFIG_HOME', '.config'), 'memonaut');
+}
+
+export function defaultDataDir(env = process.env): string {
+	if (env.MEMONAUT_DATA_DIR) return env.MEMONAUT_DATA_DIR;
+	return path.join(xdg('XDG_DATA_HOME', '.local/share'), 'memonaut');
+}
+
+export function defaultSources(): SourceConfig[] {
+	return [
+		{id: 'pi', kind: 'pi', root: path.join(os.homedir(), '.pi/agent/sessions')},
+	];
+}
+
+/**
+ * Load config, filling in defaults. Every path is resolved here so nothing
+ * downstream ever has to think about `~` or XDG again.
+ */
+export function loadConfig(env = process.env): Config {
+	const configDir = defaultConfigDir(env);
+	const dataDir = defaultDataDir(env);
+	const configPath = path.join(configDir, 'config.json');
+
+	let raw: Record<string, unknown> = {};
+	let loaded = false;
+	if (fs.existsSync(configPath)) {
+		try {
+			raw = JSON.parse(fs.readFileSync(configPath, 'utf8')) as Record<
+				string,
+				unknown
+			>;
+			loaded = true;
+		} catch (err) {
+			throw new Error(
+				`config at ${configPath} is not valid JSON: ${(err as Error).message}`,
+			);
+		}
+	}
+
+	const sources =
+		Array.isArray(raw.sources) && raw.sources.length
+			? (raw.sources as SourceConfig[])
+			: defaultSources();
+
+	const tier =
+		typeof raw.tier === 'string' && TIERS.includes(raw.tier as Tier)
+			? (raw.tier as Tier)
+			: DEFAULT_TIER;
+
+	return {
+		sources: sources.map((s) => ({
+			...s,
+			root: path.resolve(expandTilde(s.root)),
+		})),
+		ignore: (raw.ignore as string[]) ?? [],
+		private: (raw.private as string[]) ?? [],
+		tier,
+		toolResultHeadBytes: (raw.toolResultHeadBytes as number) ?? 4096,
+		toolArgsHeadBytes: (raw.toolArgsHeadBytes as number) ?? 2048,
+		configDir,
+		dataDir,
+		dbPath: env.MEMONAUT_DB ?? path.join(dataDir, 'index.db'),
+		configPath,
+		loaded,
+	};
+}
+
+/** Write a starter config, never clobbering an existing one. */
+export function writeStarterConfig(config: Config): string {
+	fs.mkdirSync(config.configDir, {recursive: true});
+	if (fs.existsSync(config.configPath)) return config.configPath;
+	const starter = {
+		sources: defaultSources(),
+		ignore: ['/tmp/**'],
+		private: [],
+		tier: DEFAULT_TIER,
+		toolResultHeadBytes: 4096,
+		toolArgsHeadBytes: 2048,
+	};
+	fs.writeFileSync(
+		config.configPath,
+		JSON.stringify(starter, null, '\t') + '\n',
+		{mode: 0o600},
+	);
+	return config.configPath;
+}

package/src/db.ts

@@ -0,0 +1,213 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {DatabaseSync} from 'node:sqlite';
+
+/**
+ * Bump when the schema or the extraction rules change. The index is a derived
+ * cache over files that are still on disk, and a full rebuild costs well under
+ * a minute, so a version mismatch simply throws the DB away rather than
+ * carrying migration code forever.
+ */
+export const SCHEMA_VERSION = 1;
+
+export type DB = DatabaseSync;
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS meta(
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+
+-- One transcript file. Also the unit of incremental indexing: (size, mtime,
+-- head_hash, offset) is the watermark, and it is written LAST so a crash can
+-- only ever cause re-work, never a silent gap.
+CREATE TABLE IF NOT EXISTS file(
+  id INTEGER PRIMARY KEY,
+  path TEXT NOT NULL UNIQUE,
+  source TEXT NOT NULL,
+  session_uuid TEXT,
+  cwd TEXT,
+  project TEXT,
+  name TEXT,
+  parent_path TEXT,
+  parent_id INTEGER,
+  lineage_id INTEGER,
+  lineage_depth INTEGER NOT NULL DEFAULT 0,
+  orphaned INTEGER NOT NULL DEFAULT 0,
+  private INTEGER NOT NULL DEFAULT 0,
+  started TEXT,
+  last_activity TEXT,
+  entry_count INTEGER NOT NULL DEFAULT 0,
+  size INTEGER,
+  mtime INTEGER,
+  head_hash TEXT,
+  offset INTEGER NOT NULL DEFAULT 0,
+  indexed_at INTEGER
+);
+CREATE INDEX IF NOT EXISTS file_lineage ON file(lineage_id);
+CREATE INDEX IF NOT EXISTS file_activity ON file(last_activity DESC);
+CREATE INDEX IF NOT EXISTS file_uuid ON file(session_uuid);
+
+-- One entry, stored ONCE per lineage even when N forks copied it. Keyed by
+-- (lineage_id, entry_key) because entry keys are 8 hex chars and collide across
+-- unrelated transcripts.
+CREATE TABLE IF NOT EXISTS entry(
+  id INTEGER PRIMARY KEY,
+  lineage_id INTEGER NOT NULL,
+  entry_key TEXT NOT NULL,
+  role TEXT,
+  tool TEXT,
+  ts TEXT,
+  first_file INTEGER,
+  byte_offset INTEGER,
+  byte_len INTEGER,
+  UNIQUE(lineage_id, entry_key)
+);
+CREATE INDEX IF NOT EXISTS entry_ts ON entry(ts);
+
+-- The fan-out edge: which threads carry this entry. A match in shared history
+-- resolves to every fork that inherited it.
+CREATE TABLE IF NOT EXISTS membership(
+  entry_id INTEGER NOT NULL,
+  file_id INTEGER NOT NULL,
+  seq INTEGER NOT NULL,
+  PRIMARY KEY(entry_id, file_id)
+) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS membership_file ON membership(file_id);
+
+CREATE TABLE IF NOT EXISTS chunk(
+  id INTEGER PRIMARY KEY,
+  entry_id INTEGER NOT NULL,
+  part INTEGER NOT NULL,
+  kind TEXT NOT NULL,
+  text TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS chunk_entry ON chunk(entry_id);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5(
+  text,
+  content='chunk',
+  content_rowid='id',
+  tokenize="unicode61 remove_diacritics 2 tokenchars '_-.$'"
+);
+
+CREATE TRIGGER IF NOT EXISTS chunk_ai AFTER INSERT ON chunk BEGIN
+  INSERT INTO chunk_fts(rowid, text) VALUES (new.id, new.text);
+END;
+CREATE TRIGGER IF NOT EXISTS chunk_ad AFTER DELETE ON chunk BEGIN
+  INSERT INTO chunk_fts(chunk_fts, rowid, text) VALUES ('delete', old.id, old.text);
+END;
+CREATE TRIGGER IF NOT EXISTS chunk_au AFTER UPDATE ON chunk BEGIN
+  INSERT INTO chunk_fts(chunk_fts, rowid, text) VALUES ('delete', old.id, old.text);
+  INSERT INTO chunk_fts(rowid, text) VALUES (new.id, new.text);
+END;
+`;
+
+export interface OpenOptions {
+	readOnly?: boolean;
+	/** Throw instead of rebuilding when the schema version differs. */
+	noRebuild?: boolean;
+}
+
+export function getMeta(db: DB, key: string): string | undefined {
+	const row = db.prepare('SELECT value FROM meta WHERE key = ?').get(key) as
+		{value?: string} | undefined;
+	return row?.value;
+}
+
+export function setMeta(db: DB, key: string, value: string): void {
+	db.prepare(
+		'INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
+	).run(key, value);
+}
+
+/**
+ * Open the index, creating it if needed.
+ *
+ * The DB file is created 0600: it concentrates every secret that ever passed
+ * through a transcript, so it must never be world-readable.
+ */
+export function openDb(dbPath: string, opts: OpenOptions = {}): DB {
+	if (opts.readOnly) {
+		if (!fs.existsSync(dbPath)) {
+			throw new Error(`no index at ${dbPath}. Run \`recall index\` first.`);
+		}
+		const db = new DatabaseSync(dbPath, {readOnly: true});
+		db.exec('PRAGMA query_only = 1');
+		return db;
+	}
+
+	fs.mkdirSync(path.dirname(dbPath), {recursive: true});
+	const fresh = !fs.existsSync(dbPath);
+	const db = new DatabaseSync(dbPath);
+	db.exec('PRAGMA journal_mode = WAL');
+	db.exec('PRAGMA synchronous = NORMAL');
+	db.exec('PRAGMA foreign_keys = OFF');
+	db.exec(SCHEMA);
+	if (fresh) {
+		try {
+			fs.chmodSync(dbPath, 0o600);
+		} catch {
+			/* best effort */
+		}
+	}
+
+	const version = getMeta(db, 'schema_version');
+	if (version !== undefined && Number(version) !== SCHEMA_VERSION) {
+		if (opts.noRebuild) {
+			throw new Error(
+				`index schema is v${version}, this build expects v${SCHEMA_VERSION}. Run \`recall index --full\`.`,
+			);
+		}
+		dropAll(db);
+		db.exec(SCHEMA);
+	}
+	setMeta(db, 'schema_version', String(SCHEMA_VERSION));
+	return db;
+}
+
+export function dropAll(db: DB): void {
+	db.exec(`
+    DROP TRIGGER IF EXISTS chunk_ai;
+    DROP TRIGGER IF EXISTS chunk_ad;
+    DROP TRIGGER IF EXISTS chunk_au;
+    DROP TABLE IF EXISTS chunk_fts;
+    DROP TABLE IF EXISTS chunk;
+    DROP TABLE IF EXISTS membership;
+    DROP TABLE IF EXISTS entry;
+    DROP TABLE IF EXISTS file;
+    DROP TABLE IF EXISTS meta;
+  `);
+}
+
+/**
+ * Drop all content for a whole lineage.
+ *
+ * Re-ingestion is a LINEAGE-level operation, never a file-level one: a forked
+ * file's membership rows point at entries owned by its parent, so re-reading
+ * one file in isolation would leave dangling references. Lineages are tiny
+ * (the vast majority are a single file; the largest measured is twelve), so
+ * this costs nothing and removes a whole class of bug.
+ */
+export function clearLineage(db: DB, lineageId: number): void {
+	db.prepare(
+		'DELETE FROM chunk WHERE entry_id IN (SELECT id FROM entry WHERE lineage_id = ?)',
+	).run(lineageId);
+	db.prepare('DELETE FROM entry WHERE lineage_id = ?').run(lineageId);
+	db.prepare(
+		'DELETE FROM membership WHERE file_id IN (SELECT id FROM file WHERE lineage_id = ?)',
+	).run(lineageId);
+	db.prepare(
+		'UPDATE file SET offset = 0, entry_count = 0, size = NULL, mtime = NULL, head_hash = NULL WHERE lineage_id = ?',
+	).run(lineageId);
+}
+
+/** Remove a file that is no longer on disk, and anything only it owned. */
+export function deleteFile(db: DB, fileId: number): void {
+	db.prepare(
+		'DELETE FROM chunk WHERE entry_id IN (SELECT id FROM entry WHERE first_file = ?)',
+	).run(fileId);
+	db.prepare('DELETE FROM entry WHERE first_file = ?').run(fileId);
+	db.prepare('DELETE FROM membership WHERE file_id = ?').run(fileId);
+	db.prepare('DELETE FROM file WHERE id = ?').run(fileId);
+}

package/src/format.ts

@@ -0,0 +1,185 @@
+import os from 'node:os';
+import type {SearchHit, ThreadRef} from './model.js';
+
+const HOME = os.homedir();
+
+export interface Style {
+	bold(s: string): string;
+	dim(s: string): string;
+	hit(s: string): string;
+	label(s: string): string;
+}
+
+export function makeStyle(enabled: boolean): Style {
+	if (!enabled) {
+		const plain = (s: string) => s;
+		return {bold: plain, dim: plain, hit: plain, label: plain};
+	}
+	return {
+		bold: (s) => `\u001b[1m${s}\u001b[0m`,
+		dim: (s) => `\u001b[2m${s}\u001b[0m`,
+		hit: (s) => `\u001b[43m\u001b[30m${s}\u001b[0m`,
+		label: (s) => `\u001b[36m${s}\u001b[0m`,
+	};
+}
+
+export function colorsEnabled(
+	stream: NodeJS.WriteStream = process.stdout,
+): boolean {
+	if (process.env.NO_COLOR !== undefined) return false;
+	if (process.env.FORCE_COLOR !== undefined) return true;
+	return Boolean(stream.isTTY);
+}
+
+export function tildify(p: string | null | undefined): string {
+	if (!p) return '';
+	return p.startsWith(HOME) ? '~' + p.slice(HOME.length) : p;
+}
+
+export function shortTime(ts: string | null | undefined): string {
+	if (!ts) return '?';
+	return ts.slice(0, 16).replace('T', ' ');
+}
+
+export function relativeTime(
+	ts: string | null | undefined,
+	now = Date.now(),
+): string {
+	if (!ts) return '?';
+	const t = Date.parse(ts);
+	if (Number.isNaN(t)) return '?';
+	const s = Math.max(0, (now - t) / 1000);
+	if (s < 90) return 'just now';
+	const m = s / 60;
+	if (m < 90) return `${Math.round(m)}m ago`;
+	const h = m / 60;
+	if (h < 36) return `${Math.round(h)}h ago`;
+	const d = h / 24;
+	if (d < 8) return `${Math.round(d)}d ago`;
+	// Past a week, "2mo ago" stops distinguishing anything. Sibling forks are
+	// usually minutes apart, and the whole point of listing them is to tell them
+	// apart, so show the actual moment instead.
+	return shortTime(ts);
+}
+
+/** Collapse a snippet to one line and paint the FTS5 highlight markers. */
+export function renderSnippet(snippet: string, style: Style): string {
+	const flat = snippet.replace(/\s+/g, ' ').trim();
+	return flat.replace(/\u0001([^\u0002]*)\u0002/g, (_m, inner: string) =>
+		style.hit(inner),
+	);
+}
+
+/**
+ * Label a thread with something you can act on.
+ *
+ * Sibling forks are created within the same second, so a truncated session UUID
+ * does NOT distinguish them (five of the twelve threads in one measured family
+ * share the first eight hex chars). The file id always does, and it is exactly
+ * what `recall show <id>` takes.
+ */
+function threadLabel(thread: ThreadRef): string {
+	if (thread.name) return `#${thread.fileId} ${thread.name}`;
+	const base = thread.path.split('/').pop() ?? thread.path;
+	const uuid = /([0-9a-f]{8}-[0-9a-f]{4})/.exec(base);
+	return `#${thread.fileId} ${uuid ? uuid[1] : base.replace(/\.jsonl$/, '')}`;
+}
+
+export interface RenderOptions {
+	style: Style;
+	now?: number;
+	showPath?: boolean;
+}
+
+export function renderHit(hit: SearchHit, opts: RenderOptions): string {
+	const {style} = opts;
+	const now = opts.now ?? Date.now();
+	const primary = hit.threads[0];
+	const project = primary?.project ?? '?';
+	const lines: string[] = [];
+
+	const head = [
+		style.bold(project),
+		style.dim('·'),
+		style.label(hit.role + (hit.tool ? `:${hit.tool}` : '')),
+		style.dim(shortTime(hit.ts)),
+	].join(' ');
+	lines.push(head);
+	lines.push('  ' + renderSnippet(hit.snippet, style));
+
+	const shared = hit.threadTotal > 1;
+	const meta: string[] = [];
+	if (shared)
+		meta.push(
+			`matched in shared history · carried by ${hit.threadTotal} threads`,
+		);
+	if (hit.otherHits > 0)
+		meta.push(
+			`${hit.otherHits} more match${hit.otherHits === 1 ? '' : 'es'} in this lineage`,
+		);
+	// The regex path reads the original transcripts, so it can match text no
+	// full-text query could reach: a kind the tier excludes (tool output is 42%
+	// of the bytes), or text past where a chunk was truncated. Saying so is the
+	// difference between a surprising result and an informative one. A match on
+	// the JSON scaffolding is NOT that, and has to say something else: the
+	// content around it is very probably indexed.
+	if (hit.matchedIn === 'transcript') {
+		const what = hit.kind ? `this ${hit.kind} text` : 'this text';
+		meta.push(`read from the transcript · ${what} is not indexed`);
+	} else if (hit.matchedIn === 'structure') {
+		meta.push('matched the transcript JSON, not the message text');
+	}
+	if (meta.length) lines.push('  ' + style.dim(meta.join(' · ')));
+
+	for (const thread of hit.threads) {
+		const bits = [
+			style.dim(shared ? '↳' : ' '),
+			threadLabel(thread),
+			style.dim(`last ${relativeTime(thread.lastActivity, now)}`),
+			style.dim(`+${thread.after} after`),
+		];
+		if (opts.showPath) bits.push(style.dim(tildify(thread.path)));
+		lines.push('  ' + bits.join(' '));
+	}
+	const hidden = hit.threadTotal - hit.threads.length;
+	if (hidden > 0)
+		lines.push(
+			'  ' +
+				style.dim(
+					`+ ${hidden} more thread${hidden === 1 ? '' : 's'} (--threads all)`,
+				),
+		);
+
+	return lines.join('\n');
+}
+
+export function renderTable(rows: Array<Record<string, unknown>>): string {
+	if (rows.length === 0) return '(no rows)';
+	const columns = Object.keys(rows[0]);
+	const widths = columns.map((c) =>
+		Math.max(
+			c.length,
+			...rows.map(
+				(r) =>
+					String(r[c] ?? '')
+						.replace(/\s+/g, ' ')
+						.slice(0, 60).length,
+			),
+		),
+	);
+	const line = (cells: string[]) =>
+		cells.map((c, i) => c.padEnd(widths[i])).join('  ');
+	const out = [line(columns), line(widths.map((w) => '-'.repeat(w)))];
+	for (const row of rows) {
+		out.push(
+			line(
+				columns.map((c) =>
+					String(row[c] ?? '')
+						.replace(/\s+/g, ' ')
+						.slice(0, 60),
+				),
+			),
+		);
+	}
+	return out.join('\n');
+}

package/src/glob.ts

@@ -0,0 +1,79 @@
+import os from 'node:os';
+import path from 'node:path';
+
+/**
+ * Tiny glob matcher for path filters. No dependency, no surprises.
+ *
+ * - `~` expands to the home directory.
+ * - `**` crosses path separators, `*` and `?` do not.
+ * - A pattern with NO wildcard matches the path itself or anything under it,
+ *   so `~/.agent-runner` behaves the way people expect without a trailing `/**`.
+ */
+export function expandTilde(pattern: string): string {
+	if (pattern === '~') return os.homedir();
+	if (pattern.startsWith('~/'))
+		return path.join(os.homedir(), pattern.slice(2));
+	return pattern;
+}
+
+const SPECIAL = /[.+^${}()|[\]\\]/g;
+
+export function globToRegExp(pattern: string): RegExp {
+	const expanded = expandTilde(pattern);
+	let out = '';
+	for (let i = 0; i < expanded.length; i++) {
+		const c = expanded[i];
+		if (c === '*') {
+			if (expanded[i + 1] === '*') {
+				// `**/` should also match zero directories, so `/a/**/b` matches `/a/b`.
+				if (expanded[i + 2] === '/') {
+					out += '(?:.*/)?';
+					i += 2;
+				} else {
+					out += '.*';
+					i += 1;
+				}
+			} else {
+				out += '[^/]*';
+			}
+		} else if (c === '?') {
+			out += '[^/]';
+		} else {
+			out += c.replace(SPECIAL, '\\$&');
+		}
+	}
+	return new RegExp(`^${out}$`);
+}
+
+export interface Matcher {
+	(value: string | null | undefined): boolean;
+	readonly empty: boolean;
+}
+
+/** Build a matcher for a list of patterns. An empty list matches nothing. */
+export function matcher(patterns: string[]): Matcher {
+	const compiled = patterns.map((p) => {
+		const expanded = expandTilde(p);
+		const hasWildcard = /[*?]/.test(expanded);
+		return hasWildcard
+			? {kind: 'glob' as const, re: globToRegExp(p)}
+			: {kind: 'prefix' as const, value: expanded.replace(/\/+$/, '')};
+	});
+	const fn = ((value: string | null | undefined) => {
+		if (!value) return false;
+		const normalized = value.length > 1 ? value.replace(/\/+$/, '') : value;
+		for (const c of compiled) {
+			if (c.kind === 'glob') {
+				if (c.re.test(value) || c.re.test(normalized)) return true;
+			} else if (
+				normalized === c.value ||
+				normalized.startsWith(c.value + '/')
+			) {
+				return true;
+			}
+		}
+		return false;
+	}) as {(value: string | null | undefined): boolean; empty: boolean};
+	fn.empty = patterns.length === 0;
+	return fn as Matcher;
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+export * from './model.js';
+export * from './config.js';
+export * from './glob.js';
+export * from './lineage.js';
+export * from './pi-source.js';
+export * from './db.js';
+export * from './indexer.js';
+export * from './search.js';
+export * from './regex.js';
+// Only what a caller of `regexSearch` needs: the errors it can throw and the
+// pre-flight check. Spawning rg is an implementation detail, not API.
+export {
+	RipgrepError,
+	RipgrepMissingError,
+	ripgrepAvailable,
+	ripgrepBinary,
+} from './ripgrep.js';
+export * from './format.js';
+export {silenceSqliteWarning} from './quiet.js';