memonaut 0.0.0 → 0.2.0

package/src/indexer.ts

@@ -0,0 +1,534 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import type {Config} from './config.js';
+import type {DB} from './db.js';
+import {clearLineage, deleteFile, getMeta, openDb, setMeta} from './db.js';
+import {matcher} from './glob.js';
+import {buildLineage, projectOf} from './lineage.js';
+import type {FileMeta, TranscriptHeader} from './model.js';
+import {
+	headFingerprint,
+	listTranscripts,
+	matchesStoredHead,
+	parseTranscript,
+	readHeader,
+} from './pi-source.js';
+
+export interface IndexOptions {
+	config: Config;
+	/** Ignore watermarks and rebuild everything. */
+	full?: boolean;
+	db?: DB;
+	onProgress?: (progress: IndexProgress) => void;
+}
+
+export interface IndexProgress {
+	phase: 'scan' | 'ingest';
+	done: number;
+	total: number;
+	current?: string;
+}
+
+export interface IndexStats {
+	filesSeen: number;
+	filesIgnored: number;
+	filesPrivate: number;
+	filesIndexed: number;
+	filesSkipped: number;
+	filesAppended: number;
+	filesRemoved: number;
+	lineagesRebuilt: number;
+	entriesInserted: number;
+	entriesShared: number;
+	chunksInserted: number;
+	malformed: number;
+	bytesRead: number;
+	durationMs: number;
+	fullRebuild: boolean;
+}
+
+interface FileRow {
+	id: number;
+	path: string;
+	size: number | null;
+	mtime: number | null;
+	head_hash: string | null;
+	offset: number;
+	entry_count: number;
+	lineage_id: number | null;
+	parent_id: number | null;
+	lineage_depth: number;
+	orphaned: number;
+	private: number;
+	session_uuid: string | null;
+	cwd: string | null;
+	parent_path: string | null;
+}
+
+function num(value: unknown): number {
+	return typeof value === 'bigint' ? Number(value) : (value as number);
+}
+
+/** One open per file: the header and the fingerprint come from the same read. */
+function probe(file: string): {
+	header: TranscriptHeader | null;
+	headHash: string;
+} {
+	return {header: readHeader(file), headHash: headFingerprint(file)};
+}
+
+export function index(opts: IndexOptions): IndexStats {
+	const started = Date.now();
+	const {config} = opts;
+	const db = opts.db ?? openDb(config.dbPath);
+	const report = opts.onProgress ?? (() => {});
+
+	const stats: IndexStats = {
+		filesSeen: 0,
+		filesIgnored: 0,
+		filesPrivate: 0,
+		filesIndexed: 0,
+		filesSkipped: 0,
+		filesAppended: 0,
+		filesRemoved: 0,
+		lineagesRebuilt: 0,
+		entriesInserted: 0,
+		entriesShared: 0,
+		chunksInserted: 0,
+		malformed: 0,
+		bytesRead: 0,
+		durationMs: 0,
+		fullRebuild: false,
+	};
+
+	// A tier change alters what text exists at all, so it forces a rebuild.
+	const storedTier = getMeta(db, 'tier');
+	let full = opts.full === true;
+	if (storedTier !== undefined && storedTier !== config.tier) full = true;
+	stats.fullRebuild = full;
+
+	const isIgnored = matcher(config.ignore);
+	const isPrivate = matcher(config.private);
+
+	// Everything already known, loaded up front. This is what lets a warm run
+	// avoid opening a single transcript: a file whose (size, mtime) still match
+	// its watermark cannot have a different header than the one already stored.
+	const existing = new Map<string, FileRow>();
+	for (const row of db
+		.prepare(
+			`SELECT id, path, size, mtime, head_hash, offset, entry_count, lineage_id,
+              parent_id, lineage_depth, orphaned, private, session_uuid, cwd, parent_path
+       FROM file`,
+		)
+		.all() as unknown as FileRow[]) {
+		existing.set(row.path, row);
+	}
+
+	// ---- scan -------------------------------------------------------------
+	const headers = new Map<string, TranscriptHeader>();
+	const metas = new Map<string, FileMeta>();
+	const discovered: Array<{file: string; source: string}> = [];
+	for (const source of config.sources) {
+		if (!fs.existsSync(source.root)) continue;
+		for (const file of listTranscripts(source.root))
+			discovered.push({file, source: source.id});
+	}
+	let scanned = 0;
+	for (const {file, source} of discovered) {
+		stats.filesSeen++;
+		if (++scanned % 250 === 0)
+			report({
+				phase: 'scan',
+				done: scanned,
+				total: discovered.length,
+				current: file,
+			});
+		let st: fs.Stats;
+		try {
+			st = fs.statSync(file);
+		} catch {
+			continue;
+		}
+		const mtime = Math.floor(st.mtimeMs);
+		const known = existing.get(file);
+		const unchanged =
+			!full &&
+			known &&
+			known.size === st.size &&
+			known.mtime === mtime &&
+			known.head_hash !== null;
+
+		let header: TranscriptHeader | null;
+		let headHash: string;
+		if (unchanged && known) {
+			// Reuse what the DB already knows instead of re-opening the file. On a
+			// 3,800-file corpus this is the difference between a ~20 ms stat pass and
+			// a ~70 ms open-and-read pass, paid on every single query.
+			header = {
+				sessionUuid: known.session_uuid ?? undefined,
+				cwd: known.cwd ?? undefined,
+				parentPath: known.parent_path ?? undefined,
+			};
+			headHash = known.head_hash as string;
+		} else {
+			const probed = probe(file);
+			header = probed.header;
+			headHash = probed.headHash;
+		}
+		if (!header) continue;
+
+		// Ignore rules match the REAL cwd from the header. The directory name is
+		// a lossy mangling of it and must never be used for a filtering decision.
+		const target = header.cwd ?? file;
+		if (isIgnored(target)) {
+			stats.filesIgnored++;
+			continue;
+		}
+		const priv = isPrivate(target);
+		if (priv) stats.filesPrivate++;
+		headers.set(file, header);
+		metas.set(file, {
+			path: file,
+			source,
+			header,
+			size: st.size,
+			mtime,
+			headHash,
+			private: priv,
+		});
+	}
+	report({phase: 'scan', done: scanned, total: discovered.length});
+
+	if (full) {
+		db.exec('BEGIN');
+		db.prepare('DELETE FROM chunk').run();
+		db.prepare('DELETE FROM entry').run();
+		db.prepare('DELETE FROM membership').run();
+		db.prepare(
+			'UPDATE file SET offset = 0, entry_count = 0, size = NULL, mtime = NULL, head_hash = NULL',
+		).run();
+		db.exec('COMMIT');
+	}
+
+	// ---- identity rows ----------------------------------------------------
+	const lineage = buildLineage(headers);
+
+	const upsertFile = db.prepare(`
+    INSERT INTO file(path, source, session_uuid, cwd, project, parent_path, private, started)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(path) DO UPDATE SET
+      source = excluded.source,
+      session_uuid = excluded.session_uuid,
+      cwd = excluded.cwd,
+      project = excluded.project,
+      parent_path = excluded.parent_path,
+      private = excluded.private,
+      started = excluded.started
+  `);
+	const selectFileId = db.prepare('SELECT id FROM file WHERE path = ?');
+	const updateLineage = db.prepare(
+		'UPDATE file SET parent_id = ?, lineage_id = ?, lineage_depth = ?, orphaned = ? WHERE id = ?',
+	);
+
+	db.exec('BEGIN');
+	const idOf = new Map<string, number>();
+	for (const meta of metas.values()) {
+		const known = existing.get(meta.path);
+		// Identity comes from the header, so for a file we did not re-read it can
+		// only change when the CONFIG changed (the private flag). Skipping the
+		// no-op upsert keeps a warm run from rewriting 3,000 identical rows.
+		if (known && known.private === (meta.private ? 1 : 0)) {
+			idOf.set(meta.path, num(known.id));
+			continue;
+		}
+		upsertFile.run(
+			meta.path,
+			meta.source,
+			meta.header.sessionUuid ?? null,
+			meta.header.cwd ?? null,
+			projectOf(meta.header.cwd),
+			meta.header.parentPath ?? null,
+			meta.private ? 1 : 0,
+			meta.header.started ?? null,
+		);
+		const row = selectFileId.get(meta.path) as {id: number} | undefined;
+		if (row) idOf.set(meta.path, num(row.id));
+	}
+	for (const [file, info] of lineage) {
+		const id = idOf.get(file);
+		if (id === undefined) continue;
+		const rootId = idOf.get(info.root) ?? id;
+		const parentId = info.parent ? (idOf.get(info.parent) ?? null) : null;
+		const orphaned = info.orphaned ? 1 : 0;
+		const known = existing.get(file);
+		// Lineage links depend on OTHER files existing, so they are recomputed on
+		// every run, but only written when they actually moved.
+		if (
+			known &&
+			num(known.parent_id ?? 0) === num(parentId ?? 0) &&
+			num(known.lineage_id ?? 0) === rootId &&
+			num(known.lineage_depth) === info.depth &&
+			num(known.orphaned) === orphaned
+		) {
+			continue;
+		}
+		updateLineage.run(parentId, rootId, info.depth, orphaned, id);
+	}
+	db.exec('COMMIT');
+
+	// Files that vanished or became ignored.
+	if (existing.size !== metas.size) {
+		db.exec('BEGIN');
+		for (const [knownPath, row] of [...existing]) {
+			if (metas.has(knownPath)) continue;
+			deleteFile(db, num(row.id));
+			existing.delete(knownPath);
+			stats.filesRemoved++;
+		}
+		db.exec('COMMIT');
+	}
+
+	// ---- decide what to do per file ---------------------------------------
+	for (const [newPath, meta] of metas) {
+		if (existing.has(newPath)) continue;
+		const id = idOf.get(newPath);
+		if (id === undefined) continue;
+		existing.set(newPath, {
+			id,
+			path: newPath,
+			size: null,
+			mtime: null,
+			head_hash: null,
+			offset: 0,
+			entry_count: 0,
+			lineage_id: null,
+			parent_id: null,
+			lineage_depth: 0,
+			orphaned: 0,
+			private: meta.private ? 1 : 0,
+			session_uuid: meta.header.sessionUuid ?? null,
+			cwd: meta.header.cwd ?? null,
+			parent_path: meta.header.parentPath ?? null,
+		});
+	}
+
+	type Action = 'clean' | 'append' | 'dirty';
+	const action = new Map<string, Action>();
+	for (const meta of metas.values()) {
+		const row = existing.get(meta.path);
+		if (!row || row.size === null || row.mtime === null) {
+			action.set(meta.path, 'dirty');
+			continue;
+		}
+		if (
+			row.head_hash === meta.headHash &&
+			row.size === meta.size &&
+			row.mtime === meta.mtime
+		) {
+			action.set(meta.path, 'clean');
+		} else if (
+			meta.size > (row.size ?? 0) &&
+			matchesStoredHead(meta.path, row.head_hash)
+		) {
+			// Grew, and everything we already read is byte-identical: append-only, so
+			// resume from the stored offset instead of re-reading the whole file.
+			action.set(meta.path, 'append');
+		} else {
+			action.set(meta.path, 'dirty');
+		}
+	}
+
+	// Re-ingestion happens per LINEAGE: a fork's membership rows point at entries
+	// owned by an ancestor, so one file can never be rebuilt alone.
+	const byLineage = new Map<string, string[]>();
+	for (const [file, info] of lineage) {
+		const list = byLineage.get(info.root);
+		if (list) list.push(file);
+		else byLineage.set(info.root, [file]);
+	}
+
+	const work: Array<{lineageRoot: string; files: string[]; rebuild: boolean}> =
+		[];
+	for (const [root, group] of byLineage) {
+		const rebuild = group.some((f) => action.get(f) === 'dirty');
+		const touched = rebuild || group.some((f) => action.get(f) === 'append');
+		if (!touched) {
+			stats.filesSkipped += group.length;
+			continue;
+		}
+		if (rebuild) stats.lineagesRebuilt++;
+		const ordered = group.slice().sort((a, b) => {
+			const da = lineage.get(a)?.depth ?? 0;
+			const db_ = lineage.get(b)?.depth ?? 0;
+			if (da !== db_) return da - db_;
+			return (metas.get(a)?.header.started ?? '').localeCompare(
+				metas.get(b)?.header.started ?? '',
+			);
+		});
+		work.push({lineageRoot: root, files: ordered, rebuild});
+	}
+
+	// ---- ingest -----------------------------------------------------------
+	const insertEntry = db.prepare(`
+    INSERT OR IGNORE INTO entry(lineage_id, entry_key, role, tool, ts, first_file, byte_offset, byte_len)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+	const findEntry = db.prepare(
+		'SELECT id FROM entry WHERE lineage_id = ? AND entry_key = ?',
+	);
+	const insertChunk = db.prepare(
+		'INSERT INTO chunk(entry_id, part, kind, text) VALUES (?, ?, ?, ?)',
+	);
+	const insertMembership = db.prepare(
+		'INSERT OR IGNORE INTO membership(entry_id, file_id, seq) VALUES (?, ?, ?)',
+	);
+	const updateFileSummary = db.prepare(
+		'UPDATE file SET name = COALESCE(?, name), last_activity = ?, entry_count = ? WHERE id = ?',
+	);
+	const updateWatermark = db.prepare(
+		'UPDATE file SET size = ?, mtime = ?, head_hash = ?, offset = ?, indexed_at = ? WHERE id = ?',
+	);
+
+	const extractOpts = {
+		tier: config.tier,
+		toolResultHeadBytes: config.toolResultHeadBytes,
+		toolArgsHeadBytes: config.toolArgsHeadBytes,
+	};
+
+	let done = 0;
+	for (const unit of work) {
+		db.exec('BEGIN');
+		try {
+			const lineageId = idOf.get(unit.lineageRoot);
+			if (unit.rebuild && lineageId !== undefined) clearLineage(db, lineageId);
+
+			for (const file of unit.files) {
+				const meta = metas.get(file);
+				const fileId = idOf.get(file);
+				if (!meta || fileId === undefined) continue;
+				const act = unit.rebuild ? 'dirty' : action.get(file);
+				if (act === 'clean') continue;
+
+				const row = existing.get(file);
+				const fromOffset =
+					unit.rebuild || act === 'dirty' ? 0 : (row?.offset ?? 0);
+				let seq = unit.rebuild || act === 'dirty' ? 0 : (row?.entry_count ?? 0);
+
+				const parsed = parseTranscript(file, fromOffset, extractOpts);
+				stats.malformed += parsed.malformed;
+				stats.bytesRead += parsed.endOffset - fromOffset;
+
+				const lid = lineageId ?? fileId;
+				let lastActivity: string | null = null;
+				let sessionName: string | null = null;
+
+				for (const entry of parsed.entries) {
+					const result = insertEntry.run(
+						lid,
+						entry.entryKey,
+						entry.role,
+						entry.tool,
+						entry.ts,
+						fileId,
+						entry.byteOffset,
+						entry.byteLength,
+					);
+					let entryId: number;
+					if (num(result.changes) > 0) {
+						entryId = num(result.lastInsertRowid);
+						stats.entriesInserted++;
+						let part = 0;
+						for (const chunk of entry.chunks) {
+							const text = chunk.text.trim();
+							if (!text) continue;
+							insertChunk.run(entryId, part++, chunk.kind, text);
+							stats.chunksInserted++;
+						}
+					} else {
+						// Already present: this entry was copied here by a fork. Store the
+						// membership edge only, never a second copy of the text.
+						const found = findEntry.get(lid, entry.entryKey) as
+							{id: number} | undefined;
+						if (!found) continue;
+						entryId = num(found.id);
+						stats.entriesShared++;
+					}
+					insertMembership.run(entryId, fileId, seq++);
+					if (entry.ts && (!lastActivity || entry.ts > lastActivity))
+						lastActivity = entry.ts;
+					if (entry.sessionName) sessionName = entry.sessionName;
+				}
+
+				updateFileSummary.run(
+					sessionName,
+					lastActivity ?? meta.header.started ?? null,
+					seq,
+					fileId,
+				);
+				// Watermark LAST: if anything above fails, the next run redoes this file
+				// rather than skipping over a gap.
+				updateWatermark.run(
+					meta.size,
+					meta.mtime,
+					meta.headHash,
+					parsed.endOffset,
+					Date.now(),
+					fileId,
+				);
+				if (act === 'append') stats.filesAppended++;
+				stats.filesIndexed++;
+			}
+			db.exec('COMMIT');
+		} catch (err) {
+			db.exec('ROLLBACK');
+			throw err;
+		}
+		done += unit.files.length;
+		report({
+			phase: 'ingest',
+			done,
+			total: work.reduce((n, u) => n + u.files.length, 0),
+			current: unit.lineageRoot,
+		});
+	}
+
+	setMeta(db, 'tier', config.tier);
+	setMeta(db, 'indexed_at', String(Date.now()));
+	setMeta(db, 'roots', JSON.stringify(config.sources.map((s) => s.root)));
+
+	stats.durationMs = Date.now() - started;
+	if (!opts.db) db.close();
+	return stats;
+}
+
+/** Absolute path of the index for a config, for messages and tooling. */
+export function describeIndex(config: Config): string {
+	return path.resolve(config.dbPath);
+}
+
+export interface SyncResult {
+	/** False when a recent enough sync meant there was nothing to do. */
+	ran: boolean;
+	stats?: IndexStats;
+}
+
+/**
+ * Catch the index up, unless it was already synced within `ttlMs`.
+ *
+ * A warm catch-up over a 3,800-file corpus costs ~120 ms, which is cheap enough
+ * to pay before a query but NOT cheap enough to pay five times in one agent
+ * turn. The TTL is what makes "always fresh" affordable for callers that query
+ * in a burst.
+ */
+export function syncIfStale(config: Config, ttlMs = 15_000): SyncResult {
+	if (!fs.existsSync(config.dbPath))
+		throw new Error(`no index at ${config.dbPath}`);
+	const db = openDb(config.dbPath);
+	try {
+		const last = Number(getMeta(db, 'indexed_at') ?? 0);
+		if (Number.isFinite(last) && Date.now() - last < ttlMs) return {ran: false};
+		return {ran: true, stats: index({config, db})};
+	} finally {
+		db.close();
+	}
+}

package/src/lineage.ts

@@ -0,0 +1,111 @@
+import path from 'node:path';
+import type {TranscriptHeader} from './model.js';
+
+export interface LineageInfo {
+	/** Resolved parent transcript path, or null when this file is a root. */
+	parent: string | null;
+	/** Path of the lineage root (may be the file itself). */
+	root: string;
+	depth: number;
+	/**
+	 * True when the header named a parent we could not find. The file is then
+	 * treated as its own root, but the flag is kept so search can say so instead
+	 * of silently pretending the history never existed.
+	 */
+	orphaned: boolean;
+}
+
+/**
+ * Resolve fork edges into a forest.
+ *
+ * `parentSession` is an ABSOLUTE path, so it breaks the moment a sessions
+ * directory is moved or a home directory is renamed. Measured on a real corpus:
+ * 1 of 133 parents was already missing. So resolution is tried three ways
+ * before giving up: exact path, path relocated under the configured roots, and
+ * finally the session UUID embedded in the filename.
+ */
+export function buildLineage(
+	headers: Map<string, TranscriptHeader>,
+): Map<string, LineageInfo> {
+	const byBasename = new Map<string, string[]>();
+	const byUuid = new Map<string, string>();
+	for (const [file, header] of headers) {
+		const base = path.basename(file);
+		const list = byBasename.get(base);
+		if (list) list.push(file);
+		else byBasename.set(base, [file]);
+		if (header.sessionUuid && !byUuid.has(header.sessionUuid))
+			byUuid.set(header.sessionUuid, file);
+	}
+
+	const resolveParent = (declared: string): string | null => {
+		const abs = path.resolve(declared);
+		if (headers.has(abs)) return abs;
+		const candidates = byBasename.get(path.basename(abs));
+		if (candidates && candidates.length === 1) return candidates[0];
+		const uuid = uuidFromFilename(abs);
+		if (uuid) {
+			const byId = byUuid.get(uuid);
+			if (byId) return byId;
+		}
+		return null;
+	};
+
+	const parents = new Map<string, {parent: string | null; declared: boolean}>();
+	for (const [file, header] of headers) {
+		if (!header.parentPath) {
+			parents.set(file, {parent: null, declared: false});
+			continue;
+		}
+		parents.set(file, {
+			parent: resolveParent(header.parentPath),
+			declared: true,
+		});
+	}
+
+	const out = new Map<string, LineageInfo>();
+	const resolve = (file: string, seen: Set<string>): LineageInfo => {
+		const cached = out.get(file);
+		if (cached) return cached;
+		const info = parents.get(file) ?? {parent: null, declared: false};
+		const orphaned = info.declared && info.parent === null;
+		if (!info.parent || seen.has(info.parent)) {
+			// No parent, an unresolvable parent, or a cycle: this file is a root.
+			const self: LineageInfo = {parent: null, root: file, depth: 0, orphaned};
+			out.set(file, self);
+			return self;
+		}
+		seen.add(file);
+		const parentInfo = resolve(info.parent, seen);
+		const self: LineageInfo = {
+			parent: info.parent,
+			root: parentInfo.root,
+			depth: parentInfo.depth + 1,
+			orphaned,
+		};
+		out.set(file, self);
+		return self;
+	};
+
+	for (const file of headers.keys()) resolve(file, new Set());
+	return out;
+}
+
+const UUID_RE =
+	/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i;
+
+export function uuidFromFilename(file: string): string | null {
+	const m = UUID_RE.exec(path.basename(file));
+	return m ? m[1].toLowerCase() : null;
+}
+
+/**
+ * Human label for a working directory: the last path segment, which is the
+ * project name in every layout anyone actually uses.
+ */
+export function projectOf(cwd: string | undefined): string | null {
+	if (!cwd) return null;
+	const trimmed = cwd.replace(/\/+$/, '');
+	const base = path.basename(trimmed);
+	return base || null;
+}