memonaut 0.0.0 → 0.2.0

package/src/model.ts

@@ -0,0 +1,157 @@
+/**
+ * Shared vocabulary.
+ *
+ * The domain has three nouns that are easy to conflate, so they are named
+ * apart here and used consistently everywhere else:
+ *
+ * - **file**: one transcript file on disk (a pi `.jsonl` session).
+ * - **entry**: one message/event inside a transcript. An entry is identified by
+ *   `(lineage, entryKey)`, NEVER by `entryKey` alone (see `lineage`).
+ * - **thread**: a file read as a root-to-leaf path. Forking copies a prefix of
+ *   entries into a new file, so ONE entry can belong to MANY threads.
+ *
+ * A **lineage** is a fork family: a root file plus every file transitively
+ * forked from it. Entry ids are only 8 hex chars and DO collide across
+ * unrelated transcripts (38 measured collisions in a 463k-entry corpus), so the
+ * lineage is what makes an entry key meaningful.
+ */
+
+/** How much of a transcript gets indexed. See `docs/DESIGN.md`. */
+export type Tier = 'slim' | 'default' | 'full';
+
+export const TIERS: Tier[] = ['slim', 'default', 'full'];
+
+/** Kinds of extracted text, used for weighting and for `--kind` filters. */
+export type ChunkKind =
+	| 'user'
+	| 'assistant'
+	| 'thinking'
+	| 'toolCall'
+	| 'toolResult'
+	| 'bash'
+	| 'summary'
+	| 'name';
+
+/** Lowest tier at which each chunk kind is indexed. */
+export const KIND_TIER: Record<ChunkKind, Tier> = {
+	user: 'slim',
+	assistant: 'slim',
+	summary: 'slim',
+	name: 'slim',
+	thinking: 'default',
+	toolCall: 'default',
+	bash: 'default',
+	toolResult: 'full',
+};
+
+/** Relevance multipliers applied on top of bm25. */
+export const KIND_WEIGHT: Record<ChunkKind, number> = {
+	name: 3,
+	user: 2.5,
+	summary: 2,
+	assistant: 1.5,
+	bash: 1,
+	toolCall: 0.6,
+	thinking: 0.5,
+	toolResult: 0.4,
+};
+
+/** The header line of a transcript file. */
+export interface TranscriptHeader {
+	/** Session UUID as recorded by the producing agent. */
+	sessionUuid?: string;
+	/** Working directory the session ran in. Authoritative; the folder name is lossy. */
+	cwd?: string;
+	/** Absolute path of the file this one was forked from, if any. */
+	parentPath?: string;
+	started?: string;
+	version?: number;
+}
+
+/** One message/event parsed out of a transcript. */
+export interface ParsedEntry {
+	/** Producer-assigned id, unique only within a lineage. */
+	entryKey: string;
+	parentKey: string | null;
+	role: string;
+	tool: string | null;
+	ts: string | null;
+	byteOffset: number;
+	byteLength: number;
+	/** Set when the entry names the session (`session_info`). */
+	sessionName?: string;
+	chunks: ExtractedChunk[];
+}
+
+export interface ExtractedChunk {
+	kind: ChunkKind;
+	text: string;
+}
+
+/** A transcript file plus everything the indexer learned from its header. */
+export interface FileMeta {
+	path: string;
+	source: string;
+	header: TranscriptHeader;
+	size: number;
+	mtime: number;
+	/** Cheap fingerprint of the first bytes; guards the append-only assumption. */
+	headHash: string;
+	private: boolean;
+}
+
+export interface SearchHit {
+	entryId: number;
+	entryKey: string;
+	lineageId: number;
+	role: string;
+	tool: string | null;
+	ts: string | null;
+	/**
+	 * Which extracted text this hit is about. Always set on the full-text path,
+	 * where a hit IS a chunk. Absent on the regex path when the match landed
+	 * outside any extracted text (see `matchedIn`), because inventing a kind
+	 * there would be a guess presented as a fact.
+	 */
+	kind?: ChunkKind;
+	snippet: string;
+	score: number;
+	/** Threads carrying this entry, most recently active first (may be capped). */
+	threads: ThreadRef[];
+	/** How many threads carry this entry in total, before any display cap. */
+	threadTotal: number;
+	/** Further matches in the same lineage that were folded into this group. */
+	otherHits: number;
+	/**
+	 * Regex path only: where the matched bytes actually live.
+	 *
+	 * - `index`: in extracted text that is also indexed, so a full-text query
+	 *   could in principle have found it too.
+	 * - `transcript`: in what someone actually said or ran, but NOT in the index.
+	 *   Either the tier excludes that kind (tool output, by default) or the text
+	 *   sits past where the chunk was truncated. This is the whole reason to
+	 *   reach for a regex search.
+	 * - `structure`: in the transcript's JSON scaffolding, a key, an id or an
+	 *   escape sequence, rather than in anything anyone wrote. Distinct from
+	 *   `transcript` on purpose: the content around it may well be indexed, and
+	 *   claiming otherwise would be exactly the dishonesty the flag exists to
+	 *   prevent.
+	 */
+	matchedIn?: 'index' | 'transcript' | 'structure';
+}
+
+export interface ThreadRef {
+	fileId: number;
+	path: string;
+	name: string | null;
+	cwd: string | null;
+	project: string | null;
+	lastActivity: string | null;
+	entryCount: number;
+	/** Position of the matched entry within this thread. */
+	seq: number;
+	/** Entries this thread accumulated after the matched entry. */
+	after: number;
+	/** True when this thread is the lineage root (not itself a fork). */
+	isRoot: boolean;
+}

package/src/pi-source.ts

@@ -0,0 +1,328 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import type {
+	ExtractedChunk,
+	ParsedEntry,
+	Tier,
+	TranscriptHeader,
+} from './model.js';
+import {KIND_TIER} from './model.js';
+
+/** Recursively collect transcript files under a root. */
+export function listTranscripts(root: string): string[] {
+	const out: string[] = [];
+	const walk = (dir: string) => {
+		let entries: fs.Dirent[];
+		try {
+			entries = fs.readdirSync(dir, {withFileTypes: true});
+		} catch {
+			return;
+		}
+		for (const e of entries) {
+			const full = path.join(dir, e.name);
+			if (e.isDirectory()) walk(full);
+			else if (e.isFile() && e.name.endsWith('.jsonl')) out.push(full);
+		}
+	};
+	walk(root);
+	out.sort();
+	return out;
+}
+
+/**
+ * FNV-1a over the first bytes of a file: guards the append-only assumption.
+ *
+ * The token embeds how many bytes were actually hashed, because a transcript
+ * shorter than the window grows past it on the next append. `matchesStoredHead`
+ * re-hashes exactly that many bytes, so "the prefix is unchanged" stays a real
+ * check instead of a coincidence.
+ */
+export function headFingerprint(file: string, bytes = HEAD_BYTES): string {
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(file, 'r');
+		const buf = Buffer.allocUnsafe(bytes);
+		const read = fs.readSync(fd, buf, 0, bytes, 0);
+		let h = 0x811c9dc5;
+		for (let i = 0; i < read; i++) {
+			h ^= buf[i];
+			h = Math.imul(h, 0x01000193) >>> 0;
+		}
+		return h.toString(16) + ':' + read;
+	} catch {
+		return '';
+	} finally {
+		if (fd !== undefined) fs.closeSync(fd);
+	}
+}
+
+/** Read only the first line of a transcript, without loading the whole file. */
+export const HEAD_BYTES = 4096;
+
+/** True when the first bytes of `file` still hash to the stored token. */
+export function matchesStoredHead(
+	file: string,
+	stored: string | null | undefined,
+): boolean {
+	if (!stored) return false;
+	const colon = stored.lastIndexOf(':');
+	if (colon < 0) return false;
+	const length = Number(stored.slice(colon + 1));
+	if (!Number.isFinite(length) || length <= 0) return false;
+	return headFingerprint(file, length) === stored;
+}
+
+export function readHeader(file: string): TranscriptHeader | null {
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(file, 'r');
+		const chunks: Buffer[] = [];
+		const buf = Buffer.allocUnsafe(8192);
+		let total = 0;
+		while (total < 1 << 20) {
+			const read = fs.readSync(fd, buf, 0, buf.length, total);
+			if (read <= 0) break;
+			const slice = buf.subarray(0, read);
+			const nl = slice.indexOf(0x0a);
+			if (nl >= 0) {
+				chunks.push(Buffer.from(slice.subarray(0, nl)));
+				break;
+			}
+			chunks.push(Buffer.from(slice));
+			total += read;
+		}
+		const line = Buffer.concat(chunks).toString('utf8');
+		if (!line.trim()) return null;
+		const obj = JSON.parse(line) as Record<string, unknown>;
+		if (obj.type !== 'session') return null;
+		return {
+			sessionUuid: typeof obj.id === 'string' ? obj.id : undefined,
+			cwd: typeof obj.cwd === 'string' ? obj.cwd : undefined,
+			parentPath:
+				typeof obj.parentSession === 'string' ? obj.parentSession : undefined,
+			started: typeof obj.timestamp === 'string' ? obj.timestamp : undefined,
+			version: typeof obj.version === 'number' ? obj.version : undefined,
+		};
+	} catch {
+		return null;
+	} finally {
+		if (fd !== undefined) fs.closeSync(fd);
+	}
+}
+
+function truncate(text: string, max: number): string {
+	if (text.length <= max) return text;
+	return text.slice(0, max) + `\n…[truncated ${text.length - max} chars]`;
+}
+
+function textOf(content: unknown): string {
+	if (typeof content === 'string') return content;
+	if (!Array.isArray(content)) return '';
+	const parts: string[] = [];
+	for (const c of content) {
+		if (typeof c === 'string') parts.push(c);
+		else if (
+			c &&
+			typeof c === 'object' &&
+			(c as {type?: string}).type === 'text'
+		) {
+			parts.push(String((c as {text?: string}).text ?? ''));
+		}
+	}
+	return parts.join('\n');
+}
+
+export interface ExtractOptions {
+	tier: Tier;
+	toolResultHeadBytes: number;
+	toolArgsHeadBytes: number;
+}
+
+const TIER_RANK: Record<Tier, number> = {slim: 0, default: 1, full: 2};
+
+function wants(kind: keyof typeof KIND_TIER, tier: Tier): boolean {
+	return TIER_RANK[KIND_TIER[kind]] <= TIER_RANK[tier];
+}
+
+/**
+ * Turn one raw transcript line into an entry plus its indexable chunks.
+ *
+ * Returns null for lines that carry no identity (the header) or nothing worth
+ * remembering. Non-message entries that still shape a session (`session_info`,
+ * `compaction`, `branch_summary`) ARE kept: a session's name and its compaction
+ * summaries are some of the highest-signal text in the whole corpus.
+ */
+export function extractEntry(
+	obj: Record<string, unknown>,
+	byteOffset: number,
+	byteLength: number,
+	opts: ExtractOptions,
+): ParsedEntry | null {
+	const entryKey = typeof obj.id === 'string' ? obj.id : null;
+	if (!entryKey) return null;
+	const type = String(obj.type ?? '');
+	const base = {
+		entryKey,
+		parentKey: typeof obj.parentId === 'string' ? obj.parentId : null,
+		ts: typeof obj.timestamp === 'string' ? obj.timestamp : null,
+		byteOffset,
+		byteLength,
+	};
+	const chunks: ExtractedChunk[] = [];
+
+	if (type === 'session_info') {
+		const name = typeof obj.name === 'string' ? obj.name : '';
+		if (name && wants('name', opts.tier))
+			chunks.push({kind: 'name', text: name});
+		return {
+			...base,
+			role: 'session_info',
+			tool: null,
+			sessionName: name || undefined,
+			chunks,
+		};
+	}
+
+	if (type === 'compaction' || type === 'branch_summary') {
+		const summary = typeof obj.summary === 'string' ? obj.summary : '';
+		if (summary && wants('summary', opts.tier))
+			chunks.push({kind: 'summary', text: summary});
+		return {...base, role: type, tool: null, chunks};
+	}
+
+	if (type === 'custom_message') {
+		const text = textOf(obj.content);
+		if (text && wants('assistant', opts.tier))
+			chunks.push({kind: 'assistant', text});
+		return {...base, role: 'custom_message', tool: null, chunks};
+	}
+
+	if (type !== 'message') {
+		// model_change, thinking_level_change, label, custom: structural only.
+		return {...base, role: type || 'unknown', tool: null, chunks};
+	}
+
+	const message = (obj.message ?? {}) as Record<string, unknown>;
+	const role = String(message.role ?? 'unknown');
+	let tool: string | null = null;
+
+	if (role === 'user') {
+		const text = textOf(message.content);
+		if (text && wants('user', opts.tier)) chunks.push({kind: 'user', text});
+	} else if (role === 'assistant') {
+		for (const c of (message.content ?? []) as Array<Record<string, unknown>>) {
+			if (!c || typeof c !== 'object') continue;
+			if (c.type === 'text' && wants('assistant', opts.tier)) {
+				chunks.push({kind: 'assistant', text: String(c.text ?? '')});
+			} else if (c.type === 'thinking' && wants('thinking', opts.tier)) {
+				chunks.push({kind: 'thinking', text: String(c.thinking ?? '')});
+			} else if (c.type === 'toolCall') {
+				tool = tool ?? (typeof c.name === 'string' ? c.name : null);
+				if (wants('toolCall', opts.tier)) {
+					const args =
+						c.arguments === undefined ? '' : JSON.stringify(c.arguments);
+					const text = `${String(c.name ?? '')} ${args}`.trim();
+					if (text)
+						chunks.push({
+							kind: 'toolCall',
+							text: truncate(text, opts.toolArgsHeadBytes),
+						});
+				}
+			}
+		}
+	} else if (role === 'toolResult') {
+		tool = typeof message.toolName === 'string' ? message.toolName : null;
+		if (wants('toolResult', opts.tier)) {
+			const text = textOf(message.content);
+			if (text)
+				chunks.push({
+					kind: 'toolResult',
+					text: truncate(text, opts.toolResultHeadBytes),
+				});
+		}
+	} else if (role === 'bashExecution') {
+		const command = String(message.command ?? '');
+		if (wants('bash', opts.tier)) {
+			const output = String(message.output ?? '');
+			const text = [command, truncate(output, opts.toolResultHeadBytes)]
+				.filter(Boolean)
+				.join('\n');
+			if (text) chunks.push({kind: 'bash', text});
+		}
+	} else if (role === 'compactionSummary' || role === 'branchSummary') {
+		const summary = String(message.summary ?? '');
+		if (summary && wants('summary', opts.tier))
+			chunks.push({kind: 'summary', text: summary});
+	} else if (role === 'custom') {
+		const text = textOf(message.content);
+		if (text && wants('assistant', opts.tier))
+			chunks.push({kind: 'assistant', text});
+	}
+
+	return {...base, role, tool, chunks};
+}
+
+export interface ParseResult {
+	entries: ParsedEntry[];
+	/** Byte offset just past the last COMPLETE line consumed. */
+	endOffset: number;
+	malformed: number;
+}
+
+/**
+ * Parse a transcript from `fromOffset` to the end.
+ *
+ * A trailing partial line (a session being written right now) is left
+ * unconsumed, so `endOffset` is always a safe resume point.
+ */
+export function parseTranscript(
+	file: string,
+	fromOffset: number,
+	opts: ExtractOptions,
+): ParseResult {
+	const buf = fs.readFileSync(file);
+	const entries: ParsedEntry[] = [];
+	let malformed = 0;
+	let offset = fromOffset;
+	let consumed = fromOffset;
+	while (offset < buf.length) {
+		const nl = buf.indexOf(0x0a, offset);
+		if (nl < 0) break; // partial trailing line: stop, resume from `consumed`
+		const line = buf.subarray(offset, nl).toString('utf8');
+		const lineStart = offset;
+		offset = nl + 1;
+		consumed = offset;
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		let obj: Record<string, unknown>;
+		try {
+			obj = JSON.parse(trimmed) as Record<string, unknown>;
+		} catch {
+			malformed++;
+			continue;
+		}
+		if (obj.type === 'session') continue;
+		const entry = extractEntry(obj, lineStart, nl - lineStart, opts);
+		if (entry) entries.push(entry);
+	}
+	return {entries, endOffset: consumed, malformed};
+}
+
+/** Read one raw line back from a transcript, for on-demand expansion of a hit. */
+export function readRawEntry(
+	file: string,
+	byteOffset: number,
+	byteLength: number,
+): unknown | null {
+	let fd: number | undefined;
+	try {
+		fd = fs.openSync(file, 'r');
+		const buf = Buffer.allocUnsafe(byteLength);
+		const read = fs.readSync(fd, buf, 0, byteLength, byteOffset);
+		return JSON.parse(buf.subarray(0, read).toString('utf8')) as unknown;
+	} catch {
+		return null;
+	} finally {
+		if (fd !== undefined) fs.closeSync(fd);
+	}
+}

package/src/quiet.ts

@@ -0,0 +1,22 @@
+/**
+ * `node:sqlite` is still flagged experimental and prints a warning on first
+ * use. That is noise in a CLI the user runs fifty times a day, so it is
+ * silenced here and ONLY here, narrowly: any other warning still gets through.
+ */
+export function silenceSqliteWarning(): void {
+	const proc = process as unknown as {
+		emit: (name: string, ...args: unknown[]) => boolean;
+	};
+	const original = proc.emit.bind(process);
+	proc.emit = (name: string, ...args: unknown[]): boolean => {
+		if (name === 'warning') {
+			const warning = args[0] as {name?: string; message?: string} | undefined;
+			if (
+				warning?.name === 'ExperimentalWarning' &&
+				/SQLite/i.test(warning.message ?? '')
+			)
+				return false;
+		}
+		return original(name, ...args);
+	};
+}