memonaut 0.1.0 → 0.3.0

package/src/cli-main.ts

@@ -11,11 +11,24 @@ import {
 	renderTable,
 	relativeTime,
 	tildify,
+	type Style,
 } from './format.js';
 import {index} from './indexer.js';
-import {TIERS, type ChunkKind, type Tier} from './model.js';
+import {TIERS, type ChunkKind, type SearchHit, type Tier} from './model.js';
 import {readRawEntry} from './pi-source.js';
+import {
+	DEFAULT_MAX_MATCHES,
+	DEFAULT_MAX_PER_FILE,
+	regexSearch,
+} from './regex.js';
 import {indexStats, readThread, resolveThread, search} from './search.js';
+import {
+	availableSkills,
+	destinationFor,
+	installSkills,
+	isInstalled,
+	type Scope,
+} from './skills.js';
 
 const USAGE = `memonaut — search your agent conversation transcripts
 
@@ -25,10 +38,12 @@ USAGE
 COMMANDS
   index                  Bring the index up to date (incremental by default)
   search <query...>      Search transcripts, grouped by fork lineage
+  search --regex <pat>   Exact regex search over the original transcripts
   show <ref>             Print a thread (file id, session uuid prefix, name or path)
   sql <query>            Run a read-only SQL query against the index
   stats                  Summarise what is indexed
   config                 Show config paths, or write a starter config
+  skills [install]       List or install the agent skill (into ~/.agents/skills)
 
 SEARCH OPTIONS
   --cwd <glob>           Filter by working directory (repeatable, globs, ~ ok)
@@ -47,6 +62,22 @@ SEARCH OPTIONS
   --json                 Machine-readable output
   --no-sync              Skip the incremental catch-up before querying
 
+REGEX OPTIONS
+  --regex <pattern>      Exact regex search, run by ripgrep over the ORIGINAL
+                         transcripts instead of the index. It can therefore
+                         match text the index does not hold at all, notably
+                         tool output (42% of the bytes, excluded by default),
+                         and it can match inside identifiers, which FTS5
+                         cannot. There is no relevance ranking: results are
+                         ordered by recency. Needs ripgrep installed.
+                         Every filter above applies, and the candidate files
+                         come from the index, so ignored and (unless --private)
+                         private transcripts are still never read.
+  -i, --ignore-case      Case-insensitive matching (put it AFTER the pattern)
+  -F, --fixed            Treat the pattern as a literal string, not a regex
+  --max-per-file <n>     Stop after n matches in one transcript (default ${DEFAULT_MAX_PER_FILE}, 0 = all)
+  --max-matches <n>      Stop the scan after n matching entries (default ${DEFAULT_MAX_MATCHES})
+
 INDEX OPTIONS
   --full                 Rebuild from scratch
   --tier <t>             slim | default | full (overrides config for this run)
@@ -57,10 +88,15 @@ QUERY SYNTAX
   match in order, OR / NOT / NEAR(a b, 5) work, and trailing * is a prefix.
   If that fails to parse, the query is retried as quoted literal tokens.
 
+SKILLS OPTIONS
+  --project              Install into ./.agents/skills instead of ~/.agents/skills
+
 EXAMPLES
   recall search steering queue --project wherever --since 30d
   recall search '"fork point" OR parentSession' --threads all
   recall search --tool bash --kind bash 'fuser -k'
+  recall search --regex 'ERR_[A-Z_]+' --since 30d
+  recall search --regex 'handleToolCall(' --fixed --project wherever
   recall sql "select project, count(*) n from file group by 1 order by n desc limit 10"
 `;
 
@@ -179,10 +215,89 @@ function cmdIndex(argv: string[]): void {
 	);
 }
 
-function cmdSearch(argv: string[]): void {
+/** Flags that only mean something on the regex path. */
+const REGEX_ONLY_FLAGS = [
+	'ignore-case',
+	'fixed',
+	'max-per-file',
+	'max-matches',
+];
+/** Flags that only mean something to the full-text ranker. */
+const FTS_ONLY_FLAGS = ['raw', 'no-recency'];
+
+/** Numeric flags and the smallest value that means anything for each. */
+const NUMERIC_FLAGS: Array<[string, number]> = [
+	['max-per-file', 0], // 0 is documented as "no per-file cap"
+	['max-matches', 1],
+];
+
+/**
+ * Everything about a search that can be judged from the arguments alone.
+ *
+ * Pure, and exported, so the rules are testable without a process: the whole
+ * point of them is to refuse silently-wrong input, and "silently" is exactly
+ * what a hand-check misses. `Number('nonsense')` is NaN, and NaN quietly fails
+ * every `> 0` guard downstream, so a mistyped cap would REMOVE the bound it was
+ * meant to tighten. Those caps are the only thing between a pathological
+ * pattern and an unbounded read.
+ */
+export function searchFlagError(
+	given: Record<string, unknown>,
+	text: string,
+	pattern: string | undefined,
+): string | null {
+	if (pattern === undefined) {
+		if (!text) return 'nothing to search for. Try `recall search <words>`';
+		const stray = REGEX_ONLY_FLAGS.find((f) => given[f] !== undefined);
+		if (stray) return `--${stray} only applies to --regex`;
+	} else {
+		if (text)
+			return `--regex takes the pattern itself, so "${text}" has nowhere to go. Try \`recall search --regex '${pattern}'\``;
+		if (!pattern.trim()) return '--regex needs a pattern';
+		const stray = FTS_ONLY_FLAGS.find((f) => given[f] === true);
+		if (stray)
+			return `--${stray} applies to full-text ranking, and a regex search has no bm25 to rank with`;
+	}
+	for (const [flag, min] of NUMERIC_FLAGS) {
+		const raw = given[flag];
+		if (raw === undefined) continue;
+		const n = Number(raw);
+		if (!Number.isInteger(n) || n < min)
+			return `--${flag} needs a whole number ${min} or more, not "${String(raw)}"`;
+	}
+	return null;
+}
+
+/** One rendered block per hit, or a plain reason there are none. */
+function writeHits(
+	hits: SearchHit[],
+	opts: {style: Style; showPath: boolean},
+	nothing: string,
+): void {
+	if (hits.length === 0) {
+		process.stdout.write(nothing + '\n');
+		return;
+	}
+	process.stdout.write(
+		hits
+			.map((hit) =>
+				renderHit(hit, {style: opts.style, showPath: opts.showPath}),
+			)
+			.join('\n\n') + '\n',
+	);
+}
+
+async function cmdSearch(argv: string[]): Promise<void> {
 	const {values, positionals} = parseArgs({
 		args: argv,
 		options: {
+			// No defaults on the mode-specific flags: `undefined` is what makes
+			// "you passed this to the wrong search" detectable.
+			regex: {type: 'string'},
+			'ignore-case': {type: 'boolean', short: 'i'},
+			fixed: {type: 'boolean', short: 'F'},
+			'max-per-file': {type: 'string'},
+			'max-matches': {type: 'string'},
 			cwd: {type: 'string', multiple: true},
 			project: {type: 'string', multiple: true},
 			role: {type: 'string', multiple: true},
@@ -202,15 +317,28 @@ function cmdSearch(argv: string[]): void {
 		allowPositionals: true,
 	});
 	const text = positionals.join(' ').trim();
-	if (!text) fail('nothing to search for. Try `recall search <words>`');
+	const pattern = values.regex as string | undefined;
+
+	// Everything that can be judged from the arguments alone is judged here,
+	// before a config is loaded, an index is synced or a file is opened.
+	const given = values as Record<string, unknown>;
+	const problem = searchFlagError(given, text, pattern);
+	if (problem) fail(problem);
+	const maxPerFile =
+		given['max-per-file'] === undefined
+			? DEFAULT_MAX_PER_FILE
+			: Number(given['max-per-file']);
+	const maxMatches =
+		given['max-matches'] === undefined
+			? DEFAULT_MAX_MATCHES
+			: Number(given['max-matches']);
 
 	const config = loadConfig();
 	if (!values['no-sync']) sync(config, Boolean(values.json));
 
 	const db = openDb(config.dbPath, {readOnly: true});
 	const threadsRaw = String(values.threads);
-	const outcome = search(db, {
-		text,
+	const common = {
 		cwd: values.cwd as string[] | undefined,
 		project: values.project as string[] | undefined,
 		role: values.role as string[] | undefined,
@@ -221,32 +349,58 @@ function cmdSearch(argv: string[]): void {
 		includePrivate: Boolean(values.private),
 		limit: Number(values.limit),
 		threadLimit: threadsRaw === 'all' ? -1 : Number(threadsRaw),
-		noRecency: Boolean(values['no-recency']),
-		raw: Boolean(values.raw),
-	});
+	};
+	const style = makeStyle(colorsEnabled());
+	const render = {style, showPath: Boolean(values.path)};
+	const json = (payload: unknown): void => {
+		process.stdout.write(JSON.stringify(payload, null, 2) + '\n');
+	};
+	const note = (line: string): void => {
+		process.stderr.write(style.dim(line + '\n'));
+	};
 
-	if (values.json) {
-		process.stdout.write(JSON.stringify(outcome, null, 2) + '\n');
-		db.close();
-		return;
-	}
+	try {
+		if (pattern !== undefined) {
+			const outcome = await regexSearch(db, {
+				...common,
+				pattern,
+				ignoreCase: Boolean(values['ignore-case']),
+				fixed: Boolean(values.fixed),
+				maxPerFile,
+				maxMatches,
+			});
+			if (values.json) return json(outcome);
+			writeHits(
+				outcome.hits,
+				render,
+				`no matches in ${outcome.filesSearched} transcript(s)`,
+			);
+			if (outcome.truncated)
+				note(
+					`(stopped at ${maxMatches} matching entries; narrow the pattern or raise --max-matches)`,
+				);
+			// The index is caught up before the query, so this means the transcripts
+			// moved underneath it: worth saying, since those lines were skipped.
+			if (outcome.unresolved > 0)
+				note(
+					`(${outcome.unresolved} matching line(s) are not in the index yet; \`recall index\` will pick them up)`,
+				);
+			return;
+		}
 
-	const style = makeStyle(colorsEnabled());
-	if (outcome.hits.length === 0) {
-		process.stdout.write('no matches\n');
+		const outcome = search(db, {
+			...common,
+			text,
+			noRecency: Boolean(values['no-recency']),
+			raw: Boolean(values.raw),
+		});
+		if (values.json) return json(outcome);
+		if (outcome.quotedFallback)
+			note(`(query retried as literal tokens: ${outcome.usedQuery})`);
+		writeHits(outcome.hits, render, 'no matches');
+	} finally {
 		db.close();
-		return;
-	}
-	if (outcome.quotedFallback) {
-		process.stderr.write(
-			style.dim(`(query retried as literal tokens: ${outcome.usedQuery})\n`),
-		);
 	}
-	const rendered = outcome.hits.map((hit) =>
-		renderHit(hit, {style, showPath: Boolean(values.path)}),
-	);
-	process.stdout.write(rendered.join('\n\n') + '\n');
-	db.close();
 }
 
 function cmdShow(argv: string[]): void {
@@ -432,7 +586,51 @@ function cmdConfig(argv: string[]): void {
 	);
 }
 
-function main(): void {
+/**
+ * `recall skills [list|install]`.
+ *
+ * Kept flag-light on purpose: the only real decision is user versus project scope.
+ */
+function cmdSkills(argv: string[]): void {
+	const scope: Scope = argv.includes('--project') ? 'project' : 'user';
+	const verb = argv.find((argument) => !argument.startsWith('-')) ?? 'list';
+	if (verb !== 'list' && verb !== 'install')
+		fail(`unknown skills command "${verb}". Try \`list\` or \`install\``);
+
+	const available = availableSkills();
+	if (available.length === 0) {
+		fail(
+			'no skills found beside this package. A published install carries them; a checkout keeps them at the repo root',
+		);
+	}
+
+	const style = makeStyle(colorsEnabled());
+	if (verb === 'list') {
+		process.stdout.write(tildify(destinationFor(scope)) + '\n\n');
+		for (const skill of available) {
+			const mark = isInstalled(skill, scope) ? 'installed' : 'not installed';
+			process.stdout.write(
+				`  ${style.bold(skill.name)}  ${style.dim(`(${mark})`)}\n    ${skill.description}\n\n`,
+			);
+		}
+		process.stdout.write('Run `recall skills install` to copy them in.\n');
+		return;
+	}
+
+	const installed = installSkills(scope);
+	for (const entry of installed) {
+		process.stdout.write(
+			`  ${entry.replaced ? 'replaced' : 'installed'}  ${tildify(entry.to)}\n`,
+		);
+	}
+	process.stdout.write(
+		style.dim(
+			`\n${installed.length} skill${installed.length === 1 ? '' : 's'} copied. They are copies, so re-run this after upgrading memonaut.\n`,
+		),
+	);
+}
+
+async function main(): Promise<void> {
 	const argv = process.argv.slice(2);
 	const command = argv[0];
 	const rest = argv.slice(1);
@@ -450,6 +648,8 @@ function main(): void {
 			return cmdStats(rest);
 		case 'config':
 			return cmdConfig(rest);
+		case 'skills':
+			return cmdSkills(rest);
 		case '--version':
 		case '-v':
 			process.stdout.write(version() + '\n');
@@ -466,9 +666,9 @@ function main(): void {
 }
 
 /** Entry point. Loaded dynamically by `cli.ts`, see the note there. */
-export function run(): void {
+export async function run(): Promise<void> {
 	try {
-		main();
+		await main();
 	} catch (err) {
 		fail((err as Error).message);
 	}

package/src/cli.ts

@@ -14,4 +14,4 @@ import {silenceSqliteWarning} from './quiet.js';
 silenceSqliteWarning();
 
 const {run} = await import('./cli-main.js');
-run();
+await run();

package/src/format.ts

@@ -117,6 +117,18 @@ export function renderHit(hit: SearchHit, opts: RenderOptions): string {
 		meta.push(
 			`${hit.otherHits} more match${hit.otherHits === 1 ? '' : 'es'} in this lineage`,
 		);
+	// The regex path reads the original transcripts, so it can match text no
+	// full-text query could reach: a kind the tier excludes (tool output is 42%
+	// of the bytes), or text past where a chunk was truncated. Saying so is the
+	// difference between a surprising result and an informative one. A match on
+	// the JSON scaffolding is NOT that, and has to say something else: the
+	// content around it is very probably indexed.
+	if (hit.matchedIn === 'transcript') {
+		const what = hit.kind ? `this ${hit.kind} text` : 'this text';
+		meta.push(`read from the transcript · ${what} is not indexed`);
+	} else if (hit.matchedIn === 'structure') {
+		meta.push('matched the transcript JSON, not the message text');
+	}
 	if (meta.length) lines.push('  ' + style.dim(meta.join(' · ')));
 
 	for (const thread of hit.threads) {

package/src/index.ts

@@ -6,5 +6,15 @@ export * from './pi-source.js';
 export * from './db.js';
 export * from './indexer.js';
 export * from './search.js';
+export * from './regex.js';
+// Only what a caller of `regexSearch` needs: the errors it can throw and the
+// pre-flight check. Spawning rg is an implementation detail, not API.
+export {
+	RipgrepError,
+	RipgrepMissingError,
+	ripgrepAvailable,
+	ripgrepBinary,
+} from './ripgrep.js';
 export * from './format.js';
+export * from './skills.js';
 export {silenceSqliteWarning} from './quiet.js';

package/src/model.ts

@@ -107,7 +107,13 @@ export interface SearchHit {
 	role: string;
 	tool: string | null;
 	ts: string | null;
-	kind: ChunkKind;
+	/**
+	 * Which extracted text this hit is about. Always set on the full-text path,
+	 * where a hit IS a chunk. Absent on the regex path when the match landed
+	 * outside any extracted text (see `matchedIn`), because inventing a kind
+	 * there would be a guess presented as a fact.
+	 */
+	kind?: ChunkKind;
 	snippet: string;
 	score: number;
 	/** Threads carrying this entry, most recently active first (may be capped). */
@@ -116,6 +122,22 @@ export interface SearchHit {
 	threadTotal: number;
 	/** Further matches in the same lineage that were folded into this group. */
 	otherHits: number;
+	/**
+	 * Regex path only: where the matched bytes actually live.
+	 *
+	 * - `index`: in extracted text that is also indexed, so a full-text query
+	 *   could in principle have found it too.
+	 * - `transcript`: in what someone actually said or ran, but NOT in the index.
+	 *   Either the tier excludes that kind (tool output, by default) or the text
+	 *   sits past where the chunk was truncated. This is the whole reason to
+	 *   reach for a regex search.
+	 * - `structure`: in the transcript's JSON scaffolding, a key, an id or an
+	 *   escape sequence, rather than in anything anyone wrote. Distinct from
+	 *   `transcript` on purpose: the content around it may well be indexed, and
+	 *   claiming otherwise would be exactly the dishonesty the flag exists to
+	 *   prevent.
+	 */
+	matchedIn?: 'index' | 'transcript' | 'structure';
 }
 
 export interface ThreadRef {