mnemonad-cli 0.2.0 → 0.3.0

package/lib/commands/buildIndex.js

@@ -0,0 +1,171 @@
+import { join } from 'node:path';
+import {
+    Indexer, TextWindowChunkingStrategy, VectorIndex, DEFAULT_DB_NAME, DEFAULT_MODEL,
+    createProvider, resolveModelName, embedderOfIndex, ModelMismatchError, MissingExtensionError,
+} from '../search/index.js';
+import { FSFolder } from '../FSFolder.js';
+import { resolvePath, DEFAULT_FS_EXCLUDES, userError } from './shared.js';
+
+/**
+ * The vector index, shared by `mnemonad index` and by `push --index` / `watch --index`.
+ * Kept in one place so all three agree on which file it is, which files it covers, and how
+ * they're chunked and embedded — an index built by `push --index` is the same index
+ * `mnemonad index` would have built.
+ *
+ * The default filename is at the folder root and not dot-prefixed: unlike `.mnemonad`, it's
+ * meant to be pushed and pulled like any other file, so a pull is searchable immediately.
+ */
+
+/** @returns {string} the index's filename (`--db`, default `search_index.db`). */
+export function indexDbName(args) {
+    return args.searchDb || DEFAULT_DB_NAME;
+}
+
+/** @returns {string[]} what the index skips: the push's own excludes, plus the index file.
+ *  The index file itself must never be walked as content to embed — explicit, not left to
+ *  chance: a SQLite file's own header/padding bytes are plausible-looking text in places,
+ *  so relying on the binary-content sniff alone to skip it would be fragile. */
+export function indexExcludes(args) {
+    const excludes = [...DEFAULT_FS_EXCLUDES, indexDbName(args)];
+    if (args.exclude) excludes.push(...args.exclude.split(',').map((s) => s.trim()));
+    return excludes;
+}
+
+function chunkingFrom(args) {
+    return new TextWindowChunkingStrategy({
+        size: args.chunkSize ?? 1000,
+        overlap: args.chunkOverlap ?? 150,
+    });
+}
+
+/**
+ * Which model to index `destPath` with. An explicit `--model` wins (an alias like `minilm`, or
+ * a full id). Without one, an existing index keeps the model it was built with — so re-indexing
+ * a folder someone indexed with MiniLM just works — and only a new index (or `--rebuild`)
+ * gets the built-in default.
+ *
+ * @returns {{model: string, embedder: ?string, dtype: ?string}}
+ */
+export function chooseModel(destPath, args) {
+    if (args.model) return { model: resolveModelName(args.model), embedder: null, dtype: null };
+    const meta = args.rebuild ? null : VectorIndex.readMeta(join(destPath, indexDbName(args)));
+    if (meta?.model) {
+        return { model: meta.model, embedder: embedderOfIndex(meta), dtype: meta.dtype || null };
+    }
+    return { model: DEFAULT_MODEL, embedder: null, dtype: null };
+}
+
+/**
+ * An Indexer configured from the CLI flags. Hold on to it across runs (as `watch --index`
+ * does) and the embedding model loads once, on first use, instead of on every run.
+ * Rejects straight away on bad chunking flags or a missing extension, before anything is
+ * read or written.
+ */
+export async function makeIndexer(destPath, args) {
+    const chunkingStrategy = chunkingFrom(args);
+    const { model, embedder, dtype } = chooseModel(destPath, args);
+    let embeddingProvider;
+    try {
+        embeddingProvider = await createProvider({ model, ...(embedder ? { embedder } : {}), dtype });
+    } catch (err) {
+        throw explainIndexError(err, destPath, args);
+    }
+    return new Indexer({ embeddingProvider, chunkingStrategy });
+}
+
+/**
+ * Builds or updates `destPath`'s index, printing the same summary `mnemonad index` does.
+ *
+ * @param {string} destPath - resolved folder path
+ * @param {Object} args - parsed CLI args (`rebuild` starts the index over)
+ * @param {Object} [opts]
+ * @param {Indexer} [opts.indexer] - reuse one (and its loaded model) across calls
+ * @param {boolean} [opts.quiet] - one summary line instead of the full report
+ * @param {boolean} [opts.rebuild] - overrides `args.rebuild` (watch rebuilds only once)
+ * @returns {Promise<{filesChanged: number, filesSkipped: number, filesRemoved: number, chunksAdded: number}>}
+ */
+export async function updateIndex(destPath, args, { indexer = null, quiet = false, rebuild = args.rebuild } = {}) {
+    indexer = indexer || await makeIndexer(destPath, { ...args, rebuild });
+    const dbName = indexDbName(args);
+    const fsFolder = new FSFolder(destPath, indexExcludes(args));
+
+    if (!quiet) {
+        console.log('  embedding model:', indexer.embeddingProvider.modelId, '(downloads and caches on first use)');
+        if (rebuild) console.log(`  rebuilding ${dbName} from scratch`);
+        console.log('  scanning for changed files...');
+    }
+    let result;
+    try {
+        result = await indexer.index(fsFolder, join(destPath, dbName), { rebuild: !!rebuild });
+    } catch (err) {
+        throw explainIndexError(err, destPath, args);
+    }
+
+    if (quiet) {
+        const changed = result.filesChanged + result.filesRemoved;
+        console.log(changed
+            ? `  index: ${result.filesChanged} file(s) re-embedded, ${result.filesRemoved} removed (${result.chunksAdded} chunks)`
+            : '  index: up to date');
+    } else {
+        console.log('  files changed:', result.filesChanged);
+        console.log('  files skipped:', result.filesSkipped, '(content unchanged since last index)');
+        console.log('  files removed:', result.filesRemoved);
+        console.log('  chunks added: ', result.chunksAdded);
+    }
+    return result;
+}
+
+/**
+ * How far `destPath`'s index has drifted from its files, without updating it — no model is
+ * loaded (not even the extension looked up) and the index is opened read-only (see
+ * Indexer.status()). Null when there's no index.
+ *
+ * @returns {Promise<?{changed: string[], added: string[], removed: string[], total: number}>}
+ */
+export async function checkIndex(destPath, args) {
+    const fsFolder = new FSFolder(destPath, indexExcludes(args));
+    // Chunking decides which files count (an empty one is never indexed), so it's configured
+    // exactly as for a real run; status() never embeds, so no provider is needed.
+    const indexer = new Indexer({ embeddingProvider: null, chunkingStrategy: chunkingFrom(args) });
+    const status = await indexer.status(fsFolder, join(destPath, indexDbName(args)));
+    if (!status) return null;
+    return { ...status, total: status.changed.length + status.added.length + status.removed.length };
+}
+
+/** Turns the two errors a user can act on into CLI messages that say what to do. */
+function explainIndexError(err, destPath, args) {
+    if (err instanceof MissingExtensionError) {
+        const lines = [err.message];
+        if (!args.model) {
+            lines.push(
+                '  Or rebuild the index with the built-in model instead:',
+                `    mnemonad index ${destPath} --rebuild`,
+            );
+        }
+        return userError(lines.join('\n'));
+    }
+    if (err instanceof ModelMismatchError) {
+        return userError(
+            `${indexDbName(args)} was built with ${err.indexModel}, but this run asked for ${err.providerModel} —\n` +
+            '  one index can\'t mix two models. Either drop --model to keep using the index\'s own model, or\n' +
+            `  add --rebuild to re-embed every file with ${err.providerModel} (the next push then uploads the\n` +
+            '  whole index again).'
+        );
+    }
+    return err;
+}
+
+/** `mnemonad index <path>` */
+export async function buildIndex(args) {
+    const destPath = resolvePath(args.path, { mustExist: true });
+    const result = await updateIndex(destPath, args);
+
+    if (result.filesChanged > 0 || result.filesRemoved > 0) {
+        // Said explicitly rather than left implicit: unlike `.mnemonad`, this file is not
+        // dot-prefixed, so the very next `push` from this folder includes it automatically —
+        // worth knowing before that push happens, not after.
+        console.log(`\n  note: ${indexDbName(args)} will be included in the next \`mnemonad push\` from this folder`);
+    } else {
+        console.log('\n  no changes — nothing new to push');
+    }
+}

package/lib/commands/burn.js

@@ -21,7 +21,7 @@ export async function burn(args) {
         throw new Error('stream-id is required for burn');
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
     const mn = new Mnemonad({

package/lib/commands/compact.js

@@ -34,7 +34,7 @@ export async function compact(args) {
     const counts = await countTree(fsFolder);
     console.log('  scanning...', counts.files, 'files,', counts.folders, 'folders');
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
     // No `password` at construction — see shared.js's unlockStream() for why.

package/lib/commands/diff.js

@@ -56,7 +56,7 @@ export async function diff(args) {
         throw new Error('stream-id is required for diff');
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
 
     const mn = new Mnemonad({
         publicClient: client.publicClient,

package/lib/commands/index.js

@@ -5,3 +5,5 @@ export { diff } from './diff.js';
 export { watch } from './watch.js';
 export { compact } from './compact.js';
 export { burn } from './burn.js';
+export { buildIndex } from './buildIndex.js';
+export { search } from './search.js';

package/lib/commands/info.js

@@ -14,7 +14,7 @@ import {
 } from './shared.js';
 
 export async function info(args) {
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
 
     console.log('  chain:          ', args.chain);
     console.log('  your wallet:    ', client.account?.address || '(none)');

package/lib/commands/pull.js

@@ -24,7 +24,7 @@ export async function pull(args) {
         throw new Error('stream-id is required for pull');
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
 
     const mn = new Mnemonad({
         publicClient: client.publicClient,

package/lib/commands/push.js

@@ -18,7 +18,9 @@ import {
     formatStreamId,
     streamIdsEqual,
     discoverStreamId,
+    userError,
 } from './shared.js';
+import { updateIndex, checkIndex, indexDbName } from './buildIndex.js';
 
 export async function push(args) {
     const destPath = resolvePath(args.path, { mustExist: true });
@@ -29,6 +31,23 @@ export async function push(args) {
     await discoverStreamId(args, destPath);
     let streamId = args.streamId;
 
+    // Before anything is scanned or signed: the push below then picks the updated index up
+    // like any other file. A failure stops the push outright — a push that silently went out
+    // without the index asked for would leave the stream unsearchable with no sign of why.
+    if (args.rebuild && !args.index) {
+        throw userError('--rebuild only applies with --index (it starts the search index over)');
+    }
+    if (args.index) {
+        console.log(args.rebuild ? '  rebuilding search index...' : '  updating search index...');
+        try {
+            await updateIndex(destPath, args, { quiet: true });
+        } catch (err) {
+            throw userError(`indexing failed, so nothing was pushed: ${err.message}`);
+        }
+    } else {
+        await warnIfIndexStale(destPath, args);
+    }
+
     const fsFolder = new FSFolder(destPath, makeExcludes(args));
 
     const counts = await countTree(fsFolder);
@@ -45,7 +64,7 @@ export async function push(args) {
         }
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
     const baseParams = {
@@ -136,3 +155,27 @@ export async function push(args) {
         await writeManifest(destPath, { streamId: formatStreamId(streamId), version: result.version, files: localFiles });
     }
 }
+
+/**
+ * A folder that already carries an index, pushed without `--index` after its files changed,
+ * would publish an index that no longer matches them — searches keep returning the old text's
+ * offsets. Cheap to catch (a hash per file, no model load, the index opened read-only) and
+ * worth a line; never a reason to fail the push itself.
+ */
+async function warnIfIndexStale(destPath, args) {
+    let status;
+    try {
+        status = await checkIndex(destPath, args);
+    } catch {
+        return;
+    }
+    if (!status || status.total === 0) return;
+    const parts = [];
+    if (status.changed.length) parts.push(`${status.changed.length} changed`);
+    if (status.added.length) parts.push(`${status.added.length} new`);
+    if (status.removed.length) parts.push(`${status.removed.length} removed`);
+    console.log(
+        `  note: ${indexDbName(args)} is out of date (${parts.join(', ')} file(s) since it was built).\n` +
+        '        Push with --index to update it first, or run `mnemonad index <path>`.'
+    );
+}

package/lib/commands/search.js

@@ -0,0 +1,91 @@
+import { join } from 'node:path';
+import {
+    Searcher, VectorIndex, DEFAULT_DB_NAME, createProvider, resolveModelName, MissingExtensionError,
+} from '../search/index.js';
+import { resolvePath, folderExists } from './shared.js';
+
+export async function search(args) {
+    if (!args.query) {
+        throw userError('a search query is required (e.g. `mnemonad search /path/to/folder "how does chunking work"`)');
+    }
+    const destPath = resolvePath(args.path, { mustExist: true });
+    const dbName = args.searchDb || DEFAULT_DB_NAME;
+    const dbPath = join(destPath, dbName);
+
+    if (!folderExists(destPath) || !(await _fileExists(dbPath))) {
+        throw userError(
+            `no ${dbName} found in ${destPath}\n` +
+            '  Run `mnemonad index <path>` first — either yours, or whoever pushed this stream\'s\n' +
+            '  (a pull already brings theirs down with the rest of the folder).'
+        );
+    }
+
+    // The query has to be embedded by the model that built the index — which the index records,
+    // so there's nothing to choose. An explicit --model that disagrees is a mistake worth
+    // naming rather than a preference to honour.
+    const indexModel = VectorIndex.readMeta(dbPath)?.model;
+    if (args.model && indexModel && resolveModelName(args.model) !== indexModel) {
+        throw userError(
+            `${dbName} was built with ${indexModel}; a search has to use the same model — drop --model\n` +
+            '  (search always uses the index\'s own), or rebuild the index with the other one:\n' +
+            `    mnemonad index ${destPath} --model ${args.model} --rebuild`
+        );
+    }
+
+    const searcher = new Searcher({ createProvider });
+    const k = args.limit ? Number(args.limit) : 5;
+    let hits;
+    try {
+        hits = await searcher.search(dbPath, args.query, k);
+    } catch (err) {
+        if (err instanceof MissingExtensionError) {
+            throw userError(
+                `this folder's index was built with ${err.model}, which ${err.message.slice(err.message.indexOf('needs'))}\n` +
+                '  Or rebuild the index with the built-in model:\n' +
+                `    mnemonad index ${destPath} --rebuild`
+            );
+        }
+        throw err;
+    }
+
+    if (hits.length === 0) {
+        console.log('  no matches');
+        return;
+    }
+
+    for (const [i, hit] of hits.entries()) {
+        console.log(`\n${i + 1}. ${hit.path} (chunk ${hit.chunkIndex}, distance ${hit.distance.toFixed(4)})`);
+        const snippet = await _readSnippet(destPath, hit);
+        if (snippet) console.log('   ' + snippet.replace(/\s+/g, ' ').slice(0, 200));
+    }
+}
+
+async function _fileExists(path) {
+    try {
+        const { stat } = await import('node:fs/promises');
+        const s = await stat(path);
+        return s.isFile();
+    } catch {
+        return false;
+    }
+}
+
+/** Re-reads the hit's snippet straight from the already-pulled source file — the index
+ *  deliberately doesn't store chunk text itself (see ../search/VectorIndex.js),
+ *  so this is the one place that cost is paid, and only for hits actually shown. */
+async function _readSnippet(destPath, hit) {
+    try {
+        const { readFile } = await import('node:fs/promises');
+        const bytes = await readFile(join(destPath, hit.path));
+        const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+        return text.slice(hit.startOffset, hit.endOffset);
+    } catch {
+        return null;
+    }
+}
+
+function userError(message) {
+    const err = new Error(message);
+    err._isUserError = true;
+    return err;
+}

package/lib/commands/shared.js

@@ -12,7 +12,7 @@ import { DoubleSync as SyncEngine, CDCStore, DoubleSyncSnapshot as SyncSnapshot
 import { FSFolder } from '../FSFolder.js';
 import { createPresignProvider } from '../presignProvider.js';
 
-const DEFAULT_FS_EXCLUDES = ['node_modules', '.git', '.env', '.DS_Store', '.mnemonad', '.claude', 'pnpm-lock.yaml', 'package-lock.json'];
+export const DEFAULT_FS_EXCLUDES = ['node_modules', '.git', '.env', '.DS_Store', '.mnemonad', '.claude', 'pnpm-lock.yaml', 'package-lock.json'];
 
 export function makeExcludes(args) {
     if (!args.exclude) return undefined;
@@ -305,7 +305,8 @@ export async function unlockStream(mn, credentials = {}) {
     return true;
 }
 
-function userError(message) {
+/** An error the CLI prints as a plain message (no stack trace) — see bin/mnemonad.js. */
+export function userError(message) {
     const err = new Error(message);
     err._isUserError = true;
     return err;

package/lib/commands/watch.js

@@ -16,7 +16,9 @@ import {
     resolveOffloadParams,
     formatStreamId,
     discoverStreamId,
+    userError,
 } from './shared.js';
+import { makeIndexer, updateIndex } from './buildIndex.js';
 
 function timestamp() {
     return new Date().toTimeString().slice(0, 8);
@@ -40,9 +42,20 @@ export async function watch(args) {
     const pushEnabled = !args.pullOnly;
     const pullEnabled = !args.pushOnly;
 
+    if (args.index && !pushEnabled) {
+        throw userError('--index only applies when watch pushes — drop --pull-only to use it');
+    }
+    if (args.rebuild && !args.index) {
+        throw userError('--rebuild only applies with --index (it starts the search index over)');
+    }
+    // One indexer for the whole session: the embedding model loads on the first change and
+    // then stays in memory, so every later re-index costs only the files that changed.
+    // Built now so bad chunking flags or a missing extension fail before watching starts.
+    const indexer = args.index ? await makeIndexer(cwd, args) : null;
+
     const excludes = makeExcludes(args);
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     if (pushEnabled) requireWalletClient(client);
 
     // Only a push-enabled session needs write-side offload capability (pinata-jwt or
@@ -79,6 +92,7 @@ export async function watch(args) {
     console.log(`  remote version: ${remoteVersion}`);
     console.log(`  push:          ${pushEnabled ? `enabled (debounce ${debounceMs}ms)` : 'disabled'}`);
     console.log(`  pull:          ${pullEnabled ? `enabled (poll every ${pollIntervalMs / 1000}s)` : 'disabled'}`);
+    if (indexer) console.log('  index:         updated before every push');
     console.log('press Ctrl+C to stop\n');
 
     async function runPush() {
@@ -88,13 +102,34 @@ export async function watch(args) {
 
         pushInFlight = true;
         try {
-            // Best-effort race guard, not an atomic one: Mnemonad's own contract has no
-            // "expected prior length" check — no atomic on-chain compare-and-abort.
-            // Re-checking here narrows the window where a concurrent push (another `watch`
-            // session, or a manual push) could make MonadSync build a diff against a base
-            // the chain has since moved past — the same failure mode monadsync's own
-            // snapshot-recovery tests exercise — but doesn't close it entirely; only an
-            // on-chain guard could.
+            // Before the push, so the push carries it. The index file's own write then fires
+            // the watcher again, but by then lastSyncedHash (taken after indexing, below)
+            // already includes it, so that second pass finds nothing new and doesn't push.
+            // A failed index skips the push rather than publishing a stale index; the next
+            // change retries both.
+            let hashToSync = newHash;
+            if (indexer) {
+                try {
+                    // Never a rebuild here — `--rebuild` applies once, to the startup pass below.
+                    await updateIndex(cwd, args, { indexer, quiet: true, rebuild: false });
+                } catch (err) {
+                    console.error(`[${timestamp()}] index update failed, not pushing:`, err.message);
+                    return;
+                }
+                hashToSync = await localTreeHash(cwd);
+            }
+
+            // Best-effort race guard, not an atomic one. Mnemonad's contract *does* check an
+            // `expectedLength` atomically (Mnemonad.sol's `_checkLength`/`LengthMismatch`) —
+            // that closes the index race outright: two writers can never land at the same
+            // position, one just reverts. What it can't check is content: it stores opaque
+            // bytes, with no way to verify a diff patch is actually consistent with the real
+            // previous item. Re-checking here narrows the separate window where a concurrent
+            // push (another `watch` session, or a manual push) lands at the *correct* index
+            // but MonadSync still builds its diff against a snapshot the chain has since
+            // moved past — the same failure mode monadsync's own snapshot-recovery tests
+            // exercise. Confirmed live as a real corruption, not just theoretical — see
+            // monadsync/README.md's "Repairing a corrupt chain".
             mn.reInitialize();
             await mn.initialize();
             if (mn.length !== remoteVersion) {
@@ -107,7 +142,7 @@ export async function watch(args) {
             const fsFolder = new FSFolder(cwd, excludes);
             console.log(`[${timestamp()}] change detected — pushing...`);
             const result = await monadSync.push(fsFolder);
-            lastSyncedHash = newHash;
+            lastSyncedHash = hashToSync;
             remoteVersion = result.version;
             console.log(`[${timestamp()}] pushed version ${result.version}`);
         } catch (err) {
@@ -169,6 +204,20 @@ export async function watch(args) {
 
     const poller = setInterval(runPoll, pollIntervalMs);
 
+    // An index that's already out of date when the session starts (files edited, or indexed
+    // for the first time) would otherwise only catch up on the next local change.
+    if (indexer) {
+        try {
+            const result = await updateIndex(cwd, args, { indexer, quiet: true, rebuild: !!args.rebuild });
+            if (result.filesChanged || result.filesRemoved) {
+                console.log(`[${timestamp()}] index was out of date — pushing the update`);
+                debounceTimer = setTimeout(runPush, 0);
+            }
+        } catch (err) {
+            console.error(`[${timestamp()}] index update failed:`, err.message);
+        }
+    }
+
     let shutdownResolve;
     const shutdownPromise = new Promise((resolve) => {
         shutdownResolve = resolve;