mnemonad-cli 0.2.0 → 0.3.1

package/lib/commands/buildIndex.js

@@ -0,0 +1,171 @@
+import { join } from 'node:path';
+import {
+    Indexer, TextWindowChunkingStrategy, VectorIndex, DEFAULT_DB_NAME, DEFAULT_MODEL,
+    createProvider, resolveModelName, embedderOfIndex, ModelMismatchError, MissingExtensionError,
+} from '../search/index.js';
+import { FSFolder } from '../FSFolder.js';
+import { resolvePath, DEFAULT_FS_EXCLUDES, userError } from './shared.js';
+
+/**
+ * The vector index, shared by `mnemonad index` and by `push --index` / `watch --index`.
+ * Kept in one place so all three agree on which file it is, which files it covers, and how
+ * they're chunked and embedded — an index built by `push --index` is the same index
+ * `mnemonad index` would have built.
+ *
+ * The default filename is at the folder root and not dot-prefixed: unlike `.mnemonad`, it's
+ * meant to be pushed and pulled like any other file, so a pull is searchable immediately.
+ */
+
+/** @returns {string} the index's filename (`--db`, default `search_index.db`). */
+export function indexDbName(args) {
+    return args.searchDb || DEFAULT_DB_NAME;
+}
+
+/** @returns {string[]} what the index skips: the push's own excludes, plus the index file.
+ *  The index file itself must never be walked as content to embed — explicit, not left to
+ *  chance: a SQLite file's own header/padding bytes are plausible-looking text in places,
+ *  so relying on the binary-content sniff alone to skip it would be fragile. */
+export function indexExcludes(args) {
+    const excludes = [...DEFAULT_FS_EXCLUDES, indexDbName(args)];
+    if (args.exclude) excludes.push(...args.exclude.split(',').map((s) => s.trim()));
+    return excludes;
+}
+
+function chunkingFrom(args) {
+    return new TextWindowChunkingStrategy({
+        size: args.chunkSize ?? 1000,
+        overlap: args.chunkOverlap ?? 150,
+    });
+}
+
+/**
+ * Which model to index `destPath` with. An explicit `--model` wins (an alias like `minilm`, or
+ * a full id). Without one, an existing index keeps the model it was built with — so re-indexing
+ * a folder someone indexed with MiniLM just works — and only a new index (or `--rebuild`)
+ * gets the built-in default.
+ *
+ * @returns {{model: string, embedder: ?string, dtype: ?string}}
+ */
+export function chooseModel(destPath, args) {
+    if (args.model) return { model: resolveModelName(args.model), embedder: null, dtype: null };
+    const meta = args.rebuild ? null : VectorIndex.readMeta(join(destPath, indexDbName(args)));
+    if (meta?.model) {
+        return { model: meta.model, embedder: embedderOfIndex(meta), dtype: meta.dtype || null };
+    }
+    return { model: DEFAULT_MODEL, embedder: null, dtype: null };
+}
+
+/**
+ * An Indexer configured from the CLI flags. Hold on to it across runs (as `watch --index`
+ * does) and the embedding model loads once, on first use, instead of on every run.
+ * Rejects straight away on bad chunking flags or a missing extension, before anything is
+ * read or written.
+ */
+export async function makeIndexer(destPath, args) {
+    const chunkingStrategy = chunkingFrom(args);
+    const { model, embedder, dtype } = chooseModel(destPath, args);
+    let embeddingProvider;
+    try {
+        embeddingProvider = await createProvider({ model, ...(embedder ? { embedder } : {}), dtype });
+    } catch (err) {
+        throw explainIndexError(err, destPath, args);
+    }
+    return new Indexer({ embeddingProvider, chunkingStrategy });
+}
+
+/**
+ * Builds or updates `destPath`'s index, printing the same summary `mnemonad index` does.
+ *
+ * @param {string} destPath - resolved folder path
+ * @param {Object} args - parsed CLI args (`rebuild` starts the index over)
+ * @param {Object} [opts]
+ * @param {Indexer} [opts.indexer] - reuse one (and its loaded model) across calls
+ * @param {boolean} [opts.quiet] - one summary line instead of the full report
+ * @param {boolean} [opts.rebuild] - overrides `args.rebuild` (watch rebuilds only once)
+ * @returns {Promise<{filesChanged: number, filesSkipped: number, filesRemoved: number, chunksAdded: number}>}
+ */
+export async function updateIndex(destPath, args, { indexer = null, quiet = false, rebuild = args.rebuild } = {}) {
+    indexer = indexer || await makeIndexer(destPath, { ...args, rebuild });
+    const dbName = indexDbName(args);
+    const fsFolder = new FSFolder(destPath, indexExcludes(args));
+
+    if (!quiet) {
+        console.log('  embedding model:', indexer.embeddingProvider.modelId, '(downloads and caches on first use)');
+        if (rebuild) console.log(`  rebuilding ${dbName} from scratch`);
+        console.log('  scanning for changed files...');
+    }
+    let result;
+    try {
+        result = await indexer.index(fsFolder, join(destPath, dbName), { rebuild: !!rebuild });
+    } catch (err) {
+        throw explainIndexError(err, destPath, args);
+    }
+
+    if (quiet) {
+        const changed = result.filesChanged + result.filesRemoved;
+        console.log(changed
+            ? `  index: ${result.filesChanged} file(s) re-embedded, ${result.filesRemoved} removed (${result.chunksAdded} chunks)`
+            : '  index: up to date');
+    } else {
+        console.log('  files changed:', result.filesChanged);
+        console.log('  files skipped:', result.filesSkipped, '(content unchanged since last index)');
+        console.log('  files removed:', result.filesRemoved);
+        console.log('  chunks added: ', result.chunksAdded);
+    }
+    return result;
+}
+
+/**
+ * How far `destPath`'s index has drifted from its files, without updating it — no model is
+ * loaded (not even the extension looked up) and the index is opened read-only (see
+ * Indexer.status()). Null when there's no index.
+ *
+ * @returns {Promise<?{changed: string[], added: string[], removed: string[], total: number}>}
+ */
+export async function checkIndex(destPath, args) {
+    const fsFolder = new FSFolder(destPath, indexExcludes(args));
+    // Chunking decides which files count (an empty one is never indexed), so it's configured
+    // exactly as for a real run; status() never embeds, so no provider is needed.
+    const indexer = new Indexer({ embeddingProvider: null, chunkingStrategy: chunkingFrom(args) });
+    const status = await indexer.status(fsFolder, join(destPath, indexDbName(args)));
+    if (!status) return null;
+    return { ...status, total: status.changed.length + status.added.length + status.removed.length };
+}
+
+/** Turns the two errors a user can act on into CLI messages that say what to do. */
+function explainIndexError(err, destPath, args) {
+    if (err instanceof MissingExtensionError) {
+        const lines = [err.message];
+        if (!args.model) {
+            lines.push(
+                '  Or rebuild the index with the built-in model instead:',
+                `    mnemonad index ${destPath} --rebuild`,
+            );
+        }
+        return userError(lines.join('\n'));
+    }
+    if (err instanceof ModelMismatchError) {
+        return userError(
+            `${indexDbName(args)} was built with ${err.indexModel}, but this run asked for ${err.providerModel} —\n` +
+            '  one index can\'t mix two models. Either drop --model to keep using the index\'s own model, or\n' +
+            `  add --rebuild to re-embed every file with ${err.providerModel} (the next push then uploads the\n` +
+            '  whole index again).'
+        );
+    }
+    return err;
+}
+
+/** `mnemonad index <path>` */
+export async function buildIndex(args) {
+    const destPath = resolvePath(args.path, { mustExist: true });
+    const result = await updateIndex(destPath, args);
+
+    if (result.filesChanged > 0 || result.filesRemoved > 0) {
+        // Said explicitly rather than left implicit: unlike `.mnemonad`, this file is not
+        // dot-prefixed, so the very next `push` from this folder includes it automatically —
+        // worth knowing before that push happens, not after.
+        console.log(`\n  note: ${indexDbName(args)} will be included in the next \`mnemonad push\` from this folder`);
+    } else {
+        console.log('\n  no changes — nothing new to push');
+    }
+}

package/lib/commands/burn.js

@@ -21,7 +21,7 @@ export async function burn(args) {
         throw new Error('stream-id is required for burn');
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
     const mn = new Mnemonad({

package/lib/commands/compact.js

@@ -12,6 +12,7 @@ import {
     resolveOffloadParams,
     discoverStreamId,
 } from './shared.js';
+import { settleStreamJobs } from '../jobs.js';
 
 /**
  * Monad has no rebate/refund concept — truncating a stream's history here gives back no
@@ -28,13 +29,14 @@ export async function compact(args) {
     if (!args.streamId) {
         throw new Error('stream-id is required for compact');
     }
+    await settleStreamJobs(args, { reason: 'before replacing the history with a fresh snapshot' });
 
     const fsFolder = new FSFolder(destPath, makeExcludes(args));
 
     const counts = await countTree(fsFolder);
     console.log('  scanning...', counts.files, 'files,', counts.folders, 'folders');
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
     // No `password` at construction — see shared.js's unlockStream() for why.

package/lib/commands/diff.js

@@ -12,6 +12,7 @@ import {
     folderExists,
     discoverStreamId,
 } from './shared.js';
+import { settleStreamJobs } from '../jobs.js';
 
 export function compareFileMaps(localFiles, remoteFiles) {
     const added = [];
@@ -55,8 +56,9 @@ export async function diff(args) {
     if (!args.streamId) {
         throw new Error('stream-id is required for diff');
     }
+    await settleStreamJobs(args, { wait: false });
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
 
     const mn = new Mnemonad({
         publicClient: client.publicClient,

package/lib/commands/index.js

@@ -5,3 +5,5 @@ export { diff } from './diff.js';
 export { watch } from './watch.js';
 export { compact } from './compact.js';
 export { burn } from './burn.js';
+export { buildIndex } from './buildIndex.js';
+export { search } from './search.js';

package/lib/commands/info.js

@@ -12,9 +12,10 @@ import {
     readManifest,
     streamIdsEqual,
 } from './shared.js';
+import { streamKey, latestJob, describeJob } from '../jobs.js';
 
 export async function info(args) {
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
 
     console.log('  chain:          ', args.chain);
     console.log('  your wallet:    ', client.account?.address || '(none)');
@@ -45,6 +46,14 @@ export async function info(args) {
         }
     }
 
+    // The last background push (`push --detach`) for this stream, whichever way its id is
+    // known — even with no --manifest, the folder's own manifest names the stream.
+    const jobStreamId = args.streamId || manifest?.streamId;
+    if (jobStreamId) {
+        const job = latestJob(streamKey({ chain: args.chain, contractAddress: args.contractAddress, streamId: jobStreamId }));
+        if (job) console.log('  background push:', describeJob(job));
+    }
+
     if (!args.streamId) {
         return;
     }

package/lib/commands/pull.js

@@ -11,6 +11,7 @@ import {
     formatStreamId,
     discoverStreamId,
 } from './shared.js';
+import { settleStreamJobs } from '../jobs.js';
 
 export async function pull(args) {
     // Resolved up front (throws if missing) so a missing path fails fast, before any chain
@@ -24,7 +25,11 @@ export async function pull(args) {
         throw new Error('stream-id is required for pull');
     }
 
-    const client = makeChainClient(args);
+    // Pulling while a background push of this folder is still sending would restore the
+    // version before it — and delete the very files that push is uploading.
+    await settleStreamJobs(args, { reason: 'so the pull includes it' });
+
+    const client = await makeChainClient(args);
 
     const mn = new Mnemonad({
         publicClient: client.publicClient,

package/lib/commands/push.js

@@ -18,7 +18,13 @@ import {
     formatStreamId,
     streamIdsEqual,
     discoverStreamId,
+    userError,
+    formatBytes,
+    formatPath,
 } from './shared.js';
+import { updateIndex, checkIndex, indexDbName } from './buildIndex.js';
+import { streamKey, waitForJobs, settleStreamJobs, startPushJob } from '../jobs.js';
+import { formatEther } from 'viem';
 
 export async function push(args) {
     const destPath = resolvePath(args.path, { mustExist: true });
@@ -29,6 +35,42 @@ export async function push(args) {
     await discoverStreamId(args, destPath);
     let streamId = args.streamId;
 
+    if (args.detach) {
+        // Creating a stream prints the id the user has to keep, and needs signatures that have
+        // to be seen to succeed — nothing worth handing off. --passkey's signer is a browser
+        // tab that closes when this process exits.
+        if (!streamId) {
+            throw userError(
+                '--detach only pushes to an existing stream. Pass its id, or --manifest on a folder\n' +
+                '  that has a .mnemonad, or push once without --detach to create the stream first.'
+            );
+        }
+        if (args.passkey) {
+            throw userError('--detach can\'t be combined with --passkey: the browser tab signing for it closes when this command exits.');
+        }
+    }
+
+    // A background push still sending to this stream means its next version isn't on chain
+    // yet — building on top of the current tip now would only lose the race to it.
+    await settleStreamJobs(args, { reason: 'before building the next version on top of it' });
+
+    // Before anything is scanned or signed: the push below then picks the updated index up
+    // like any other file. A failure stops the push outright — a push that silently went out
+    // without the index asked for would leave the stream unsearchable with no sign of why.
+    if (args.rebuild && !args.index) {
+        throw userError('--rebuild only applies with --index (it starts the search index over)');
+    }
+    if (args.index) {
+        console.log(args.rebuild ? '  rebuilding search index...' : '  updating search index...');
+        try {
+            await updateIndex(destPath, args, { quiet: true });
+        } catch (err) {
+            throw userError(`indexing failed, so nothing was pushed: ${err.message}`);
+        }
+    } else {
+        await warnIfIndexStale(destPath, args);
+    }
+
     const fsFolder = new FSFolder(destPath, makeExcludes(args));
 
     const counts = await countTree(fsFolder);
@@ -45,9 +87,15 @@ export async function push(args) {
         }
     }
 
-    const client = makeChainClient(args);
+    const client = await makeChainClient(args);
     requireWalletClient(client);
 
+    // Same wallet, different stream: two processes sending at once would race for one nonce.
+    await waitForJobs(
+        { signer: client.account.address, chain: args.chain },
+        { reason: 'so the two pushes don\'t race for the same wallet nonce' },
+    );
+
     const baseParams = {
         publicClient: client.publicClient,
         walletClient: client.walletClient,
@@ -120,6 +168,11 @@ export async function push(args) {
         }
     }
 
+    if (args.detach) {
+        await detachPush({ args, mn, monadSync, fsFolder, client, streamId, destPath, existingVersions });
+        return;
+    }
+
     console.log('  building patch + pushing to chain...');
     const result = await monadSync.push(fsFolder);
 
@@ -136,3 +189,101 @@ export async function push(args) {
         await writeManifest(destPath, { streamId: formatStreamId(streamId), version: result.version, files: localFiles });
     }
 }
+
+/**
+ * `push --detach`: everything up to a finished patch runs here, so every error a user can act
+ * on still surfaces before this command exits. Only the sending — transactions, receipts, an
+ * IPFS upload — goes to a detached child (see lib/jobs.js), and this process exits as soon
+ * as it has started.
+ *
+ * The patch is frozen now: files edited after this command returns go into the next push,
+ * not this one. The manifest's file hashes are taken at the same moment for the same reason.
+ */
+async function detachPush({ args, mn, monadSync, fsFolder, client, streamId, destPath, existingVersions }) {
+    console.log('  building patch...');
+    const prepared = await monadSync.preparePush(fsFolder);
+
+    // A single-item patch is the common case and the only one estimatePushCost models; a
+    // segmented one (over 8 MB) just goes without an estimate rather than a wrong one.
+    let cost = null;
+    if (prepared.itemCount === 1) {
+        try {
+            cost = await mn.estimatePushCost(prepared.patchBytes);
+        } catch {
+            // No estimate is not a reason to stop — the child surfaces any real send error.
+        }
+    }
+    if (cost) {
+        const balance = await client.publicClient.getBalance({ address: client.account.address });
+        if (balance < cost.total) {
+            throw userError(
+                `not enough MON to push: about ${formatEther(cost.total)} needed, ` +
+                `${formatEther(balance)} in ${client.account.address}. Nothing was sent.`
+            );
+        }
+    }
+
+    const files = args.manifest ? await hashLocalTree(fsFolder) : null;
+    const job = await startPushJob(
+        {
+            streamId: formatStreamId(streamId),
+            streamKey: streamKey({ chain: args.chain, contractAddress: args.contractAddress, streamId }),
+            chain: args.chain,
+            signer: client.account.address,
+            folder: destPath,
+            baseLength: prepared.baseLength,
+            version: prepared.version,
+            kind: prepared.kind,
+            bytes: prepared.patchBytes.length,
+            manifest: files ? { files } : null,
+            // Everything the child needs to rebuild the same chain client — never a secret.
+            args: {
+                chain: args.chain,
+                rpcUrl: args.rpcUrl,
+                contractAddress: args.contractAddress,
+                gatewayUrl: args.gatewayUrl,
+                presignUrl: args.presignUrl,
+            },
+        },
+        prepared.patchBytes,
+        {
+            key: args.key || process.env.MNEMONAD_KEY || null,
+            phrase: args.phrase,
+            password: args.password,
+            pinataJwt: args.pinataJwt,
+        },
+    );
+
+    const patchType = existingVersions === 0 || prepared.kind === 'patch' ? 'full snapshot' : 'diff';
+    const compressedNote = args.compress !== false ? ', gzip compressed' : '';
+    console.log('  version', prepared.version, 'prepared (' + patchType + compressedNote + ', ' + formatBytes(prepared.patchBytes.length) + ')');
+    if (cost) console.log('  estimated cost:', formatEther(cost.total), 'MON');
+    console.log('  stream:', formatStreamId(streamId));
+    console.log('  sending in the background: job', job.id, '(pid ' + job.pid + ')');
+    console.log('  log:  ', formatPath(job.dir + '/push.log'));
+    console.log('  check: mnemonad info', formatPath(destPath) + (args.manifest ? ' --manifest' : ''));
+}
+
+/**
+ * A folder that already carries an index, pushed without `--index` after its files changed,
+ * would publish an index that no longer matches them — searches keep returning the old text's
+ * offsets. Cheap to catch (a hash per file, no model load, the index opened read-only) and
+ * worth a line; never a reason to fail the push itself.
+ */
+async function warnIfIndexStale(destPath, args) {
+    let status;
+    try {
+        status = await checkIndex(destPath, args);
+    } catch {
+        return;
+    }
+    if (!status || status.total === 0) return;
+    const parts = [];
+    if (status.changed.length) parts.push(`${status.changed.length} changed`);
+    if (status.added.length) parts.push(`${status.added.length} new`);
+    if (status.removed.length) parts.push(`${status.removed.length} removed`);
+    console.log(
+        `  note: ${indexDbName(args)} is out of date (${parts.join(', ')} file(s) since it was built).\n` +
+        '        Push with --index to update it first, or run `mnemonad index <path>`.'
+    );
+}

package/lib/commands/search.js

@@ -0,0 +1,91 @@
+import { join } from 'node:path';
+import {
+    Searcher, VectorIndex, DEFAULT_DB_NAME, createProvider, resolveModelName, MissingExtensionError,
+} from '../search/index.js';
+import { resolvePath, folderExists } from './shared.js';
+
+export async function search(args) {
+    if (!args.query) {
+        throw userError('a search query is required (e.g. `mnemonad search /path/to/folder "how does chunking work"`)');
+    }
+    const destPath = resolvePath(args.path, { mustExist: true });
+    const dbName = args.searchDb || DEFAULT_DB_NAME;
+    const dbPath = join(destPath, dbName);
+
+    if (!folderExists(destPath) || !(await _fileExists(dbPath))) {
+        throw userError(
+            `no ${dbName} found in ${destPath}\n` +
+            '  Run `mnemonad index <path>` first — either yours, or whoever pushed this stream\'s\n' +
+            '  (a pull already brings theirs down with the rest of the folder).'
+        );
+    }
+
+    // The query has to be embedded by the model that built the index — which the index records,
+    // so there's nothing to choose. An explicit --model that disagrees is a mistake worth
+    // naming rather than a preference to honour.
+    const indexModel = VectorIndex.readMeta(dbPath)?.model;
+    if (args.model && indexModel && resolveModelName(args.model) !== indexModel) {
+        throw userError(
+            `${dbName} was built with ${indexModel}; a search has to use the same model — drop --model\n` +
+            '  (search always uses the index\'s own), or rebuild the index with the other one:\n' +
+            `    mnemonad index ${destPath} --model ${args.model} --rebuild`
+        );
+    }
+
+    const searcher = new Searcher({ createProvider });
+    const k = args.limit ? Number(args.limit) : 5;
+    let hits;
+    try {
+        hits = await searcher.search(dbPath, args.query, k);
+    } catch (err) {
+        if (err instanceof MissingExtensionError) {
+            throw userError(
+                `this folder's index was built with ${err.model}, which ${err.message.slice(err.message.indexOf('needs'))}\n` +
+                '  Or rebuild the index with the built-in model:\n' +
+                `    mnemonad index ${destPath} --rebuild`
+            );
+        }
+        throw err;
+    }
+
+    if (hits.length === 0) {
+        console.log('  no matches');
+        return;
+    }
+
+    for (const [i, hit] of hits.entries()) {
+        console.log(`\n${i + 1}. ${hit.path} (chunk ${hit.chunkIndex}, distance ${hit.distance.toFixed(4)})`);
+        const snippet = await _readSnippet(destPath, hit);
+        if (snippet) console.log('   ' + snippet.replace(/\s+/g, ' ').slice(0, 200));
+    }
+}
+
+async function _fileExists(path) {
+    try {
+        const { stat } = await import('node:fs/promises');
+        const s = await stat(path);
+        return s.isFile();
+    } catch {
+        return false;
+    }
+}
+
+/** Re-reads the hit's snippet straight from the already-pulled source file — the index
+ *  deliberately doesn't store chunk text itself (see ../search/VectorIndex.js),
+ *  so this is the one place that cost is paid, and only for hits actually shown. */
+async function _readSnippet(destPath, hit) {
+    try {
+        const { readFile } = await import('node:fs/promises');
+        const bytes = await readFile(join(destPath, hit.path));
+        const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+        return text.slice(hit.startOffset, hit.endOffset);
+    } catch {
+        return null;
+    }
+}
+
+function userError(message) {
+    const err = new Error(message);
+    err._isUserError = true;
+    return err;
+}

package/lib/commands/shared.js

@@ -12,7 +12,7 @@ import { DoubleSync as SyncEngine, CDCStore, DoubleSyncSnapshot as SyncSnapshot
 import { FSFolder } from '../FSFolder.js';
 import { createPresignProvider } from '../presignProvider.js';
 
-const DEFAULT_FS_EXCLUDES = ['node_modules', '.git', '.env', '.DS_Store', '.mnemonad', '.claude', 'pnpm-lock.yaml', 'package-lock.json'];
+export const DEFAULT_FS_EXCLUDES = ['node_modules', '.git', '.env', '.DS_Store', '.mnemonad', '.claude', 'pnpm-lock.yaml', 'package-lock.json'];
 
 export function makeExcludes(args) {
     if (!args.exclude) return undefined;
@@ -305,7 +305,8 @@ export async function unlockStream(mn, credentials = {}) {
     return true;
 }
 
-function userError(message) {
+/** An error the CLI prints as a plain message (no stack trace) — see bin/mnemonad.js. */
+export function userError(message) {
     const err = new Error(message);
     err._isUserError = true;
     return err;