mnemonad-cli 0.3.1 → 0.3.2

package/README.md

@@ -118,7 +118,7 @@ Every command prints the installed CLI version as its first line of output.
 | `--pinata-jwt <jwt>` | Pinata JWT for IPFS offload of large files (or `PINATA_JWT` env var) |
 | `--manifest` | Write/use `.mnemonad` manifest for faster change detection. With no stream-id given, every command recovers it from an existing `.mnemonad` in the folder instead of requiring it explicitly |
 | `--force-snapshot` | Push a full snapshot regardless of prior history (repairs a corrupt stream) |
-| `--detach` | `push`: build the next version here, send it from a background process, and return right away. Existing streams only. See "Push in the background" below |
+| `--detach` | `push`: build the next version here, send it from a background process, and return right away. With `--index`, indexing and building move to the background too. Existing streams only. See "Push in the background" below |
 | `--index` | `push`/`watch`: update the folder's search index before every push (takes `--db`/`--model`/`--chunk-size`/`--chunk-overlap`); if indexing fails, nothing is pushed |
 | `--poll-interval <s>` | `watch`: seconds between remote version checks (default: `2`) |
 | `--debounce <ms>` | `watch`: quiet period in ms before pushing after a local change (default: `1000`) |
@@ -231,7 +231,7 @@ mnemonad pull 123 ~/restored --version 1
 ### Inspect a stream
 
 ```bash
-# check your wallet + chain (no stream id needed)
+# check your wallet, its MON balance, and the chain (no stream id needed)
 mnemonad info
 
 # full stream metadata + local sync status
@@ -241,6 +241,7 @@ mnemonad info 123 ~/my-data
 ```
   chain:           testnet
   your wallet:     0x06bc6420b37a4898424429dcfa236f0065e12279
+  balance:         4.2183 MON
 
 stream: 0x68d9736257f0d2da3c848666a4b36c4b9f18eda7e4dfba655599c47560d8a4b5
   owner:           0x06bc6420b37a4898424429dcfa236f0065e12279 (you)
@@ -389,6 +390,27 @@ mnemonad push ~/my-data --manifest --detach
   IPFS upload, can't be resumed. A paid but unused upload can be refunded after the gateway's
   time limit, with the SDK's `MnemonadUploadGateway.refund()` (the CLI has no command for it).
 
+**With `--index`, indexing goes to the background too.** The updated index is part of the
+version being pushed, so the version can only be built once indexing is done, and sent once
+it is built. `push --detach --index` therefore hands off all three, and returns after only the
+checks that need no folder scan: the signer owns the stream, the credential unlocks it, and the
+wallet isn't empty.
+
+```bash
+mnemonad push ~/my-data --manifest --index --detach
+#   stream: 0x68d9…
+#   indexing, building and sending in the background: job 3b9e07c1 (pid 41388)
+```
+
+The trade-offs, compared with a plain `--detach`:
+
+- No version number, size or cost estimate is printed. `mnemonad info` shows them once the job
+  has built the version, and shows "no changes, nothing pushed" if the folder already matched.
+- An indexing error, a model download that fails, and "no changes" show up in the job's status,
+  not in the terminal.
+- The folder is not frozen when the command runs. The job reads it when it starts building, so
+  edits made before then are part of this version.
+
 Jobs live in `~/.cache/mnemonad/jobs/` (set `MNEMONAD_JOBS_DIR` to move them). Finished ones are
 removed after 14 days.
 

package/bin/mnemonad.js

@@ -108,7 +108,8 @@ Options:
   --detach                  push: build the next version here, send it from a background process,
                             and return right away. Existing streams only (an id, or --manifest).
                             Later pushes and pulls of that stream wait for it; \`mnemonad info\`
-                            shows how it went
+                            shows how it went. With --index, indexing and building the version
+                            move to the background too
   --index                   push/watch: update the folder's search index (search_index.db)
                             before every push, so the stream is searchable right after a pull.
                             Takes --db/--model/--chunk-size/--chunk-overlap like \`index\`

package/lib/commands/info.js

@@ -1,5 +1,6 @@
 import Mnemonad, { kekMethodOf } from 'mnemonad';
 import { MonadSync } from 'monadsync';
+import { formatEther } from 'viem';
 import { makeChainClient } from '../chainClient.js';
 import {
     resolvePath,
@@ -19,6 +20,9 @@ export async function info(args) {
 
     console.log('  chain:          ', args.chain);
     console.log('  your wallet:    ', client.account?.address || '(none)');
+    if (client.account) {
+        console.log('  balance:        ', await walletBalance(client));
+    }
 
     // Peek at .mnemonad whenever a path is given, regardless of --manifest and regardless
     // of whether an id was given — info is read-only, so surfacing what's there costs
@@ -170,3 +174,18 @@ export async function info(args) {
         console.log('  latest remote:   version', totalVersions);
     }
 }
+
+/**
+ * The signing wallet's native balance, in the chain's own unit (MON on Monad, ETH on a local
+ * Hardhat node). A failed RPC read is shown as such rather than failing `info` — the balance
+ * is a convenience next to the stream report, not something the rest depends on.
+ */
+async function walletBalance(client) {
+    const symbol = client.chain.nativeCurrency?.symbol || 'MON';
+    try {
+        const wei = await client.publicClient.getBalance({ address: client.account.address });
+        return `${formatEther(wei)} ${symbol}`;
+    } catch {
+        return '(could not read)';
+    }
+}

package/lib/commands/push.js

@@ -60,6 +60,10 @@ export async function push(args) {
     if (args.rebuild && !args.index) {
         throw userError('--rebuild only applies with --index (it starts the search index over)');
     }
+    if (args.detach && args.index) {
+        await detachPrepare(args, destPath, streamId);
+        return;
+    }
     if (args.index) {
         console.log(args.rebuild ? '  rebuilding search index...' : '  updating search index...');
         try {
@@ -190,6 +194,85 @@ export async function push(args) {
     }
 }
 
+/**
+ * `push --detach --index`: indexing can't run here and still be "detached" — the updated index
+ * is part of the version being pushed, so building the patch has to wait for it, and the
+ * sending waits for the patch. So this hands off all three, and keeps only the checks that
+ * need no folder scan and no chain replay: the signer owns the stream, the credential unlocks
+ * it, and the wallet isn't empty. Everything else — "no changes", an indexing error, the
+ * version number — shows up in the job (`mnemonad info`).
+ *
+ * Unlike a plain `--detach`, the folder isn't frozen here: the job reads it when its own turn
+ * comes, so edits made before then go into this version.
+ */
+async function detachPrepare(args, destPath, streamId) {
+    const client = await makeChainClient(args);
+    requireWalletClient(client);
+    await waitForJobs(
+        { signer: client.account.address, chain: args.chain },
+        { reason: 'so the two pushes don\'t race for the same wallet nonce' },
+    );
+
+    // Flag combinations are checked here too, not left for the job to trip over.
+    resolveEncryptMethod(args);
+
+    const mn = new Mnemonad({
+        publicClient: client.publicClient,
+        walletClient: client.walletClient,
+        contractAddress: client.contractAddress,
+        id: streamId,
+        ...resolveOffloadParams(args, client.account),
+    });
+    await assertStreamOwner(mn, client.account.address);
+    await unlockStream(mn, { password: args.password, signer: client.walletClient });
+
+    const balance = await client.publicClient.getBalance({ address: client.account.address });
+    if (balance === 0n) {
+        throw userError(`${client.account.address} has no MON to pay for the push. Nothing was started.`);
+    }
+
+    const job = await startPushJob(
+        {
+            mode: 'prepare',
+            streamId: formatStreamId(streamId),
+            streamKey: streamKey({ chain: args.chain, contractAddress: args.contractAddress, streamId }),
+            chain: args.chain,
+            signer: client.account.address,
+            folder: destPath,
+            baseLength: null,
+            version: null,
+            manifest: args.manifest ? {} : null,
+            args: {
+                chain: args.chain,
+                rpcUrl: args.rpcUrl,
+                contractAddress: args.contractAddress,
+                gatewayUrl: args.gatewayUrl,
+                presignUrl: args.presignUrl,
+                exclude: args.exclude,
+                compress: args.compress,
+                forceSnapshot: args.forceSnapshot,
+                searchDb: args.searchDb,
+                model: args.model,
+                chunkSize: args.chunkSize,
+                chunkOverlap: args.chunkOverlap,
+                rebuild: args.rebuild,
+            },
+        },
+        null,
+        {
+            key: args.key || process.env.MNEMONAD_KEY || null,
+            phrase: args.phrase,
+            password: args.password,
+            pinataJwt: args.pinataJwt,
+        },
+    );
+
+    console.log('  stream:', formatStreamId(streamId));
+    console.log('  indexing, building and sending in the background: job', job.id, '(pid ' + job.pid + ')');
+    console.log('  log:  ', formatPath(job.dir + '/push.log'));
+    console.log('  check: mnemonad info', formatPath(destPath) + (args.manifest ? ' --manifest' : ''));
+}
+
 /**
  * `push --detach`: everything up to a finished patch runs here, so every error a user can act
  * on still surfaces before this command exits. Only the sending — transactions, receipts, an

package/lib/jobs.js

@@ -29,22 +29,36 @@ import { fileURLToPath } from 'node:url';
 import Mnemonad from 'mnemonad';
 import { MonadSync } from 'monadsync';
 import { makeChainClient, requireWalletClient } from './chainClient.js';
+import { FSFolder } from './FSFolder.js';
 import {
     formatStreamId,
     formatPath,
     unlockStream,
+    assertStreamOwner,
     resolveOffloadParams,
     writeManifest,
+    makeExcludes,
+    hashLocalTree,
+    localTreeHash,
     userError,
 } from './commands/shared.js';
+import { updateIndex } from './commands/buildIndex.js';
 
 const BIN_PATH = join(dirname(fileURLToPath(import.meta.url)), '..', 'bin', 'mnemonad.js');
 
 /** pending: written, child not started yet · waiting: queued behind an earlier job from the
- *  same signer · sending: transactions in flight · done / failed: finished. */
-export const JOB_STATUS = { PENDING: 'pending', WAITING: 'waiting', SENDING: 'sending', DONE: 'done', FAILED: 'failed' };
+ *  same signer · preparing: indexing and building the patch (`push --detach --index` only) ·
+ *  sending: transactions in flight · done / failed: finished. */
+export const JOB_STATUS = {
+    PENDING: 'pending', WAITING: 'waiting', PREPARING: 'preparing', SENDING: 'sending', DONE: 'done', FAILED: 'failed',
+};
 
-const ACTIVE = new Set([JOB_STATUS.PENDING, JOB_STATUS.WAITING, JOB_STATUS.SENDING]);
+const ACTIVE = new Set([JOB_STATUS.PENDING, JOB_STATUS.WAITING, JOB_STATUS.PREPARING, JOB_STATUS.SENDING]);
+
+/** "version 8", or what a job that hasn't built its version yet is doing instead. */
+function versionLabel(job) {
+    return job.version != null ? `version ${job.version}` : 'the next version';
+}
 
 /** A job still `pending` with no pid after this long never got its child started. */
 const SPAWN_GRACE_MS = 60_000;
@@ -176,7 +190,7 @@ export async function waitForJobs(selector, { reason = 'before continuing' } = {
 
     const job = first[first.length - 1];
     console.log(
-        `  waiting for background push ${job.id} (version ${job.version}) to finish ${reason}...\n` +
+        `  waiting for background push ${job.id} (${versionLabel(job)}) to finish ${reason}...\n` +
         `  log: ${formatPath(join(job.dir, 'push.log'))}`
     );
     while (first.length) {
@@ -195,7 +209,7 @@ export function reportFailedJob(key) {
     const job = latestJob(key);
     if (!job || job.status !== JOB_STATUS.FAILED || job.reported) return;
     console.log(
-        `  warning: background push ${job.id} (version ${job.version}) failed: ${job.error}\n` +
+        `  warning: background push ${job.id} (${versionLabel(job)}) failed: ${job.error}\n` +
         `           nothing from it reached the stream. Log: ${formatPath(join(job.dir, 'push.log'))}`
     );
     writeJobSync(job.dir, { ...readJobSync(job.dir), reported: true });
@@ -224,7 +238,7 @@ export async function settleStreamJobs(args, { wait = true, reason } = {}) {
     if (running.length) {
         const job = running[running.length - 1];
         console.log(
-            `  note: background push ${job.id} (version ${job.version}) is still sending —\n` +
+            `  note: background push ${job.id} (${versionLabel(job)}) is still running —\n` +
             '        this compares against the chain as it is before that version lands'
         );
     }
@@ -235,13 +249,17 @@ export function describeJob(job) {
     const log = formatPath(join(job.dir, 'push.log'));
     switch (job.status) {
         case JOB_STATUS.DONE:
-            return `version ${job.version} sent ${job.finishedAt} (job ${job.id})`;
+            return job.noChanges
+                ? `no changes, nothing pushed ${job.finishedAt} (job ${job.id})`
+                : `version ${job.version} sent ${job.finishedAt} (job ${job.id})`;
         case JOB_STATUS.FAILED:
-            return `version ${job.version} FAILED: ${job.error} (job ${job.id}, log: ${log})`;
+            return `${versionLabel(job)} FAILED: ${job.error} (job ${job.id}, log: ${log})`;
         case JOB_STATUS.WAITING:
-            return `version ${job.version} queued behind another push from the same wallet (job ${job.id}, log: ${log})`;
+            return `${versionLabel(job)} queued behind another push from the same wallet (job ${job.id}, log: ${log})`;
+        case JOB_STATUS.PREPARING:
+            return `indexing and building the next version (job ${job.id}, pid ${job.pid ?? '?'}, log: ${log})`;
         default:
-            return `version ${job.version} still sending (job ${job.id}, pid ${job.pid ?? '?'}, log: ${log})`;
+            return `${versionLabel(job)} still sending (job ${job.id}, pid ${job.pid ?? '?'}, log: ${log})`;
     }
 }
 
@@ -257,8 +275,9 @@ function pruneFinishedJobs() {
 /**
  * Writes a job and starts its detached child.
  *
- * @param {Object} job - everything the child needs except secrets; see push.js's detachPush
- * @param {Uint8Array} patchBytes
+ * @param {Object} job - everything the child needs except secrets; see push.js's detachPush.
+ *   `mode: 'prepare'` (push.js's detachPrepare) has the child index and build the patch too.
+ * @param {?Uint8Array} patchBytes - the prepared patch; null in `prepare` mode
  * @param {{key?: ?string, phrase?: ?string, password?: ?string, pinataJwt?: ?string}} secrets
  * @returns {Promise<{id: string, dir: string, pid: number}>}
  */
@@ -268,7 +287,7 @@ export async function startPushJob(job, patchBytes, secrets) {
     const id = randomBytes(4).toString('hex');
     const dir = join(jobsRoot(), id);
     await mkdir(dir, { recursive: true });
-    await writeFile(join(dir, 'patch.bin'), patchBytes);
+    if (patchBytes) await writeFile(join(dir, 'patch.bin'), patchBytes);
     writeJobSync(dir, { ...job, id, status: JOB_STATUS.PENDING, pid: null, createdAt: new Date().toISOString() });
 
     const logFd = openSync(join(dir, 'push.log'), 'a');
@@ -331,9 +350,6 @@ export async function runPushJob(dir) {
             while (earlier().length) await new Promise((r) => setTimeout(r, POLL_MS));
         }
 
-        update({ status: JOB_STATUS.SENDING, startedAt: new Date().toISOString() });
-        log(`sending version ${job.version} of ${job.streamId} on ${job.chain}...`);
-
         const args = { ...job.args, key: secrets.key ?? null, phrase: secrets.phrase ?? null, password: secrets.password ?? null, pinataJwt: secrets.pinataJwt ?? null, passkey: false };
         const client = await makeChainClient(args);
         requireWalletClient(client);
@@ -347,11 +363,31 @@ export async function runPushJob(dir) {
         });
         await unlockStream(mn, { password: args.password, signer: client.walletClient });
 
-        const patchBytes = new Uint8Array(await readFile(join(dir, 'patch.bin')));
-        const result = await new MonadSync({ mnemonad: mn }).commitPush({ patchBytes, baseLength: job.baseLength });
+        let sync;
+        let prepared;
+        let files = job.manifest?.files || null;
+        if (job.mode === 'prepare') {
+            update({ status: JOB_STATUS.PREPARING, startedAt: new Date().toISOString() });
+            await assertStreamOwner(mn, client.account.address);
+            const built = await prepareInBackground(job, args, mn);
+            if (!built) {
+                update({ status: JOB_STATUS.DONE, noChanges: true, version: mn.length, finishedAt: new Date().toISOString() });
+                log('no changes, nothing to push');
+                return;
+            }
+            ({ sync, prepared, files } = built);
+            update({ version: prepared.version, baseLength: prepared.baseLength, kind: prepared.kind, bytes: prepared.patchBytes.length });
+        } else {
+            sync = new MonadSync({ mnemonad: mn });
+            prepared = { patchBytes: new Uint8Array(await readFile(join(dir, 'patch.bin'))), baseLength: job.baseLength };
+        }
+
+        update({ status: JOB_STATUS.SENDING, sendingAt: new Date().toISOString() });
+        log(`sending version ${job.version} of ${job.streamId} on ${job.chain}...`);
+        const result = await sync.commitPush(prepared);
 
         if (job.manifest) {
-            await writeManifest(job.folder, { streamId: job.streamId, version: result.version, files: job.manifest.files });
+            await writeManifest(job.folder, { streamId: job.streamId, version: result.version, files });
         }
         update({ status: JOB_STATUS.DONE, version: result.version, finishedAt: new Date().toISOString() });
         log(`version ${result.version} pushed`);
@@ -367,6 +403,49 @@ export async function runPushJob(dir) {
     }
 }
 
+/**
+ * The part of a foreground push that `push --detach --index` hands off along with the sending:
+ * update the search index, replay the chain, and build the patch — same steps, same order, as
+ * push.js. Runs after any earlier job from the same wallet has landed, so it builds on the
+ * real tip.
+ *
+ * @returns {Promise<?{sync: MonadSync, prepared: Object, files: ?Object}>} null when the folder
+ *   already matches the latest version
+ */
+async function prepareInBackground(job, args, mn) {
+    log('updating search index...');
+    try {
+        await updateIndex(job.folder, args, { quiet: true });
+    } catch (err) {
+        throw new Error(`indexing failed, so nothing was pushed: ${err.message}`);
+    }
+
+    const fsFolder = new FSFolder(job.folder, makeExcludes(args));
+    const sync = new MonadSync({ mnemonad: mn, compress: args.compress === false ? false : 'gzip' });
+
+    if (args.forceSnapshot) {
+        // Same escape hatch push.js uses — see monadsync/README.md "Repairing a corrupt chain".
+        await mn.initialize();
+        sync._isInitialized = true;
+        sync._lastSnapshot = null;
+        sync._replayedCount = mn.length;
+    } else {
+        await sync.initialize();
+        if (mn.length > 0) {
+            const localHash = await localTreeHash(job.folder);
+            const remoteHash = await sync.getTreeHash(mn.length);
+            if (localHash && remoteHash.length === localHash.length && localHash.every((b, i) => b === remoteHash[i])) {
+                return null;
+            }
+        }
+    }
+
+    log('building patch...');
+    const prepared = await sync.preparePush(fsFolder);
+    const files = job.manifest ? await hashLocalTree(fsFolder) : null;
+    return { sync, prepared, files };
+}
+
 /** For tests: blocks until a job has finished, and returns it. */
 export async function waitForJob(dir, { timeoutMs = 60_000 } = {}) {
     const deadline = Date.now() + timeoutMs;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mnemonad-cli",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "CLI to sync local folders to versioned, diffed on-chain streams on Monad, backed by Mnemonad + monadsync.",
   "repository": {
     "type": "git",