mnemonad-cli 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +173 -19
- package/bin/mnemonad.js +164 -11
- package/lib/bridgeCodec.js +12 -0
- package/lib/chainClient.js +28 -7
- package/lib/commands/buildIndex.js +171 -0
- package/lib/commands/burn.js +59 -0
- package/lib/commands/compact.js +7 -2
- package/lib/commands/diff.js +25 -6
- package/lib/commands/index.js +3 -0
- package/lib/commands/info.js +42 -2
- package/lib/commands/pull.js +13 -6
- package/lib/commands/push.js +61 -7
- package/lib/commands/search.js +91 -0
- package/lib/commands/shared.js +137 -11
- package/lib/commands/watch.js +66 -12
- package/lib/passkeyBridge.js +203 -0
- package/lib/search/Indexer.js +204 -0
- package/lib/search/Searcher.js +68 -0
- package/lib/search/VectorIndex.js +225 -0
- package/lib/search/browser/BrowserSearcher.js +131 -0
- package/lib/search/browser/WasmIndexReader.js +93 -0
- package/lib/search/browser.js +13 -0
- package/lib/search/chunking/ChunkingStrategy.js +23 -0
- package/lib/search/chunking/TextWindowChunkingStrategy.js +69 -0
- package/lib/search/embeddings/EmbeddingProvider.js +40 -0
- package/lib/search/embeddings/StaticEmbeddingProvider.js +180 -0
- package/lib/search/embeddings/modelFiles.browser.js +28 -0
- package/lib/search/embeddings/modelFiles.node.js +41 -0
- package/lib/search/embeddings/modelFiles.shared.js +44 -0
- package/lib/search/index.js +18 -0
- package/lib/search/providers.js +153 -0
- package/lib/search/schema.js +103 -0
- package/mnemonad.config.js +19 -5
- package/package.json +18 -3
package/lib/chainClient.js
CHANGED
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
import { createPublicClient, createWalletClient, defineChain, http } from 'viem';
|
|
9
9
|
import { privateKeyToAccount, mnemonicToAccount } from 'viem/accounts';
|
|
10
10
|
import { monad, monadTestnet } from 'viem/chains';
|
|
11
|
+
import { openPasskeyBridge } from './passkeyBridge.js';
|
|
12
|
+
import { userError } from './commands/shared.js';
|
|
11
13
|
|
|
12
14
|
/**
|
|
13
15
|
* CLI `--chain` name -> { viem chain object, Mnemonad network shorthand for
|
|
@@ -45,10 +47,20 @@ function normalizeKey(key) {
|
|
|
45
47
|
return key.startsWith('0x') ? key : `0x${key}`;
|
|
46
48
|
}
|
|
47
49
|
|
|
48
|
-
|
|
50
|
+
/**
|
|
51
|
+
* `--key`/`--phrase`/`MNEMONAD_KEY` resolve synchronously; `--passkey` doesn't (it opens a
|
|
52
|
+
* browser tab and waits for a human to complete a WebAuthn prompt) — see
|
|
53
|
+
* passkeyBridge.js. That's the one thing that makes this function, and therefore
|
|
54
|
+
* `makeChainClient`, async rather than the plain synchronous lookup it used to be.
|
|
55
|
+
*/
|
|
56
|
+
async function resolveAccount(args) {
|
|
49
57
|
const key = args.key || process.env.MNEMONAD_KEY;
|
|
58
|
+
if (args.passkey && (key || args.phrase)) {
|
|
59
|
+
throw userError('--passkey cannot be combined with --key/--phrase/MNEMONAD_KEY — pick one signing method.');
|
|
60
|
+
}
|
|
50
61
|
if (key) return privateKeyToAccount(normalizeKey(key));
|
|
51
62
|
if (args.phrase) return mnemonicToAccount(args.phrase);
|
|
63
|
+
if (args.passkey) return openPasskeyBridge(args);
|
|
52
64
|
return null;
|
|
53
65
|
}
|
|
54
66
|
|
|
@@ -60,16 +72,17 @@ function resolveAccount(args) {
|
|
|
60
72
|
* `requireWalletClient` themselves, so the "no key" error only surfaces where it's
|
|
61
73
|
* actually required.
|
|
62
74
|
*
|
|
63
|
-
* @param {Object} args - parsed CLI args: `chain`, `rpcUrl`, `key`, `phrase`, `
|
|
64
|
-
*
|
|
75
|
+
* @param {Object} args - parsed CLI args: `chain`, `rpcUrl`, `key`, `phrase`, `passkey`,
|
|
76
|
+
* `authOrigin`, `contractAddress`
|
|
77
|
+
* @returns {Promise<{
|
|
65
78
|
* publicClient: Object,
|
|
66
79
|
* walletClient: ?Object,
|
|
67
80
|
* account: ?Object,
|
|
68
81
|
* contractAddress: string,
|
|
69
82
|
* chain: Object,
|
|
70
|
-
* }}
|
|
83
|
+
* }>}
|
|
71
84
|
*/
|
|
72
|
-
export function makeChainClient(args) {
|
|
85
|
+
export async function makeChainClient(args) {
|
|
73
86
|
let chain, contractAddress;
|
|
74
87
|
if (args.chain === 'local') {
|
|
75
88
|
// Dev/test only — see localChain() above.
|
|
@@ -95,7 +108,7 @@ export function makeChainClient(args) {
|
|
|
95
108
|
const transport = http(args.rpcUrl);
|
|
96
109
|
const publicClient = createPublicClient({ chain, transport });
|
|
97
110
|
|
|
98
|
-
const account = resolveAccount(args);
|
|
111
|
+
const account = await resolveAccount(args);
|
|
99
112
|
const walletClient = account ? createWalletClient({ chain, transport, account }) : null;
|
|
100
113
|
|
|
101
114
|
return { publicClient, walletClient, account, contractAddress, chain };
|
|
@@ -107,6 +120,14 @@ export function makeChainClient(args) {
|
|
|
107
120
|
*/
|
|
108
121
|
export function requireWalletClient(client) {
|
|
109
122
|
if (!client.walletClient) {
|
|
110
|
-
|
|
123
|
+
// Marked as a user error so the CLI boundary prints the message alone: this is the
|
|
124
|
+
// first thing anyone hits before configuring a key, and a stack trace through
|
|
125
|
+
// chainClient makes an actionable one-liner look like a crash.
|
|
126
|
+
const err = new Error(
|
|
127
|
+
'No signing key. Set the MNEMONAD_KEY env var to a private key, pass --key / --phrase, ' +
|
|
128
|
+
'or pass --passkey to sign in with a passkey via the browser.'
|
|
129
|
+
);
|
|
130
|
+
err._isUserError = true;
|
|
131
|
+
throw err;
|
|
111
132
|
}
|
|
112
133
|
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import {
|
|
3
|
+
Indexer, TextWindowChunkingStrategy, VectorIndex, DEFAULT_DB_NAME, DEFAULT_MODEL,
|
|
4
|
+
createProvider, resolveModelName, embedderOfIndex, ModelMismatchError, MissingExtensionError,
|
|
5
|
+
} from '../search/index.js';
|
|
6
|
+
import { FSFolder } from '../FSFolder.js';
|
|
7
|
+
import { resolvePath, DEFAULT_FS_EXCLUDES, userError } from './shared.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The vector index, shared by `mnemonad index` and by `push --index` / `watch --index`.
|
|
11
|
+
* Kept in one place so all three agree on which file it is, which files it covers, and how
|
|
12
|
+
* they're chunked and embedded — an index built by `push --index` is the same index
|
|
13
|
+
* `mnemonad index` would have built.
|
|
14
|
+
*
|
|
15
|
+
* The default filename is at the folder root and not dot-prefixed: unlike `.mnemonad`, it's
|
|
16
|
+
* meant to be pushed and pulled like any other file, so a pull is searchable immediately.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** @returns {string} the index's filename (`--db`, default `search_index.db`). */
|
|
20
|
+
export function indexDbName(args) {
|
|
21
|
+
return args.searchDb || DEFAULT_DB_NAME;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** @returns {string[]} what the index skips: the push's own excludes, plus the index file.
|
|
25
|
+
* The index file itself must never be walked as content to embed — explicit, not left to
|
|
26
|
+
* chance: a SQLite file's own header/padding bytes are plausible-looking text in places,
|
|
27
|
+
* so relying on the binary-content sniff alone to skip it would be fragile. */
|
|
28
|
+
export function indexExcludes(args) {
|
|
29
|
+
const excludes = [...DEFAULT_FS_EXCLUDES, indexDbName(args)];
|
|
30
|
+
if (args.exclude) excludes.push(...args.exclude.split(',').map((s) => s.trim()));
|
|
31
|
+
return excludes;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function chunkingFrom(args) {
|
|
35
|
+
return new TextWindowChunkingStrategy({
|
|
36
|
+
size: args.chunkSize ?? 1000,
|
|
37
|
+
overlap: args.chunkOverlap ?? 150,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Which model to index `destPath` with. An explicit `--model` wins (an alias like `minilm`, or
|
|
43
|
+
* a full id). Without one, an existing index keeps the model it was built with — so re-indexing
|
|
44
|
+
* a folder someone indexed with MiniLM just works — and only a new index (or `--rebuild`)
|
|
45
|
+
* gets the built-in default.
|
|
46
|
+
*
|
|
47
|
+
* @returns {{model: string, embedder: ?string, dtype: ?string}}
|
|
48
|
+
*/
|
|
49
|
+
export function chooseModel(destPath, args) {
|
|
50
|
+
if (args.model) return { model: resolveModelName(args.model), embedder: null, dtype: null };
|
|
51
|
+
const meta = args.rebuild ? null : VectorIndex.readMeta(join(destPath, indexDbName(args)));
|
|
52
|
+
if (meta?.model) {
|
|
53
|
+
return { model: meta.model, embedder: embedderOfIndex(meta), dtype: meta.dtype || null };
|
|
54
|
+
}
|
|
55
|
+
return { model: DEFAULT_MODEL, embedder: null, dtype: null };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* An Indexer configured from the CLI flags. Hold on to it across runs (as `watch --index`
|
|
60
|
+
* does) and the embedding model loads once, on first use, instead of on every run.
|
|
61
|
+
* Rejects straight away on bad chunking flags or a missing extension, before anything is
|
|
62
|
+
* read or written.
|
|
63
|
+
*/
|
|
64
|
+
export async function makeIndexer(destPath, args) {
|
|
65
|
+
const chunkingStrategy = chunkingFrom(args);
|
|
66
|
+
const { model, embedder, dtype } = chooseModel(destPath, args);
|
|
67
|
+
let embeddingProvider;
|
|
68
|
+
try {
|
|
69
|
+
embeddingProvider = await createProvider({ model, ...(embedder ? { embedder } : {}), dtype });
|
|
70
|
+
} catch (err) {
|
|
71
|
+
throw explainIndexError(err, destPath, args);
|
|
72
|
+
}
|
|
73
|
+
return new Indexer({ embeddingProvider, chunkingStrategy });
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Builds or updates `destPath`'s index, printing the same summary `mnemonad index` does.
|
|
78
|
+
*
|
|
79
|
+
* @param {string} destPath - resolved folder path
|
|
80
|
+
* @param {Object} args - parsed CLI args (`rebuild` starts the index over)
|
|
81
|
+
* @param {Object} [opts]
|
|
82
|
+
* @param {Indexer} [opts.indexer] - reuse one (and its loaded model) across calls
|
|
83
|
+
* @param {boolean} [opts.quiet] - one summary line instead of the full report
|
|
84
|
+
* @param {boolean} [opts.rebuild] - overrides `args.rebuild` (watch rebuilds only once)
|
|
85
|
+
* @returns {Promise<{filesChanged: number, filesSkipped: number, filesRemoved: number, chunksAdded: number}>}
|
|
86
|
+
*/
|
|
87
|
+
export async function updateIndex(destPath, args, { indexer = null, quiet = false, rebuild = args.rebuild } = {}) {
|
|
88
|
+
indexer = indexer || await makeIndexer(destPath, { ...args, rebuild });
|
|
89
|
+
const dbName = indexDbName(args);
|
|
90
|
+
const fsFolder = new FSFolder(destPath, indexExcludes(args));
|
|
91
|
+
|
|
92
|
+
if (!quiet) {
|
|
93
|
+
console.log(' embedding model:', indexer.embeddingProvider.modelId, '(downloads and caches on first use)');
|
|
94
|
+
if (rebuild) console.log(` rebuilding ${dbName} from scratch`);
|
|
95
|
+
console.log(' scanning for changed files...');
|
|
96
|
+
}
|
|
97
|
+
let result;
|
|
98
|
+
try {
|
|
99
|
+
result = await indexer.index(fsFolder, join(destPath, dbName), { rebuild: !!rebuild });
|
|
100
|
+
} catch (err) {
|
|
101
|
+
throw explainIndexError(err, destPath, args);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (quiet) {
|
|
105
|
+
const changed = result.filesChanged + result.filesRemoved;
|
|
106
|
+
console.log(changed
|
|
107
|
+
? ` index: ${result.filesChanged} file(s) re-embedded, ${result.filesRemoved} removed (${result.chunksAdded} chunks)`
|
|
108
|
+
: ' index: up to date');
|
|
109
|
+
} else {
|
|
110
|
+
console.log(' files changed:', result.filesChanged);
|
|
111
|
+
console.log(' files skipped:', result.filesSkipped, '(content unchanged since last index)');
|
|
112
|
+
console.log(' files removed:', result.filesRemoved);
|
|
113
|
+
console.log(' chunks added: ', result.chunksAdded);
|
|
114
|
+
}
|
|
115
|
+
return result;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* How far `destPath`'s index has drifted from its files, without updating it — no model is
|
|
120
|
+
* loaded (not even the extension looked up) and the index is opened read-only (see
|
|
121
|
+
* Indexer.status()). Null when there's no index.
|
|
122
|
+
*
|
|
123
|
+
* @returns {Promise<?{changed: string[], added: string[], removed: string[], total: number}>}
|
|
124
|
+
*/
|
|
125
|
+
export async function checkIndex(destPath, args) {
|
|
126
|
+
const fsFolder = new FSFolder(destPath, indexExcludes(args));
|
|
127
|
+
// Chunking decides which files count (an empty one is never indexed), so it's configured
|
|
128
|
+
// exactly as for a real run; status() never embeds, so no provider is needed.
|
|
129
|
+
const indexer = new Indexer({ embeddingProvider: null, chunkingStrategy: chunkingFrom(args) });
|
|
130
|
+
const status = await indexer.status(fsFolder, join(destPath, indexDbName(args)));
|
|
131
|
+
if (!status) return null;
|
|
132
|
+
return { ...status, total: status.changed.length + status.added.length + status.removed.length };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Turns the two errors a user can act on into CLI messages that say what to do. */
|
|
136
|
+
function explainIndexError(err, destPath, args) {
|
|
137
|
+
if (err instanceof MissingExtensionError) {
|
|
138
|
+
const lines = [err.message];
|
|
139
|
+
if (!args.model) {
|
|
140
|
+
lines.push(
|
|
141
|
+
' Or rebuild the index with the built-in model instead:',
|
|
142
|
+
` mnemonad index ${destPath} --rebuild`,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
return userError(lines.join('\n'));
|
|
146
|
+
}
|
|
147
|
+
if (err instanceof ModelMismatchError) {
|
|
148
|
+
return userError(
|
|
149
|
+
`${indexDbName(args)} was built with ${err.indexModel}, but this run asked for ${err.providerModel} —\n` +
|
|
150
|
+
' one index can\'t mix two models. Either drop --model to keep using the index\'s own model, or\n' +
|
|
151
|
+
` add --rebuild to re-embed every file with ${err.providerModel} (the next push then uploads the\n` +
|
|
152
|
+
' whole index again).'
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
return err;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** `mnemonad index <path>` */
|
|
159
|
+
export async function buildIndex(args) {
|
|
160
|
+
const destPath = resolvePath(args.path, { mustExist: true });
|
|
161
|
+
const result = await updateIndex(destPath, args);
|
|
162
|
+
|
|
163
|
+
if (result.filesChanged > 0 || result.filesRemoved > 0) {
|
|
164
|
+
// Said explicitly rather than left implicit: unlike `.mnemonad`, this file is not
|
|
165
|
+
// dot-prefixed, so the very next `push` from this folder includes it automatically —
|
|
166
|
+
// worth knowing before that push happens, not after.
|
|
167
|
+
console.log(`\n note: ${indexDbName(args)} will be included in the next \`mnemonad push\` from this folder`);
|
|
168
|
+
} else {
|
|
169
|
+
console.log('\n no changes — nothing new to push');
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import Mnemonad from 'mnemonad';
|
|
2
|
+
import { makeChainClient, requireWalletClient } from '../chainClient.js';
|
|
3
|
+
import { assertStreamOwner, formatStreamId, formatBytes } from './shared.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Destroys a stream permanently: burns the ERC-721 token and clears its metadata and
|
|
7
|
+
* wrapped encryption key (`contracts/src/Mnemonad.sol`'s own `burn()`). Irreversible, and
|
|
8
|
+
* re-creating a stream with the same id afterward does NOT resurrect the old data — ids are
|
|
9
|
+
* deterministic, so the contract deliberately marks a burned id as permanently used rather
|
|
10
|
+
* than letting a fresh `create()` silently inherit old item records.
|
|
11
|
+
*
|
|
12
|
+
* Deployed CODE-tier item bytes physically survive on-chain regardless of burn — this
|
|
13
|
+
* removes the stream as a usable, readable thing, not necessarily every trace of its bytes.
|
|
14
|
+
*
|
|
15
|
+
* Requires `--yes`: the one command here that destroys something with no way back, so it
|
|
16
|
+
* refuses to run without an explicit, unambiguous opt-in rather than an interactive prompt
|
|
17
|
+
* a script could accidentally click through.
|
|
18
|
+
*/
|
|
19
|
+
export async function burn(args) {
|
|
20
|
+
if (!args.streamId) {
|
|
21
|
+
throw new Error('stream-id is required for burn');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const client = await makeChainClient(args);
|
|
25
|
+
requireWalletClient(client);
|
|
26
|
+
|
|
27
|
+
const mn = new Mnemonad({
|
|
28
|
+
publicClient: client.publicClient,
|
|
29
|
+
walletClient: client.walletClient,
|
|
30
|
+
contractAddress: client.contractAddress,
|
|
31
|
+
id: args.streamId,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// Ownership checked before printing anything about the stream's contents — same
|
|
35
|
+
// reasoning as every other write command's own assertStreamOwner call.
|
|
36
|
+
await assertStreamOwner(mn, client.account.address);
|
|
37
|
+
await mn.initialize();
|
|
38
|
+
|
|
39
|
+
console.log('stream: ', formatStreamId(mn.id));
|
|
40
|
+
console.log('chain: ', args.chain);
|
|
41
|
+
console.log('versions: ', mn.length);
|
|
42
|
+
console.log('binary size: ', formatBytes(mn.binaryLength));
|
|
43
|
+
|
|
44
|
+
if (!args.yes) {
|
|
45
|
+
const err = new Error(
|
|
46
|
+
'This permanently destroys the stream above: the ERC-721 token and its metadata\n' +
|
|
47
|
+
' are gone, and re-creating a stream with this id afterward does NOT recover the\n' +
|
|
48
|
+
' old data — there is no undo. (Deployed CODE-tier chunks physically survive on\n' +
|
|
49
|
+
' chain regardless; this only removes the stream as a usable, readable thing.)\n' +
|
|
50
|
+
' Re-run with --yes to actually burn it.'
|
|
51
|
+
);
|
|
52
|
+
err._isUserError = true;
|
|
53
|
+
throw err;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
console.log('burning...');
|
|
57
|
+
await mn.burn();
|
|
58
|
+
console.log(' done —', formatStreamId(mn.id), 'is burned.');
|
|
59
|
+
}
|
package/lib/commands/compact.js
CHANGED
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
assertStreamOwner,
|
|
11
11
|
unlockStream,
|
|
12
12
|
resolveOffloadParams,
|
|
13
|
+
discoverStreamId,
|
|
13
14
|
} from './shared.js';
|
|
14
15
|
|
|
15
16
|
/**
|
|
@@ -19,17 +20,21 @@ import {
|
|
|
19
20
|
* to it.
|
|
20
21
|
*/
|
|
21
22
|
export async function compact(args) {
|
|
23
|
+
const destPath = resolvePath(args.path, { mustExist: true });
|
|
24
|
+
|
|
25
|
+
// With --manifest and no id, a folder pulled/pushed before already has one recorded —
|
|
26
|
+
// no need to require it again.
|
|
27
|
+
await discoverStreamId(args, destPath);
|
|
22
28
|
if (!args.streamId) {
|
|
23
29
|
throw new Error('stream-id is required for compact');
|
|
24
30
|
}
|
|
25
31
|
|
|
26
|
-
const destPath = resolvePath(args.path);
|
|
27
32
|
const fsFolder = new FSFolder(destPath, makeExcludes(args));
|
|
28
33
|
|
|
29
34
|
const counts = await countTree(fsFolder);
|
|
30
35
|
console.log(' scanning...', counts.files, 'files,', counts.folders, 'folders');
|
|
31
36
|
|
|
32
|
-
const client = makeChainClient(args);
|
|
37
|
+
const client = await makeChainClient(args);
|
|
33
38
|
requireWalletClient(client);
|
|
34
39
|
|
|
35
40
|
// No `password` at construction — see shared.js's unlockStream() for why.
|
package/lib/commands/diff.js
CHANGED
|
@@ -8,6 +8,9 @@ import {
|
|
|
8
8
|
formatPath,
|
|
9
9
|
hashLocalTree,
|
|
10
10
|
unlockStream,
|
|
11
|
+
formatStreamId,
|
|
12
|
+
folderExists,
|
|
13
|
+
discoverStreamId,
|
|
11
14
|
} from './shared.js';
|
|
12
15
|
|
|
13
16
|
export function compareFileMaps(localFiles, remoteFiles) {
|
|
@@ -36,14 +39,24 @@ export function compareFileMaps(localFiles, remoteFiles) {
|
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
export async function diff(args) {
|
|
39
|
-
if (!args.streamId) {
|
|
40
|
-
throw new Error('stream-id is required for diff');
|
|
41
|
-
}
|
|
42
42
|
// Resolved up front (throws if missing) so a missing path fails fast, before any chain
|
|
43
43
|
// work — see resolvePath()'s own comment for why this is never defaulted to cwd.
|
|
44
|
+
//
|
|
45
|
+
// Deliberately NOT `{ mustExist: true }`, unlike push/compact/watch: diffing a stream
|
|
46
|
+
// against a folder you haven't pulled yet is a legitimate question ("what would I get?"),
|
|
47
|
+
// and this command writes nothing either way. It is called out explicitly below though —
|
|
48
|
+
// "everything deleted" for a folder that never existed reads as data loss rather than as
|
|
49
|
+
// a wrong path.
|
|
44
50
|
const destPath = resolvePath(args.path);
|
|
45
51
|
|
|
46
|
-
|
|
52
|
+
// With --manifest and no id, a folder pulled/pushed before already has one recorded —
|
|
53
|
+
// no need to require it again.
|
|
54
|
+
await discoverStreamId(args, destPath);
|
|
55
|
+
if (!args.streamId) {
|
|
56
|
+
throw new Error('stream-id is required for diff');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const client = await makeChainClient(args);
|
|
47
60
|
|
|
48
61
|
const mn = new Mnemonad({
|
|
49
62
|
publicClient: client.publicClient,
|
|
@@ -61,7 +74,7 @@ export async function diff(args) {
|
|
|
61
74
|
const totalVersions = mn.length;
|
|
62
75
|
|
|
63
76
|
if (totalVersions === 0) {
|
|
64
|
-
console.log('stream', args.streamId, 'has no versions yet — nothing to diff');
|
|
77
|
+
console.log('stream', formatStreamId(args.streamId), 'has no versions yet — nothing to diff');
|
|
65
78
|
return;
|
|
66
79
|
}
|
|
67
80
|
|
|
@@ -70,7 +83,13 @@ export async function diff(args) {
|
|
|
70
83
|
throw new Error('version must be between 1 and ' + totalVersions);
|
|
71
84
|
}
|
|
72
85
|
|
|
73
|
-
console.log('diff:', formatPath(destPath), 'vs', args.streamId, '(version ' + version + ' of ' + totalVersions + ')');
|
|
86
|
+
console.log('diff:', formatPath(destPath), 'vs', formatStreamId(args.streamId), '(version ' + version + ' of ' + totalVersions + ')');
|
|
87
|
+
// Said up front, before the file list: without it, a mistyped path produces a list of
|
|
88
|
+
// every file in the stream marked "deleted", which reads as data loss rather than as
|
|
89
|
+
// a folder that was never there.
|
|
90
|
+
if (!folderExists(destPath)) {
|
|
91
|
+
console.log(' note: local folder does not exist — everything below is listed as deleted');
|
|
92
|
+
}
|
|
74
93
|
console.log(' restoring chain version...');
|
|
75
94
|
|
|
76
95
|
let remoteFolder;
|
package/lib/commands/index.js
CHANGED
|
@@ -4,3 +4,6 @@ export { info } from './info.js';
|
|
|
4
4
|
export { diff } from './diff.js';
|
|
5
5
|
export { watch } from './watch.js';
|
|
6
6
|
export { compact } from './compact.js';
|
|
7
|
+
export { burn } from './burn.js';
|
|
8
|
+
export { buildIndex } from './buildIndex.js';
|
|
9
|
+
export { search } from './search.js';
|
package/lib/commands/info.js
CHANGED
|
@@ -6,14 +6,45 @@ import {
|
|
|
6
6
|
formatBytes,
|
|
7
7
|
localTreeHash,
|
|
8
8
|
unlockStream,
|
|
9
|
+
formatStreamId,
|
|
10
|
+
folderExists,
|
|
11
|
+
discoverStreamId,
|
|
12
|
+
readManifest,
|
|
13
|
+
streamIdsEqual,
|
|
9
14
|
} from './shared.js';
|
|
10
15
|
|
|
11
16
|
export async function info(args) {
|
|
12
|
-
const client = makeChainClient(args);
|
|
17
|
+
const client = await makeChainClient(args);
|
|
13
18
|
|
|
14
19
|
console.log(' chain: ', args.chain);
|
|
15
20
|
console.log(' your wallet: ', client.account?.address || '(none)');
|
|
16
21
|
|
|
22
|
+
// Peek at .mnemonad whenever a path is given, regardless of --manifest and regardless
|
|
23
|
+
// of whether an id was given — info is read-only, so surfacing what's there costs
|
|
24
|
+
// nothing, and it's the only command where "you have a manifest but forgot the flag" or
|
|
25
|
+
// "you gave an id that doesn't match your own manifest" are worth a note rather than a
|
|
26
|
+
// silent mismatch.
|
|
27
|
+
const manifest = args.path ? await readManifest(resolvePath(args.path)) : null;
|
|
28
|
+
|
|
29
|
+
if (manifest?.streamId) {
|
|
30
|
+
if (!args.streamId) {
|
|
31
|
+
if (args.manifest) {
|
|
32
|
+
// Logs its own "using stream id from .mnemonad manifest: ..." line.
|
|
33
|
+
await discoverStreamId(args, resolvePath(args.path));
|
|
34
|
+
} else {
|
|
35
|
+
console.log(
|
|
36
|
+
' manifest: ', formatStreamId(manifest.streamId),
|
|
37
|
+
'(.mnemonad found — pass --manifest to use it without an id)'
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
} else if (!streamIdsEqual(manifest.streamId, args.streamId)) {
|
|
41
|
+
console.log(
|
|
42
|
+
' manifest: ', formatStreamId(manifest.streamId),
|
|
43
|
+
'(differs from the id given:', formatStreamId(args.streamId) + ')'
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
17
48
|
if (!args.streamId) {
|
|
18
49
|
return;
|
|
19
50
|
}
|
|
@@ -33,7 +64,7 @@ export async function info(args) {
|
|
|
33
64
|
|
|
34
65
|
const totalVersions = mn.length;
|
|
35
66
|
|
|
36
|
-
console.log('\nstream:', args.streamId);
|
|
67
|
+
console.log('\nstream:', formatStreamId(args.streamId));
|
|
37
68
|
|
|
38
69
|
const owner = await mn.owner().catch(() => null) || '(unknown)';
|
|
39
70
|
const isOwner = client.account && owner !== '(unknown)' && client.account.address.toLowerCase() === owner.toLowerCase();
|
|
@@ -63,7 +94,16 @@ export async function info(args) {
|
|
|
63
94
|
console.log(' history starts: ', 'index', mn.firstAvailableIndex, '(earlier history truncated — see `compact`)');
|
|
64
95
|
}
|
|
65
96
|
|
|
97
|
+
// Deliberately NOT `{ mustExist: true }` like every other command that reads a folder:
|
|
98
|
+
// `info` is a read-only report, and "you haven't pulled this anywhere yet" is a state
|
|
99
|
+
// worth describing rather than refusing over — the stream metadata above is already
|
|
100
|
+
// printed and still useful on its own.
|
|
66
101
|
const destPath = resolvePath(args.path);
|
|
102
|
+
if (!folderExists(destPath)) {
|
|
103
|
+
console.log('\n local folder: missing (run `mnemonad pull` to fetch it)');
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
67
107
|
const localHash = await localTreeHash(destPath);
|
|
68
108
|
|
|
69
109
|
if (!localHash) {
|
package/lib/commands/pull.js
CHANGED
|
@@ -8,17 +8,23 @@ import {
|
|
|
8
8
|
syncMemoryFolderToDisk,
|
|
9
9
|
writeManifest,
|
|
10
10
|
unlockStream,
|
|
11
|
+
formatStreamId,
|
|
12
|
+
discoverStreamId,
|
|
11
13
|
} from './shared.js';
|
|
12
14
|
|
|
13
15
|
export async function pull(args) {
|
|
14
|
-
if (!args.streamId) {
|
|
15
|
-
throw new Error('stream-id is required for pull');
|
|
16
|
-
}
|
|
17
16
|
// Resolved up front (throws if missing) so a missing path fails fast, before any chain
|
|
18
17
|
// work — see resolvePath()'s own comment for why this is never defaulted to cwd.
|
|
19
18
|
const destPath = resolvePath(args.path);
|
|
20
19
|
|
|
21
|
-
|
|
20
|
+
// With --manifest and no id, a folder pulled/pushed before already has one recorded —
|
|
21
|
+
// no need to require it again.
|
|
22
|
+
await discoverStreamId(args, destPath);
|
|
23
|
+
if (!args.streamId) {
|
|
24
|
+
throw new Error('stream-id is required for pull');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const client = await makeChainClient(args);
|
|
22
28
|
|
|
23
29
|
const mn = new Mnemonad({
|
|
24
30
|
publicClient: client.publicClient,
|
|
@@ -35,7 +41,7 @@ export async function pull(args) {
|
|
|
35
41
|
await unlockStream(mn, { password: args.password, signer: client.walletClient });
|
|
36
42
|
|
|
37
43
|
const version = args.version != null ? Number(args.version) : undefined;
|
|
38
|
-
console.log('restoring', args.streamId, version != null ? 'at version ' + version : '(latest)', '...');
|
|
44
|
+
console.log('restoring', formatStreamId(args.streamId), version != null ? 'at version ' + version : '(latest)', '...');
|
|
39
45
|
|
|
40
46
|
let folder;
|
|
41
47
|
try {
|
|
@@ -56,7 +62,8 @@ export async function pull(args) {
|
|
|
56
62
|
if (args.manifest) {
|
|
57
63
|
await mn.initialize();
|
|
58
64
|
const pulledVersion = version ?? mn.length;
|
|
59
|
-
|
|
65
|
+
// Canonical form stored — see push.js's own writeManifest call for why.
|
|
66
|
+
await writeManifest(destPath, { streamId: formatStreamId(args.streamId), version: pulledVersion, files: newFiles });
|
|
60
67
|
}
|
|
61
68
|
|
|
62
69
|
console.log(' ', stats.written, 'written,', stats.skipped, 'unchanged,',
|
package/lib/commands/push.js
CHANGED
|
@@ -15,12 +15,39 @@ import {
|
|
|
15
15
|
unlockStream,
|
|
16
16
|
resolveEncryptMethod,
|
|
17
17
|
resolveOffloadParams,
|
|
18
|
+
formatStreamId,
|
|
19
|
+
streamIdsEqual,
|
|
20
|
+
discoverStreamId,
|
|
21
|
+
userError,
|
|
18
22
|
} from './shared.js';
|
|
23
|
+
import { updateIndex, checkIndex, indexDbName } from './buildIndex.js';
|
|
19
24
|
|
|
20
25
|
export async function push(args) {
|
|
26
|
+
const destPath = resolvePath(args.path, { mustExist: true });
|
|
27
|
+
|
|
28
|
+
// With --manifest and no id, a folder pushed before already has one recorded — reuse it
|
|
29
|
+
// rather than falling into the "no id" branch below and creating a second, unrelated
|
|
30
|
+
// stream for the same folder.
|
|
31
|
+
await discoverStreamId(args, destPath);
|
|
21
32
|
let streamId = args.streamId;
|
|
22
33
|
|
|
23
|
-
|
|
34
|
+
// Before anything is scanned or signed: the push below then picks the updated index up
|
|
35
|
+
// like any other file. A failure stops the push outright — a push that silently went out
|
|
36
|
+
// without the index asked for would leave the stream unsearchable with no sign of why.
|
|
37
|
+
if (args.rebuild && !args.index) {
|
|
38
|
+
throw userError('--rebuild only applies with --index (it starts the search index over)');
|
|
39
|
+
}
|
|
40
|
+
if (args.index) {
|
|
41
|
+
console.log(args.rebuild ? ' rebuilding search index...' : ' updating search index...');
|
|
42
|
+
try {
|
|
43
|
+
await updateIndex(destPath, args, { quiet: true });
|
|
44
|
+
} catch (err) {
|
|
45
|
+
throw userError(`indexing failed, so nothing was pushed: ${err.message}`);
|
|
46
|
+
}
|
|
47
|
+
} else {
|
|
48
|
+
await warnIfIndexStale(destPath, args);
|
|
49
|
+
}
|
|
50
|
+
|
|
24
51
|
const fsFolder = new FSFolder(destPath, makeExcludes(args));
|
|
25
52
|
|
|
26
53
|
const counts = await countTree(fsFolder);
|
|
@@ -29,7 +56,7 @@ export async function push(args) {
|
|
|
29
56
|
if (args.manifest) {
|
|
30
57
|
const manifest = await readManifest(destPath);
|
|
31
58
|
const localFiles = await hashLocalTree(fsFolder);
|
|
32
|
-
if (streamId && manifest && manifest.streamId
|
|
59
|
+
if (streamId && manifest && streamIdsEqual(manifest.streamId, streamId) && manifest.files) {
|
|
33
60
|
if (hashMapsEqual(localFiles, manifest.files)) {
|
|
34
61
|
console.log(' no changes since last pull, nothing to push');
|
|
35
62
|
return;
|
|
@@ -37,7 +64,7 @@ export async function push(args) {
|
|
|
37
64
|
}
|
|
38
65
|
}
|
|
39
66
|
|
|
40
|
-
const client = makeChainClient(args);
|
|
67
|
+
const client = await makeChainClient(args);
|
|
41
68
|
requireWalletClient(client);
|
|
42
69
|
|
|
43
70
|
const baseParams = {
|
|
@@ -73,7 +100,7 @@ export async function push(args) {
|
|
|
73
100
|
console.log('creating new stream on', args.chain, how, '...');
|
|
74
101
|
mn = await Mnemonad.create(createParams);
|
|
75
102
|
streamId = mn.id.toString();
|
|
76
|
-
console.log(' created:', streamId);
|
|
103
|
+
console.log(' created:', formatStreamId(streamId));
|
|
77
104
|
} else {
|
|
78
105
|
mn = new Mnemonad({ ...baseParams, id: streamId });
|
|
79
106
|
await assertStreamOwner(mn, client.account.address);
|
|
@@ -83,7 +110,7 @@ export async function push(args) {
|
|
|
83
110
|
}
|
|
84
111
|
}
|
|
85
112
|
|
|
86
|
-
console.log('syncing ./ →', streamId);
|
|
113
|
+
console.log('syncing ./ →', formatStreamId(streamId));
|
|
87
114
|
|
|
88
115
|
const monadSync = new MonadSync({ mnemonad: mn, compress: args.compress || 'gzip' });
|
|
89
116
|
|
|
@@ -118,10 +145,37 @@ export async function push(args) {
|
|
|
118
145
|
const patchType = args.forceSnapshot || existingVersions === 0 ? 'full snapshot' : 'diff';
|
|
119
146
|
const compressedNote = args.compress !== false ? ', gzip compressed' : '';
|
|
120
147
|
console.log(' version', result.version, 'pushed (' + patchType + compressedNote + ')');
|
|
121
|
-
console.log(' stream:', streamId);
|
|
148
|
+
console.log(' stream:', formatStreamId(streamId));
|
|
122
149
|
|
|
123
150
|
if (args.manifest) {
|
|
124
151
|
const localFiles = await hashLocalTree(fsFolder);
|
|
125
|
-
|
|
152
|
+
// Stored canonical (see formatStreamId) rather than whatever form the caller typed —
|
|
153
|
+
// so a manifest written by a hex-id push and read back by a decimal-id one (or vice
|
|
154
|
+
// versa) still matches; streamIdsEqual() above is the belt to this suspenders.
|
|
155
|
+
await writeManifest(destPath, { streamId: formatStreamId(streamId), version: result.version, files: localFiles });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* A folder that already carries an index, pushed without `--index` after its files changed,
|
|
161
|
+
* would publish an index that no longer matches them — searches keep returning the old text's
|
|
162
|
+
* offsets. Cheap to catch (a hash per file, no model load, the index opened read-only) and
|
|
163
|
+
* worth a line; never a reason to fail the push itself.
|
|
164
|
+
*/
|
|
165
|
+
async function warnIfIndexStale(destPath, args) {
|
|
166
|
+
let status;
|
|
167
|
+
try {
|
|
168
|
+
status = await checkIndex(destPath, args);
|
|
169
|
+
} catch {
|
|
170
|
+
return;
|
|
126
171
|
}
|
|
172
|
+
if (!status || status.total === 0) return;
|
|
173
|
+
const parts = [];
|
|
174
|
+
if (status.changed.length) parts.push(`${status.changed.length} changed`);
|
|
175
|
+
if (status.added.length) parts.push(`${status.added.length} new`);
|
|
176
|
+
if (status.removed.length) parts.push(`${status.removed.length} removed`);
|
|
177
|
+
console.log(
|
|
178
|
+
` note: ${indexDbName(args)} is out of date (${parts.join(', ')} file(s) since it was built).\n` +
|
|
179
|
+
' Push with --index to update it first, or run `mnemonad index <path>`.'
|
|
180
|
+
);
|
|
127
181
|
}
|