mnemonad-cli 0.2.0 → 0.3.0

package/lib/passkeyBridge.js

@@ -0,0 +1,203 @@
+import { randomBytes } from 'node:crypto';
+import { createServer } from 'node:http';
+import { WebSocketServer } from 'ws';
+import open from 'open';
+import { toAccount } from 'viem/accounts';
+import { userError } from './commands/shared.js';
+import { encodeForBridge } from './bridgeCodec.js';
+
+const CONNECT_TIMEOUT_MS = 3 * 60 * 1000;
+
+// One bridge per CLI process — `--passkey` resolves at most one signing account per
+// invocation (see resolveAccount in chainClient.js), so there's nothing to key this by.
+let active = null;
+
+/** A one-line "what's this for" the browser tab shows while it waits — just the command
+ *  shape, not the flags, so it stays short. */
+function describeCommand(args) {
+	const parts = ['mnemonad', args.command];
+	if (args.streamId) parts.push(args.streamId);
+	if (args.path) parts.push(args.path);
+	return parts.join(' ');
+}
+
+/**
+ * The CLI half of `--passkey`: opens the explorer's `/cli-auth` page in the system browser
+ * to run the WebAuthn ceremony there (this process has no `navigator.credentials` — passkeys
+ * only exist in a browser), then relays every signing request to that tab for the rest of
+ * this command. The raw key never enters this process, only signatures do — see
+ * docs/wallet/passkey-accounts.md's CLI section for why it has to be that domain specifically
+ * (rp.id) and why this is a one-shot-per-command bridge rather than something that persists
+ * across separate CLI invocations.
+ *
+ * @param {Object} args - parsed CLI args: `authOrigin` (the explorer deployment to open, e.g.
+ *   https://mnemonad.vercel.app), plus `command`/`streamId`/`path` to build the one-line
+ *   description the page shows while it waits.
+ * @returns {Promise<Object>} a viem Account whose signMessage/signTransaction proxy to the tab
+ */
+export async function openPasskeyBridge(args) {
+	if (active) return active.account;
+
+	const token = randomBytes(24).toString('hex');
+	// Read by /ping below — the one thing the page can't get from the WS (it doesn't exist
+	// yet at that point, see docs/wallet/passkey-accounts.md's CLI section) but still wants
+	// live: how much of the connect-timeout is actually left, and confirmation the CLI is
+	// still there at all.
+	const deadlineAt = Date.now() + CONNECT_TIMEOUT_MS;
+	const httpServer = createServer((req, res) => {
+		const url = new URL(req.url, 'http://127.0.0.1');
+		if (url.pathname === '/ping' && url.searchParams.get('token') === token) {
+			res.writeHead(200, {
+				'Content-Type': 'application/json',
+				// The page is served from the explorer's own origin, not this loopback one —
+				// an ordinary fetch() response is otherwise unreadable cross-origin. The token
+				// above is what actually gates this, same as the WS connection itself; the
+				// wildcard just matches that (no meaningful "origin" to restrict to when the
+				// caller could be any explorer deployment this token was ever handed to).
+				'Access-Control-Allow-Origin': '*',
+			});
+			res.end(JSON.stringify({ ttl: Math.max(0, Math.round((deadlineAt - Date.now()) / 1000)) }));
+			return;
+		}
+		res.writeHead(404);
+		res.end();
+	});
+	const wss = new WebSocketServer({ server: httpServer });
+
+	await new Promise((resolve, reject) => {
+		httpServer.once('error', reject);
+		httpServer.listen(0, '127.0.0.1', resolve);
+	});
+	const port = httpServer.address().port;
+
+	const pending = new Map();
+	let nextId = 1;
+	let socket = null;
+
+	const connected = new Promise((resolve, reject) => {
+		const timer = setTimeout(() => {
+			// Nothing ever reached the point (below) that assigns `active`, so
+			// closePasskeyBridge() has nothing to close on this path — without this, the
+			// server/socket this function opened above would sit open forever, and the CLI
+			// process would hang right after printing the error below rather than exiting.
+			wss.close();
+			httpServer.close();
+			reject(userError(
+				'Timed out waiting for the browser passkey sign-in. Complete the prompt in the\n' +
+				'  tab that opened, or re-run with --passkey to try again.'
+			));
+		}, CONNECT_TIMEOUT_MS);
+
+		wss.on('connection', (ws, req) => {
+			const url = new URL(req.url, 'http://127.0.0.1');
+			if (url.searchParams.get('token') !== token) {
+				ws.close(4001, 'bad token');
+				return;
+			}
+			socket = ws;
+
+			ws.on('message', (raw) => {
+				let msg;
+				try {
+					msg = JSON.parse(raw.toString());
+				} catch {
+					return;
+				}
+				if (msg.type === 'hello') {
+					clearTimeout(timer);
+					resolve(msg.address);
+				} else if (msg.type === 'sign-result' || msg.type === 'sign-error') {
+					const p = pending.get(msg.id);
+					if (!p) return;
+					pending.delete(msg.id);
+					if (msg.type === 'sign-result') p.resolve(msg.result);
+					else p.reject(new Error(msg.error || 'signing failed'));
+				}
+			});
+
+			ws.on('close', () => {
+				if (socket === ws) socket = null;
+				for (const p of pending.values()) p.reject(userError('Browser tab disconnected before signing finished.'));
+				pending.clear();
+			});
+		});
+	});
+
+	const desc = describeCommand(args);
+	const ttl = Math.floor(CONNECT_TIMEOUT_MS / 1000);
+	// Lets the result screen turn a stream id in its summary line into a link to the
+	// explorer's own stream page — only for a real deployment id (testnet/mainnet); --chain
+	// local has no explorer route to link to, so the page just falls back to plain text.
+	const network = args.chain === 'testnet' || args.chain === 'mainnet' ? `&network=${args.chain}` : '';
+	const authUrl = `${args.authOrigin.replace(/\/$/, '')}/cli-auth?port=${port}&token=${token}&desc=${encodeURIComponent(desc)}&ttl=${ttl}${network}`;
+	console.log('  opening', authUrl, 'to sign in with a passkey...');
+	try {
+		await open(authUrl);
+	} catch {
+		console.log('  could not open a browser automatically — open this URL yourself:', authUrl);
+	}
+
+	const address = await connected;
+	console.log('  connected:', address);
+
+	function request(type, payload) {
+		if (!socket) {
+			return Promise.reject(userError('Browser tab disconnected. Re-run with --passkey to reconnect.'));
+		}
+		return new Promise((resolve, reject) => {
+			const id = nextId++;
+			pending.set(id, { resolve, reject });
+			socket.send(JSON.stringify({ type, id, ...payload }));
+		});
+	}
+
+	const account = toAccount({
+		address,
+		async signMessage({ message }) {
+			return request('sign-message', { message });
+		},
+		async signTransaction(transaction) {
+			return request('sign-transaction', { transaction: encodeForBridge(transaction) });
+		},
+		async signTypedData() {
+			// Not used anywhere in this codebase (personal_sign only — see the "Wallet-signature
+			// encryption keys" plan's locked-in decision) — a real relay implementation would
+			// need the same BigInt-safe encoding sign-transaction already has, not worth building
+			// for a path nothing calls.
+			throw userError('Signing typed data is not supported over --passkey.');
+		},
+	});
+
+	active = { wss, httpServer, account };
+	return account;
+}
+
+/**
+ * Called once, at the very end of the CLI's command dispatch (bin/mnemonad.js), whether the
+ * command succeeded or failed — a no-op when --passkey was never used. Tells the browser tab
+ * it's done so it can end its session and zero the key, mirroring the explorer's own
+ * disconnect() lifecycle instead of just abandoning the connection.
+ *
+ * `result` lets the tab show something more useful than a bare "done" before it closes —
+ * whether the command actually succeeded, and the one line of output that says what
+ * happened (bin/mnemonad.js captures its own last console.log line for this rather than
+ * every command needing to return a summary explicitly).
+ *
+ * @param {{ok: boolean, summary: string}} [result]
+ */
+export function closePasskeyBridge(result) {
+	if (!active) return;
+	const { wss, httpServer } = active;
+	active = null;
+	const payload = JSON.stringify({ type: 'done', ok: result?.ok ?? true, summary: result?.summary || '' });
+	for (const ws of wss.clients) {
+		try {
+			ws.send(payload);
+		} catch {
+			// Nothing to tell a socket that's already gone.
+		}
+		ws.close();
+	}
+	wss.close();
+	httpServer.close();
+}

package/lib/search/Indexer.js

@@ -0,0 +1,204 @@
+import { existsSync, rmSync } from 'node:fs';
+import VectorIndex from './VectorIndex.js';
+import TextWindowChunkingStrategy from './chunking/TextWindowChunkingStrategy.js';
+
+/** First N bytes checked for a null byte — the standard, cheap heuristic (same one git and
+ *  most diff tools use) for "this is almost certainly binary, don't try to embed it as text". */
+const BINARY_SNIFF_BYTES = 8000;
+
+function looksLikeText(bytes) {
+	const n = Math.min(bytes.length, BINARY_SNIFF_BYTES);
+	for (let i = 0; i < n; i++) {
+		if (bytes[i] === 0) return false;
+	}
+	return true;
+}
+
+function toHex(bytes) {
+	return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * Builds/updates a `VectorIndex` from a folder — anything shaped like
+ * `@fizzyflow/doublesync`'s `DoubleSyncFolder` (has an async `walk()` generator), which
+ * covers the CLI's real-disk `FSFolder` and any in-memory folder the same way. Incremental by
+ * construction: a file whose content hash already matches what's stored is skipped entirely,
+ * no re-embedding, no re-chunking, not even a read past the hash check.
+ */
+export default class Indexer {
+	/**
+	 * @param {Object} [params]
+	 * @param {?import('./embeddings/EmbeddingProvider.js').default} [params.embeddingProvider] -
+	 *   required by `index()`; `status()` never embeds, so it can go without one (see
+	 *   providers.js's createProvider for the usual way to get one).
+	 * @param {import('./chunking/ChunkingStrategy.js').default} [params.chunkingStrategy]
+	 */
+	constructor({ embeddingProvider = null, chunkingStrategy } = {}) {
+		this.embeddingProvider = embeddingProvider;
+		this.chunkingStrategy = chunkingStrategy || new TextWindowChunkingStrategy();
+	}
+
+	/**
+	 * What `index()` would do to `dbPath`, without doing it: no embedding, no model load, and
+	 * the database opened read-only. Uses the same walk and the same rules as `index()` (see
+	 * `_scan`), so it agrees exactly on which files count — a binary or empty file, which
+	 * `index()` never records, isn't reported as missing from the index.
+	 *
+	 * @param {import('@fizzyflow/doublesync').DoubleSyncFolder} folder
+	 * @param {string} dbPath
+	 * @returns {Promise<?{changed: string[], added: string[], removed: string[]}>} null when
+	 *   there's no index at `dbPath` at all.
+	 */
+	async status(folder, dbPath) {
+		if (!existsSync(dbPath)) return null;
+		const index = new VectorIndex(dbPath, { readonly: true });
+		try {
+			const indexed = new Set(index.listFiles());
+			const changed = [];
+			const added = [];
+			const seen = new Set();
+			for await (const entry of this._scan(folder, index)) {
+				// An emptied file is left out of `seen`, so it comes out as removed below —
+				// exactly what index() would do to it.
+				if (!entry.unchanged && !entry.chunks) continue;
+				seen.add(entry.relPath);
+				if (entry.unchanged) continue;
+				(indexed.has(entry.relPath) ? changed : added).push(entry.relPath);
+			}
+			const removed = [...indexed].filter((p) => !seen.has(p));
+			return { changed, added, removed };
+		} finally {
+			index.close();
+		}
+	}
+
+	/**
+	 * The walk both `index()` and `status()` run. Yields one entry per text file:
+	 * `{relPath, unchanged: true}` when the index already has it at this content hash, or
+	 * `{relPath, hash, chunks}` otherwise (`chunks` null for a file with no text to embed).
+	 */
+	async *_scan(folder, index) {
+		for await (const { path, file } of folder.walk()) {
+			const relPath = path.join('/');
+			const bytes = await file.getContent();
+			if (!looksLikeText(bytes)) continue;
+
+			const hash = toHex(await file.getFingerprint());
+			if (index.getFileHash(relPath) === hash) {
+				yield { relPath, unchanged: true };
+				continue;
+			}
+
+			const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+			const chunks = this.chunkingStrategy.chunk(text);
+			yield { relPath, hash, chunks: chunks.length > 0 ? chunks : null };
+		}
+	}
+
+	/**
+	 * @param {import('@fizzyflow/doublesync').DoubleSyncFolder} folder
+	 * @param {string} dbPath
+	 * @param {Object} [opts]
+	 * @param {boolean} [opts.rebuild] - start from an empty index instead of updating this one:
+	 *   every file is embedded again. The one way to switch an index to a different model — and,
+	 *   since it rewrites the whole file, the next push carries the full index again.
+	 * @returns {Promise<{filesChanged: number, filesSkipped: number, filesRemoved: number, chunksAdded: number}>}
+	 */
+	async index(folder, dbPath, { rebuild = false } = {}) {
+		if (!this.embeddingProvider) throw new Error('Indexer.index: no embeddingProvider configured');
+		if (rebuild) {
+			rmSync(dbPath, { force: true });
+			rmSync(`${dbPath}-journal`, { force: true });
+		} else {
+			assertSameModel(VectorIndex.readMeta(dbPath), this.embeddingProvider, dbPath);
+		}
+		const index = new VectorIndex(dbPath);
+		try {
+			const seenPaths = new Set();
+			// One entry per changed/new file, chunked but not yet embedded — embedding happens
+			// in one batched call across every changed file below, rather than one call per
+			// file, since a single call amortizes the model's own fixed per-call overhead.
+			const pending = [];
+			let filesSkipped = 0;
+
+			for await (const entry of this._scan(folder, index)) {
+				// A file with no text to embed (emptied since last time) is deliberately left out
+				// of seenPaths, so the removal pass below drops its old chunks like a deleted file's.
+				if (entry.unchanged) {
+					seenPaths.add(entry.relPath);
+					filesSkipped++;
+				} else if (entry.chunks) {
+					seenPaths.add(entry.relPath);
+					pending.push(entry);
+				}
+			}
+
+			let chunksAdded = 0;
+			if (pending.length > 0) {
+				const allTexts = pending.flatMap((p) => p.chunks.map((c) => c.text));
+				const allVectors = await this.embeddingProvider.embed(allTexts);
+				index.initVectorColumn(this.embeddingProvider.dimension);
+
+				let cursor = 0;
+				for (const p of pending) {
+					const vectors = allVectors.slice(cursor, cursor + p.chunks.length);
+					cursor += p.chunks.length;
+					index.replaceFileChunks(p.relPath, p.hash, p.chunks.map((c, i) => ({
+						startOffset: c.startOffset,
+						endOffset: c.endOffset,
+						embedding: vectors[i],
+					})));
+					chunksAdded += p.chunks.length;
+				}
+
+				index.setMeta('model', this.embeddingProvider.modelId);
+				if (this.embeddingProvider.embedder) index.setMeta('embedder', this.embeddingProvider.embedder);
+				index.setMeta('dimension', String(this.embeddingProvider.dimension));
+				if (this.embeddingProvider.dtype) index.setMeta('dtype', this.embeddingProvider.dtype);
+				index.setMeta('chunk_size', String(this.chunkingStrategy.size ?? ''));
+				index.setMeta('chunk_overlap', String(this.chunkingStrategy.overlap ?? ''));
+			} else {
+				// Nothing changed this run — if the index already has content from a previous
+				// run, the vector column still needs registering on this fresh connection (see
+				// VectorIndex.initVectorColumn's own doc) before removeFile()'s bookkeeping below
+				// touches the same tables. Not required for a genuinely empty/new index — there's
+				// no dimension to know yet, and nothing to query either.
+				const existingDim = index.getMeta('dimension');
+				if (existingDim) index.initVectorColumn(Number(existingDim));
+			}
+
+			let filesRemoved = 0;
+			for (const indexedPath of index.listFiles()) {
+				if (!seenPaths.has(indexedPath)) {
+					index.removeFile(indexedPath);
+					filesRemoved++;
+				}
+			}
+
+			return { filesChanged: pending.length, filesSkipped, filesRemoved, chunksAdded };
+		} finally {
+			index.close();
+		}
+	}
+}
+
+/**
+ * Thrown when `index()` would add one model's vectors to an index built with another — two
+ * models' vectors aren't comparable (and usually aren't even the same length), so mixing them
+ * would quietly break every search. `rebuild` is the way through.
+ */
+export class ModelMismatchError extends Error {
+	constructor(dbPath, indexModel, providerModel) {
+		super(
+			`${dbPath} was built with ${indexModel}, not ${providerModel} — one index can't mix two models.`
+		);
+		this.name = 'ModelMismatchError';
+		this.indexModel = indexModel;
+		this.providerModel = providerModel;
+	}
+}
+
+function assertSameModel(meta, provider, dbPath) {
+	if (!meta?.model) return;
+	if (meta.model !== provider.modelId) throw new ModelMismatchError(dbPath, meta.model, provider.modelId);
+}

package/lib/search/Searcher.js

@@ -0,0 +1,68 @@
+import VectorIndex from './VectorIndex.js';
+import { LEGACY_DTYPE, EMBEDDER_TRANSFORMERS, embedderOfIndex } from './schema.js';
+
+/**
+ * Queries an already-built `VectorIndex`. Deliberately the only piece that needs an embedding
+ * model loaded on the *searcher's* machine — and only to embed the query string itself, never
+ * the corpus, which is the entire point of pushing the index alongside the content: whoever
+ * pulls it gets every embedding for free, already paid for by whoever ran `Indexer` first.
+ */
+export default class Searcher {
+	/**
+	 * @param {Object} params
+	 * @param {(opts: {model: string, embedder: string, dtype: ?string}) => Promise<any>} [params.createProvider] -
+	 *   builds the provider the index's own metadata asks for (see providers.js) — the common
+	 *   case, opening someone else's index.
+	 * @param {any} [params.embeddingProvider] - use this one instead, whatever the index recorded
+	 *   (tests; a custom provider). Its vectors must still match the index's dimension.
+	 */
+	constructor({ createProvider = null, embeddingProvider = null } = {}) {
+		if (!createProvider && !embeddingProvider) {
+			throw new Error('Searcher: pass createProvider or embeddingProvider');
+		}
+		this._createProvider = createProvider;
+		this._embeddingProvider = embeddingProvider;
+	}
+
+	/**
+	 * @param {string} dbPath
+	 * @param {string} query
+	 * @param {number} [k=5]
+	 * @returns {Promise<{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}[]>}
+	 */
+	async search(dbPath, query, k = 5) {
+		// Read-only: the index is a file that gets pushed, and searching it must not change it.
+		const index = new VectorIndex(dbPath, { readonly: true });
+		try {
+			const dim = Number(index.getMeta('dimension'));
+			if (!dim) {
+				throw new Error(`${dbPath}: no index metadata found — run \`mnemonad index\` first`);
+			}
+			index.initVectorColumn(dim);
+
+			const provider = this._embeddingProvider || await this._createProvider(describeIndexModel(index));
+			const [queryVector] = await provider.embed([query]);
+			if (queryVector.length !== dim) {
+				throw new Error(
+					`Query embedding is ${queryVector.length}-dimensional but this index is ${dim}-dimensional — ` +
+					`the embedding model must match the one used to build it (recorded model: ${index.getMeta('model')})`
+				);
+			}
+
+			return index.search(queryVector, k);
+		} finally {
+			index.close();
+		}
+	}
+}
+
+/** The model an index says it was built with, in the shape createProvider takes. */
+export function describeIndexModel(index) {
+	const meta = { model: index.getMeta('model'), embedder: index.getMeta('embedder') };
+	const embedder = embedderOfIndex(meta);
+	return {
+		model: meta.model,
+		embedder,
+		dtype: embedder === EMBEDDER_TRANSFORMERS ? (index.getMeta('dtype') || LEGACY_DTYPE) : null,
+	};
+}