mnemonad-cli 0.1.1 → 0.3.0

package/lib/search/providers.js

@@ -0,0 +1,153 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import StaticEmbeddingProvider from './embeddings/StaticEmbeddingProvider.js';
+import { nodeModelFiles } from './embeddings/modelFiles.node.js';
+import { EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS, embedderForModel } from './schema.js';
+
+/**
+ * Embedding providers for Node — the built-in static one, or one from the optional
+ * transformers extension.
+ *
+ * The extension (`mnemonad-search-transformers`) is deliberately *not* a dependency of the CLI:
+ * it brings ONNX Runtime, ~450 MB installed, for the one feature (MiniLM-style neural models)
+ * most users never need. It's installed separately, and found at runtime:
+ *
+ *  1. A plain `import()`. Node resolves it by walking up from this file's own folder, which
+ *     reaches it for a local project install, `npx -p … -p …`, and npm's global installs —
+ *     global packages sit side by side in `<prefix>/lib/node_modules`, which the walk reaches.
+ *  2. Failing that, the package managers' own global folders: `npm root -g` (for layouts the
+ *     walk doesn't reach), then `pnpm root -g`. pnpm 11 installs every global package into its
+ *     own isolated folder (`<root>/<hash>/node_modules/<pkg>`), so there the CLI can never see
+ *     a sibling by walking up; each of those folders is checked instead, newest first.
+ */
+
+export const EXTENSION_PACKAGE = 'mnemonad-search-transformers';
+
+/** What this CLI expects the extension's interface to look like; the extension exports its own. */
+export const EXTENSION_API_VERSION = 1;
+
+export const EXTENSION_INSTALL = `npm install -g ${EXTENSION_PACKAGE}   (or: pnpm add -g ${EXTENSION_PACKAGE})`;
+
+export class MissingExtensionError extends Error {
+	constructor(model) {
+		super(
+			`${model} needs the optional transformers extension (about 450 MB installed, not included by default):\n` +
+			`    ${EXTENSION_INSTALL}`
+		);
+		this.name = 'MissingExtensionError';
+		this.model = model;
+	}
+}
+
+/**
+ * @param {Object} [opts] - both overridable for tests
+ * @param {(specifier: string) => Promise<any>} [opts.importer]
+ * @param {() => string[]} [opts.globalRoots] - global package folders to look in (see above)
+ * @returns {Promise<?any>} the extension module, or null when it isn't installed
+ */
+export async function findTransformersExtension({ importer = (s) => import(s), globalRoots = defaultGlobalRoots } = {}) {
+	try {
+		return await importer(EXTENSION_PACKAGE);
+	} catch (err) {
+		if (!isNotFound(err)) throw err;
+	}
+	for (const root of globalRoots()) {
+		const pkgJson = extensionIn(root);
+		if (!pkgJson) continue;
+		// Resolved from inside the package itself (a self-reference through its own `exports`),
+		// so the entry point is whatever the package declares, not a guessed file name.
+		const entry = createRequire(pkgJson).resolve(EXTENSION_PACKAGE);
+		return importer(pathToFileURL(entry).href);
+	}
+	return null;
+}
+
+/**
+ * The extension's package.json under one global folder, or null. Only these exact places —
+ * never a module-resolution walk, which could wander out of the folder and pick up some
+ * unrelated copy:
+ *  - `<root>/<package>` — npm, and older pnpm, where global packages sit side by side;
+ *  - `<root>/<dir>/node_modules/<package>` — pnpm 11's isolated per-package folders; newest
+ *    first, since reinstalling leaves an older folder behind until pnpm prunes it.
+ */
+function extensionIn(root) {
+	const direct = join(root, EXTENSION_PACKAGE, 'package.json');
+	if (existsSync(direct)) return direct;
+	let dirs;
+	try {
+		dirs = readdirSync(root, { withFileTypes: true }).filter((d) => d.isDirectory());
+	} catch {
+		return null;
+	}
+	const found = dirs
+		.map((d) => join(root, d.name, 'node_modules', EXTENSION_PACKAGE, 'package.json'))
+		.filter((p) => existsSync(p))
+		.map((p) => ({ p, mtime: statSync(p).mtimeMs }))
+		.sort((a, b) => b.mtime - a.mtime);
+	return found.length ? found[0].p : null;
+}
+
+/**
+ * @param {string} model - for the error message
+ * @param {Object} [opts] - see findTransformersExtension
+ */
+export async function loadTransformersExtension(model, opts) {
+	const ext = await findTransformersExtension(opts);
+	if (!ext) throw new MissingExtensionError(model);
+	if (ext.apiVersion !== EXTENSION_API_VERSION) {
+		throw new Error(
+			`${EXTENSION_PACKAGE} speaks extension API v${ext.apiVersion ?? '?'}, but this mnemonad-cli expects ` +
+			`v${EXTENSION_API_VERSION} — update both: npm install -g mnemonad-cli@latest ${EXTENSION_PACKAGE}@latest`
+		);
+	}
+	return ext;
+}
+
+/**
+ * @param {Object} params
+ * @param {string} params.model - full model id
+ * @param {string} [params.embedder] - defaults to what the model needs (see embedderForModel)
+ * @param {?string} [params.dtype] - transformers models only
+ * @param {(event: Object) => void} [params.onProgress]
+ * @param {Object} [params.extension] - passed to loadTransformersExtension (tests)
+ */
+export async function createProvider({ model, embedder = embedderForModel(model), dtype = null, onProgress = null, extension } = {}) {
+	if (embedder === EMBEDDER_STATIC) {
+		return new StaticEmbeddingProvider({ model, files: nodeModelFiles(model, { onProgress }) });
+	}
+	if (embedder === EMBEDDER_TRANSFORMERS) {
+		const ext = await loadTransformersExtension(model, extension);
+		return ext.createProvider({ model, ...(dtype ? { dtype } : {}), onProgress });
+	}
+	throw new Error(`unknown embedder '${embedder}' (expected ${EMBEDDER_STATIC} or ${EMBEDDER_TRANSFORMERS})`);
+}
+
+function isNotFound(err) {
+	return !!err
+		&& (err.code === 'ERR_MODULE_NOT_FOUND' || err.code === 'MODULE_NOT_FOUND')
+		// Only the extension itself being absent — a missing dependency *inside* an installed
+		// extension is a broken install, and should surface as such.
+		&& String(err.message).includes(`'${EXTENSION_PACKAGE}'`);
+}
+
+/** npm's and pnpm's global package folders — whichever of the two is installed. Asked only
+ *  when the plain import has already missed, so a normal run never spawns either. */
+function defaultGlobalRoots() {
+	return ['npm', 'pnpm'].map(globalRootOf).filter(Boolean);
+}
+
+function globalRootOf(tool) {
+	try {
+		return execFileSync(tool, ['root', '-g'], {
+			encoding: 'utf8',
+			stdio: ['ignore', 'pipe', 'ignore'],
+			timeout: 10_000,
+			shell: process.platform === 'win32',
+		}).trim() || null;
+	} catch {
+		return null;
+	}
+}

package/lib/search/schema.js

@@ -0,0 +1,103 @@
+/**
+ * Everything about the on-disk index format that more than one reader has to agree on — the
+ * Node `VectorIndex` (better-sqlite3 + the native sqlite-vector extension) and the browser's
+ * `WasmIndexReader` (@sqliteai/sqlite-wasm, same sqlite-vector compiled in). Kept in one place
+ * so the two can't drift: a search from the explorer must rank exactly like `mnemonad search`
+ * on the same file.
+ *
+ * Deliberately dependency-free (no `node:*`, no native modules) — the browser entry imports it.
+ */
+
+export const DEFAULT_DB_NAME = 'search_index.db';
+
+/**
+ * Two kinds of embedding model, recorded per index as `embedder` in its metadata:
+ *
+ *  - 'model2vec' — static embeddings (a token → vector lookup table, averaged). Built in: pure
+ *    JS, no native code, runs the same in Node and the browser. The default.
+ *  - 'transformers' — a neural model run by transformers.js on ONNX Runtime (MiniLM, say).
+ *    Better on paraphrased queries, but ONNX Runtime is ~450 MB installed, so the CLI loads it
+ *    only from the optional `mnemonad-search-transformers` package.
+ */
+export const EMBEDDER_STATIC = 'model2vec';
+export const EMBEDDER_TRANSFORMERS = 'transformers';
+
+export const DEFAULT_MODEL = 'minishlab/potion-base-8M';
+
+/** Short names accepted wherever a model is (`--model minilm`). */
+export const MODEL_ALIASES = {
+	potion: 'minishlab/potion-base-8M',
+	minilm: 'Xenova/all-MiniLM-L6-v2',
+};
+
+/** @returns {string} the full model id for an alias, or the name unchanged. */
+export function resolveModelName(name) {
+	return MODEL_ALIASES[name] || name;
+}
+
+/** Which embedder a model needs. model2vec models live under the `minishlab/` organisation
+ *  on the Hugging Face hub; everything else is taken to be a transformers.js model. */
+export function embedderForModel(model) {
+	return /^minishlab\//.test(model) ? EMBEDDER_STATIC : EMBEDDER_TRANSFORMERS;
+}
+
+/**
+ * Which embedder built an existing index. Indexes from before `embedder` was recorded were all
+ * built with MiniLM through transformers.js — the only option at the time.
+ *
+ * @param {{embedder?: ?string, model?: ?string}} meta
+ */
+export function embedderOfIndex(meta) {
+	if (meta.embedder) return meta.embedder;
+	return meta.model ? embedderForModel(meta.model) : EMBEDDER_TRANSFORMERS;
+}
+
+/** What a transformers-built index recorded before `dtype` was written to its meta was built
+ *  with — the Node default at the time, since the CLI was the only thing that could build one. */
+export const LEGACY_DTYPE = 'fp32';
+
+export const TABLE_CHUNKS = 'search_index';
+export const TABLE_FILES = 'search_index_files';
+export const TABLE_META = 'search_index_meta';
+export const VECTOR_COLUMN = 'embedding';
+
+/** @param {number} dim @param {'FLOAT32'|'INT8'} [type='FLOAT32'] */
+export function vectorInitOptions(dim, type = 'FLOAT32') {
+	return `type=${type},dimension=${dim},distance=COSINE`;
+}
+
+export const VECTOR_INIT_SQL = `SELECT vector_init('${TABLE_CHUNKS}', '${VECTOR_COLUMN}', ?)`;
+
+/**
+ * Nearest-neighbor search, current chunks only. Binds, in order: query vector (BLOB), overfetch
+ * count, k. `vector_full_scan` is exact brute force — appropriate for the corpus sizes a synced
+ * folder actually has, and an ANN/partitioning index would break the diffability the whole
+ * format depends on. Some of its top candidates may be stale (superseded by a later reindex of
+ * the same file, or belonging to a removed file), which the join against the files table drops
+ * — hence the overfetch, see `overfetchFor`.
+ */
+export const SEARCH_SQL = `
+	SELECT s.path, s.chunk_index, s.start_offset, s.end_offset, v.distance
+	FROM vector_full_scan('${TABLE_CHUNKS}', '${VECTOR_COLUMN}', ?, ?) v
+	JOIN ${TABLE_CHUNKS} s ON s.id = v.rowid
+	JOIN ${TABLE_FILES} f ON f.path = s.path AND f.content_hash = s.content_hash
+	ORDER BY v.distance ASC
+	LIMIT ?
+`;
+
+/** A generous overfetch — cheap (still one exact scan) and simple, rather than looping with a
+ *  growing k until enough candidates survive the staleness join. */
+export function overfetchFor(k) {
+	return Math.max(k * 8, 64);
+}
+
+/** @returns {{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}} */
+export function mapSearchRow(r) {
+	return {
+		path: r.path,
+		chunkIndex: r.chunk_index,
+		startOffset: r.start_offset,
+		endOffset: r.end_offset,
+		distance: r.distance,
+	};
+}

package/mnemonad.config.js

@@ -13,9 +13,23 @@ export default {
     // their own. Overridable per-run with --presign-url or MNEMONAD_PRESIGN_URL.
     presignUrl: 'https://presign-server.vercel.app',
 
-    // Public IPFS gateway for *reading* externally-offloaded items (pull/diff/info/watch)
-    // — same default the explorer dApp uses (explorer/src/mnemonad/presignProvider.js), so
-    // a stream pushed through either tool reads back the same way with no setup. Overridable
-    // per-run with --gateway-url or MNEMONAD_IPFS_GATEWAY.
-    gatewayUrl: 'https://gateway.pinata.cloud/ipfs/',
+    // IPFS gateway for *reading* externally-offloaded items (pull/diff/info/watch) — same
+    // default the explorer dApp uses (explorer/src/mnemonad/presignProvider.js), so a stream
+    // pushed through either tool reads back the same way with no setup. Overridable per-run
+    // with --gateway-url or MNEMONAD_IPFS_GATEWAY.
+    //
+    // A Pinata *dedicated* gateway, not the shared `gateway.pinata.cloud` — measured live: a
+    // burst of 20 rapid reads of the same CID got 20/20 429 on the shared gateway and 20/20
+    // (then 50/50) 200 here, fully unauthenticated. Safe to commit: it's a hostname, not a
+    // credential, and the content behind it is public IPFS either way. List yours with
+    // `GET https://api.pinata.cloud/v3/ipfs/gateways`.
+    gatewayUrl: 'https://azure-casual-firefly-850.mypinata.cloud/ipfs/',
+
+    // Where --passkey opens the browser to run the WebAuthn ceremony (see
+    // lib/passkeyBridge.js and docs/wallet/passkey-accounts.md's CLI section). Has to be this
+    // exact domain, not a placeholder or a local dev server — a passkey is bound to whichever
+    // domain created it (rp.id), so this only ever works against the same deployment the dApp
+    // itself runs on. Overridable per-run with --auth-origin or MNEMONAD_AUTH_ORIGIN, mainly
+    // for pointing at a different deployment during development of this feature itself.
+    authOrigin: 'https://mnemonad.vercel.app',
 };

package/package.json

@@ -1,8 +1,16 @@
 {
   "name": "mnemonad-cli",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "CLI to sync local folders to versioned, diffed on-chain streams on Monad, backed by Mnemonad + monadsync.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jeka911/mnemonad.git",
+    "directory": "cli"
+  },
   "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
   "license": "AGPL-3.0",
   "bin": {
     "mnemonad": "./bin/mnemonad.js"
@@ -14,14 +22,21 @@
     "README.md"
   ],
   "dependencies": {
+    "@huggingface/tokenizers": "^0.2.0",
+    "@sqliteai/sqlite-vector": "^1.1.2",
+    "better-sqlite3": "^13.0.3",
     "mnemonad": "^0.0.1",
     "monadsync": "^0.0.1",
-    "viem": "^2.56.3"
+    "open": "^11.0.4",
+    "viem": "^2.56.3",
+    "ws": "^8.22.0"
   },
   "devDependencies": {
+    "@sqliteai/sqlite-wasm": "3.50.4-wasm.1.0.0-sync.1.1.3-vector.1.1.2-memory.1.3.5",
+    "mnemonad-search-transformers": "^0.1.0",
     "vitest": "^4.1.11"
   },
   "scripts": {
-    "test": "vitest run ./test/*.test.js"
+    "test": "vitest run ./test/*.test.js ./test/search/*.test.js"
   }
 }