mnemonad-cli 0.2.0 → 0.3.1

package/lib/search/embeddings/StaticEmbeddingProvider.js

@@ -0,0 +1,180 @@
+import { Tokenizer } from '@huggingface/tokenizers';
+import EmbeddingProvider from './EmbeddingProvider.js';
+import { DEFAULT_MODEL, EMBEDDER_STATIC } from '../schema.js';
+
+/**
+ * The built-in embedder: static (model2vec) embeddings — `minishlab/potion-base-8M` by
+ * default. A static model is a lookup table with one vector per token; a text's embedding is
+ * the mean of its tokens' vectors, normalized. No neural network runs, so there's no ONNX
+ * Runtime, no native code and no WASM: plain JS that behaves identically in Node and in the
+ * browser, embedding a whole folder in milliseconds.
+ *
+ * The trade-off is quality on paraphrased queries — word order and context are invisible to
+ * it. See docs/search.md for how it compares with MiniLM on this project's own corpora; MiniLM
+ * stays available through the optional `mnemonad-search-transformers` package.
+ *
+ * Mirrors model2vec's own `StaticModel.encode` exactly: tokenize without special tokens, drop
+ * the unknown token, cap at `MAX_TOKENS`, average the rows, normalize when the model's config
+ * says so. Models with a separate weights vector or a token mapping (newer model2vec
+ * features) aren't supported; potion-base-8M uses neither.
+ *
+ * The model's files come from a `files` source, so this class never touches a filesystem or a
+ * cache itself — see modelFiles.node.js (disk cache) and modelFiles.browser.js (Cache API).
+ */
+
+/** model2vec's own default cap (StaticModel's max_length). */
+const MAX_TOKENS = 512;
+
+export const MODEL_FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json', 'model.safetensors'];
+
+export default class StaticEmbeddingProvider extends EmbeddingProvider {
+	/**
+	 * @param {Object} params
+	 * @param {{get(name: string): Promise<Uint8Array>}} params.files - the model's files, by name
+	 *   (see MODEL_FILES)
+	 * @param {string} [params.model='minishlab/potion-base-8M']
+	 */
+	constructor({ files, model = DEFAULT_MODEL } = {}) {
+		super();
+		if (!files) throw new Error('StaticEmbeddingProvider: a `files` source is required');
+		this._files = files;
+		this._model = model;
+		this._table = null;
+		this._dim = null;
+		this._loading = null;
+	}
+
+	get dimension() {
+		if (this._dim === null) {
+			throw new Error('StaticEmbeddingProvider: call load() or embed() before reading dimension');
+		}
+		return this._dim;
+	}
+
+	get modelId() {
+		return this._model;
+	}
+
+	get embedder() {
+		return EMBEDDER_STATIC;
+	}
+
+	get isLoaded() {
+		return this._table !== null;
+	}
+
+	/** Safe to call more than once, and concurrently: one shared load. */
+	async load() {
+		if (this._table) return;
+		if (!this._loading) {
+			this._loading = this._load();
+			this._loading.catch(() => {}).finally(() => { this._loading = null; });
+		}
+		await this._loading;
+	}
+
+	async _load() {
+		const decoder = new TextDecoder();
+		const [configBytes, tokenizerBytes, tokenizerConfigBytes, weights] = await Promise.all(
+			MODEL_FILES.map((name) => this._files.get(name))
+		);
+		const config = JSON.parse(decoder.decode(configBytes));
+		const tokenizerJson = JSON.parse(decoder.decode(tokenizerBytes));
+		const tokenizerConfig = JSON.parse(decoder.decode(tokenizerConfigBytes));
+
+		this._tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
+		this._unkId = unknownTokenId(tokenizerJson);
+		this._normalize = config.normalize !== false;
+		const { table, rows, dim } = readEmbeddings(weights);
+		this._rows = rows;
+		this._dim = dim;
+		this._table = table;
+	}
+
+	async embed(texts) {
+		await this.load();
+		return texts.map((text) => this._embedOne(text));
+	}
+
+	_embedOne(text) {
+		const dim = this._dim;
+		const out = new Float32Array(dim);
+		// Truncated first, then the unknown token dropped — the same order as model2vec (its
+		// tokenizer truncates, its tokenize() filters afterwards).
+		const ids = this._tokenizer.encode(text, { add_special_tokens: false }).ids.slice(0, MAX_TOKENS);
+		let count = 0;
+		for (const id of ids) {
+			if (id === this._unkId || id < 0 || id >= this._rows) continue;
+			const base = id * dim;
+			for (let j = 0; j < dim; j++) out[j] += this._table[base + j];
+			count++;
+		}
+		if (count === 0) return out;
+		let norm = 0;
+		for (let j = 0; j < dim; j++) {
+			out[j] /= count;
+			norm += out[j] * out[j];
+		}
+		if (this._normalize && norm > 0) {
+			norm = Math.sqrt(norm);
+			for (let j = 0; j < dim; j++) out[j] /= norm;
+		}
+		return out;
+	}
+}
+
+/** The vocabulary id of the tokenizer's unknown token, or null if it has none. */
+function unknownTokenId(tokenizerJson) {
+	const model = tokenizerJson.model || {};
+	const unk = model.unk_token;
+	if (unk == null) return null;
+	const vocab = model.vocab;
+	if (Array.isArray(vocab)) {
+		// Unigram vocabularies are [token, score] pairs.
+		const index = vocab.findIndex((entry) => entry[0] === unk);
+		return index >= 0 ? index : null;
+	}
+	return vocab && Object.hasOwn(vocab, unk) ? vocab[unk] : null;
+}
+
+/**
+ * Reads the `embeddings` tensor out of a safetensors file: an 8-byte little-endian header
+ * length, a JSON header naming each tensor's dtype, shape and byte range, then the raw data.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {{table: Float32Array, rows: number, dim: number}}
+ */
+export function readEmbeddings(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const headerLength = Number(view.getBigUint64(0, true));
+	const header = JSON.parse(new TextDecoder().decode(bytes.subarray(8, 8 + headerLength)));
+	const tensor = header.embeddings;
+	if (!tensor) throw new Error('model.safetensors has no `embeddings` tensor — not a model2vec model?');
+	const [rows, dim] = tensor.shape;
+	const [start, end] = tensor.data_offsets;
+	const data = bytes.subarray(8 + headerLength + start, 8 + headerLength + end);
+
+	if (tensor.dtype === 'F32') {
+		// Copied rather than viewed: a Float32Array needs 4-byte alignment, which the data
+		// start (right after a variable-length header) doesn't guarantee.
+		const table = new Float32Array(rows * dim);
+		new Uint8Array(table.buffer).set(data);
+		return { table, rows, dim };
+	}
+	if (tensor.dtype === 'F16') {
+		const table = new Float32Array(rows * dim);
+		const halves = new DataView(data.buffer, data.byteOffset, data.byteLength);
+		for (let i = 0; i < table.length; i++) table[i] = halfToFloat(halves.getUint16(i * 2, true));
+		return { table, rows, dim };
+	}
+	throw new Error(`model.safetensors: unsupported dtype ${tensor.dtype} (expected F32 or F16)`);
+}
+
+function halfToFloat(h) {
+	const sign = h & 0x8000 ? -1 : 1;
+	const exp = (h >> 10) & 0x1f;
+	const frac = h & 0x03ff;
+	if (exp === 0) return sign * 2 ** -14 * (frac / 1024);
+	if (exp === 0x1f) return frac ? NaN : sign * Infinity;
+	return sign * 2 ** (exp - 15) * (1 + frac / 1024);
+}

package/lib/search/embeddings/modelFiles.browser.js

@@ -0,0 +1,28 @@
+import { downloadWithProgress, hubUrl } from './modelFiles.shared.js';
+
+/**
+ * A model's files for StaticEmbeddingProvider in a browser (or a worker): fetched from the
+ * Hugging Face hub once, then served from the Cache API, so revisits don't download again.
+ * Falls back to plain fetching where the Cache API isn't available (an insecure context, say).
+ *
+ * @param {string} model - e.g. 'minishlab/potion-base-8M'
+ * @param {Object} [opts]
+ * @param {(event: Object) => void} [opts.onProgress] - `{status: 'progress', file, loaded, total}`
+ */
+export function browserModelFiles(model, { onProgress = null, cacheName = 'mnemonad-models' } = {}) {
+	return {
+		async get(name) {
+			const url = hubUrl(model, name);
+			const cache = typeof caches !== 'undefined' ? await caches.open(cacheName).catch(() => null) : null;
+			const hit = cache && await cache.match(url);
+			if (hit) return new Uint8Array(await hit.arrayBuffer());
+
+			const bytes = await downloadWithProgress(url, name, onProgress);
+			if (cache) {
+				// Best effort: a full quota just means downloading again next time.
+				await cache.put(url, new Response(bytes, { headers: { 'content-length': String(bytes.byteLength) } })).catch(() => {});
+			}
+			return bytes;
+		},
+	};
+}

package/lib/search/embeddings/modelFiles.node.js

@@ -0,0 +1,41 @@
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { downloadWithProgress, hubUrl } from './modelFiles.shared.js';
+
+/**
+ * A model's files for StaticEmbeddingProvider under Node: downloaded from the Hugging Face hub
+ * once, then read from a local cache — `$MNEMONAD_CACHE_DIR`, else `$XDG_CACHE_HOME/mnemonad`,
+ * else `~/.cache/mnemonad`, under `models/<model id>/`. Written through a temp file and a
+ * rename, so an interrupted download never leaves a truncated file behind to be read as valid.
+ *
+ * @param {string} model - e.g. 'minishlab/potion-base-8M'
+ * @param {Object} [opts]
+ * @param {(event: Object) => void} [opts.onProgress] - `{status: 'progress', file, loaded, total}`
+ * @param {string} [opts.cacheDir]
+ */
+export function nodeModelFiles(model, { onProgress = null, cacheDir = defaultCacheDir() } = {}) {
+	const dir = join(cacheDir, 'models', ...model.split('/'));
+	return {
+		async get(name) {
+			const path = join(dir, name);
+			try {
+				return new Uint8Array(await readFile(path));
+			} catch (err) {
+				if (err.code !== 'ENOENT') throw err;
+			}
+			const bytes = await downloadWithProgress(hubUrl(model, name), name, onProgress);
+			await mkdir(dir, { recursive: true });
+			const tmp = `${path}.${process.pid}.tmp`;
+			await writeFile(tmp, bytes);
+			await rename(tmp, path);
+			return bytes;
+		},
+	};
+}
+
+export function defaultCacheDir() {
+	if (process.env.MNEMONAD_CACHE_DIR) return process.env.MNEMONAD_CACHE_DIR;
+	if (process.env.XDG_CACHE_HOME) return join(process.env.XDG_CACHE_HOME, 'mnemonad');
+	return join(homedir(), '.cache', 'mnemonad');
+}

package/lib/search/embeddings/modelFiles.shared.js

@@ -0,0 +1,44 @@
+/**
+ * The download half of a model file source, shared by Node (modelFiles.node.js) and the browser
+ * (modelFiles.browser.js) — plain `fetch`, which both have.
+ */
+
+const HUB = 'https://huggingface.co';
+
+/** @returns {string} where the hub serves `name` from `model`'s main branch. */
+export function hubUrl(model, name) {
+	return `${HUB}/${model}/resolve/main/${name}`;
+}
+
+/**
+ * Fetches `url` into memory, reporting progress in the same shape transformers.js does
+ * (`{status: 'progress', file, loaded, total}`), so one progress display works for either
+ * kind of model.
+ *
+ * @returns {Promise<Uint8Array>}
+ */
+export async function downloadWithProgress(url, name, onProgress) {
+	const res = await fetch(url);
+	if (!res.ok) throw new Error(`couldn't download ${name} (${res.status} ${res.statusText}) from ${url}`);
+	const total = Number(res.headers.get('content-length')) || 0;
+	if (!res.body || !onProgress) return new Uint8Array(await res.arrayBuffer());
+
+	const reader = res.body.getReader();
+	const parts = [];
+	let loaded = 0;
+	for (;;) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		parts.push(value);
+		loaded += value.byteLength;
+		onProgress({ status: 'progress', file: name, loaded, total: total || loaded });
+	}
+	const bytes = new Uint8Array(loaded);
+	let offset = 0;
+	for (const part of parts) {
+		bytes.set(part, offset);
+		offset += part.byteLength;
+	}
+	onProgress({ status: 'done', file: name, loaded, total: total || loaded });
+	return bytes;
+}

package/lib/search/index.js

@@ -0,0 +1,18 @@
+// The search engine behind `mnemonad index` / `search` / `--index` — Node side. The browser
+// half (for the explorer) is ./browser.js, which must stay free of native and `node:*` imports.
+export { default as VectorIndex } from './VectorIndex.js';
+export { default as Indexer, ModelMismatchError } from './Indexer.js';
+export { default as Searcher, describeIndexModel } from './Searcher.js';
+export { default as EmbeddingProvider } from './embeddings/EmbeddingProvider.js';
+export { default as StaticEmbeddingProvider } from './embeddings/StaticEmbeddingProvider.js';
+export { nodeModelFiles } from './embeddings/modelFiles.node.js';
+export { default as ChunkingStrategy } from './chunking/ChunkingStrategy.js';
+export { default as TextWindowChunkingStrategy } from './chunking/TextWindowChunkingStrategy.js';
+export {
+	createProvider, loadTransformersExtension, findTransformersExtension, MissingExtensionError,
+	EXTENSION_PACKAGE, EXTENSION_INSTALL,
+} from './providers.js';
+export {
+	DEFAULT_DB_NAME, DEFAULT_MODEL, MODEL_ALIASES, resolveModelName, embedderForModel, embedderOfIndex,
+	EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS,
+} from './schema.js';

package/lib/search/providers.js

@@ -0,0 +1,153 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import StaticEmbeddingProvider from './embeddings/StaticEmbeddingProvider.js';
+import { nodeModelFiles } from './embeddings/modelFiles.node.js';
+import { EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS, embedderForModel } from './schema.js';
+
+/**
+ * Embedding providers for Node — the built-in static one, or one from the optional
+ * transformers extension.
+ *
+ * The extension (`mnemonad-search-transformers`) is deliberately *not* a dependency of the CLI:
+ * it brings ONNX Runtime, ~450 MB installed, for the one feature (MiniLM-style neural models)
+ * most users never need. It's installed separately, and found at runtime:
+ *
+ *  1. A plain `import()`. Node resolves it by walking up from this file's own folder, which
+ *     reaches it for a local project install, `npx -p … -p …`, and npm's global installs —
+ *     global packages sit side by side in `<prefix>/lib/node_modules`, which the walk reaches.
+ *  2. Failing that, the package managers' own global folders: `npm root -g` (for layouts the
+ *     walk doesn't reach), then `pnpm root -g`. pnpm 11 installs every global package into its
+ *     own isolated folder (`<root>/<hash>/node_modules/<pkg>`), so there the CLI can never see
+ *     a sibling by walking up; each of those folders is checked instead, newest first.
+ */
+
+export const EXTENSION_PACKAGE = 'mnemonad-search-transformers';
+
+/** What this CLI expects the extension's interface to look like; the extension exports its own. */
+export const EXTENSION_API_VERSION = 1;
+
+export const EXTENSION_INSTALL = `npm install -g ${EXTENSION_PACKAGE}   (or: pnpm add -g ${EXTENSION_PACKAGE})`;
+
+export class MissingExtensionError extends Error {
+	constructor(model) {
+		super(
+			`${model} needs the optional transformers extension (about 450 MB installed, not included by default):\n` +
+			`    ${EXTENSION_INSTALL}`
+		);
+		this.name = 'MissingExtensionError';
+		this.model = model;
+	}
+}
+
+/**
+ * @param {Object} [opts] - both overridable for tests
+ * @param {(specifier: string) => Promise<any>} [opts.importer]
+ * @param {() => string[]} [opts.globalRoots] - global package folders to look in (see above)
+ * @returns {Promise<?any>} the extension module, or null when it isn't installed
+ */
+export async function findTransformersExtension({ importer = (s) => import(s), globalRoots = defaultGlobalRoots } = {}) {
+	try {
+		return await importer(EXTENSION_PACKAGE);
+	} catch (err) {
+		if (!isNotFound(err)) throw err;
+	}
+	for (const root of globalRoots()) {
+		const pkgJson = extensionIn(root);
+		if (!pkgJson) continue;
+		// Resolved from inside the package itself (a self-reference through its own `exports`),
+		// so the entry point is whatever the package declares, not a guessed file name.
+		const entry = createRequire(pkgJson).resolve(EXTENSION_PACKAGE);
+		return importer(pathToFileURL(entry).href);
+	}
+	return null;
+}
+
+/**
+ * The extension's package.json under one global folder, or null. Only these exact places —
+ * never a module-resolution walk, which could wander out of the folder and pick up some
+ * unrelated copy:
+ *  - `<root>/<package>` — npm, and older pnpm, where global packages sit side by side;
+ *  - `<root>/<dir>/node_modules/<package>` — pnpm 11's isolated per-package folders; newest
+ *    first, since reinstalling leaves an older folder behind until pnpm prunes it.
+ */
+function extensionIn(root) {
+	const direct = join(root, EXTENSION_PACKAGE, 'package.json');
+	if (existsSync(direct)) return direct;
+	let dirs;
+	try {
+		dirs = readdirSync(root, { withFileTypes: true }).filter((d) => d.isDirectory());
+	} catch {
+		return null;
+	}
+	const found = dirs
+		.map((d) => join(root, d.name, 'node_modules', EXTENSION_PACKAGE, 'package.json'))
+		.filter((p) => existsSync(p))
+		.map((p) => ({ p, mtime: statSync(p).mtimeMs }))
+		.sort((a, b) => b.mtime - a.mtime);
+	return found.length ? found[0].p : null;
+}
+
+/**
+ * @param {string} model - for the error message
+ * @param {Object} [opts] - see findTransformersExtension
+ */
+export async function loadTransformersExtension(model, opts) {
+	const ext = await findTransformersExtension(opts);
+	if (!ext) throw new MissingExtensionError(model);
+	if (ext.apiVersion !== EXTENSION_API_VERSION) {
+		throw new Error(
+			`${EXTENSION_PACKAGE} speaks extension API v${ext.apiVersion ?? '?'}, but this mnemonad-cli expects ` +
+			`v${EXTENSION_API_VERSION} — update both: npm install -g mnemonad-cli@latest ${EXTENSION_PACKAGE}@latest`
+		);
+	}
+	return ext;
+}
+
+/**
+ * @param {Object} params
+ * @param {string} params.model - full model id
+ * @param {string} [params.embedder] - defaults to what the model needs (see embedderForModel)
+ * @param {?string} [params.dtype] - transformers models only
+ * @param {(event: Object) => void} [params.onProgress]
+ * @param {Object} [params.extension] - passed to loadTransformersExtension (tests)
+ */
+export async function createProvider({ model, embedder = embedderForModel(model), dtype = null, onProgress = null, extension } = {}) {
+	if (embedder === EMBEDDER_STATIC) {
+		return new StaticEmbeddingProvider({ model, files: nodeModelFiles(model, { onProgress }) });
+	}
+	if (embedder === EMBEDDER_TRANSFORMERS) {
+		const ext = await loadTransformersExtension(model, extension);
+		return ext.createProvider({ model, ...(dtype ? { dtype } : {}), onProgress });
+	}
+	throw new Error(`unknown embedder '${embedder}' (expected ${EMBEDDER_STATIC} or ${EMBEDDER_TRANSFORMERS})`);
+}
+
+function isNotFound(err) {
+	return !!err
+		&& (err.code === 'ERR_MODULE_NOT_FOUND' || err.code === 'MODULE_NOT_FOUND')
+		// Only the extension itself being absent — a missing dependency *inside* an installed
+		// extension is a broken install, and should surface as such.
+		&& String(err.message).includes(`'${EXTENSION_PACKAGE}'`);
+}
+
+/** npm's and pnpm's global package folders — whichever of the two is installed. Asked only
+ *  when the plain import has already missed, so a normal run never spawns either. */
+function defaultGlobalRoots() {
+	return ['npm', 'pnpm'].map(globalRootOf).filter(Boolean);
+}
+
+function globalRootOf(tool) {
+	try {
+		return execFileSync(tool, ['root', '-g'], {
+			encoding: 'utf8',
+			stdio: ['ignore', 'pipe', 'ignore'],
+			timeout: 10_000,
+			shell: process.platform === 'win32',
+		}).trim() || null;
+	} catch {
+		return null;
+	}
+}

package/lib/search/schema.js

@@ -0,0 +1,103 @@
+/**
+ * Everything about the on-disk index format that more than one reader has to agree on — the
+ * Node `VectorIndex` (better-sqlite3 + the native sqlite-vector extension) and the browser's
+ * `WasmIndexReader` (@sqliteai/sqlite-wasm, same sqlite-vector compiled in). Kept in one place
+ * so the two can't drift: a search from the explorer must rank exactly like `mnemonad search`
+ * on the same file.
+ *
+ * Deliberately dependency-free (no `node:*`, no native modules) — the browser entry imports it.
+ */
+
+export const DEFAULT_DB_NAME = 'search_index.db';
+
+/**
+ * Two kinds of embedding model, recorded per index as `embedder` in its metadata:
+ *
+ *  - 'model2vec' — static embeddings (a token → vector lookup table, averaged). Built in: pure
+ *    JS, no native code, runs the same in Node and the browser. The default.
+ *  - 'transformers' — a neural model run by transformers.js on ONNX Runtime (MiniLM, say).
+ *    Better on paraphrased queries, but ONNX Runtime is ~450 MB installed, so the CLI loads it
+ *    only from the optional `mnemonad-search-transformers` package.
+ */
+export const EMBEDDER_STATIC = 'model2vec';
+export const EMBEDDER_TRANSFORMERS = 'transformers';
+
+export const DEFAULT_MODEL = 'minishlab/potion-base-8M';
+
+/** Short names accepted wherever a model is (`--model minilm`). */
+export const MODEL_ALIASES = {
+	potion: 'minishlab/potion-base-8M',
+	minilm: 'Xenova/all-MiniLM-L6-v2',
+};
+
+/** @returns {string} the full model id for an alias, or the name unchanged. */
+export function resolveModelName(name) {
+	return MODEL_ALIASES[name] || name;
+}
+
+/** Which embedder a model needs. model2vec models live under the `minishlab/` organisation
+ *  on the Hugging Face hub; everything else is taken to be a transformers.js model. */
+export function embedderForModel(model) {
+	return /^minishlab\//.test(model) ? EMBEDDER_STATIC : EMBEDDER_TRANSFORMERS;
+}
+
+/**
+ * Which embedder built an existing index. Indexes from before `embedder` was recorded were all
+ * built with MiniLM through transformers.js — the only option at the time.
+ *
+ * @param {{embedder?: ?string, model?: ?string}} meta
+ */
+export function embedderOfIndex(meta) {
+	if (meta.embedder) return meta.embedder;
+	return meta.model ? embedderForModel(meta.model) : EMBEDDER_TRANSFORMERS;
+}
+
+/** What a transformers-built index recorded before `dtype` was written to its meta was built
+ *  with — the Node default at the time, since the CLI was the only thing that could build one. */
+export const LEGACY_DTYPE = 'fp32';
+
+export const TABLE_CHUNKS = 'search_index';
+export const TABLE_FILES = 'search_index_files';
+export const TABLE_META = 'search_index_meta';
+export const VECTOR_COLUMN = 'embedding';
+
+/** @param {number} dim @param {'FLOAT32'|'INT8'} [type='FLOAT32'] */
+export function vectorInitOptions(dim, type = 'FLOAT32') {
+	return `type=${type},dimension=${dim},distance=COSINE`;
+}
+
+export const VECTOR_INIT_SQL = `SELECT vector_init('${TABLE_CHUNKS}', '${VECTOR_COLUMN}', ?)`;
+
+/**
+ * Nearest-neighbor search, current chunks only. Binds, in order: query vector (BLOB), overfetch
+ * count, k. `vector_full_scan` is exact brute force — appropriate for the corpus sizes a synced
+ * folder actually has, and an ANN/partitioning index would break the diffability the whole
+ * format depends on. Some of its top candidates may be stale (superseded by a later reindex of
+ * the same file, or belonging to a removed file), which the join against the files table drops
+ * — hence the overfetch, see `overfetchFor`.
+ */
+export const SEARCH_SQL = `
+	SELECT s.path, s.chunk_index, s.start_offset, s.end_offset, v.distance
+	FROM vector_full_scan('${TABLE_CHUNKS}', '${VECTOR_COLUMN}', ?, ?) v
+	JOIN ${TABLE_CHUNKS} s ON s.id = v.rowid
+	JOIN ${TABLE_FILES} f ON f.path = s.path AND f.content_hash = s.content_hash
+	ORDER BY v.distance ASC
+	LIMIT ?
+`;
+
+/** A generous overfetch — cheap (still one exact scan) and simple, rather than looping with a
+ *  growing k until enough candidates survive the staleness join. */
+export function overfetchFor(k) {
+	return Math.max(k * 8, 64);
+}
+
+/** @returns {{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}} */
+export function mapSearchRow(r) {
+	return {
+		path: r.path,
+		chunkIndex: r.chunk_index,
+		startOffset: r.start_offset,
+		endOffset: r.end_offset,
+		distance: r.distance,
+	};
+}

package/mnemonad.config.js

@@ -13,9 +13,23 @@ export default {
     // their own. Overridable per-run with --presign-url or MNEMONAD_PRESIGN_URL.
     presignUrl: 'https://presign-server.vercel.app',
 
-    // Public IPFS gateway for *reading* externally-offloaded items (pull/diff/info/watch)
-    // — same default the explorer dApp uses (explorer/src/mnemonad/presignProvider.js), so
-    // a stream pushed through either tool reads back the same way with no setup. Overridable
-    // per-run with --gateway-url or MNEMONAD_IPFS_GATEWAY.
-    gatewayUrl: 'https://gateway.pinata.cloud/ipfs/',
+    // IPFS gateway for *reading* externally-offloaded items (pull/diff/info/watch) — same
+    // default the explorer dApp uses (explorer/src/mnemonad/presignProvider.js), so a stream
+    // pushed through either tool reads back the same way with no setup. Overridable per-run
+    // with --gateway-url or MNEMONAD_IPFS_GATEWAY.
+    //
+    // A Pinata *dedicated* gateway, not the shared `gateway.pinata.cloud` — measured live: a
+    // burst of 20 rapid reads of the same CID got 20/20 429 on the shared gateway and 20/20
+    // (then 50/50) 200 here, fully unauthenticated. Safe to commit: it's a hostname, not a
+    // credential, and the content behind it is public IPFS either way. List yours with
+    // `GET https://api.pinata.cloud/v3/ipfs/gateways`.
+    gatewayUrl: 'https://azure-casual-firefly-850.mypinata.cloud/ipfs/',
+
+    // Where --passkey opens the browser to run the WebAuthn ceremony (see
+    // lib/passkeyBridge.js and docs/wallet/passkey-accounts.md's CLI section). Has to be this
+    // exact domain, not a placeholder or a local dev server — a passkey is bound to whichever
+    // domain created it (rp.id), so this only ever works against the same deployment the dApp
+    // itself runs on. Overridable per-run with --auth-origin or MNEMONAD_AUTH_ORIGIN, mainly
+    // for pointing at a different deployment during development of this feature itself.
+    authOrigin: 'https://mnemonad.vercel.app',
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mnemonad-cli",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "CLI to sync local folders to versioned, diffed on-chain streams on Monad, backed by Mnemonad + monadsync.",
   "repository": {
     "type": "git",
@@ -8,6 +8,9 @@
     "directory": "cli"
   },
   "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
   "license": "AGPL-3.0",
   "bin": {
     "mnemonad": "./bin/mnemonad.js"
@@ -19,14 +22,21 @@
     "README.md"
   ],
   "dependencies": {
+    "@huggingface/tokenizers": "^0.2.0",
+    "@sqliteai/sqlite-vector": "^1.1.2",
+    "better-sqlite3": "^13.0.3",
     "mnemonad": "^0.0.1",
-    "monadsync": "^0.0.1",
-    "viem": "^2.56.3"
+    "monadsync": "^0.0.2",
+    "open": "^11.0.4",
+    "viem": "^2.56.3",
+    "ws": "^8.22.0"
   },
   "devDependencies": {
+    "@sqliteai/sqlite-wasm": "3.50.4-wasm.1.0.0-sync.1.1.3-vector.1.1.2-memory.1.3.5",
+    "mnemonad-search-transformers": "^0.1.0",
     "vitest": "^4.1.11"
   },
   "scripts": {
-    "test": "vitest run ./test/*.test.js"
+    "test": "vitest run ./test/*.test.js ./test/search/*.test.js"
   }
 }