mnemonad-cli 0.1.1 → 0.3.0

package/lib/search/browser/WasmIndexReader.js

@@ -0,0 +1,93 @@
+import {
+	TABLE_FILES, TABLE_META, VECTOR_INIT_SQL, SEARCH_SQL,
+	vectorInitOptions, overfetchFor, mapSearchRow,
+} from '../schema.js';
+
+/**
+ * Read-only view of a `search_index.db` held in memory — the browser counterpart to
+ * `VectorIndex`, for a page that already has the file's bytes (the explorer does: after a
+ * restore, the whole folder, index included, lives in memory) and no filesystem to open it from.
+ *
+ * Runs on @sqliteai/sqlite-wasm, which ships the same sqlite-vector build the Node side loads
+ * as a native extension, so it answers with the same `vector_full_scan` query (shared through
+ * schema.js) and returns the same distances — not a second, JS-side scoring that could drift.
+ *
+ * The sqlite3 module is passed in rather than imported here: it's a ~3 MB WASM payload the
+ * caller decides when (and whether) to load, and it keeps this package free of a hard browser-
+ * only dependency: `(await import('@sqliteai/sqlite-wasm')).default()` is the usual way to get one.
+ */
+export default class WasmIndexReader {
+	/**
+	 * @param {any} sqlite3 - an initialized module from `@sqliteai/sqlite-wasm`'s default export.
+	 * @param {Uint8Array} bytes - the whole database file.
+	 */
+	constructor(sqlite3, bytes) {
+		this._sqlite3 = sqlite3;
+		this._db = new sqlite3.oo1.DB(':memory:');
+		// sqlite3_deserialize takes ownership of a WASM-heap copy (FREEONCLOSE), so the caller's
+		// buffer is never referenced again — safe to hand over bytes that later get replaced by a
+		// live poll. READONLY: nothing here should ever write, and a stray write would otherwise
+		// silently diverge this copy from the file actually in the stream.
+		const ptr = sqlite3.wasm.allocFromTypedArray(bytes);
+		const { capi } = sqlite3;
+		const rc = capi.sqlite3_deserialize(
+			this._db.pointer, 'main', ptr, bytes.byteLength, bytes.byteLength,
+			capi.SQLITE_DESERIALIZE_FREEONCLOSE | capi.SQLITE_DESERIALIZE_READONLY,
+		);
+		if (rc !== 0) {
+			this._db.close();
+			throw new Error(`WasmIndexReader: couldn't open the index (sqlite3_deserialize rc=${rc})`);
+		}
+		this._dim = null;
+	}
+
+	/** @returns {?string} */
+	getMeta(key) {
+		const value = this._db.selectValue(`SELECT value FROM ${TABLE_META} WHERE key = ?`, [key]);
+		return value ?? null;
+	}
+
+	/** @returns {Map<string, string>} every currently-indexed path → the content hash it was
+	 *  indexed at — what a caller compares against the live files to spot stale hits. */
+	getFileHashes() {
+		const rows = this._db.exec({
+			sql: `SELECT path, content_hash FROM ${TABLE_FILES}`,
+			rowMode: 'object',
+			returnValue: 'resultRows',
+		});
+		return new Map(rows.map((r) => [r.path, r.content_hash]));
+	}
+
+	/** Same role as `VectorIndex.initVectorColumn` — required once per connection before search. */
+	initVectorColumn(dim, type = 'FLOAT32') {
+		this._db.exec({ sql: VECTOR_INIT_SQL, bind: [vectorInitOptions(dim, type)] });
+		this._dim = dim;
+	}
+
+	/**
+	 * @param {Float32Array} queryEmbedding
+	 * @param {number} k
+	 * @returns {{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}[]}
+	 */
+	search(queryEmbedding, k) {
+		if (this._dim === null) {
+			const dim = Number(this.getMeta('dimension'));
+			if (!dim) throw new Error('WasmIndexReader: index has no dimension recorded — is it empty?');
+			this.initVectorColumn(dim);
+		}
+		const queryBlob = new Uint8Array(queryEmbedding.buffer, queryEmbedding.byteOffset, queryEmbedding.byteLength);
+		// Plain numbers are fine here, unlike better-sqlite3 (see VectorIndex.search): the WASM
+		// binding binds whole JS numbers as INTEGER, which vector_full_scan requires.
+		const rows = this._db.exec({
+			sql: SEARCH_SQL,
+			bind: [queryBlob, overfetchFor(k), k],
+			rowMode: 'object',
+			returnValue: 'resultRows',
+		});
+		return rows.map(mapSearchRow);
+	}
+
+	close() {
+		this._db.close();
+	}
+}

package/lib/search/browser.js

@@ -0,0 +1,13 @@
+// Browser-safe entry (the explorer imports this through a Vite alias): nothing here imports
+// better-sqlite3, the native sqlite-vector extension or any `node:*` module. Searching only —
+// building an index stays a Node job (`mnemonad index`), since that's where the folder's files
+// live on disk and where embedding the whole corpus belongs; the browser only ever embeds a
+// query string.
+export { default as WasmIndexReader } from './browser/WasmIndexReader.js';
+export { default as BrowserSearcher } from './browser/BrowserSearcher.js';
+export { default as EmbeddingProvider } from './embeddings/EmbeddingProvider.js';
+export { default as StaticEmbeddingProvider } from './embeddings/StaticEmbeddingProvider.js';
+export { browserModelFiles } from './embeddings/modelFiles.browser.js';
+export {
+	DEFAULT_DB_NAME, DEFAULT_MODEL, LEGACY_DTYPE, EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS, embedderOfIndex,
+} from './schema.js';

package/lib/search/chunking/ChunkingStrategy.js

@@ -0,0 +1,23 @@
+/**
+ * Base class for splitting one file's text content into embeddable chunks. A subclass fills
+ * in `chunk()`; everything else here is just the shared shape `Indexer` depends on.
+ *
+ * Deliberately separate from Mnemonad's own content-defined chunking (`FastCDC`, used for
+ * on-chain diffing): that one finds boundaries by content hash to maximize byte-level dedup
+ * between versions, which has nothing to do with what makes a good *semantic* unit to embed.
+ * The two chunkers operate on the same bytes for entirely unrelated reasons and should never
+ * be conflated.
+ */
+export default class ChunkingStrategy {
+	/**
+	 * @param {string} text - decoded file content (chunking only ever runs on text files —
+	 *   the caller is responsible for skipping binaries before this is reached).
+	 * @returns {{startOffset: number, endOffset: number, text: string}[]} offsets are UTF-16
+	 *   code-unit offsets into `text` (i.e. plain JS string indices — `text.slice(startOffset,
+	 *   endOffset)` round-trips), not byte offsets — cheap to compute, and exact enough to
+	 *   locate a snippet back in the source file at display time.
+	 */
+	chunk(_text) {
+		throw new Error(`${this.constructor.name}: chunk() not implemented`);
+	}
+}

package/lib/search/chunking/TextWindowChunkingStrategy.js

@@ -0,0 +1,69 @@
+import ChunkingStrategy from './ChunkingStrategy.js';
+
+/**
+ * Default chunker: fixed-size character windows with overlap, breaking on the nearest
+ * paragraph/sentence/word boundary before the target size rather than mid-word, when one is
+ * available within a reasonable lookback. Character-based rather than token-based
+ * deliberately — a token-accurate chunker needs a loaded tokenizer first, coupling chunking
+ * to whichever embedding model happens to be configured; character windows chunk perfectly
+ * well standalone, synchronously, with no model dependency, and 1000 characters is a
+ * conservative-enough stand-in for typical embedding models' token limits (roughly 4
+ * characters/token in English prose, so ~250 tokens — comfortably under the 256-512 token
+ * budget most small embedding models expect). A token-aware `ChunkingStrategy` reusing the
+ * configured `EmbeddingProvider`'s own tokenizer is a reasonable later swap-in — this stays
+ * the default because it works content-agnostically with zero setup.
+ */
+export default class TextWindowChunkingStrategy extends ChunkingStrategy {
+	/**
+	 * @param {Object} [params]
+	 * @param {number} [params.size=1000] - target chunk size, in characters
+	 * @param {number} [params.overlap=150] - characters of overlap between consecutive chunks
+	 */
+	constructor({ size = 1000, overlap = 150 } = {}) {
+		super();
+		if (overlap >= size) {
+			throw new Error('TextWindowChunkingStrategy: overlap must be smaller than size');
+		}
+		this.size = size;
+		this.overlap = overlap;
+	}
+
+	chunk(text) {
+		if (!text) return [];
+		const chunks = [];
+		const step = this.size - this.overlap;
+		let start = 0;
+		while (start < text.length) {
+			let end = Math.min(start + this.size, text.length);
+			// Prefer breaking at a paragraph/sentence/word boundary over a hard mid-word cut —
+			// searched backward from `end`, but never past `start` (a very long unbroken run of
+			// text, e.g. a minified file, just gets a hard cut, which is fine: it wasn't going to
+			// chunk meaningfully either way).
+			if (end < text.length) {
+				const lookback = text.slice(start, end);
+				const boundary = _lastBoundary(lookback);
+				if (boundary > 0) end = start + boundary;
+			}
+			chunks.push({ startOffset: start, endOffset: end, text: text.slice(start, end) });
+			if (end >= text.length) break;
+			start = end - this.overlap;
+		}
+		return chunks;
+	}
+}
+
+/**
+ * Index just past the best break point found in `s` (paragraph > sentence > word), searched
+ * from the end backward, or -1 if none exists worth preferring over a hard cut.
+ * @param {string} s
+ * @returns {number}
+ */
+function _lastBoundary(s) {
+	const paragraph = s.lastIndexOf('\n\n');
+	if (paragraph > s.length * 0.5) return paragraph + 2;
+	const sentence = Math.max(s.lastIndexOf('. '), s.lastIndexOf('.\n'));
+	if (sentence > s.length * 0.5) return sentence + 1;
+	const word = s.lastIndexOf(' ');
+	if (word > s.length * 0.5) return word + 1;
+	return -1;
+}

package/lib/search/embeddings/EmbeddingProvider.js

@@ -0,0 +1,40 @@
+/**
+ * Base class for turning text into vectors. A subclass fills in `embed()`/`dimension`;
+ * everything downstream (`VectorIndex`, `Indexer`, `Searcher`) only ever talks to this
+ * interface, never to a specific model or runtime — swapping the default local model for an
+ * API-based one later needs no change outside a new subclass.
+ */
+export default class EmbeddingProvider {
+	/** @returns {number} vector length this provider produces — fixed for its lifetime. */
+	get dimension() {
+		throw new Error(`${this.constructor.name}: dimension getter not implemented`);
+	}
+
+	/** @returns {string} identifies this provider + model in `search_index_meta`, so a later
+	 *  `search` (possibly a different process entirely) knows what it's matching against. */
+	get modelId() {
+		throw new Error(`${this.constructor.name}: modelId getter not implemented`);
+	}
+
+	/** @returns {?string} which kind of embedder this is ('model2vec', 'transformers'), recorded
+	 *  in the index so a later search builds the same kind (see schema.js). */
+	get embedder() {
+		return null;
+	}
+
+	/** @returns {?string} weight precision (e.g. 'q8', 'fp32') when the model comes in more than
+	 *  one, recorded alongside `modelId` so a searcher loads the same weights. Null when the
+	 *  provider has no such choice. */
+	get dtype() {
+		return null;
+	}
+
+	/**
+	 * @param {string[]} texts
+	 * @returns {Promise<Float32Array[]>} one vector per input text, same order, each of length
+	 *   `this.dimension`.
+	 */
+	async embed(_texts) {
+		throw new Error(`${this.constructor.name}: embed() not implemented`);
+	}
+}

package/lib/search/embeddings/StaticEmbeddingProvider.js

@@ -0,0 +1,180 @@
+import { Tokenizer } from '@huggingface/tokenizers';
+import EmbeddingProvider from './EmbeddingProvider.js';
+import { DEFAULT_MODEL, EMBEDDER_STATIC } from '../schema.js';
+
+/**
+ * The built-in embedder: static (model2vec) embeddings — `minishlab/potion-base-8M` by
+ * default. A static model is a lookup table with one vector per token; a text's embedding is
+ * the mean of its tokens' vectors, normalized. No neural network runs, so there's no ONNX
+ * Runtime, no native code and no WASM: plain JS that behaves identically in Node and in the
+ * browser, embedding a whole folder in milliseconds.
+ *
+ * The trade-off is quality on paraphrased queries — word order and context are invisible to
+ * it. See docs/search.md for how it compares with MiniLM on this project's own corpora; MiniLM
+ * stays available through the optional `mnemonad-search-transformers` package.
+ *
+ * Mirrors model2vec's own `StaticModel.encode` exactly: tokenize without special tokens, drop
+ * the unknown token, cap at `MAX_TOKENS`, average the rows, normalize when the model's config
+ * says so. Models with a separate weights vector or a token mapping (newer model2vec
+ * features) aren't supported; potion-base-8M uses neither.
+ *
+ * The model's files come from a `files` source, so this class never touches a filesystem or a
+ * cache itself — see modelFiles.node.js (disk cache) and modelFiles.browser.js (Cache API).
+ */
+
+/** model2vec's own default cap (StaticModel's max_length). */
+const MAX_TOKENS = 512;
+
+export const MODEL_FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json', 'model.safetensors'];
+
+export default class StaticEmbeddingProvider extends EmbeddingProvider {
+	/**
+	 * @param {Object} params
+	 * @param {{get(name: string): Promise<Uint8Array>}} params.files - the model's files, by name
+	 *   (see MODEL_FILES)
+	 * @param {string} [params.model='minishlab/potion-base-8M']
+	 */
+	constructor({ files, model = DEFAULT_MODEL } = {}) {
+		super();
+		if (!files) throw new Error('StaticEmbeddingProvider: a `files` source is required');
+		this._files = files;
+		this._model = model;
+		this._table = null;
+		this._dim = null;
+		this._loading = null;
+	}
+
+	get dimension() {
+		if (this._dim === null) {
+			throw new Error('StaticEmbeddingProvider: call load() or embed() before reading dimension');
+		}
+		return this._dim;
+	}
+
+	get modelId() {
+		return this._model;
+	}
+
+	get embedder() {
+		return EMBEDDER_STATIC;
+	}
+
+	get isLoaded() {
+		return this._table !== null;
+	}
+
+	/** Safe to call more than once, and concurrently: one shared load. */
+	async load() {
+		if (this._table) return;
+		if (!this._loading) {
+			this._loading = this._load();
+			this._loading.catch(() => {}).finally(() => { this._loading = null; });
+		}
+		await this._loading;
+	}
+
+	async _load() {
+		const decoder = new TextDecoder();
+		const [configBytes, tokenizerBytes, tokenizerConfigBytes, weights] = await Promise.all(
+			MODEL_FILES.map((name) => this._files.get(name))
+		);
+		const config = JSON.parse(decoder.decode(configBytes));
+		const tokenizerJson = JSON.parse(decoder.decode(tokenizerBytes));
+		const tokenizerConfig = JSON.parse(decoder.decode(tokenizerConfigBytes));
+
+		this._tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
+		this._unkId = unknownTokenId(tokenizerJson);
+		this._normalize = config.normalize !== false;
+		const { table, rows, dim } = readEmbeddings(weights);
+		this._rows = rows;
+		this._dim = dim;
+		this._table = table;
+	}
+
+	async embed(texts) {
+		await this.load();
+		return texts.map((text) => this._embedOne(text));
+	}
+
+	_embedOne(text) {
+		const dim = this._dim;
+		const out = new Float32Array(dim);
+		// Truncated first, then the unknown token dropped — the same order as model2vec (its
+		// tokenizer truncates, its tokenize() filters afterwards).
+		const ids = this._tokenizer.encode(text, { add_special_tokens: false }).ids.slice(0, MAX_TOKENS);
+		let count = 0;
+		for (const id of ids) {
+			if (id === this._unkId || id < 0 || id >= this._rows) continue;
+			const base = id * dim;
+			for (let j = 0; j < dim; j++) out[j] += this._table[base + j];
+			count++;
+		}
+		if (count === 0) return out;
+		let norm = 0;
+		for (let j = 0; j < dim; j++) {
+			out[j] /= count;
+			norm += out[j] * out[j];
+		}
+		if (this._normalize && norm > 0) {
+			norm = Math.sqrt(norm);
+			for (let j = 0; j < dim; j++) out[j] /= norm;
+		}
+		return out;
+	}
+}
+
+/** The vocabulary id of the tokenizer's unknown token, or null if it has none. */
+function unknownTokenId(tokenizerJson) {
+	const model = tokenizerJson.model || {};
+	const unk = model.unk_token;
+	if (unk == null) return null;
+	const vocab = model.vocab;
+	if (Array.isArray(vocab)) {
+		// Unigram vocabularies are [token, score] pairs.
+		const index = vocab.findIndex((entry) => entry[0] === unk);
+		return index >= 0 ? index : null;
+	}
+	return vocab && Object.hasOwn(vocab, unk) ? vocab[unk] : null;
+}
+
+/**
+ * Reads the `embeddings` tensor out of a safetensors file: an 8-byte little-endian header
+ * length, a JSON header naming each tensor's dtype, shape and byte range, then the raw data.
+ *
+ * @param {Uint8Array} bytes
+ * @returns {{table: Float32Array, rows: number, dim: number}}
+ */
+export function readEmbeddings(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	const headerLength = Number(view.getBigUint64(0, true));
+	const header = JSON.parse(new TextDecoder().decode(bytes.subarray(8, 8 + headerLength)));
+	const tensor = header.embeddings;
+	if (!tensor) throw new Error('model.safetensors has no `embeddings` tensor — not a model2vec model?');
+	const [rows, dim] = tensor.shape;
+	const [start, end] = tensor.data_offsets;
+	const data = bytes.subarray(8 + headerLength + start, 8 + headerLength + end);
+
+	if (tensor.dtype === 'F32') {
+		// Copied rather than viewed: a Float32Array needs 4-byte alignment, which the data
+		// start (right after a variable-length header) doesn't guarantee.
+		const table = new Float32Array(rows * dim);
+		new Uint8Array(table.buffer).set(data);
+		return { table, rows, dim };
+	}
+	if (tensor.dtype === 'F16') {
+		const table = new Float32Array(rows * dim);
+		const halves = new DataView(data.buffer, data.byteOffset, data.byteLength);
+		for (let i = 0; i < table.length; i++) table[i] = halfToFloat(halves.getUint16(i * 2, true));
+		return { table, rows, dim };
+	}
+	throw new Error(`model.safetensors: unsupported dtype ${tensor.dtype} (expected F32 or F16)`);
+}
+
+function halfToFloat(h) {
+	const sign = h & 0x8000 ? -1 : 1;
+	const exp = (h >> 10) & 0x1f;
+	const frac = h & 0x03ff;
+	if (exp === 0) return sign * 2 ** -14 * (frac / 1024);
+	if (exp === 0x1f) return frac ? NaN : sign * Infinity;
+	return sign * 2 ** (exp - 15) * (1 + frac / 1024);
+}

package/lib/search/embeddings/modelFiles.browser.js

@@ -0,0 +1,28 @@
+import { downloadWithProgress, hubUrl } from './modelFiles.shared.js';
+
+/**
+ * A model's files for StaticEmbeddingProvider in a browser (or a worker): fetched from the
+ * Hugging Face hub once, then served from the Cache API, so revisits don't download again.
+ * Falls back to plain fetching where the Cache API isn't available (an insecure context, say).
+ *
+ * @param {string} model - e.g. 'minishlab/potion-base-8M'
+ * @param {Object} [opts]
+ * @param {(event: Object) => void} [opts.onProgress] - `{status: 'progress', file, loaded, total}`
+ */
+export function browserModelFiles(model, { onProgress = null, cacheName = 'mnemonad-models' } = {}) {
+	return {
+		async get(name) {
+			const url = hubUrl(model, name);
+			const cache = typeof caches !== 'undefined' ? await caches.open(cacheName).catch(() => null) : null;
+			const hit = cache && await cache.match(url);
+			if (hit) return new Uint8Array(await hit.arrayBuffer());
+
+			const bytes = await downloadWithProgress(url, name, onProgress);
+			if (cache) {
+				// Best effort: a full quota just means downloading again next time.
+				await cache.put(url, new Response(bytes, { headers: { 'content-length': String(bytes.byteLength) } })).catch(() => {});
+			}
+			return bytes;
+		},
+	};
+}

package/lib/search/embeddings/modelFiles.node.js

@@ -0,0 +1,41 @@
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { downloadWithProgress, hubUrl } from './modelFiles.shared.js';
+
+/**
+ * A model's files for StaticEmbeddingProvider under Node: downloaded from the Hugging Face hub
+ * once, then read from a local cache — `$MNEMONAD_CACHE_DIR`, else `$XDG_CACHE_HOME/mnemonad`,
+ * else `~/.cache/mnemonad`, under `models/<model id>/`. Written through a temp file and a
+ * rename, so an interrupted download never leaves a truncated file behind to be read as valid.
+ *
+ * @param {string} model - e.g. 'minishlab/potion-base-8M'
+ * @param {Object} [opts]
+ * @param {(event: Object) => void} [opts.onProgress] - `{status: 'progress', file, loaded, total}`
+ * @param {string} [opts.cacheDir]
+ */
+export function nodeModelFiles(model, { onProgress = null, cacheDir = defaultCacheDir() } = {}) {
+	const dir = join(cacheDir, 'models', ...model.split('/'));
+	return {
+		async get(name) {
+			const path = join(dir, name);
+			try {
+				return new Uint8Array(await readFile(path));
+			} catch (err) {
+				if (err.code !== 'ENOENT') throw err;
+			}
+			const bytes = await downloadWithProgress(hubUrl(model, name), name, onProgress);
+			await mkdir(dir, { recursive: true });
+			const tmp = `${path}.${process.pid}.tmp`;
+			await writeFile(tmp, bytes);
+			await rename(tmp, path);
+			return bytes;
+		},
+	};
+}
+
+export function defaultCacheDir() {
+	if (process.env.MNEMONAD_CACHE_DIR) return process.env.MNEMONAD_CACHE_DIR;
+	if (process.env.XDG_CACHE_HOME) return join(process.env.XDG_CACHE_HOME, 'mnemonad');
+	return join(homedir(), '.cache', 'mnemonad');
+}

package/lib/search/embeddings/modelFiles.shared.js

@@ -0,0 +1,44 @@
+/**
+ * The download half of a model file source, shared by Node (modelFiles.node.js) and the browser
+ * (modelFiles.browser.js) — plain `fetch`, which both have.
+ */
+
+const HUB = 'https://huggingface.co';
+
+/** @returns {string} where the hub serves `name` from `model`'s main branch. */
+export function hubUrl(model, name) {
+	return `${HUB}/${model}/resolve/main/${name}`;
+}
+
+/**
+ * Fetches `url` into memory, reporting progress in the same shape transformers.js does
+ * (`{status: 'progress', file, loaded, total}`), so one progress display works for either
+ * kind of model.
+ *
+ * @returns {Promise<Uint8Array>}
+ */
+export async function downloadWithProgress(url, name, onProgress) {
+	const res = await fetch(url);
+	if (!res.ok) throw new Error(`couldn't download ${name} (${res.status} ${res.statusText}) from ${url}`);
+	const total = Number(res.headers.get('content-length')) || 0;
+	if (!res.body || !onProgress) return new Uint8Array(await res.arrayBuffer());
+
+	const reader = res.body.getReader();
+	const parts = [];
+	let loaded = 0;
+	for (;;) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		parts.push(value);
+		loaded += value.byteLength;
+		onProgress({ status: 'progress', file: name, loaded, total: total || loaded });
+	}
+	const bytes = new Uint8Array(loaded);
+	let offset = 0;
+	for (const part of parts) {
+		bytes.set(part, offset);
+		offset += part.byteLength;
+	}
+	onProgress({ status: 'done', file: name, loaded, total: total || loaded });
+	return bytes;
+}

package/lib/search/index.js

@@ -0,0 +1,18 @@
+// The search engine behind `mnemonad index` / `search` / `--index` — Node side. The browser
+// half (for the explorer) is ./browser.js, which must stay free of native and `node:*` imports.
+export { default as VectorIndex } from './VectorIndex.js';
+export { default as Indexer, ModelMismatchError } from './Indexer.js';
+export { default as Searcher, describeIndexModel } from './Searcher.js';
+export { default as EmbeddingProvider } from './embeddings/EmbeddingProvider.js';
+export { default as StaticEmbeddingProvider } from './embeddings/StaticEmbeddingProvider.js';
+export { nodeModelFiles } from './embeddings/modelFiles.node.js';
+export { default as ChunkingStrategy } from './chunking/ChunkingStrategy.js';
+export { default as TextWindowChunkingStrategy } from './chunking/TextWindowChunkingStrategy.js';
+export {
+	createProvider, loadTransformersExtension, findTransformersExtension, MissingExtensionError,
+	EXTENSION_PACKAGE, EXTENSION_INSTALL,
+} from './providers.js';
+export {
+	DEFAULT_DB_NAME, DEFAULT_MODEL, MODEL_ALIASES, resolveModelName, embedderForModel, embedderOfIndex,
+	EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS,
+} from './schema.js';