mnemonad-cli 0.2.0 → 0.3.1

package/lib/search/VectorIndex.js

@@ -0,0 +1,225 @@
+import { existsSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import Database from 'better-sqlite3';
+import { VECTOR_INIT_SQL, SEARCH_SQL, vectorInitOptions, overfetchFor, mapSearchRow } from './schema.js';
+
+// @sqliteai/sqlite-vector's own ESM build (dist/index.mjs) cannot find its platform binary
+// package from a genuine ESM context — it tries a bare `require(packageName)` via a bundler-
+// injected shim that only works when a real CJS `require` is already in scope, which never
+// happens in native ESM (confirmed live: `import { getExtensionPath } from
+// '@sqliteai/sqlite-vector'` throws `ExtensionNotFoundError` even when the platform package
+// is correctly installed). `createRequire` is the standard Node workaround — it resolves the
+// package's CJS build instead, where a real `require` exists and the same lookup works.
+const require = createRequire(import.meta.url);
+const { getExtensionPath } = require('@sqliteai/sqlite-vector');
+
+/**
+ * A single, diff-friendly SQLite file holding embeddings for one synced folder — meant to be
+ * pushed alongside the folder's own content (a plain file, not dot-prefixed, so it rides
+ * through `push`/`pull`/`diff` unmodified) so a pull is immediately searchable with no
+ * re-embedding.
+ *
+ * The whole schema is designed around one constraint, verified empirically (not assumed):
+ * inserting rows into a plain table with a monotonic rowid touches only new/near-full pages,
+ * which content-defined chunking recognizes as mostly-unchanged on a re-push. A real
+ * measurement (200 existing rows, 5 new ones) showed ~2.9x overhead versus the new data's
+ * raw size — nowhere near a full-file rewrite, but *not* literally zero-touch either. What
+ * actually breaks this property: `VACUUM` (copies and rewrites every byte — never call it
+ * during routine reindexing) and `UPDATE`/`DELETE` on the (large, embedding-heavy)
+ * `search_index` table. `search_index_files` is deliberately the one place this class *does*
+ * mutate rows in place — it holds one row per file, so its churn is proportional to how many
+ * files actually changed, not to corpus size, and paying that small cost is what makes
+ * "which chunks are current" a correct query instead of a guess (see search_index_files' own
+ * doc below).
+ */
+export default class VectorIndex {
+	/**
+	 * @param {string} dbPath - path to the SQLite file. Created if it doesn't exist yet
+	 *   (unless `readonly`).
+	 * @param {Object} [options]
+	 * @param {boolean} [options.readonly=false] - open an existing index without writing
+	 *   anything to it — not even the schema check below. For inspecting an index that's about
+	 *   to be pushed (see Indexer.status()), where touching the file would change what's pushed.
+	 */
+	constructor(dbPath, { readonly = false } = {}) {
+		this.path = dbPath;
+		this._db = readonly
+			? new Database(dbPath, { readonly: true, fileMustExist: true })
+			: new Database(dbPath);
+		// Deliberately not enabling WAL mode: WAL leaves a separate `-wal`/`-shm` file holding
+		// uncommitted pages, which would need an explicit checkpoint before every push (a real
+		// operational footgun for anyone forgetting it). Left at SQLite's own default rollback
+		// journal instead — this class always closes the database (see close()) before anything
+		// downstream would push it, so there's never a mid-transaction state to leave behind.
+		this._db.loadExtension(getExtensionPath());
+		if (!readonly) this._ensureSchema();
+	}
+
+	_ensureSchema() {
+		this._db.exec(`
+			CREATE TABLE IF NOT EXISTS search_index (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				path TEXT NOT NULL,
+				chunk_index INTEGER NOT NULL,
+				content_hash TEXT NOT NULL,
+				start_offset INTEGER NOT NULL,
+				end_offset INTEGER NOT NULL,
+				embedding BLOB NOT NULL
+			)
+		`);
+		// One row per file, deliberately mutable (see class doc) — "content_hash" here is what
+		// decides which rows in the (append-only) search_index table above actually count as
+		// current: a chunk only counts if its own content_hash matches this table's row for the
+		// same path. That's what lets a stale reindex's chunks keep existing physically (pure
+		// insert, never deleted) while still being correctly excluded from search results —
+		// see search()'s own join.
+		this._db.exec(`
+			CREATE TABLE IF NOT EXISTS search_index_files (
+				path TEXT PRIMARY KEY,
+				content_hash TEXT NOT NULL,
+				chunk_count INTEGER NOT NULL
+			)
+		`);
+		this._db.exec(`
+			CREATE TABLE IF NOT EXISTS search_index_meta (
+				key TEXT PRIMARY KEY,
+				value TEXT NOT NULL
+			)
+		`);
+	}
+
+	/**
+	 * Registers `search_index`/`embedding` as a vector column with sqlite-vector — required on
+	 * every fresh connection before `vector_full_scan` etc. work (confirmed live: omitting this
+	 * fails with "vector_full_scan: unable to retrieve context", not a clearer error). Safe to
+	 * call more than once; last call wins.
+	 *
+	 * @param {number} dim
+	 * @param {'FLOAT32'|'INT8'} [type='FLOAT32']
+	 */
+	initVectorColumn(dim, type = 'FLOAT32') {
+		this._dim = dim;
+		this._vectorType = type;
+		this._db.prepare(VECTOR_INIT_SQL).run(vectorInitOptions(dim, type));
+	}
+
+	/** @returns {?string} the file's currently-indexed content hash, or null if never indexed. */
+	getFileHash(path) {
+		const row = this._db.prepare('SELECT content_hash FROM search_index_files WHERE path = ?').get(path);
+		return row?.content_hash ?? null;
+	}
+
+	/** @returns {string[]} every path currently recorded in the index (indexed, not necessarily still on disk). */
+	listFiles() {
+		return this._db.prepare('SELECT path FROM search_index_files').all().map((r) => r.path);
+	}
+
+	/**
+	 * Replaces one file's chunks. Never touches the file's *old* rows in `search_index` — it
+	 * only ever inserts new ones (see class doc for why) — and upserts the single
+	 * `search_index_files` row that makes the old ones stop counting as current.
+	 *
+	 * @param {string} path
+	 * @param {string} contentHash - the file's new content hash (same hashing this package's
+	 *   caller already uses to detect the file changed in the first place).
+	 * @param {{startOffset: number, endOffset: number, embedding: Float32Array}[]} chunks
+	 */
+	replaceFileChunks(path, contentHash, chunks) {
+		const insert = this._db.prepare(`
+			INSERT INTO search_index (path, chunk_index, content_hash, start_offset, end_offset, embedding)
+			VALUES (?, ?, ?, ?, ?, ?)
+		`);
+		const upsertFile = this._db.prepare(`
+			INSERT INTO search_index_files (path, content_hash, chunk_count) VALUES (?, ?, ?)
+			ON CONFLICT(path) DO UPDATE SET content_hash = excluded.content_hash, chunk_count = excluded.chunk_count
+		`);
+		const tx = this._db.transaction(() => {
+			chunks.forEach((chunk, index) => {
+				insert.run(path, index, contentHash, chunk.startOffset, chunk.endOffset, Buffer.from(chunk.embedding.buffer, chunk.embedding.byteOffset, chunk.embedding.byteLength));
+			});
+			upsertFile.run(path, contentHash, chunks.length);
+		});
+		tx();
+	}
+
+	/** A file no longer exists in the tree — stop counting its (still physically present) chunks as current. */
+	removeFile(path) {
+		this._db.prepare('DELETE FROM search_index_files WHERE path = ?').run(path);
+	}
+
+	setMeta(key, value) {
+		this._db.prepare(`
+			INSERT INTO search_index_meta (key, value) VALUES (?, ?)
+			ON CONFLICT(key) DO UPDATE SET value = excluded.value
+		`).run(key, String(value));
+	}
+
+	/**
+	 * An existing index's metadata, read without writing anything — or null when there's no
+	 * index at `dbPath` (or it has no metadata yet).
+	 * @returns {?Object<string, string>}
+	 */
+	static readMeta(dbPath) {
+		if (!existsSync(dbPath)) return null;
+		const index = new VectorIndex(dbPath, { readonly: true });
+		try {
+			const rows = index._db.prepare('SELECT key, value FROM search_index_meta').all();
+			return rows.length ? Object.fromEntries(rows.map((r) => [r.key, r.value])) : null;
+		} catch {
+			// No meta table: not an index this package built.
+			return null;
+		} finally {
+			index.close();
+		}
+	}
+
+	/** @returns {?string} */
+	getMeta(key) {
+		return this._db.prepare('SELECT value FROM search_index_meta WHERE key = ?').get(key)?.value ?? null;
+	}
+
+	/**
+	 * Nearest-neighbor search, current chunks only. Over-fetches from `vector_full_scan` (exact,
+	 * brute-force — appropriate for the corpus sizes a synced folder actually has; no ANN/
+	 * partitioning index, which would break the diffability this whole design depends on) since
+	 * some of its top candidates may turn out to be stale once joined against
+	 * `search_index_files`, and stops short of the caller's requested `k` otherwise.
+	 *
+	 * @param {Float32Array} queryEmbedding
+	 * @param {number} k
+	 * @returns {{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}[]}
+	 */
+	search(queryEmbedding, k) {
+		const queryBuf = Buffer.from(queryEmbedding.buffer, queryEmbedding.byteOffset, queryEmbedding.byteLength);
+		// BigInt, not a plain number: better-sqlite3 binds every JS `number` parameter as SQLite
+		// REAL (confirmed live — even a whole number like 64), but vector_full_scan's own count
+		// argument strictly requires INTEGER and rejects REAL outright rather than coercing it.
+		// (The WASM build binds whole numbers as INTEGER on its own — see WasmIndexReader.)
+		return this._db
+			.prepare(SEARCH_SQL)
+			.all(queryBuf, BigInt(overfetchFor(k)), BigInt(k))
+			.map(mapSearchRow);
+	}
+
+	/**
+	 * The deliberate, rare, full-rewrite operation — drops every chunk that isn't current
+	 * (superseded by a later reindex of the same file, or belonging to a file removed from the
+	 * index entirely) and reclaims the space. Costs a CDC-diff equivalent to a full snapshot —
+	 * same trade routine reindexing exists specifically to avoid — so this is opt-in, not run
+	 * automatically. Mirrors the CLI's own `compact` command for stream history.
+	 */
+	compact() {
+		this._db.exec(`
+			DELETE FROM search_index
+			WHERE NOT EXISTS (
+				SELECT 1 FROM search_index_files f
+				WHERE f.path = search_index.path AND f.content_hash = search_index.content_hash
+			)
+		`);
+		this._db.exec('VACUUM');
+	}
+
+	close() {
+		this._db.close();
+	}
+}

package/lib/search/browser/BrowserSearcher.js

@@ -0,0 +1,131 @@
+import WasmIndexReader from './WasmIndexReader.js';
+import { LEGACY_DTYPE, EMBEDDER_TRANSFORMERS, embedderOfIndex } from '../schema.js';
+
+/**
+ * The browser's `Searcher`: holds one open index (from bytes, not a path) and the embedding
+ * model that matches it, so repeated queries pay for neither again. `open()` again with new
+ * bytes when the index file changes; the model is only reloaded if the new index was built
+ * with a different one.
+ *
+ * Which provider to build for which index is the caller's job (`createProvider`), so this
+ * class — and the browser entry around it — never imports a model runtime itself: the
+ * explorer builds the static provider from its own files source, and loads transformers.js
+ * only for an index that needs it.
+ */
+export default class BrowserSearcher {
+	/**
+	 * @param {Object} params
+	 * @param {any} params.sqlite3 - an initialized `@sqliteai/sqlite-wasm` module.
+	 * @param {(opts: {model: string, embedder: string, dtype: ?string, onProgress: ?Function}) => (any|Promise<any>)} [params.createProvider] -
+	 *   builds the provider an index's metadata asks for.
+	 * @param {any} [params.embeddingProvider] - use this one for every index instead (tests).
+	 * @param {(event: Object) => void} [params.onModelProgress] - passed to createProvider.
+	 */
+	constructor({ sqlite3, createProvider = null, embeddingProvider = null, onModelProgress = null } = {}) {
+		if (!sqlite3) throw new Error('BrowserSearcher: sqlite3 module is required');
+		if (!createProvider && !embeddingProvider) {
+			throw new Error('BrowserSearcher: pass createProvider or embeddingProvider');
+		}
+		this._sqlite3 = sqlite3;
+		this._createProvider = createProvider;
+		this._onModelProgress = onModelProgress;
+		this._fixedProvider = embeddingProvider;
+		this._provider = embeddingProvider;
+		this._providerKey = null;
+		this._reader = null;
+	}
+
+	/** @param {Uint8Array} bytes - the whole search_index.db file. */
+	open(bytes) {
+		const reader = new WasmIndexReader(this._sqlite3, bytes);
+		const dim = Number(reader.getMeta('dimension'));
+		if (!dim) {
+			reader.close();
+			throw new Error('This search index has no metadata — it was never built, or is empty.');
+		}
+		reader.initVectorColumn(dim);
+		this.close();
+		this._reader = reader;
+		this._dim = dim;
+		this._model = reader.getMeta('model');
+		this._embedder = embedderOfIndex({ model: this._model, embedder: reader.getMeta('embedder') });
+		this._dtype = this._embedder === EMBEDDER_TRANSFORMERS ? (reader.getMeta('dtype') || LEGACY_DTYPE) : null;
+	}
+
+	get isOpen() {
+		return this._reader !== null;
+	}
+
+	/** @returns {{model: ?string, embedder: string, dtype: ?string, dimension: number}} */
+	get info() {
+		return { model: this._model, embedder: this._embedder, dtype: this._dtype, dimension: this._dim };
+	}
+
+	/** @returns {Map<string, string>} see WasmIndexReader.getFileHashes */
+	getFileHashes() {
+		this._requireOpen();
+		return this._reader.getFileHashes();
+	}
+
+	/** @returns {boolean} whether the model the open index needs is already loaded. */
+	get isModelLoaded() {
+		if (!this._reader) return false;
+		if (this._fixedProvider) return true;
+		return this._providerKey === this._key() && !!this._provider?.isLoaded;
+	}
+
+	/** Loads the embedding model now (the slow, one-time part — a download on first use) so a
+	 *  caller can show that as its own step instead of as a slow first query. */
+	async loadModel() {
+		this._requireOpen();
+		const provider = await this._providerForIndex();
+		if (typeof provider.load === 'function') await provider.load();
+	}
+
+	/**
+	 * @param {string} query
+	 * @param {number} [k=5]
+	 */
+	async search(query, k = 5) {
+		this._requireOpen();
+		const provider = await this._providerForIndex();
+		const [queryVector] = await provider.embed([query]);
+		if (queryVector.length !== this._dim) {
+			throw new Error(
+				`Query embedding is ${queryVector.length}-dimensional but this index is ${this._dim}-dimensional — ` +
+				`the embedding model must match the one used to build it (recorded model: ${this._model})`
+			);
+		}
+		return this._reader.search(queryVector, k);
+	}
+
+	close() {
+		if (this._reader) {
+			this._reader.close();
+			this._reader = null;
+		}
+	}
+
+	_key() {
+		return `${this._embedder}|${this._model}|${this._dtype}`;
+	}
+
+	async _providerForIndex() {
+		if (this._fixedProvider) return this._fixedProvider;
+		const key = this._key();
+		if (this._providerKey !== key) {
+			this._provider = await this._createProvider({
+				model: this._model,
+				embedder: this._embedder,
+				dtype: this._dtype,
+				onProgress: this._onModelProgress,
+			});
+			this._providerKey = key;
+		}
+		return this._provider;
+	}
+
+	_requireOpen() {
+		if (!this._reader) throw new Error('BrowserSearcher: call open(bytes) first');
+	}
+}

package/lib/search/browser/WasmIndexReader.js

@@ -0,0 +1,93 @@
+import {
+	TABLE_FILES, TABLE_META, VECTOR_INIT_SQL, SEARCH_SQL,
+	vectorInitOptions, overfetchFor, mapSearchRow,
+} from '../schema.js';
+
+/**
+ * Read-only view of a `search_index.db` held in memory — the browser counterpart to
+ * `VectorIndex`, for a page that already has the file's bytes (the explorer does: after a
+ * restore, the whole folder, index included, lives in memory) and no filesystem to open it from.
+ *
+ * Runs on @sqliteai/sqlite-wasm, which ships the same sqlite-vector build the Node side loads
+ * as a native extension, so it answers with the same `vector_full_scan` query (shared through
+ * schema.js) and returns the same distances — not a second, JS-side scoring that could drift.
+ *
+ * The sqlite3 module is passed in rather than imported here: it's a ~3 MB WASM payload the
+ * caller decides when (and whether) to load, and it keeps this package free of a hard browser-
+ * only dependency: `(await import('@sqliteai/sqlite-wasm')).default()` is the usual way to get one.
+ */
+export default class WasmIndexReader {
+	/**
+	 * @param {any} sqlite3 - an initialized module from `@sqliteai/sqlite-wasm`'s default export.
+	 * @param {Uint8Array} bytes - the whole database file.
+	 */
+	constructor(sqlite3, bytes) {
+		this._sqlite3 = sqlite3;
+		this._db = new sqlite3.oo1.DB(':memory:');
+		// sqlite3_deserialize takes ownership of a WASM-heap copy (FREEONCLOSE), so the caller's
+		// buffer is never referenced again — safe to hand over bytes that later get replaced by a
+		// live poll. READONLY: nothing here should ever write, and a stray write would otherwise
+		// silently diverge this copy from the file actually in the stream.
+		const ptr = sqlite3.wasm.allocFromTypedArray(bytes);
+		const { capi } = sqlite3;
+		const rc = capi.sqlite3_deserialize(
+			this._db.pointer, 'main', ptr, bytes.byteLength, bytes.byteLength,
+			capi.SQLITE_DESERIALIZE_FREEONCLOSE | capi.SQLITE_DESERIALIZE_READONLY,
+		);
+		if (rc !== 0) {
+			this._db.close();
+			throw new Error(`WasmIndexReader: couldn't open the index (sqlite3_deserialize rc=${rc})`);
+		}
+		this._dim = null;
+	}
+
+	/** @returns {?string} */
+	getMeta(key) {
+		const value = this._db.selectValue(`SELECT value FROM ${TABLE_META} WHERE key = ?`, [key]);
+		return value ?? null;
+	}
+
+	/** @returns {Map<string, string>} every currently-indexed path → the content hash it was
+	 *  indexed at — what a caller compares against the live files to spot stale hits. */
+	getFileHashes() {
+		const rows = this._db.exec({
+			sql: `SELECT path, content_hash FROM ${TABLE_FILES}`,
+			rowMode: 'object',
+			returnValue: 'resultRows',
+		});
+		return new Map(rows.map((r) => [r.path, r.content_hash]));
+	}
+
+	/** Same role as `VectorIndex.initVectorColumn` — required once per connection before search. */
+	initVectorColumn(dim, type = 'FLOAT32') {
+		this._db.exec({ sql: VECTOR_INIT_SQL, bind: [vectorInitOptions(dim, type)] });
+		this._dim = dim;
+	}
+
+	/**
+	 * @param {Float32Array} queryEmbedding
+	 * @param {number} k
+	 * @returns {{path: string, chunkIndex: number, startOffset: number, endOffset: number, distance: number}[]}
+	 */
+	search(queryEmbedding, k) {
+		if (this._dim === null) {
+			const dim = Number(this.getMeta('dimension'));
+			if (!dim) throw new Error('WasmIndexReader: index has no dimension recorded — is it empty?');
+			this.initVectorColumn(dim);
+		}
+		const queryBlob = new Uint8Array(queryEmbedding.buffer, queryEmbedding.byteOffset, queryEmbedding.byteLength);
+		// Plain numbers are fine here, unlike better-sqlite3 (see VectorIndex.search): the WASM
+		// binding binds whole JS numbers as INTEGER, which vector_full_scan requires.
+		const rows = this._db.exec({
+			sql: SEARCH_SQL,
+			bind: [queryBlob, overfetchFor(k), k],
+			rowMode: 'object',
+			returnValue: 'resultRows',
+		});
+		return rows.map(mapSearchRow);
+	}
+
+	close() {
+		this._db.close();
+	}
+}

package/lib/search/browser.js

@@ -0,0 +1,13 @@
+// Browser-safe entry (the explorer imports this through a Vite alias): nothing here imports
+// better-sqlite3, the native sqlite-vector extension or any `node:*` module. Searching only —
+// building an index stays a Node job (`mnemonad index`), since that's where the folder's files
+// live on disk and where embedding the whole corpus belongs; the browser only ever embeds a
+// query string.
+export { default as WasmIndexReader } from './browser/WasmIndexReader.js';
+export { default as BrowserSearcher } from './browser/BrowserSearcher.js';
+export { default as EmbeddingProvider } from './embeddings/EmbeddingProvider.js';
+export { default as StaticEmbeddingProvider } from './embeddings/StaticEmbeddingProvider.js';
+export { browserModelFiles } from './embeddings/modelFiles.browser.js';
+export {
+	DEFAULT_DB_NAME, DEFAULT_MODEL, LEGACY_DTYPE, EMBEDDER_STATIC, EMBEDDER_TRANSFORMERS, embedderOfIndex,
+} from './schema.js';

package/lib/search/chunking/ChunkingStrategy.js

@@ -0,0 +1,23 @@
+/**
+ * Base class for splitting one file's text content into embeddable chunks. A subclass fills
+ * in `chunk()`; everything else here is just the shared shape `Indexer` depends on.
+ *
+ * Deliberately separate from Mnemonad's own content-defined chunking (`FastCDC`, used for
+ * on-chain diffing): that one finds boundaries by content hash to maximize byte-level dedup
+ * between versions, which has nothing to do with what makes a good *semantic* unit to embed.
+ * The two chunkers operate on the same bytes for entirely unrelated reasons and should never
+ * be conflated.
+ */
+export default class ChunkingStrategy {
+	/**
+	 * @param {string} text - decoded file content (chunking only ever runs on text files —
+	 *   the caller is responsible for skipping binaries before this is reached).
+	 * @returns {{startOffset: number, endOffset: number, text: string}[]} offsets are UTF-16
+	 *   code-unit offsets into `text` (i.e. plain JS string indices — `text.slice(startOffset,
+	 *   endOffset)` round-trips), not byte offsets — cheap to compute, and exact enough to
+	 *   locate a snippet back in the source file at display time.
+	 */
+	chunk(_text) {
+		throw new Error(`${this.constructor.name}: chunk() not implemented`);
+	}
+}

package/lib/search/chunking/TextWindowChunkingStrategy.js

@@ -0,0 +1,69 @@
+import ChunkingStrategy from './ChunkingStrategy.js';
+
+/**
+ * Default chunker: fixed-size character windows with overlap, breaking on the nearest
+ * paragraph/sentence/word boundary before the target size rather than mid-word, when one is
+ * available within a reasonable lookback. Character-based rather than token-based
+ * deliberately — a token-accurate chunker needs a loaded tokenizer first, coupling chunking
+ * to whichever embedding model happens to be configured; character windows chunk perfectly
+ * well standalone, synchronously, with no model dependency, and 1000 characters is a
+ * conservative-enough stand-in for typical embedding models' token limits (roughly 4
+ * characters/token in English prose, so ~250 tokens — comfortably under the 256-512 token
+ * budget most small embedding models expect). A token-aware `ChunkingStrategy` reusing the
+ * configured `EmbeddingProvider`'s own tokenizer is a reasonable later swap-in — this stays
+ * the default because it works content-agnostically with zero setup.
+ */
+export default class TextWindowChunkingStrategy extends ChunkingStrategy {
+	/**
+	 * @param {Object} [params]
+	 * @param {number} [params.size=1000] - target chunk size, in characters
+	 * @param {number} [params.overlap=150] - characters of overlap between consecutive chunks
+	 */
+	constructor({ size = 1000, overlap = 150 } = {}) {
+		super();
+		if (overlap >= size) {
+			throw new Error('TextWindowChunkingStrategy: overlap must be smaller than size');
+		}
+		this.size = size;
+		this.overlap = overlap;
+	}
+
+	chunk(text) {
+		if (!text) return [];
+		const chunks = [];
+		const step = this.size - this.overlap;
+		let start = 0;
+		while (start < text.length) {
+			let end = Math.min(start + this.size, text.length);
+			// Prefer breaking at a paragraph/sentence/word boundary over a hard mid-word cut —
+			// searched backward from `end`, but never past `start` (a very long unbroken run of
+			// text, e.g. a minified file, just gets a hard cut, which is fine: it wasn't going to
+			// chunk meaningfully either way).
+			if (end < text.length) {
+				const lookback = text.slice(start, end);
+				const boundary = _lastBoundary(lookback);
+				if (boundary > 0) end = start + boundary;
+			}
+			chunks.push({ startOffset: start, endOffset: end, text: text.slice(start, end) });
+			if (end >= text.length) break;
+			start = end - this.overlap;
+		}
+		return chunks;
+	}
+}
+
+/**
+ * Index just past the best break point found in `s` (paragraph > sentence > word), searched
+ * from the end backward, or -1 if none exists worth preferring over a hard cut.
+ * @param {string} s
+ * @returns {number}
+ */
+function _lastBoundary(s) {
+	const paragraph = s.lastIndexOf('\n\n');
+	if (paragraph > s.length * 0.5) return paragraph + 2;
+	const sentence = Math.max(s.lastIndexOf('. '), s.lastIndexOf('.\n'));
+	if (sentence > s.length * 0.5) return sentence + 1;
+	const word = s.lastIndexOf(' ');
+	if (word > s.length * 0.5) return word + 1;
+	return -1;
+}

package/lib/search/embeddings/EmbeddingProvider.js

@@ -0,0 +1,40 @@
+/**
+ * Base class for turning text into vectors. A subclass fills in `embed()`/`dimension`;
+ * everything downstream (`VectorIndex`, `Indexer`, `Searcher`) only ever talks to this
+ * interface, never to a specific model or runtime — swapping the default local model for an
+ * API-based one later needs no change outside a new subclass.
+ */
+export default class EmbeddingProvider {
+	/** @returns {number} vector length this provider produces — fixed for its lifetime. */
+	get dimension() {
+		throw new Error(`${this.constructor.name}: dimension getter not implemented`);
+	}
+
+	/** @returns {string} identifies this provider + model in `search_index_meta`, so a later
+	 *  `search` (possibly a different process entirely) knows what it's matching against. */
+	get modelId() {
+		throw new Error(`${this.constructor.name}: modelId getter not implemented`);
+	}
+
+	/** @returns {?string} which kind of embedder this is ('model2vec', 'transformers'), recorded
+	 *  in the index so a later search builds the same kind (see schema.js). */
+	get embedder() {
+		return null;
+	}
+
+	/** @returns {?string} weight precision (e.g. 'q8', 'fp32') when the model comes in more than
+	 *  one, recorded alongside `modelId` so a searcher loads the same weights. Null when the
+	 *  provider has no such choice. */
+	get dtype() {
+		return null;
+	}
+
+	/**
+	 * @param {string[]} texts
+	 * @returns {Promise<Float32Array[]>} one vector per input text, same order, each of length
+	 *   `this.dimension`.
+	 */
+	async embed(_texts) {
+		throw new Error(`${this.constructor.name}: embed() not implemented`);
+	}
+}