@statewalker/indexer-sqlite 0.0.0-stage → 0.1.0

package/dist/fts-provider.js

@@ -0,0 +1,143 @@
+import { pathPrefixSql } from "./path-sql.js";
+import { newEnumeration } from "./enumeration.js";
+import { toAsyncIterable } from "@statewalker/indexer-core";
+import { inTransaction } from "@statewalker/db-sqlite";
+import { FULL_TEXT_TYPE } from "@statewalker/indexer-fulltext";
+//#region src/fts-provider.ts
+/** `en` gets the Porter stemmer; any other language (known or not) gets plain `unicode61`. */
+function tokenizerFor(language) {
+	return language === "en" ? "porter unicode61 remove_diacritics 2" : "unicode61 remove_diacritics 2";
+}
+/**
+* One FTS5 MATCH expression per query string: every word with a letter or a
+* digit, double-quoted (so `AND`, `*`, `NEAR(` are plain words), OR-joined so
+* partial matches are returned. `null` when nothing is searchable.
+*/
+function toMatchExpression(query) {
+	const words = query.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).map((w) => `"${w.replaceAll("\"", "\"\"")}"`);
+	return words.length > 0 ? words.join(" OR ") : null;
+}
+function newSqliteFullTextProvider() {
+	return {
+		type: FULL_TEXT_TYPE,
+		async drop({ db, prefix }) {
+			await db.exec(`DROP TABLE IF EXISTS ${prefix}fts`);
+		},
+		async open({ db, prefix, blockIds }, config) {
+			const table = `${prefix}fts`;
+			const blocks = blockIds.table;
+			await db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`);
+			const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`);
+			let closed = false;
+			const ensureOpen = () => {
+				if (closed) throw new Error("FullTextIndex is closed");
+			};
+			const collectSelectors = async (selectors) => {
+				const all = [];
+				for await (const s of toAsyncIterable(selectors)) all.push(s);
+				return all;
+			};
+			return {
+				async getIndexInfo() {
+					return {
+						language: config.language,
+						metadata: config.metadata
+					};
+				},
+				async *search(params) {
+					ensureOpen();
+					const topK = params.topK ?? 100;
+					const scope = pathPrefixSql("b.path", params.paths ?? []);
+					const merged = /* @__PURE__ */ new Map();
+					for (const query of params.queries ?? []) {
+						const match = toMatchExpression(query);
+						if (!match) continue;
+						const rows = await db.query(`SELECT ${table}.rowid AS id, -bm25(${table}) AS score, snippet(${table}, 0, '', '', '…', 16) AS snippet FROM ${table} JOIN ${blocks} b ON b.id = ${table}.rowid WHERE ${table} MATCH ? AND ${scope.sql} ORDER BY score DESC LIMIT ?`, [
+							match,
+							...scope.params,
+							topK
+						]);
+						for (const r of rows) {
+							const id = Number(r.id);
+							const prev = merged.get(id);
+							if (!prev) merged.set(id, {
+								score: r.score,
+								best: r.score,
+								snippet: r.snippet
+							});
+							else {
+								prev.score += r.score;
+								if (r.score > prev.best) Object.assign(prev, {
+									best: r.score,
+									snippet: r.snippet
+								});
+							}
+						}
+					}
+					const top = [...merged.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, topK);
+					const refs = await blockIds.refs(db, top.map(([id]) => id));
+					for (const [id, hit] of top) {
+						const ref = refs.get(id);
+						if (ref) yield {
+							...ref,
+							score: hit.score,
+							snippet: hit.snippet
+						};
+					}
+				},
+				async addDocument(blocksIn) {
+					ensureOpen();
+					if (blocksIn.length === 0) return;
+					await inTransaction(db, async (tx) => {
+						const ids = await blockIds.ensure(tx, blocksIn);
+						for (const [i, id] of ids.entries()) {
+							const block = blocksIn[i];
+							await tx.query(`DELETE FROM ${table} WHERE rowid = ?`, [id]);
+							await tx.query(`INSERT INTO ${table} (rowid, content, metadata) VALUES (?, ?, ?)`, [
+								id,
+								block.content,
+								block.metadata ? JSON.stringify(block.metadata) : null
+							]);
+						}
+					});
+				},
+				async addDocuments(batches) {
+					for await (const batch of batches) await this.addDocument(batch);
+				},
+				async deleteDocuments(selectors) {
+					ensureOpen();
+					const sels = await collectSelectors(selectors);
+					await inTransaction(db, async (tx) => {
+						const ids = await blockIds.select(tx, sels);
+						await tx.query(`DELETE FROM ${table} WHERE rowid IN (SELECT value FROM json_each(?))`, [JSON.stringify(ids)]);
+					});
+				},
+				getSize: (p) => list.size(p),
+				getDocumentPaths: (p) => list.paths(p),
+				getDocumentBlocksRefs: (p) => list.refs(p),
+				async *getDocumentsBlocks(p) {
+					const scope = pathPrefixSql("b.path", p ? [p] : []);
+					const rows = await db.query(`SELECT b.path, b.block_id, f.content, f.metadata FROM ${table} f JOIN ${blocks} b ON b.id = f.rowid WHERE ${scope.sql} ORDER BY b.id`, scope.params);
+					for (const r of rows) yield {
+						path: r.path,
+						blockId: r.block_id,
+						content: r.content,
+						...r.metadata ? { metadata: JSON.parse(r.metadata) } : {}
+					};
+				},
+				async close() {
+					closed = true;
+				},
+				async flush() {},
+				async deleteIndex() {
+					closed = true;
+					await db.exec(`DROP TABLE IF EXISTS ${table}`);
+				}
+			};
+		}
+	};
+}
+//#endregion
+export { newSqliteFullTextProvider, toMatchExpression };
+
+//# sourceMappingURL=fts-provider.js.map

package/dist/fts-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts-provider.js","names":[],"sources":["../src/fts-provider.ts"],"sourcesContent":["import { inTransaction } from \"@statewalker/db-sqlite\";\nimport type { DocumentPath, Metadata, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  FULL_TEXT_TYPE,\n  type FullTextBlock,\n  type FullTextConfig,\n  type FullTextIndex,\n  type FulltextQuery,\n  type FulltextResult,\n} from \"@statewalker/indexer-fulltext\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/** `en` gets the Porter stemmer; any other language (known or not) gets plain `unicode61`. */\nfunction tokenizerFor(language: string): string {\n  return language === \"en\"\n    ? \"porter unicode61 remove_diacritics 2\"\n    : \"unicode61 remove_diacritics 2\";\n}\n\n/**\n * One FTS5 MATCH expression per query string: every word with a letter or a\n * digit, double-quoted (so `AND`, `*`, `NEAR(` are plain words), OR-joined so\n * partial matches are returned. `null` when nothing is searchable.\n */\nexport function toMatchExpression(query: string): string | null {\n  const words = query\n    .split(/\\s+/)\n    .filter((w) => /[\\p{L}\\p{N}]/u.test(w))\n    .map((w) => `\"${w.replaceAll('\"', '\"\"')}\"`);\n  return words.length > 0 ? words.join(\" OR \") : null;\n}\n\nexport function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConfig, FullTextIndex> {\n  return {\n    type: FULL_TEXT_TYPE,\n\n    async drop({ db, prefix }) {\n      await db.exec(`DROP TABLE IF EXISTS ${prefix}fts`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const table = `${prefix}fts`;\n      const blocks = blockIds.table;\n      await db.exec(\n        `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`,\n      );\n      const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"FullTextIndex is closed\");\n      };\n      const collectSelectors = async (selectors: PathSelector[] | AsyncIterable<PathSelector>) => {\n        const all: PathSelector[] = [];\n        for await (const s of toAsyncIterable(selectors)) all.push(s);\n        return all;\n      };\n\n      const index: FullTextIndex = {\n        async getIndexInfo() {\n          return { language: config.language, metadata: config.metadata };\n        },\n\n        async *search(params: FulltextQuery): AsyncGenerator<FulltextResult> {\n          ensureOpen();\n          const topK = params.topK ?? 100;\n          const scope = pathPrefixSql(\"b.path\", params.paths ?? []);\n          // Several queries accumulate (SUM): a block matching more of them ranks higher.\n          const merged = new Map<number, { score: number; best: number; snippet: string }>();\n          for (const query of params.queries ?? []) {\n            const match = toMatchExpression(query);\n            if (!match) continue;\n            const rows = await db.query<{ id: number; score: number; snippet: string }>(\n              `SELECT ${table}.rowid AS id, -bm25(${table}) AS score, snippet(${table}, 0, '', '', '…', 16) AS snippet FROM ${table} JOIN ${blocks} b ON b.id = ${table}.rowid WHERE ${table} MATCH ? AND ${scope.sql} ORDER BY score DESC LIMIT ?`,\n              [match, ...scope.params, topK],\n            );\n            for (const r of rows) {\n              const id = Number(r.id);\n              const prev = merged.get(id);\n              if (!prev) merged.set(id, { score: r.score, best: r.score, snippet: r.snippet });\n              else {\n                prev.score += r.score;\n                if (r.score > prev.best) Object.assign(prev, { best: r.score, snippet: r.snippet });\n              }\n            }\n          }\n          const top = [...merged.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, topK);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, hit] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score: hit.score, snippet: hit.snippet };\n          }\n        },\n\n        async addDocument(blocksIn: FullTextBlock[]) {\n          ensureOpen();\n          if (blocksIn.length === 0) return;\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.ensure(tx, blocksIn);\n            for (const [i, id] of ids.entries()) {\n              const block = blocksIn[i] as FullTextBlock;\n              // FTS5 has no UPSERT: delete then insert, atomically inside the transaction.\n              await tx.query(`DELETE FROM ${table} WHERE rowid = ?`, [id]);\n              await tx.query(`INSERT INTO ${table} (rowid, content, metadata) VALUES (?, ?, ?)`, [\n                id,\n                block.content,\n                block.metadata ? JSON.stringify(block.metadata) : null,\n              ]);\n            }\n          });\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels = await collectSelectors(selectors);\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.select(tx, sels);\n            await tx.query(`DELETE FROM ${table} WHERE rowid IN (SELECT value FROM json_each(?))`, [\n              JSON.stringify(ids),\n            ]);\n          });\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<FullTextBlock> {\n          const scope = pathPrefixSql(\"b.path\", p ? [p] : []);\n          const rows = await db.query<{\n            path: string;\n            block_id: string;\n            content: string;\n            metadata: string | null;\n          }>(\n            `SELECT b.path, b.block_id, f.content, f.metadata FROM ${table} f JOIN ${blocks} b ON b.id = f.rowid WHERE ${scope.sql} ORDER BY b.id`,\n            scope.params,\n          );\n          for (const r of rows) {\n            yield {\n              path: r.path as DocumentPath,\n              blockId: r.block_id,\n              content: r.content,\n              ...(r.metadata ? { metadata: JSON.parse(r.metadata) as Metadata } : {}),\n            };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await db.exec(`DROP TABLE IF EXISTS ${table}`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;AAgBA,SAAS,aAAa,UAA0B;CAC9C,OAAO,aAAa,OAChB,yCACA;AACN;;;;;;AAOA,SAAgB,kBAAkB,OAA8B;CAC9D,MAAM,QAAQ,MACX,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,gBAAgB,KAAK,CAAC,CAAC,CAAC,CACtC,KAAK,MAAM,IAAI,EAAE,WAAW,MAAK,MAAI,EAAE,EAAE;CAC5C,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,MAAM,IAAI;AACjD;AAEA,SAAgB,4BAAmF;CACjG,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,GAAG,KAAK,wBAAwB,OAAO,IAAI;EACnD;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,QAAQ,GAAG,OAAO;GACxB,MAAM,SAAS,SAAS;GACxB,MAAM,GAAG,KACP,sCAAsC,MAAM,qDAAqD,aAAa,OAAO,QAAQ,EAAE,GACjI;GACA,MAAM,OAAO,eAAe,IAAI,UAAU,2BAA2B,OAAO;GAC5E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,yBAAyB;GACvD;GACA,MAAM,mBAAmB,OAAO,cAA4D;IAC1F,MAAM,MAAsB,CAAC;IAC7B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,IAAI,KAAK,CAAC;IAC5D,OAAO;GACT;GA4GA,OAAO;IAzGL,MAAM,eAAe;KACnB,OAAO;MAAE,UAAU,OAAO;MAAU,UAAU,OAAO;KAAS;IAChE;IAEA,OAAO,OAAO,QAAuD;KACnE,WAAW;KACX,MAAM,OAAO,OAAO,QAAQ;KAC5B,MAAM,QAAQ,cAAc,UAAU,OAAO,SAAS,CAAC,CAAC;KAExD,MAAM,yBAAS,IAAI,IAA8D;KACjF,KAAK,MAAM,SAAS,OAAO,WAAW,CAAC,GAAG;MACxC,MAAM,QAAQ,kBAAkB,KAAK;MACrC,IAAI,CAAC,OAAO;MACZ,MAAM,OAAO,MAAM,GAAG,MACpB,UAAU,MAAM,sBAAsB,MAAM,sBAAsB,MAAM,wCAAwC,MAAM,QAAQ,OAAO,eAAe,MAAM,eAAe,MAAM,eAAe,MAAM,IAAI,+BACxM;OAAC;OAAO,GAAG,MAAM;OAAQ;MAAI,CAC/B;MACA,KAAK,MAAM,KAAK,MAAM;OACpB,MAAM,KAAK,OAAO,EAAE,EAAE;OACtB,MAAM,OAAO,OAAO,IAAI,EAAE;OAC1B,IAAI,CAAC,MAAM,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE;QAAO,MAAM,EAAE;QAAO,SAAS,EAAE;OAAQ,CAAC;YAC1E;QACH,KAAK,SAAS,EAAE;QAChB,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM;SAAE,MAAM,EAAE;SAAO,SAAS,EAAE;QAAQ,CAAC;OACpF;MACF;KACF;KACA,MAAM,MAAM,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI;KACvF,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,QAAQ,KAAK;MAC3B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK,OAAO,IAAI;OAAO,SAAS,IAAI;MAAQ;KAClE;IACF;IAEA,MAAM,YAAY,UAA2B;KAC3C,WAAW;KACX,IAAI,SAAS,WAAW,GAAG;KAC3B,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,QAAQ;MAC9C,KAAK,MAAM,CAAC,GAAG,OAAO,IAAI,QAAQ,GAAG;OACnC,MAAM,QAAQ,SAAS;OAEvB,MAAM,GAAG,MAAM,eAAe,MAAM,mBAAmB,CAAC,EAAE,CAAC;OAC3D,MAAM,GAAG,MAAM,eAAe,MAAM,+CAA+C;QACjF;QACA,MAAM;QACN,MAAM,WAAW,KAAK,UAAU,MAAM,QAAQ,IAAI;OACpD,CAAC;MACH;KACF,CAAC;IACH;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAO,MAAM,iBAAiB,SAAS;KAC7C,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;MAC1C,MAAM,GAAG,MAAM,eAAe,MAAM,mDAAmD,CACrF,KAAK,UAAU,GAAG,CACpB,CAAC;KACH,CAAC;IACH;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAAiD;KACzE,MAAM,QAAQ,cAAc,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;KAClD,MAAM,OAAO,MAAM,GAAG,MAMpB,yDAAyD,MAAM,UAAU,OAAO,6BAA6B,MAAM,IAAI,iBACvH,MAAM,MACR;KACA,KAAK,MAAM,KAAK,MACd,MAAM;MACJ,MAAM,EAAE;MACR,SAAS,EAAE;MACX,SAAS,EAAE;MACX,GAAI,EAAE,WAAW,EAAE,UAAU,KAAK,MAAM,EAAE,QAAQ,EAAc,IAAI,CAAC;KACvE;IAEJ;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,GAAG,KAAK,wBAAwB,OAAO;IAC/C;GAES;EACb;CACF;AACF"}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+import { BlockIds, openBlockIds } from "./block-ids.js";
+import { SqliteModalityContext, SqliteModalityProvider } from "./types.js";
+import { SqliteIndexerOptions, createSqliteIndexer } from "./create-sqlite-indexer.js";
+import { BlockEntry, Enumeration, newEnumeration } from "./enumeration.js";
+import { newSqliteFullTextProvider } from "./fts-provider.js";
+import { newSqliteNgramProvider } from "./ngram-provider.js";
+import { pathPrefixSql } from "./path-sql.js";
+import { newSqliteVectorProvider } from "./vector-provider.js";
+export { type BlockEntry, type BlockIds, type Enumeration, type SqliteIndexerOptions, type SqliteModalityContext, type SqliteModalityProvider, createSqliteIndexer, newEnumeration, newSqliteFullTextProvider, newSqliteNgramProvider, newSqliteVectorProvider, openBlockIds, pathPrefixSql };

package/dist/index.js

@@ -0,0 +1,8 @@
+import { pathPrefixSql } from "./path-sql.js";
+import { openBlockIds } from "./block-ids.js";
+import { createSqliteIndexer } from "./create-sqlite-indexer.js";
+import { newEnumeration } from "./enumeration.js";
+import { newSqliteFullTextProvider } from "./fts-provider.js";
+import { newSqliteNgramProvider } from "./ngram-provider.js";
+import { newSqliteVectorProvider } from "./vector-provider.js";
+export { createSqliteIndexer, newEnumeration, newSqliteFullTextProvider, newSqliteNgramProvider, newSqliteVectorProvider, openBlockIds, pathPrefixSql };

package/dist/ngram-provider.d.ts

@@ -0,0 +1,12 @@
+import { SqliteModalityProvider } from "./types.js";
+import { NgramConfig, NgramIndex } from "@statewalker/indexer-ngram";
+//#region src/ngram-provider.d.ts
+/**
+ * Wraps the id-based `NgramStore` (own transactions; never call it inside
+ * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join
+ * the store's `idTable`. Score = match count; ties keep the store's
+ * ascending-id order.
+ */
+export declare function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex>;
+//#endregion
+//# sourceMappingURL=ngram-provider.d.ts.map

package/dist/ngram-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ngram-provider.d.ts","names":[],"sources":["../src/ngram-provider.ts"],"mappings":";;;;;;;;;wBA6CgB,0BAA0B,uBAAuB,aAAa"}

package/dist/ngram-provider.js

@@ -0,0 +1,142 @@
+import { newEnumeration } from "./enumeration.js";
+import { toAsyncIterable } from "@statewalker/indexer-core";
+import { NgramStore } from "@statewalker/db-sqlite-index/ngram";
+import { NGRAM_TYPE } from "@statewalker/indexer-ngram";
+//#region src/ngram-provider.ts
+/**
+* Blocks per `NgramStore.upsert`: each upsert is one transaction, and a
+* Durable Object retries a write that runs past its ~15 s deadline.
+*/
+const UPSERT_BATCH = 500;
+/** 1-based line of each offset, from the positions of `\n` in `text`. */
+function withLines(text, matches) {
+	const breaks = [];
+	for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) breaks.push(i);
+	let line = 1;
+	let b = 0;
+	return [...matches].sort((x, y) => x.start - y.start).map(({ start, end }) => {
+		while (b < breaks.length && breaks[b] < start) {
+			b++;
+			line++;
+		}
+		return {
+			start,
+			end,
+			line
+		};
+	});
+}
+/**
+* Wraps the id-based `NgramStore` (own transactions; never call it inside
+* `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join
+* the store's `idTable`. Score = match count; ties keep the store's
+* ascending-id order.
+*/
+function newSqliteNgramProvider() {
+	return {
+		type: NGRAM_TYPE,
+		async drop({ db, prefix }) {
+			await new NgramStore({
+				db,
+				prefix: `${prefix}ng_`
+			}).drop();
+		},
+		async open({ db, prefix, blockIds }, config) {
+			const storeKind = config.store ?? "fts5";
+			const store = new NgramStore({
+				db,
+				prefix: `${prefix}ng_`,
+				store: storeKind
+			});
+			await store.init();
+			const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`);
+			let closed = false;
+			const ensureOpen = () => {
+				if (closed) throw new Error("NgramIndex is closed");
+			};
+			return {
+				async getIndexInfo() {
+					return {
+						store: storeKind,
+						...config.metadata ? { metadata: config.metadata } : {}
+					};
+				},
+				async *search(params) {
+					ensureOpen();
+					const allowed = params.paths?.length ? new Set(await blockIds.select(db, params.paths.map((path) => ({ path })))) : void 0;
+					const result = await store.search(params.pattern, {
+						regex: params.regex ?? false,
+						caseSensitive: params.caseSensitive ?? true,
+						limit: params.topK ?? 100,
+						...allowed ? { filter: (id) => allowed.has(id) } : {}
+					});
+					const hits = result.hits.filter((h) => h.matches.length > 0);
+					const ids = hits.map((h) => h.id);
+					const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);
+					const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);
+					for (const h of ranked) {
+						const ref = refs.get(h.id);
+						if (!ref) continue;
+						const count = h.matches.length;
+						yield {
+							...ref,
+							score: count,
+							count,
+							matches: withLines(texts.get(h.id) ?? "", h.matches)
+						};
+					}
+					return { truncated: result.truncated };
+				},
+				async addDocument(blocks) {
+					ensureOpen();
+					for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {
+						const batch = blocks.slice(i, i + UPSERT_BATCH);
+						const ids = await blockIds.ensure(db, batch);
+						await store.upsert(ids.map((id, j) => ({
+							id,
+							text: batch[j].content
+						})));
+					}
+				},
+				async addDocuments(batches) {
+					for await (const batch of batches) await this.addDocument(batch);
+				},
+				async deleteDocuments(selectors) {
+					ensureOpen();
+					const sels = [];
+					for await (const s of toAsyncIterable(selectors)) sels.push(s);
+					const ids = await blockIds.select(db, sels);
+					if (ids.length === 0) return;
+					await store.delete(ids);
+				},
+				getSize: (p) => list.size(p),
+				getDocumentPaths: (p) => list.paths(p),
+				getDocumentBlocksRefs: (p) => list.refs(p),
+				async *getDocumentsBlocks(p) {
+					const entries = await list.entries(p);
+					const texts = await store.getTexts(entries.map((e) => e.id));
+					for (const e of entries) {
+						const content = texts.get(e.id);
+						if (content !== void 0) yield {
+							path: e.path,
+							blockId: e.blockId,
+							content
+						};
+					}
+				},
+				async close() {
+					closed = true;
+				},
+				async flush() {},
+				async deleteIndex() {
+					closed = true;
+					await store.drop();
+				}
+			};
+		}
+	};
+}
+//#endregion
+export { newSqliteNgramProvider };
+
+//# sourceMappingURL=ngram-provider.js.map

package/dist/ngram-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  NGRAM_TYPE,\n  type NgramBlock,\n  type NgramConfig,\n  type NgramIndex,\n  type NgramMatch,\n  type NgramQuery,\n  type NgramResult,\n  type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n  const breaks: number[] = [];\n  for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n  let line = 1;\n  let b = 0;\n  return [...matches]\n    .sort((x, y) => x.start - y.start)\n    .map(({ start, end }) => {\n      while (b < breaks.length && (breaks[b] as number) < start) {\n        b++;\n        line++;\n      }\n      return { start, end, line };\n    });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n  return {\n    type: NGRAM_TYPE,\n\n    async drop({ db, prefix }) {\n      await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const storeKind = config.store ?? \"fts5\";\n      const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n      await store.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"NgramIndex is closed\");\n      };\n\n      const index: NgramIndex = {\n        async getIndexInfo() {\n          return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n        },\n\n        async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n          ensureOpen();\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const result = await store.search(params.pattern, {\n            regex: params.regex ?? false,\n            caseSensitive: params.caseSensitive ?? true,\n            limit: params.topK ?? 100,\n            ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n          });\n          const hits = result.hits.filter((h) => h.matches.length > 0);\n          const ids = hits.map((h) => h.id);\n          const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n          // Array sort is stable: equal counts keep the store's ascending-id order.\n          const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n          for (const h of ranked) {\n            const ref = refs.get(h.id);\n            if (!ref) continue;\n            const count = h.matches.length;\n            yield {\n              ...ref,\n              score: count,\n              count,\n              matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n            };\n          }\n          return { truncated: result.truncated };\n        },\n\n        async addDocument(blocks: NgramBlock[]) {\n          ensureOpen();\n          for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n            const batch = blocks.slice(i, i + UPSERT_BATCH);\n            const ids = await blockIds.ensure(db, batch);\n            await store.upsert(\n              ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n            );\n          }\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await store.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n          const entries = await list.entries(p);\n          const texts = await store.getTexts(entries.map((e) => e.id));\n          for (const e of entries) {\n            const content = texts.get(e.id);\n            if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await store.drop();\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,SAAS;GAC3E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAwFA,OAAO;IArFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}

package/dist/path-sql.d.ts

@@ -0,0 +1,15 @@
+//#region src/path-sql.d.ts
+/**
+ * SQL condition matching `column` under any of `prefixes`, on path-component
+ * boundaries (`/foo` matches `/foo` and `/foo/x`, never `/foobar`) and
+ * case-sensitively. Uses `substr` rather than `LIKE`: SQLite's `LIKE` ignores
+ * ASCII case, and `PRAGMA case_sensitive_like` is not authorised in Durable
+ * Objects. `column` must be table-qualified (`b.path`): `json_each` has a `path`
+ * column of its own. No prefixes → `1` (no constraint).
+ */
+export declare function pathPrefixSql(column: string, prefixes: readonly string[]): {
+  sql: string;
+  params: string[];
+};
+//#endregion
+//# sourceMappingURL=path-sql.d.ts.map

package/dist/path-sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"path-sql.d.ts","names":[],"sources":["../src/path-sql.ts"],"mappings":";;;;;;;;;wBAQgB,cACd,gBACA;EACG;EAAa"}

package/dist/path-sql.js

@@ -0,0 +1,23 @@
+//#region src/path-sql.ts
+/**
+* SQL condition matching `column` under any of `prefixes`, on path-component
+* boundaries (`/foo` matches `/foo` and `/foo/x`, never `/foobar`) and
+* case-sensitively. Uses `substr` rather than `LIKE`: SQLite's `LIKE` ignores
+* ASCII case, and `PRAGMA case_sensitive_like` is not authorised in Durable
+* Objects. `column` must be table-qualified (`b.path`): `json_each` has a `path`
+* column of its own. No prefixes → `1` (no constraint).
+*/
+function pathPrefixSql(column, prefixes) {
+	if (prefixes.length === 0) return {
+		sql: "1",
+		params: []
+	};
+	return {
+		sql: `EXISTS (SELECT 1 FROM json_each(?) p WHERE CASE WHEN substr(p.value, -1) = '/' THEN substr(${column}, 1, length(p.value)) = p.value ELSE ${column} = p.value OR substr(${column}, 1, length(p.value) + 1) = p.value || '/' END)`,
+		params: [JSON.stringify(prefixes)]
+	};
+}
+//#endregion
+export { pathPrefixSql };
+
+//# sourceMappingURL=path-sql.js.map

package/dist/path-sql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"path-sql.js","names":[],"sources":["../src/path-sql.ts"],"sourcesContent":["/**\n * SQL condition matching `column` under any of `prefixes`, on path-component\n * boundaries (`/foo` matches `/foo` and `/foo/x`, never `/foobar`) and\n * case-sensitively. Uses `substr` rather than `LIKE`: SQLite's `LIKE` ignores\n * ASCII case, and `PRAGMA case_sensitive_like` is not authorised in Durable\n * Objects. `column` must be table-qualified (`b.path`): `json_each` has a `path`\n * column of its own. No prefixes → `1` (no constraint).\n */\nexport function pathPrefixSql(\n  column: string,\n  prefixes: readonly string[],\n): { sql: string; params: string[] } {\n  if (prefixes.length === 0) return { sql: \"1\", params: [] };\n  // One JSON parameter, however many prefixes: Durable Objects cap bound variables at 100.\n  const sql = `EXISTS (SELECT 1 FROM json_each(?) p WHERE CASE WHEN substr(p.value, -1) = '/' THEN substr(${column}, 1, length(p.value)) = p.value ELSE ${column} = p.value OR substr(${column}, 1, length(p.value) + 1) = p.value || '/' END)`;\n  return { sql, params: [JSON.stringify(prefixes)] };\n}\n"],"mappings":";;;;;;;;;AAQA,SAAgB,cACd,QACA,UACmC;CACnC,IAAI,SAAS,WAAW,GAAG,OAAO;EAAE,KAAK;EAAK,QAAQ,CAAC;CAAE;CAGzD,OAAO;EAAE,KAAA,8FADiG,OAAO,uCAAuC,OAAO,uBAAuB,OAAO;EAC/K,QAAQ,CAAC,KAAK,UAAU,QAAQ,CAAC;CAAE;AACnD"}

package/dist/types.d.ts

@@ -0,0 +1,24 @@
+import { BlockIds } from "./block-ids.js";
+import { Db } from "@statewalker/db-api";
+import { ScoredHit, SearchIndex } from "@statewalker/indexer-api";
+//#region src/types.d.ts
+export interface SqliteModalityContext {
+  /** The database. Inside `inTransaction(db, (tx) => …)` use `tx` and `ctx.blockIds` methods with `tx`. */
+  db: Db;
+  /** Unique table-name prefix for this sub-index, e.g. `idx_docs__body_`. */
+  prefix: string;
+  /** The Index's shared block-id table. */
+  blockIds: BlockIds;
+}
+/**
+ * A modality backed by tables in one SQLite `Db`. Unlike `ModalityProvider`,
+ * opening is async (tables are created, existing data is adopted) and the
+ * provider can drop its tables without an open instance.
+ */
+export interface SqliteModalityProvider<C = unknown, I extends SearchIndex<unknown, unknown, ScoredHit> = SearchIndex<unknown, unknown, ScoredHit>> {
+  readonly type: string;
+  open(ctx: SqliteModalityContext, config: C): Promise<I>;
+  drop(ctx: SqliteModalityContext): Promise<void>;
+}
+//#endregion
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;iBAIiB;;EAEf,IAAI;;EAEJ;;EAEA,UAAU;;;;;;;iBAQK,uBACf,aACA,UAAU,8BAA8B,aAAa,8BAA8B;WAE1E;EACT,KAAK,KAAK,uBAAuB,QAAQ,IAAI,QAAQ;EACrD,KAAK,KAAK,wBAAwB"}

package/dist/vector-provider.d.ts

@@ -0,0 +1,12 @@
+import { SqliteModalityProvider } from "./types.js";
+import { VectorConfig, VectorIndex } from "@statewalker/indexer-vector";
+//#region src/vector-provider.d.ts
+/**
+ * Wraps the id-based `VectorIndex` engine. The engine owns its tables under
+ * `{prefix}vec_` and its own transactions (never call it inside `inTransaction`);
+ * this provider adds only the `{path, blockId}` mapping. Size and enumeration
+ * join the engine's `idTable`, so they list exactly what the engine holds.
+ */
+export declare function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig, VectorIndex>;
+//#endregion
+//# sourceMappingURL=vector-provider.d.ts.map

package/dist/vector-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-provider.d.ts","names":[],"sources":["../src/vector-provider.ts"],"mappings":";;;;;;;;;wBAoBgB,2BAA2B,uBAAuB,cAAc"}

package/dist/vector-provider.js

@@ -0,0 +1,114 @@
+import { newEnumeration } from "./enumeration.js";
+import { toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core";
+import { VectorIndex, dropVectorIndex } from "@statewalker/db-sqlite-index/vector";
+import { VECTOR_TYPE } from "@statewalker/indexer-vector";
+//#region src/vector-provider.ts
+/**
+* Wraps the id-based `VectorIndex` engine. The engine owns its tables under
+* `{prefix}vec_` and its own transactions (never call it inside `inTransaction`);
+* this provider adds only the `{path, blockId}` mapping. Size and enumeration
+* join the engine's `idTable`, so they list exactly what the engine holds.
+*/
+function newSqliteVectorProvider() {
+	return {
+		type: VECTOR_TYPE,
+		async drop({ db, prefix }) {
+			await dropVectorIndex(db, `${prefix}vec_`);
+		},
+		async open({ db, prefix, blockIds }, config) {
+			const engine = new VectorIndex({
+				db,
+				prefix: `${prefix}vec_`,
+				dims: config.dimensionality,
+				metric: "cosine"
+			});
+			await engine.init();
+			const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`);
+			let closed = false;
+			const ensureOpen = () => {
+				if (closed) throw new Error("VectorIndex is closed");
+			};
+			const info = {
+				dimensionality: config.dimensionality,
+				model: config.model,
+				metadata: config.metadata
+			};
+			return {
+				async getIndexInfo() {
+					return { ...info };
+				},
+				async *search(params) {
+					ensureOpen();
+					const k = params.topK ?? 100;
+					const allowed = params.paths?.length ? new Set(await blockIds.select(db, params.paths.map((path) => ({ path })))) : void 0;
+					const filter = allowed ? (id) => allowed.has(id) : void 0;
+					const best = /* @__PURE__ */ new Map();
+					for (const q of params.embeddings ?? []) for (const hit of await engine.search(q, {
+						k,
+						...filter ? { filter } : {}
+					})) {
+						const score = 1 - hit.distance;
+						if (score > (best.get(hit.id) ?? Number.NEGATIVE_INFINITY)) best.set(hit.id, score);
+					}
+					const top = [...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, k);
+					const refs = await blockIds.refs(db, top.map(([id]) => id));
+					for (const [id, score] of top) {
+						const ref = refs.get(id);
+						if (ref) yield {
+							...ref,
+							score
+						};
+					}
+				},
+				async addDocument(blocks) {
+					ensureOpen();
+					if (blocks.length === 0) return;
+					for (const b of blocks) validateDimensionality(info, b.embedding);
+					const ids = await blockIds.ensure(db, blocks);
+					await engine.upsert(ids.map((id, i) => ({
+						id,
+						vector: blocks[i].embedding
+					})));
+				},
+				async addDocuments(batches) {
+					for await (const batch of batches) await this.addDocument(batch);
+				},
+				async deleteDocuments(selectors) {
+					ensureOpen();
+					const sels = [];
+					for await (const s of toAsyncIterable(selectors)) sels.push(s);
+					const ids = await blockIds.select(db, sels);
+					if (ids.length === 0) return;
+					await engine.delete(ids);
+				},
+				getSize: (p) => list.size(p),
+				getDocumentPaths: (p) => list.paths(p),
+				getDocumentBlocksRefs: (p) => list.refs(p),
+				async *getDocumentsBlocks(p) {
+					const entries = await list.entries(p);
+					const vectors = await engine.getVectors(entries.map((e) => e.id));
+					for (const e of entries) {
+						const embedding = vectors.get(e.id);
+						if (embedding) yield {
+							path: e.path,
+							blockId: e.blockId,
+							embedding
+						};
+					}
+				},
+				async close() {
+					closed = true;
+				},
+				async flush() {},
+				async deleteIndex() {
+					closed = true;
+					await dropVectorIndex(db, `${prefix}vec_`);
+				}
+			};
+		}
+	};
+}
+//#endregion
+export { newSqliteVectorProvider };
+
+//# sourceMappingURL=vector-provider.js.map

package/dist/vector-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-provider.js","names":["VectorEngine"],"sources":["../src/vector-provider.ts"],"sourcesContent":["import { dropVectorIndex, VectorIndex as VectorEngine } from \"@statewalker/db-sqlite-index/vector\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable, validateDimensionality } from \"@statewalker/indexer-core\";\nimport {\n  VECTOR_TYPE,\n  type VectorBlock,\n  type VectorConfig,\n  type VectorIndex,\n  type VectorQuery,\n  type VectorResult,\n} from \"@statewalker/indexer-vector\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Wraps the id-based `VectorIndex` engine. The engine owns its tables under\n * `{prefix}vec_` and its own transactions (never call it inside `inTransaction`);\n * this provider adds only the `{path, blockId}` mapping. Size and enumeration\n * join the engine's `idTable`, so they list exactly what the engine holds.\n */\nexport function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig, VectorIndex> {\n  return {\n    type: VECTOR_TYPE,\n\n    async drop({ db, prefix }) {\n      await dropVectorIndex(db, `${prefix}vec_`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const engine = new VectorEngine({\n        db,\n        prefix: `${prefix}vec_`,\n        dims: config.dimensionality,\n        metric: \"cosine\",\n      });\n      await engine.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"VectorIndex is closed\");\n      };\n      const info = {\n        dimensionality: config.dimensionality,\n        model: config.model,\n        metadata: config.metadata,\n      };\n\n      const index: VectorIndex = {\n        async getIndexInfo() {\n          return { ...info };\n        },\n\n        async *search(params: VectorQuery): AsyncGenerator<VectorResult> {\n          ensureOpen();\n          const k = params.topK ?? 100;\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const filter = allowed ? (id: number) => allowed.has(id) : undefined;\n          // Several embeddings merge by max score, like every other backend.\n          const best = new Map<number, number>();\n          for (const q of params.embeddings ?? []) {\n            for (const hit of await engine.search(q, { k, ...(filter ? { filter } : {}) })) {\n              const score = 1 - hit.distance;\n              if (score > (best.get(hit.id) ?? Number.NEGATIVE_INFINITY)) best.set(hit.id, score);\n            }\n          }\n          const top = [...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, k);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, score] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score };\n          }\n        },\n\n        async addDocument(blocks: VectorBlock[]) {\n          ensureOpen();\n          if (blocks.length === 0) return;\n          for (const b of blocks) validateDimensionality(info, b.embedding);\n          const ids = await blockIds.ensure(db, blocks);\n          await engine.upsert(\n            ids.map((id, i) => ({ id, vector: (blocks[i] as VectorBlock).embedding })),\n          );\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await engine.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<VectorBlock> {\n          const entries = await list.entries(p);\n          const vectors = await engine.getVectors(entries.map((e) => e.id));\n          for (const e of entries) {\n            const embedding = vectors.get(e.id);\n            if (embedding) yield { path: e.path, blockId: e.blockId, embedding };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await dropVectorIndex(db, `${prefix}vec_`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;AAoBA,SAAgB,0BAA6E;CAC3F,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;EAC3C;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,SAAS,IAAIA,YAAa;IAC9B;IACA,QAAQ,GAAG,OAAO;IAClB,MAAM,OAAO;IACb,QAAQ;GACV,CAAC;GACD,MAAM,OAAO,KAAK;GAClB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,OAAO,SAAS;GAC5E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;GACrD;GACA,MAAM,OAAO;IACX,gBAAgB,OAAO;IACvB,OAAO,OAAO;IACd,UAAU,OAAO;GACnB;GAmFA,OAAO;IAhFL,MAAM,eAAe;KACnB,OAAO,EAAE,GAAG,KAAK;IACnB;IAEA,OAAO,OAAO,QAAmD;KAC/D,WAAW;KACX,MAAM,IAAI,OAAO,QAAQ;KACzB,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,WAAW,OAAe,QAAQ,IAAI,EAAE,IAAI,KAAA;KAE3D,MAAM,uBAAO,IAAI,IAAoB;KACrC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC,GACpC,KAAK,MAAM,OAAO,MAAM,OAAO,OAAO,GAAG;MAAE;MAAG,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;KAAG,CAAC,GAAG;MAC9E,MAAM,QAAQ,IAAI,IAAI;MACtB,IAAI,SAAS,KAAK,IAAI,IAAI,EAAE,KAAK,OAAO,oBAAoB,KAAK,IAAI,IAAI,IAAI,KAAK;KACpF;KAEF,MAAM,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;KACtE,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,UAAU,KAAK;MAC7B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK;MAAM;KACjC;IACF;IAEA,MAAM,YAAY,QAAuB;KACvC,WAAW;KACX,IAAI,OAAO,WAAW,GAAG;KACzB,KAAK,MAAM,KAAK,QAAQ,uBAAuB,MAAM,EAAE,SAAS;KAChE,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM;KAC5C,MAAM,OAAO,OACX,IAAI,KAAK,IAAI,OAAO;MAAE;MAAI,QAAS,OAAO,EAAE,CAAiB;KAAU,EAAE,CAC3E;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,OAAO,OAAO,GAAG;IACzB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA+C;KACvE,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,UAAU,MAAM,OAAO,WAAW,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAChE,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,YAAY,QAAQ,IAAI,EAAE,EAAE;MAClC,IAAI,WAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAU;KACrE;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;IAC3C;GAES;EACb;CACF;AACF"}