@statewalker/indexer-sqlite 0.1.0 → 0.1.2

package/dist/create-sqlite-indexer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-sqlite-indexer.d.ts","names":[],"sources":["../src/create-sqlite-indexer.ts"],"mappings":";;;;iBAaiB;;EAEf,IAAI;;EAEJ,WAAW;;;;;;;wBAqBS,oBAAoB,SAAS,uBAAuB,QAAQ"}
+{"version":3,"file":"create-sqlite-indexer.d.ts","names":[],"sources":["../src/create-sqlite-indexer.ts"],"mappings":";;;;iBAciB;;EAEf,IAAI;;EAEJ,WAAW;;;;;;;wBAuBS,oBAAoB,SAAS,uBAAuB,QAAQ"}

package/dist/create-sqlite-indexer.js

@@ -38,7 +38,7 @@ async function createSqliteIndexer(options) {
 		await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
 		manifest.delete(name);
 	}
-	async function openIndex(name, subIndexes, options) {
+	async function openIndex(name, subIndexes, options, metadata) {
 		const onMissing = options?.onMissingProvider ?? "throw";
 		const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
 		const bindings = [];
@@ -68,25 +68,33 @@ async function createSqliteIndexer(options) {
 			});
 			effective[sub] = spec;
 		}
+		const index = createCompositeIndex({
+			name,
+			metadata,
+			bindings,
+			onClose: async () => {
+				if (indexes.get(name) === index) indexes.delete(name);
+			},
+			onDeleteIndex: async () => {
+				for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
+				await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
+				await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
+				manifest.delete(name);
+				indexes.delete(name);
+			}
+		});
 		return {
-			index: createCompositeIndex({
-				name,
-				bindings,
-				onDeleteIndex: async () => {
-					for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
-					await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
-					await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
-					manifest.delete(name);
-					indexes.delete(name);
-				}
-			}),
+			index,
 			effective
 		};
 	}
 	return {
 		async getIndexNames() {
 			ensureOpen();
-			return [...manifest.keys()].map((name) => ({ name }));
+			return [...manifest.entries()].map(([name, entry]) => entry.metadata !== void 0 ? {
+				name,
+				metadata: entry.metadata
+			} : { name });
 		},
 		createIndex(params) {
 			return runExclusive(async () => {
@@ -100,9 +108,11 @@ async function createSqliteIndexer(options) {
 					indexes.delete(name);
 					await dropIndexTables(name, existing?.subIndexes ?? {});
 				}
-				const { index, effective } = await openIndex(name, subIndexes);
-				await writeManifest(name, { subIndexes: effective });
-				if (metadata !== void 0) index.metadata = metadata;
+				const { index, effective } = await openIndex(name, subIndexes, void 0, metadata);
+				await writeManifest(name, {
+					subIndexes: effective,
+					...metadata !== void 0 ? { metadata } : {}
+				});
 				indexes.set(name, index);
 				return index;
 			});
@@ -114,8 +124,11 @@ async function createSqliteIndexer(options) {
 				if (cached) return cached;
 				const entry = manifest.get(name);
 				if (!entry) return null;
-				const { index, effective } = await openIndex(name, entry.subIndexes, options);
-				if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) await writeManifest(name, { subIndexes: effective });
+				const { index, effective } = await openIndex(name, entry.subIndexes, options, entry.metadata);
+				if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) await writeManifest(name, {
+					...entry,
+					subIndexes: effective
+				});
 				indexes.set(name, index);
 				return index;
 			});
@@ -124,25 +137,31 @@ async function createSqliteIndexer(options) {
 			ensureOpen();
 			return manifest.has(name);
 		},
-		async deleteIndex(name) {
-			ensureOpen();
-			const live = indexes.get(name);
-			if (live) {
-				await live.deleteIndex();
-				return;
-			}
-			const entry = manifest.get(name);
-			if (entry) await dropIndexTables(name, entry.subIndexes);
+		deleteIndex(name) {
+			return runExclusive(async () => {
+				ensureOpen();
+				const live = indexes.get(name);
+				if (live) {
+					await live.deleteIndex();
+					return;
+				}
+				const entry = manifest.get(name);
+				if (entry) await dropIndexTables(name, entry.subIndexes);
+			});
 		},
-		async flush() {
-			ensureOpen();
-			for (const index of indexes.values()) await index.flush();
+		flush() {
+			return runExclusive(async () => {
+				ensureOpen();
+				for (const index of indexes.values()) await index.flush();
+			});
 		},
-		async close() {
-			if (closed) return;
-			closed = true;
-			for (const index of indexes.values()) await index.close().catch(() => void 0);
-			indexes.clear();
+		close() {
+			return runExclusive(async () => {
+				if (closed) return;
+				closed = true;
+				for (const index of [...indexes.values()]) await index.close().catch(() => void 0);
+				indexes.clear();
+			});
 		}
 	};
 }

package/dist/create-sqlite-indexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-sqlite-indexer.js","names":[],"sources":["../src/create-sqlite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type {\n  AnySubIndexBinding,\n  CreateIndexParams,\n  GetIndexOptions,\n  Index,\n  Indexer,\n  IndexInfo,\n} from \"@statewalker/indexer-api\";\nimport { createCompositeIndex, createSerialiser } from \"@statewalker/indexer-core\";\nimport { type BlockIds, openBlockIds } from \"./block-ids.js\";\nimport type { SqliteModalityContext, SqliteModalityProvider } from \"./types.js\";\n\nexport interface SqliteIndexerOptions {\n  /** The database. The indexer never closes it. */\n  db: Db;\n  /** One provider per modality type, e.g. FTS5, vector, n-gram. */\n  providers: SqliteModalityProvider[];\n}\n\ntype SubSpec = { type: string } & Record<string, unknown>;\ninterface IndexManifest {\n  subIndexes: Record<string, SubSpec>;\n}\n\nconst MANIFEST_TABLE = \"__indexer_manifest\";\n\n/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so \"Docs\" and \"docs\" must not collide. */\nconst sanitizePrefix = (name: string) =>\n  name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);\n\nconst indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;\n\n/**\n * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared\n * `{indexPrefix}blocks` id table per Index, and opens each sub-index through\n * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).\n */\nexport async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {\n  const { db } = options;\n  const providers = new Map(options.providers.map((p) => [p.type, p]));\n  const indexes = new Map<string, Index>();\n  const manifest = new Map<string, IndexManifest>();\n  const runExclusive = createSerialiser();\n  let closed = false;\n\n  await db.exec(\n    `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,\n  );\n  for (const row of await db.query<{ name: string; config: string }>(\n    `SELECT name, config FROM ${MANIFEST_TABLE}`,\n  )) {\n    manifest.set(row.name, JSON.parse(row.config) as IndexManifest);\n  }\n\n  const ensureOpen = () => {\n    if (closed) throw new Error(\"Indexer is closed\");\n  };\n\n  const contextFor = (name: string, sub: string, blockIds: BlockIds): SqliteModalityContext => ({\n    db,\n    prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,\n    blockIds,\n  });\n\n  async function writeManifest(name: string, entry: IndexManifest): Promise<void> {\n    await db.query(\n      `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,\n      [name, JSON.stringify(entry)],\n    );\n    manifest.set(name, entry);\n  }\n\n  async function dropIndexTables(name: string, subIndexes: Record<string, SubSpec>): Promise<void> {\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    for (const [sub, spec] of Object.entries(subIndexes)) {\n      await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n    }\n    await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n    await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n    manifest.delete(name);\n  }\n\n  async function openIndex(\n    name: string,\n    subIndexes: Record<string, SubSpec>,\n    options?: GetIndexOptions,\n  ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {\n    const onMissing = options?.onMissingProvider ?? \"throw\";\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    const bindings: AnySubIndexBinding[] = [];\n    const effective: Record<string, SubSpec> = {};\n\n    for (const [sub, saved] of Object.entries(subIndexes)) {\n      const action = options?.subIndexes?.[sub];\n      if (action && \"skip\" in action && action.skip === true) {\n        effective[sub] = saved;\n        continue;\n      }\n      const ctx = contextFor(name, sub, blockIds);\n      const spec = action && !(\"skip\" in action) ? (action as SubSpec) : saved;\n      const provider = providers.get(spec.type);\n      if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);\n      if (!provider) {\n        if (onMissing === \"throw\") {\n          throw new Error(\n            `Cannot load index: sub-index \"${sub}\" requires modality \"${spec.type}\" but only [${[\n              ...providers.keys(),\n            ]\n              .map((t) => `\"${t}\"`)\n              .join(\", \")}] are registered.`,\n          );\n        }\n        const g = globalThis as { console?: { warn(...args: unknown[]): void } };\n        g.console?.warn(`Sub-index \"${sub}\" (type \"${spec.type}\") has no provider; skipping.`);\n        effective[sub] = saved;\n        continue;\n      }\n      const { type, ...config } = spec;\n      bindings.push({ name: sub, type, config, index: await provider.open(ctx, config) });\n      effective[sub] = spec;\n    }\n\n    const index = createCompositeIndex({\n      name,\n      bindings,\n      onDeleteIndex: async () => {\n        // Bound sub-indexes dropped themselves; skipped ones still own tables.\n        for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {\n          await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n        }\n        await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n        await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n        manifest.delete(name);\n        indexes.delete(name);\n      },\n    });\n    return { index, effective };\n  }\n\n  return {\n    async getIndexNames(): Promise<IndexInfo[]> {\n      ensureOpen();\n      return [...manifest.keys()].map((name) => ({ name }));\n    },\n\n    createIndex(params: CreateIndexParams): Promise<Index> {\n      return runExclusive(async () => {\n        ensureOpen();\n        const { name, subIndexes, overwrite, metadata } = params;\n        if (!subIndexes || Object.keys(subIndexes).length === 0) {\n          throw new Error(\"Cannot create index without any subIndexes\");\n        }\n        const existing = manifest.get(name);\n        if (existing || indexes.has(name)) {\n          if (!overwrite) throw new Error(`Index \"${name}\" already exists`);\n          await indexes\n            .get(name)\n            ?.close()\n            .catch(() => undefined);\n          indexes.delete(name);\n          await dropIndexTables(name, existing?.subIndexes ?? {});\n        }\n        const { index, effective } = await openIndex(name, subIndexes);\n        await writeManifest(name, { subIndexes: effective });\n        if (metadata !== undefined) (index as { metadata?: unknown }).metadata = metadata;\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    getIndex(name: string, options?: GetIndexOptions): Promise<Index | null> {\n      // Serialised with createIndex so concurrent first opens open once\n      // (a second reinit would otherwise drop tables the first just wrote).\n      return runExclusive(async () => {\n        ensureOpen();\n        const cached = indexes.get(name);\n        if (cached) return cached;\n        const entry = manifest.get(name);\n        if (!entry) return null;\n        const { index, effective } = await openIndex(name, entry.subIndexes, options);\n        if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {\n          await writeManifest(name, { subIndexes: effective });\n        }\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    async hasIndex(name: string): Promise<boolean> {\n      ensureOpen();\n      return manifest.has(name);\n    },\n\n    async deleteIndex(name: string): Promise<void> {\n      ensureOpen();\n      const live = indexes.get(name);\n      if (live) {\n        await live.deleteIndex();\n        return;\n      }\n      const entry = manifest.get(name);\n      if (entry) await dropIndexTables(name, entry.subIndexes);\n    },\n\n    async flush(): Promise<void> {\n      ensureOpen();\n      for (const index of indexes.values()) await index.flush();\n    },\n\n    async close(): Promise<void> {\n      if (closed) return;\n      closed = true;\n      for (const index of indexes.values()) await index.close().catch(() => undefined);\n      indexes.clear();\n    },\n  } satisfies Indexer;\n}\n"],"mappings":";;;AAyBA,MAAM,iBAAiB;;AAGvB,MAAM,kBAAkB,SACtB,KAAK,QAAQ,eAAe,OAAO,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;AAE5D,MAAM,eAAe,SAAiB,OAAO,eAAe,IAAI,EAAE;;;;;;AAOlE,eAAsB,oBAAoB,SAAiD;CACzF,MAAM,EAAE,OAAO;CACf,MAAM,YAAY,IAAI,IAAI,QAAQ,UAAU,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;CACnE,MAAM,0BAAU,IAAI,IAAmB;CACvC,MAAM,2BAAW,IAAI,IAA2B;CAChD,MAAM,eAAe,iBAAiB;CACtC,IAAI,SAAS;CAEb,MAAM,GAAG,KACP,8BAA8B,eAAe,+CAC/C;CACA,KAAK,MAAM,OAAO,MAAM,GAAG,MACzB,4BAA4B,gBAC9B,GACE,SAAS,IAAI,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAkB;CAGhE,MAAM,mBAAmB;EACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,mBAAmB;CACjD;CAEA,MAAM,cAAc,MAAc,KAAa,cAA+C;EAC5F;EACA,QAAQ,GAAG,YAAY,IAAI,IAAI,eAAe,GAAG,EAAE;EACnD;CACF;CAEA,eAAe,cAAc,MAAc,OAAqC;EAC9E,MAAM,GAAG,MACP,eAAe,eAAe,0FAC9B,CAAC,MAAM,KAAK,UAAU,KAAK,CAAC,CAC9B;EACA,SAAS,IAAI,MAAM,KAAK;CAC1B;CAEA,eAAe,gBAAgB,MAAc,YAAoD;EAC/F,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAU,GACjD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;EAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;EACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;EACrE,SAAS,OAAO,IAAI;CACtB;CAEA,eAAe,UACb,MACA,YACA,SAC+D;EAC/D,MAAM,YAAY,SAAS,qBAAqB;EAChD,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,MAAM,WAAiC,CAAC;EACxC,MAAM,YAAqC,CAAC;EAE5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,UAAU,GAAG;GACrD,MAAM,SAAS,SAAS,aAAa;GACrC,IAAI,UAAU,UAAU,UAAU,OAAO,SAAS,MAAM;IACtD,UAAU,OAAO;IACjB;GACF;GACA,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ;GAC1C,MAAM,OAAO,UAAU,EAAE,UAAU,UAAW,SAAqB;GACnE,MAAM,WAAW,UAAU,IAAI,KAAK,IAAI;GACxC,IAAI,YAAY,SAAS,OAAO,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,EAAE,KAAK,GAAG;GACzE,IAAI,CAAC,UAAU;IACb,IAAI,cAAc,SAChB,MAAM,IAAI,MACR,iCAAiC,IAAI,uBAAuB,KAAK,KAAK,cAAc,CAClF,GAAG,UAAU,KAAK,CACpB,CAAC,CACE,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CACpB,KAAK,IAAI,EAAE,kBAChB;IAGF,WAAE,SAAS,KAAK,cAAc,IAAI,WAAW,KAAK,KAAK,8BAA8B;IACrF,UAAU,OAAO;IACjB;GACF;GACA,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,SAAS,KAAK;IAAE,MAAM;IAAK;IAAM;IAAQ,OAAO,MAAM,SAAS,KAAK,KAAK,MAAM;GAAE,CAAC;GAClF,UAAU,OAAO;EACnB;EAgBA,OAAO;GAAE,OAdK,qBAAqB;IACjC;IACA;IACA,eAAe,YAAY;KAEzB,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,IAAI,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC,GAC3E,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;KAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;KACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;KACrE,SAAS,OAAO,IAAI;KACpB,QAAQ,OAAO,IAAI;IACrB;GACF,CACa;GAAG;EAAU;CAC5B;CAEA,OAAO;EACL,MAAM,gBAAsC;GAC1C,WAAW;GACX,OAAO,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,KAAK,EAAE;EACtD;EAEA,YAAY,QAA2C;GACrD,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,EAAE,MAAM,YAAY,WAAW,aAAa;IAClD,IAAI,CAAC,cAAc,OAAO,KAAK,UAAU,CAAC,CAAC,WAAW,GACpD,MAAM,IAAI,MAAM,4CAA4C;IAE9D,MAAM,WAAW,SAAS,IAAI,IAAI;IAClC,IAAI,YAAY,QAAQ,IAAI,IAAI,GAAG;KACjC,IAAI,CAAC,WAAW,MAAM,IAAI,MAAM,UAAU,KAAK,iBAAiB;KAChE,MAAM,QACH,IAAI,IAAI,CAAC,EACR,MAAM,CAAC,CACR,YAAY,KAAA,CAAS;KACxB,QAAQ,OAAO,IAAI;KACnB,MAAM,gBAAgB,MAAM,UAAU,cAAc,CAAC,CAAC;IACxD;IACA,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,UAAU;IAC7D,MAAM,cAAc,MAAM,EAAE,YAAY,UAAU,CAAC;IACnD,IAAI,aAAa,KAAA,GAAW,MAAkC,WAAW;IACzE,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,SAAS,MAAc,SAAkD;GAGvE,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,SAAS,QAAQ,IAAI,IAAI;IAC/B,IAAI,QAAQ,OAAO;IACnB,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,CAAC,OAAO,OAAO;IACnB,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,MAAM,YAAY,OAAO;IAC5E,IAAI,KAAK,UAAU,SAAS,MAAM,KAAK,UAAU,MAAM,UAAU,GAC/D,MAAM,cAAc,MAAM,EAAE,YAAY,UAAU,CAAC;IAErD,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,MAAM,SAAS,MAAgC;GAC7C,WAAW;GACX,OAAO,SAAS,IAAI,IAAI;EAC1B;EAEA,MAAM,YAAY,MAA6B;GAC7C,WAAW;GACX,MAAM,OAAO,QAAQ,IAAI,IAAI;GAC7B,IAAI,MAAM;IACR,MAAM,KAAK,YAAY;IACvB;GACF;GACA,MAAM,QAAQ,SAAS,IAAI,IAAI;GAC/B,IAAI,OAAO,MAAM,gBAAgB,MAAM,MAAM,UAAU;EACzD;EAEA,MAAM,QAAuB;GAC3B,WAAW;GACX,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM;EAC1D;EAEA,MAAM,QAAuB;GAC3B,IAAI,QAAQ;GACZ,SAAS;GACT,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E,QAAQ,MAAM;EAChB;CACF;AACF"}
+{"version":3,"file":"create-sqlite-indexer.js","names":[],"sources":["../src/create-sqlite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type {\n  AnySubIndexBinding,\n  CreateIndexParams,\n  GetIndexOptions,\n  Index,\n  Indexer,\n  IndexInfo,\n  Metadata,\n} from \"@statewalker/indexer-api\";\nimport { createCompositeIndex, createSerialiser } from \"@statewalker/indexer-core\";\nimport { type BlockIds, openBlockIds } from \"./block-ids.js\";\nimport type { SqliteModalityContext, SqliteModalityProvider } from \"./types.js\";\n\nexport interface SqliteIndexerOptions {\n  /** The database. The indexer never closes it. */\n  db: Db;\n  /** One provider per modality type, e.g. FTS5, vector, n-gram. */\n  providers: SqliteModalityProvider[];\n}\n\ntype SubSpec = { type: string } & Record<string, unknown>;\ninterface IndexManifest {\n  subIndexes: Record<string, SubSpec>;\n  /** Index-level metadata from `createIndex` (absent in manifests written before it was saved). */\n  metadata?: Metadata;\n}\n\nconst MANIFEST_TABLE = \"__indexer_manifest\";\n\n/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so \"Docs\" and \"docs\" must not collide. */\nconst sanitizePrefix = (name: string) =>\n  name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);\n\nconst indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;\n\n/**\n * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared\n * `{indexPrefix}blocks` id table per Index, and opens each sub-index through\n * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).\n */\nexport async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {\n  const { db } = options;\n  const providers = new Map(options.providers.map((p) => [p.type, p]));\n  const indexes = new Map<string, Index>();\n  const manifest = new Map<string, IndexManifest>();\n  const runExclusive = createSerialiser();\n  let closed = false;\n\n  await db.exec(\n    `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,\n  );\n  for (const row of await db.query<{ name: string; config: string }>(\n    `SELECT name, config FROM ${MANIFEST_TABLE}`,\n  )) {\n    manifest.set(row.name, JSON.parse(row.config) as IndexManifest);\n  }\n\n  const ensureOpen = () => {\n    if (closed) throw new Error(\"Indexer is closed\");\n  };\n\n  const contextFor = (name: string, sub: string, blockIds: BlockIds): SqliteModalityContext => ({\n    db,\n    prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,\n    blockIds,\n  });\n\n  async function writeManifest(name: string, entry: IndexManifest): Promise<void> {\n    await db.query(\n      `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,\n      [name, JSON.stringify(entry)],\n    );\n    manifest.set(name, entry);\n  }\n\n  async function dropIndexTables(name: string, subIndexes: Record<string, SubSpec>): Promise<void> {\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    for (const [sub, spec] of Object.entries(subIndexes)) {\n      await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n    }\n    await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n    await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n    manifest.delete(name);\n  }\n\n  async function openIndex(\n    name: string,\n    subIndexes: Record<string, SubSpec>,\n    options?: GetIndexOptions,\n    metadata?: Metadata,\n  ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {\n    const onMissing = options?.onMissingProvider ?? \"throw\";\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    const bindings: AnySubIndexBinding[] = [];\n    const effective: Record<string, SubSpec> = {};\n\n    for (const [sub, saved] of Object.entries(subIndexes)) {\n      const action = options?.subIndexes?.[sub];\n      if (action && \"skip\" in action && action.skip === true) {\n        effective[sub] = saved;\n        continue;\n      }\n      const ctx = contextFor(name, sub, blockIds);\n      const spec = action && !(\"skip\" in action) ? (action as SubSpec) : saved;\n      const provider = providers.get(spec.type);\n      if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);\n      if (!provider) {\n        if (onMissing === \"throw\") {\n          throw new Error(\n            `Cannot load index: sub-index \"${sub}\" requires modality \"${spec.type}\" but only [${[\n              ...providers.keys(),\n            ]\n              .map((t) => `\"${t}\"`)\n              .join(\", \")}] are registered.`,\n          );\n        }\n        const g = globalThis as { console?: { warn(...args: unknown[]): void } };\n        g.console?.warn(`Sub-index \"${sub}\" (type \"${spec.type}\") has no provider; skipping.`);\n        effective[sub] = saved;\n        continue;\n      }\n      const { type, ...config } = spec;\n      bindings.push({ name: sub, type, config, index: await provider.open(ctx, config) });\n      effective[sub] = spec;\n    }\n\n    const index: Index = createCompositeIndex({\n      name,\n      metadata,\n      bindings,\n      // A closed instance leaves the cache, so getIndex opens a new one.\n      onClose: async () => {\n        if (indexes.get(name) === index) indexes.delete(name);\n      },\n      onDeleteIndex: async () => {\n        // Bound sub-indexes dropped themselves; skipped ones still own tables.\n        for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {\n          await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n        }\n        await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n        await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n        manifest.delete(name);\n        indexes.delete(name);\n      },\n    });\n    return { index, effective };\n  }\n\n  return {\n    async getIndexNames(): Promise<IndexInfo[]> {\n      ensureOpen();\n      return [...manifest.entries()].map(([name, entry]) =>\n        entry.metadata !== undefined ? { name, metadata: entry.metadata } : { name },\n      );\n    },\n\n    createIndex(params: CreateIndexParams): Promise<Index> {\n      return runExclusive(async () => {\n        ensureOpen();\n        const { name, subIndexes, overwrite, metadata } = params;\n        if (!subIndexes || Object.keys(subIndexes).length === 0) {\n          throw new Error(\"Cannot create index without any subIndexes\");\n        }\n        const existing = manifest.get(name);\n        if (existing || indexes.has(name)) {\n          if (!overwrite) throw new Error(`Index \"${name}\" already exists`);\n          await indexes\n            .get(name)\n            ?.close()\n            .catch(() => undefined);\n          indexes.delete(name);\n          await dropIndexTables(name, existing?.subIndexes ?? {});\n        }\n        const { index, effective } = await openIndex(name, subIndexes, undefined, metadata);\n        await writeManifest(name, {\n          subIndexes: effective,\n          ...(metadata !== undefined ? { metadata } : {}),\n        });\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    getIndex(name: string, options?: GetIndexOptions): Promise<Index | null> {\n      // Serialised with createIndex so concurrent first opens open once\n      // (a second reinit would otherwise drop tables the first just wrote).\n      return runExclusive(async () => {\n        ensureOpen();\n        const cached = indexes.get(name);\n        if (cached) return cached;\n        const entry = manifest.get(name);\n        if (!entry) return null;\n        const { index, effective } = await openIndex(\n          name,\n          entry.subIndexes,\n          options,\n          entry.metadata,\n        );\n        if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {\n          await writeManifest(name, { ...entry, subIndexes: effective });\n        }\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    async hasIndex(name: string): Promise<boolean> {\n      ensureOpen();\n      return manifest.has(name);\n    },\n\n    deleteIndex(name: string): Promise<void> {\n      return runExclusive(async () => {\n        ensureOpen();\n        const live = indexes.get(name);\n        if (live) {\n          await live.deleteIndex();\n          return;\n        }\n        const entry = manifest.get(name);\n        if (entry) await dropIndexTables(name, entry.subIndexes);\n      });\n    },\n\n    flush(): Promise<void> {\n      return runExclusive(async () => {\n        ensureOpen();\n        for (const index of indexes.values()) await index.flush();\n      });\n    },\n\n    close(): Promise<void> {\n      return runExclusive(async () => {\n        if (closed) return;\n        closed = true;\n        for (const index of [...indexes.values()]) await index.close().catch(() => undefined);\n        indexes.clear();\n      });\n    },\n  } satisfies Indexer;\n}\n"],"mappings":";;;AA4BA,MAAM,iBAAiB;;AAGvB,MAAM,kBAAkB,SACtB,KAAK,QAAQ,eAAe,OAAO,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;AAE5D,MAAM,eAAe,SAAiB,OAAO,eAAe,IAAI,EAAE;;;;;;AAOlE,eAAsB,oBAAoB,SAAiD;CACzF,MAAM,EAAE,OAAO;CACf,MAAM,YAAY,IAAI,IAAI,QAAQ,UAAU,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;CACnE,MAAM,0BAAU,IAAI,IAAmB;CACvC,MAAM,2BAAW,IAAI,IAA2B;CAChD,MAAM,eAAe,iBAAiB;CACtC,IAAI,SAAS;CAEb,MAAM,GAAG,KACP,8BAA8B,eAAe,+CAC/C;CACA,KAAK,MAAM,OAAO,MAAM,GAAG,MACzB,4BAA4B,gBAC9B,GACE,SAAS,IAAI,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAkB;CAGhE,MAAM,mBAAmB;EACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,mBAAmB;CACjD;CAEA,MAAM,cAAc,MAAc,KAAa,cAA+C;EAC5F;EACA,QAAQ,GAAG,YAAY,IAAI,IAAI,eAAe,GAAG,EAAE;EACnD;CACF;CAEA,eAAe,cAAc,MAAc,OAAqC;EAC9E,MAAM,GAAG,MACP,eAAe,eAAe,0FAC9B,CAAC,MAAM,KAAK,UAAU,KAAK,CAAC,CAC9B;EACA,SAAS,IAAI,MAAM,KAAK;CAC1B;CAEA,eAAe,gBAAgB,MAAc,YAAoD;EAC/F,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAU,GACjD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;EAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;EACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;EACrE,SAAS,OAAO,IAAI;CACtB;CAEA,eAAe,UACb,MACA,YACA,SACA,UAC+D;EAC/D,MAAM,YAAY,SAAS,qBAAqB;EAChD,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,MAAM,WAAiC,CAAC;EACxC,MAAM,YAAqC,CAAC;EAE5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,UAAU,GAAG;GACrD,MAAM,SAAS,SAAS,aAAa;GACrC,IAAI,UAAU,UAAU,UAAU,OAAO,SAAS,MAAM;IACtD,UAAU,OAAO;IACjB;GACF;GACA,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ;GAC1C,MAAM,OAAO,UAAU,EAAE,UAAU,UAAW,SAAqB;GACnE,MAAM,WAAW,UAAU,IAAI,KAAK,IAAI;GACxC,IAAI,YAAY,SAAS,OAAO,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,EAAE,KAAK,GAAG;GACzE,IAAI,CAAC,UAAU;IACb,IAAI,cAAc,SAChB,MAAM,IAAI,MACR,iCAAiC,IAAI,uBAAuB,KAAK,KAAK,cAAc,CAClF,GAAG,UAAU,KAAK,CACpB,CAAC,CACE,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CACpB,KAAK,IAAI,EAAE,kBAChB;IAGF,WAAE,SAAS,KAAK,cAAc,IAAI,WAAW,KAAK,KAAK,8BAA8B;IACrF,UAAU,OAAO;IACjB;GACF;GACA,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,SAAS,KAAK;IAAE,MAAM;IAAK;IAAM;IAAQ,OAAO,MAAM,SAAS,KAAK,KAAK,MAAM;GAAE,CAAC;GAClF,UAAU,OAAO;EACnB;EAEA,MAAM,QAAe,qBAAqB;GACxC;GACA;GACA;GAEA,SAAS,YAAY;IACnB,IAAI,QAAQ,IAAI,IAAI,MAAM,OAAO,QAAQ,OAAO,IAAI;GACtD;GACA,eAAe,YAAY;IAEzB,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,IAAI,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC,GAC3E,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;IAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;IACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;IACrE,SAAS,OAAO,IAAI;IACpB,QAAQ,OAAO,IAAI;GACrB;EACF,CAAC;EACD,OAAO;GAAE;GAAO;EAAU;CAC5B;CAEA,OAAO;EACL,MAAM,gBAAsC;GAC1C,WAAW;GACX,OAAO,CAAC,GAAG,SAAS,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,WACzC,MAAM,aAAa,KAAA,IAAY;IAAE;IAAM,UAAU,MAAM;GAAS,IAAI,EAAE,KAAK,CAC7E;EACF;EAEA,YAAY,QAA2C;GACrD,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,EAAE,MAAM,YAAY,WAAW,aAAa;IAClD,IAAI,CAAC,cAAc,OAAO,KAAK,UAAU,CAAC,CAAC,WAAW,GACpD,MAAM,IAAI,MAAM,4CAA4C;IAE9D,MAAM,WAAW,SAAS,IAAI,IAAI;IAClC,IAAI,YAAY,QAAQ,IAAI,IAAI,GAAG;KACjC,IAAI,CAAC,WAAW,MAAM,IAAI,MAAM,UAAU,KAAK,iBAAiB;KAChE,MAAM,QACH,IAAI,IAAI,CAAC,EACR,MAAM,CAAC,CACR,YAAY,KAAA,CAAS;KACxB,QAAQ,OAAO,IAAI;KACnB,MAAM,gBAAgB,MAAM,UAAU,cAAc,CAAC,CAAC;IACxD;IACA,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,YAAY,KAAA,GAAW,QAAQ;IAClF,MAAM,cAAc,MAAM;KACxB,YAAY;KACZ,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;IAC/C,CAAC;IACD,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,SAAS,MAAc,SAAkD;GAGvE,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,SAAS,QAAQ,IAAI,IAAI;IAC/B,IAAI,QAAQ,OAAO;IACnB,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,CAAC,OAAO,OAAO;IACnB,MAAM,EAAE,OAAO,cAAc,MAAM,UACjC,MACA,MAAM,YACN,SACA,MAAM,QACR;IACA,IAAI,KAAK,UAAU,SAAS,MAAM,KAAK,UAAU,MAAM,UAAU,GAC/D,MAAM,cAAc,MAAM;KAAE,GAAG;KAAO,YAAY;IAAU,CAAC;IAE/D,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,MAAM,SAAS,MAAgC;GAC7C,WAAW;GACX,OAAO,SAAS,IAAI,IAAI;EAC1B;EAEA,YAAY,MAA6B;GACvC,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,OAAO,QAAQ,IAAI,IAAI;IAC7B,IAAI,MAAM;KACR,MAAM,KAAK,YAAY;KACvB;IACF;IACA,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,OAAO,MAAM,gBAAgB,MAAM,MAAM,UAAU;GACzD,CAAC;EACH;EAEA,QAAuB;GACrB,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM;GAC1D,CAAC;EACH;EAEA,QAAuB;GACrB,OAAO,aAAa,YAAY;IAC9B,IAAI,QAAQ;IACZ,SAAS;IACT,KAAK,MAAM,SAAS,CAAC,GAAG,QAAQ,OAAO,CAAC,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IACpF,QAAQ,MAAM;GAChB,CAAC;EACH;CACF;AACF"}

package/dist/enumeration.d.ts

@@ -12,7 +12,10 @@ export interface Enumeration {
   paths(prefix?: DocumentPath): AsyncGenerator<DocumentPath>;
   refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;
 }
-/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */
-export declare function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string): Enumeration;
+/**
+ * `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`.
+ * `ensureOpen` runs before every call, so a closed sub-index throws like the other backends.
+ */
+export declare function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string, ensureOpen?: () => void): Enumeration;
 //#endregion
 //# sourceMappingURL=enumeration.d.ts.map

package/dist/enumeration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"enumeration.d.ts","names":[],"sources":["../src/enumeration.ts"],"mappings":";;;;iBAKiB,mBAAmB;EAClC;;;iBAIe;EACf,KAAK,SAAS,eAAe;EAC7B,QAAQ,SAAS,eAAe,QAAQ;EACxC,MAAM,SAAS,eAAe,eAAe;EAC7C,KAAK,SAAS,eAAe,eAAe;;;wBAI9B,eAAe,IAAI,IAAI,UAAU,UAAU,uBAAuB"}
+{"version":3,"file":"enumeration.d.ts","names":[],"sources":["../src/enumeration.ts"],"mappings":";;;;iBAKiB,mBAAmB;EAClC;;;iBAIe;EACf,KAAK,SAAS,eAAe;EAC7B,QAAQ,SAAS,eAAe,QAAQ;EACxC,MAAM,SAAS,eAAe,eAAe;EAC7C,KAAK,SAAS,eAAe,eAAe;;;;;;wBAO9B,eACd,IAAI,IACJ,UAAU,UACV,sBACA,0BACC"}

package/dist/enumeration.js

@@ -1,7 +1,10 @@
 import { pathPrefixSql } from "./path-sql.js";
 //#region src/enumeration.ts
-/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */
-function newEnumeration(db, blockIds, memberIdsSql) {
+/**
+* `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`.
+* `ensureOpen` runs before every call, so a closed sub-index throws like the other backends.
+*/
+function newEnumeration(db, blockIds, memberIdsSql, ensureOpen = () => {}) {
 	const from = (prefix) => {
 		const scope = pathPrefixSql("b.path", prefix ? [prefix] : []);
 		return {
@@ -11,11 +14,13 @@ function newEnumeration(db, blockIds, memberIdsSql) {
 	};
 	return {
 		async size(prefix) {
+			ensureOpen();
 			const f = from(prefix);
 			const rows = await db.query(`SELECT COUNT(*) AS n ${f.sql}`, f.params);
 			return Number(rows[0]?.n ?? 0);
 		},
 		async entries(prefix) {
+			ensureOpen();
 			const f = from(prefix);
 			return (await db.query(`SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`, f.params)).map((r) => ({
 				id: Number(r.id),
@@ -24,6 +29,7 @@ function newEnumeration(db, blockIds, memberIdsSql) {
 			}));
 		},
 		async *paths(prefix) {
+			ensureOpen();
 			const f = from(prefix);
 			const rows = await db.query(`SELECT DISTINCT b.path ${f.sql}`, f.params);
 			for (const r of rows) yield r.path;

package/dist/enumeration.js.map

@@ -1 +1 @@
-{"version":3,"file":"enumeration.js","names":[],"sources":["../src/enumeration.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { BlockReference, DocumentPath } from \"@statewalker/indexer-api\";\nimport type { BlockIds } from \"./block-ids.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\n\nexport interface BlockEntry extends BlockReference {\n  id: number;\n}\n\n/** `SearchIndex` enumeration over a sub-index's member ids joined with the blocks table. */\nexport interface Enumeration {\n  size(prefix?: DocumentPath): Promise<number>;\n  entries(prefix?: DocumentPath): Promise<BlockEntry[]>;\n  paths(prefix?: DocumentPath): AsyncGenerator<DocumentPath>;\n  refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;\n}\n\n/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */\nexport function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string): Enumeration {\n  const from = (prefix?: DocumentPath) => {\n    const scope = pathPrefixSql(\"b.path\", prefix ? [prefix] : []);\n    return {\n      sql: `FROM (${memberIdsSql}) m JOIN ${blockIds.table} b ON b.id = m.id WHERE ${scope.sql}`,\n      params: scope.params,\n    };\n  };\n  return {\n    async size(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ n: number }>(`SELECT COUNT(*) AS n ${f.sql}`, f.params);\n      return Number(rows[0]?.n ?? 0);\n    },\n    async entries(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ id: number; path: string; block_id: string }>(\n        `SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`,\n        f.params,\n      );\n      return rows.map((r) => ({\n        id: Number(r.id),\n        path: r.path as DocumentPath,\n        blockId: r.block_id,\n      }));\n    },\n    async *paths(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ path: string }>(`SELECT DISTINCT b.path ${f.sql}`, f.params);\n      for (const r of rows) yield r.path as DocumentPath;\n    },\n    async *refs(prefix) {\n      for (const e of await this.entries(prefix)) yield { path: e.path, blockId: e.blockId };\n    },\n  };\n}\n"],"mappings":";;;AAkBA,SAAgB,eAAe,IAAQ,UAAoB,cAAmC;CAC5F,MAAM,QAAQ,WAA0B;EACtC,MAAM,QAAQ,cAAc,UAAU,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;EAC5D,OAAO;GACL,KAAK,SAAS,aAAa,WAAW,SAAS,MAAM,0BAA0B,MAAM;GACrF,QAAQ,MAAM;EAChB;CACF;CACA,OAAO;EACL,MAAM,KAAK,QAAQ;GACjB,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAqB,wBAAwB,EAAE,OAAO,EAAE,MAAM;GACpF,OAAO,OAAO,KAAK,EAAE,EAAE,KAAK,CAAC;EAC/B;EACA,MAAM,QAAQ,QAAQ;GACpB,MAAM,IAAI,KAAK,MAAM;GAKrB,QAAO,MAJY,GAAG,MACpB,mCAAmC,EAAE,IAAI,iBACzC,EAAE,MACJ,EAAA,CACY,KAAK,OAAO;IACtB,IAAI,OAAO,EAAE,EAAE;IACf,MAAM,EAAE;IACR,SAAS,EAAE;GACb,EAAE;EACJ;EACA,OAAO,MAAM,QAAQ;GACnB,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAwB,0BAA0B,EAAE,OAAO,EAAE,MAAM;GACzF,KAAK,MAAM,KAAK,MAAM,MAAM,EAAE;EAChC;EACA,OAAO,KAAK,QAAQ;GAClB,KAAK,MAAM,KAAK,MAAM,KAAK,QAAQ,MAAM,GAAG,MAAM;IAAE,MAAM,EAAE;IAAM,SAAS,EAAE;GAAQ;EACvF;CACF;AACF"}
+{"version":3,"file":"enumeration.js","names":[],"sources":["../src/enumeration.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { BlockReference, DocumentPath } from \"@statewalker/indexer-api\";\nimport type { BlockIds } from \"./block-ids.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\n\nexport interface BlockEntry extends BlockReference {\n  id: number;\n}\n\n/** `SearchIndex` enumeration over a sub-index's member ids joined with the blocks table. */\nexport interface Enumeration {\n  size(prefix?: DocumentPath): Promise<number>;\n  entries(prefix?: DocumentPath): Promise<BlockEntry[]>;\n  paths(prefix?: DocumentPath): AsyncGenerator<DocumentPath>;\n  refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;\n}\n\n/**\n * `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`.\n * `ensureOpen` runs before every call, so a closed sub-index throws like the other backends.\n */\nexport function newEnumeration(\n  db: Db,\n  blockIds: BlockIds,\n  memberIdsSql: string,\n  ensureOpen: () => void = () => {},\n): Enumeration {\n  const from = (prefix?: DocumentPath) => {\n    const scope = pathPrefixSql(\"b.path\", prefix ? [prefix] : []);\n    return {\n      sql: `FROM (${memberIdsSql}) m JOIN ${blockIds.table} b ON b.id = m.id WHERE ${scope.sql}`,\n      params: scope.params,\n    };\n  };\n  return {\n    async size(prefix) {\n      ensureOpen();\n      const f = from(prefix);\n      const rows = await db.query<{ n: number }>(`SELECT COUNT(*) AS n ${f.sql}`, f.params);\n      return Number(rows[0]?.n ?? 0);\n    },\n    async entries(prefix) {\n      ensureOpen();\n      const f = from(prefix);\n      const rows = await db.query<{ id: number; path: string; block_id: string }>(\n        `SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`,\n        f.params,\n      );\n      return rows.map((r) => ({\n        id: Number(r.id),\n        path: r.path as DocumentPath,\n        blockId: r.block_id,\n      }));\n    },\n    async *paths(prefix) {\n      ensureOpen();\n      const f = from(prefix);\n      const rows = await db.query<{ path: string }>(`SELECT DISTINCT b.path ${f.sql}`, f.params);\n      for (const r of rows) yield r.path as DocumentPath;\n    },\n    async *refs(prefix) {\n      for (const e of await this.entries(prefix)) yield { path: e.path, blockId: e.blockId };\n    },\n  };\n}\n"],"mappings":";;;;;;AAqBA,SAAgB,eACd,IACA,UACA,cACA,mBAA+B,CAAC,GACnB;CACb,MAAM,QAAQ,WAA0B;EACtC,MAAM,QAAQ,cAAc,UAAU,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;EAC5D,OAAO;GACL,KAAK,SAAS,aAAa,WAAW,SAAS,MAAM,0BAA0B,MAAM;GACrF,QAAQ,MAAM;EAChB;CACF;CACA,OAAO;EACL,MAAM,KAAK,QAAQ;GACjB,WAAW;GACX,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAqB,wBAAwB,EAAE,OAAO,EAAE,MAAM;GACpF,OAAO,OAAO,KAAK,EAAE,EAAE,KAAK,CAAC;EAC/B;EACA,MAAM,QAAQ,QAAQ;GACpB,WAAW;GACX,MAAM,IAAI,KAAK,MAAM;GAKrB,QAAO,MAJY,GAAG,MACpB,mCAAmC,EAAE,IAAI,iBACzC,EAAE,MACJ,EAAA,CACY,KAAK,OAAO;IACtB,IAAI,OAAO,EAAE,EAAE;IACf,MAAM,EAAE;IACR,SAAS,EAAE;GACb,EAAE;EACJ;EACA,OAAO,MAAM,QAAQ;GACnB,WAAW;GACX,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAwB,0BAA0B,EAAE,OAAO,EAAE,MAAM;GACzF,KAAK,MAAM,KAAK,MAAM,MAAM,EAAE;EAChC;EACA,OAAO,KAAK,QAAQ;GAClB,KAAK,MAAM,KAAK,MAAM,KAAK,QAAQ,MAAM,GAAG,MAAM;IAAE,MAAM,EAAE;IAAM,SAAS,EAAE;GAAQ;EACvF;CACF;AACF"}

package/dist/fts-provider.js

@@ -27,7 +27,7 @@ function newSqliteFullTextProvider() {
 			const table = `${prefix}fts`;
 			const blocks = blockIds.table;
 			await db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`);
-			const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`);
+			const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`, () => ensureOpen());
 			let closed = false;
 			const ensureOpen = () => {
 				if (closed) throw new Error("FullTextIndex is closed");
@@ -116,6 +116,7 @@ function newSqliteFullTextProvider() {
 				getDocumentPaths: (p) => list.paths(p),
 				getDocumentBlocksRefs: (p) => list.refs(p),
 				async *getDocumentsBlocks(p) {
+					ensureOpen();
 					const scope = pathPrefixSql("b.path", p ? [p] : []);
 					const rows = await db.query(`SELECT b.path, b.block_id, f.content, f.metadata FROM ${table} f JOIN ${blocks} b ON b.id = f.rowid WHERE ${scope.sql} ORDER BY b.id`, scope.params);
 					for (const r of rows) yield {

package/dist/fts-provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"fts-provider.js","names":[],"sources":["../src/fts-provider.ts"],"sourcesContent":["import { inTransaction } from \"@statewalker/db-sqlite\";\nimport type { DocumentPath, Metadata, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  FULL_TEXT_TYPE,\n  type FullTextBlock,\n  type FullTextConfig,\n  type FullTextIndex,\n  type FulltextQuery,\n  type FulltextResult,\n} from \"@statewalker/indexer-fulltext\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/** `en` gets the Porter stemmer; any other language (known or not) gets plain `unicode61`. */\nfunction tokenizerFor(language: string): string {\n  return language === \"en\"\n    ? \"porter unicode61 remove_diacritics 2\"\n    : \"unicode61 remove_diacritics 2\";\n}\n\n/**\n * One FTS5 MATCH expression per query string: every word with a letter or a\n * digit, double-quoted (so `AND`, `*`, `NEAR(` are plain words), OR-joined so\n * partial matches are returned. `null` when nothing is searchable.\n */\nexport function toMatchExpression(query: string): string | null {\n  const words = query\n    .split(/\\s+/)\n    .filter((w) => /[\\p{L}\\p{N}]/u.test(w))\n    .map((w) => `\"${w.replaceAll('\"', '\"\"')}\"`);\n  return words.length > 0 ? words.join(\" OR \") : null;\n}\n\nexport function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConfig, FullTextIndex> {\n  return {\n    type: FULL_TEXT_TYPE,\n\n    async drop({ db, prefix }) {\n      await db.exec(`DROP TABLE IF EXISTS ${prefix}fts`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const table = `${prefix}fts`;\n      const blocks = blockIds.table;\n      await db.exec(\n        `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`,\n      );\n      const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"FullTextIndex is closed\");\n      };\n      const collectSelectors = async (selectors: PathSelector[] | AsyncIterable<PathSelector>) => {\n        const all: PathSelector[] = [];\n        for await (const s of toAsyncIterable(selectors)) all.push(s);\n        return all;\n      };\n\n      const index: FullTextIndex = {\n        async getIndexInfo() {\n          return { language: config.language, metadata: config.metadata };\n        },\n\n        async *search(params: FulltextQuery): AsyncGenerator<FulltextResult> {\n          ensureOpen();\n          const topK = params.topK ?? 100;\n          const scope = pathPrefixSql(\"b.path\", params.paths ?? []);\n          // Several queries accumulate (SUM): a block matching more of them ranks higher.\n          const merged = new Map<number, { score: number; best: number; snippet: string }>();\n          for (const query of params.queries ?? []) {\n            const match = toMatchExpression(query);\n            if (!match) continue;\n            const rows = await db.query<{ id: number; score: number; snippet: string }>(\n              `SELECT ${table}.rowid AS id, -bm25(${table}) AS score, snippet(${table}, 0, '', '', '…', 16) AS snippet FROM ${table} JOIN ${blocks} b ON b.id = ${table}.rowid WHERE ${table} MATCH ? AND ${scope.sql} ORDER BY score DESC LIMIT ?`,\n              [match, ...scope.params, topK],\n            );\n            for (const r of rows) {\n              const id = Number(r.id);\n              const prev = merged.get(id);\n              if (!prev) merged.set(id, { score: r.score, best: r.score, snippet: r.snippet });\n              else {\n                prev.score += r.score;\n                if (r.score > prev.best) Object.assign(prev, { best: r.score, snippet: r.snippet });\n              }\n            }\n          }\n          const top = [...merged.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, topK);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, hit] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score: hit.score, snippet: hit.snippet };\n          }\n        },\n\n        async addDocument(blocksIn: FullTextBlock[]) {\n          ensureOpen();\n          if (blocksIn.length === 0) return;\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.ensure(tx, blocksIn);\n            for (const [i, id] of ids.entries()) {\n              const block = blocksIn[i] as FullTextBlock;\n              // FTS5 has no UPSERT: delete then insert, atomically inside the transaction.\n              await tx.query(`DELETE FROM ${table} WHERE rowid = ?`, [id]);\n              await tx.query(`INSERT INTO ${table} (rowid, content, metadata) VALUES (?, ?, ?)`, [\n                id,\n                block.content,\n                block.metadata ? JSON.stringify(block.metadata) : null,\n              ]);\n            }\n          });\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels = await collectSelectors(selectors);\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.select(tx, sels);\n            await tx.query(`DELETE FROM ${table} WHERE rowid IN (SELECT value FROM json_each(?))`, [\n              JSON.stringify(ids),\n            ]);\n          });\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<FullTextBlock> {\n          const scope = pathPrefixSql(\"b.path\", p ? [p] : []);\n          const rows = await db.query<{\n            path: string;\n            block_id: string;\n            content: string;\n            metadata: string | null;\n          }>(\n            `SELECT b.path, b.block_id, f.content, f.metadata FROM ${table} f JOIN ${blocks} b ON b.id = f.rowid WHERE ${scope.sql} ORDER BY b.id`,\n            scope.params,\n          );\n          for (const r of rows) {\n            yield {\n              path: r.path as DocumentPath,\n              blockId: r.block_id,\n              content: r.content,\n              ...(r.metadata ? { metadata: JSON.parse(r.metadata) as Metadata } : {}),\n            };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await db.exec(`DROP TABLE IF EXISTS ${table}`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;AAgBA,SAAS,aAAa,UAA0B;CAC9C,OAAO,aAAa,OAChB,yCACA;AACN;;;;;;AAOA,SAAgB,kBAAkB,OAA8B;CAC9D,MAAM,QAAQ,MACX,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,gBAAgB,KAAK,CAAC,CAAC,CAAC,CACtC,KAAK,MAAM,IAAI,EAAE,WAAW,MAAK,MAAI,EAAE,EAAE;CAC5C,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,MAAM,IAAI;AACjD;AAEA,SAAgB,4BAAmF;CACjG,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,GAAG,KAAK,wBAAwB,OAAO,IAAI;EACnD;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,QAAQ,GAAG,OAAO;GACxB,MAAM,SAAS,SAAS;GACxB,MAAM,GAAG,KACP,sCAAsC,MAAM,qDAAqD,aAAa,OAAO,QAAQ,EAAE,GACjI;GACA,MAAM,OAAO,eAAe,IAAI,UAAU,2BAA2B,OAAO;GAC5E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,yBAAyB;GACvD;GACA,MAAM,mBAAmB,OAAO,cAA4D;IAC1F,MAAM,MAAsB,CAAC;IAC7B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,IAAI,KAAK,CAAC;IAC5D,OAAO;GACT;GA4GA,OAAO;IAzGL,MAAM,eAAe;KACnB,OAAO;MAAE,UAAU,OAAO;MAAU,UAAU,OAAO;KAAS;IAChE;IAEA,OAAO,OAAO,QAAuD;KACnE,WAAW;KACX,MAAM,OAAO,OAAO,QAAQ;KAC5B,MAAM,QAAQ,cAAc,UAAU,OAAO,SAAS,CAAC,CAAC;KAExD,MAAM,yBAAS,IAAI,IAA8D;KACjF,KAAK,MAAM,SAAS,OAAO,WAAW,CAAC,GAAG;MACxC,MAAM,QAAQ,kBAAkB,KAAK;MACrC,IAAI,CAAC,OAAO;MACZ,MAAM,OAAO,MAAM,GAAG,MACpB,UAAU,MAAM,sBAAsB,MAAM,sBAAsB,MAAM,wCAAwC,MAAM,QAAQ,OAAO,eAAe,MAAM,eAAe,MAAM,eAAe,MAAM,IAAI,+BACxM;OAAC;OAAO,GAAG,MAAM;OAAQ;MAAI,CAC/B;MACA,KAAK,MAAM,KAAK,MAAM;OACpB,MAAM,KAAK,OAAO,EAAE,EAAE;OACtB,MAAM,OAAO,OAAO,IAAI,EAAE;OAC1B,IAAI,CAAC,MAAM,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE;QAAO,MAAM,EAAE;QAAO,SAAS,EAAE;OAAQ,CAAC;YAC1E;QACH,KAAK,SAAS,EAAE;QAChB,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM;SAAE,MAAM,EAAE;SAAO,SAAS,EAAE;QAAQ,CAAC;OACpF;MACF;KACF;KACA,MAAM,MAAM,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI;KACvF,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,QAAQ,KAAK;MAC3B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK,OAAO,IAAI;OAAO,SAAS,IAAI;MAAQ;KAClE;IACF;IAEA,MAAM,YAAY,UAA2B;KAC3C,WAAW;KACX,IAAI,SAAS,WAAW,GAAG;KAC3B,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,QAAQ;MAC9C,KAAK,MAAM,CAAC,GAAG,OAAO,IAAI,QAAQ,GAAG;OACnC,MAAM,QAAQ,SAAS;OAEvB,MAAM,GAAG,MAAM,eAAe,MAAM,mBAAmB,CAAC,EAAE,CAAC;OAC3D,MAAM,GAAG,MAAM,eAAe,MAAM,+CAA+C;QACjF;QACA,MAAM;QACN,MAAM,WAAW,KAAK,UAAU,MAAM,QAAQ,IAAI;OACpD,CAAC;MACH;KACF,CAAC;IACH;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAO,MAAM,iBAAiB,SAAS;KAC7C,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;MAC1C,MAAM,GAAG,MAAM,eAAe,MAAM,mDAAmD,CACrF,KAAK,UAAU,GAAG,CACpB,CAAC;KACH,CAAC;IACH;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAAiD;KACzE,MAAM,QAAQ,cAAc,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;KAClD,MAAM,OAAO,MAAM,GAAG,MAMpB,yDAAyD,MAAM,UAAU,OAAO,6BAA6B,MAAM,IAAI,iBACvH,MAAM,MACR;KACA,KAAK,MAAM,KAAK,MACd,MAAM;MACJ,MAAM,EAAE;MACR,SAAS,EAAE;MACX,SAAS,EAAE;MACX,GAAI,EAAE,WAAW,EAAE,UAAU,KAAK,MAAM,EAAE,QAAQ,EAAc,IAAI,CAAC;KACvE;IAEJ;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,GAAG,KAAK,wBAAwB,OAAO;IAC/C;GAES;EACb;CACF;AACF"}
+{"version":3,"file":"fts-provider.js","names":[],"sources":["../src/fts-provider.ts"],"sourcesContent":["import { inTransaction } from \"@statewalker/db-sqlite\";\nimport type { DocumentPath, Metadata, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  FULL_TEXT_TYPE,\n  type FullTextBlock,\n  type FullTextConfig,\n  type FullTextIndex,\n  type FulltextQuery,\n  type FulltextResult,\n} from \"@statewalker/indexer-fulltext\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/** `en` gets the Porter stemmer; any other language (known or not) gets plain `unicode61`. */\nfunction tokenizerFor(language: string): string {\n  return language === \"en\"\n    ? \"porter unicode61 remove_diacritics 2\"\n    : \"unicode61 remove_diacritics 2\";\n}\n\n/**\n * One FTS5 MATCH expression per query string: every word with a letter or a\n * digit, double-quoted (so `AND`, `*`, `NEAR(` are plain words), OR-joined so\n * partial matches are returned. `null` when nothing is searchable.\n */\nexport function toMatchExpression(query: string): string | null {\n  const words = query\n    .split(/\\s+/)\n    .filter((w) => /[\\p{L}\\p{N}]/u.test(w))\n    .map((w) => `\"${w.replaceAll('\"', '\"\"')}\"`);\n  return words.length > 0 ? words.join(\" OR \") : null;\n}\n\nexport function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConfig, FullTextIndex> {\n  return {\n    type: FULL_TEXT_TYPE,\n\n    async drop({ db, prefix }) {\n      await db.exec(`DROP TABLE IF EXISTS ${prefix}fts`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const table = `${prefix}fts`;\n      const blocks = blockIds.table;\n      await db.exec(\n        `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`,\n      );\n      const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`, () =>\n        ensureOpen(),\n      );\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"FullTextIndex is closed\");\n      };\n      const collectSelectors = async (selectors: PathSelector[] | AsyncIterable<PathSelector>) => {\n        const all: PathSelector[] = [];\n        for await (const s of toAsyncIterable(selectors)) all.push(s);\n        return all;\n      };\n\n      const index: FullTextIndex = {\n        async getIndexInfo() {\n          return { language: config.language, metadata: config.metadata };\n        },\n\n        async *search(params: FulltextQuery): AsyncGenerator<FulltextResult> {\n          ensureOpen();\n          const topK = params.topK ?? 100;\n          const scope = pathPrefixSql(\"b.path\", params.paths ?? []);\n          // Several queries accumulate (SUM): a block matching more of them ranks higher.\n          const merged = new Map<number, { score: number; best: number; snippet: string }>();\n          for (const query of params.queries ?? []) {\n            const match = toMatchExpression(query);\n            if (!match) continue;\n            const rows = await db.query<{ id: number; score: number; snippet: string }>(\n              `SELECT ${table}.rowid AS id, -bm25(${table}) AS score, snippet(${table}, 0, '', '', '…', 16) AS snippet FROM ${table} JOIN ${blocks} b ON b.id = ${table}.rowid WHERE ${table} MATCH ? AND ${scope.sql} ORDER BY score DESC LIMIT ?`,\n              [match, ...scope.params, topK],\n            );\n            for (const r of rows) {\n              const id = Number(r.id);\n              const prev = merged.get(id);\n              if (!prev) merged.set(id, { score: r.score, best: r.score, snippet: r.snippet });\n              else {\n                prev.score += r.score;\n                if (r.score > prev.best) Object.assign(prev, { best: r.score, snippet: r.snippet });\n              }\n            }\n          }\n          const top = [...merged.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, topK);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, hit] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score: hit.score, snippet: hit.snippet };\n          }\n        },\n\n        async addDocument(blocksIn: FullTextBlock[]) {\n          ensureOpen();\n          if (blocksIn.length === 0) return;\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.ensure(tx, blocksIn);\n            for (const [i, id] of ids.entries()) {\n              const block = blocksIn[i] as FullTextBlock;\n              // FTS5 has no UPSERT: delete then insert, atomically inside the transaction.\n              await tx.query(`DELETE FROM ${table} WHERE rowid = ?`, [id]);\n              await tx.query(`INSERT INTO ${table} (rowid, content, metadata) VALUES (?, ?, ?)`, [\n                id,\n                block.content,\n                block.metadata ? JSON.stringify(block.metadata) : null,\n              ]);\n            }\n          });\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels = await collectSelectors(selectors);\n          await inTransaction(db, async (tx) => {\n            const ids = await blockIds.select(tx, sels);\n            await tx.query(`DELETE FROM ${table} WHERE rowid IN (SELECT value FROM json_each(?))`, [\n              JSON.stringify(ids),\n            ]);\n          });\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<FullTextBlock> {\n          ensureOpen();\n          const scope = pathPrefixSql(\"b.path\", p ? [p] : []);\n          const rows = await db.query<{\n            path: string;\n            block_id: string;\n            content: string;\n            metadata: string | null;\n          }>(\n            `SELECT b.path, b.block_id, f.content, f.metadata FROM ${table} f JOIN ${blocks} b ON b.id = f.rowid WHERE ${scope.sql} ORDER BY b.id`,\n            scope.params,\n          );\n          for (const r of rows) {\n            yield {\n              path: r.path as DocumentPath,\n              blockId: r.block_id,\n              content: r.content,\n              ...(r.metadata ? { metadata: JSON.parse(r.metadata) as Metadata } : {}),\n            };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await db.exec(`DROP TABLE IF EXISTS ${table}`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;AAgBA,SAAS,aAAa,UAA0B;CAC9C,OAAO,aAAa,OAChB,yCACA;AACN;;;;;;AAOA,SAAgB,kBAAkB,OAA8B;CAC9D,MAAM,QAAQ,MACX,MAAM,KAAK,CAAC,CACZ,QAAQ,MAAM,gBAAgB,KAAK,CAAC,CAAC,CAAC,CACtC,KAAK,MAAM,IAAI,EAAE,WAAW,MAAK,MAAI,EAAE,EAAE;CAC5C,OAAO,MAAM,SAAS,IAAI,MAAM,KAAK,MAAM,IAAI;AACjD;AAEA,SAAgB,4BAAmF;CACjG,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,GAAG,KAAK,wBAAwB,OAAO,IAAI;EACnD;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,QAAQ,GAAG,OAAO;GACxB,MAAM,SAAS,SAAS;GACxB,MAAM,GAAG,KACP,sCAAsC,MAAM,qDAAqD,aAAa,OAAO,QAAQ,EAAE,GACjI;GACA,MAAM,OAAO,eAAe,IAAI,UAAU,2BAA2B,eACnE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,yBAAyB;GACvD;GACA,MAAM,mBAAmB,OAAO,cAA4D;IAC1F,MAAM,MAAsB,CAAC;IAC7B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,IAAI,KAAK,CAAC;IAC5D,OAAO;GACT;GA6GA,OAAO;IA1GL,MAAM,eAAe;KACnB,OAAO;MAAE,UAAU,OAAO;MAAU,UAAU,OAAO;KAAS;IAChE;IAEA,OAAO,OAAO,QAAuD;KACnE,WAAW;KACX,MAAM,OAAO,OAAO,QAAQ;KAC5B,MAAM,QAAQ,cAAc,UAAU,OAAO,SAAS,CAAC,CAAC;KAExD,MAAM,yBAAS,IAAI,IAA8D;KACjF,KAAK,MAAM,SAAS,OAAO,WAAW,CAAC,GAAG;MACxC,MAAM,QAAQ,kBAAkB,KAAK;MACrC,IAAI,CAAC,OAAO;MACZ,MAAM,OAAO,MAAM,GAAG,MACpB,UAAU,MAAM,sBAAsB,MAAM,sBAAsB,MAAM,wCAAwC,MAAM,QAAQ,OAAO,eAAe,MAAM,eAAe,MAAM,eAAe,MAAM,IAAI,+BACxM;OAAC;OAAO,GAAG,MAAM;OAAQ;MAAI,CAC/B;MACA,KAAK,MAAM,KAAK,MAAM;OACpB,MAAM,KAAK,OAAO,EAAE,EAAE;OACtB,MAAM,OAAO,OAAO,IAAI,EAAE;OAC1B,IAAI,CAAC,MAAM,OAAO,IAAI,IAAI;QAAE,OAAO,EAAE;QAAO,MAAM,EAAE;QAAO,SAAS,EAAE;OAAQ,CAAC;YAC1E;QACH,KAAK,SAAS,EAAE;QAChB,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,OAAO,MAAM;SAAE,MAAM,EAAE;SAAO,SAAS,EAAE;QAAQ,CAAC;OACpF;MACF;KACF;KACA,MAAM,MAAM,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI;KACvF,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,QAAQ,KAAK;MAC3B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK,OAAO,IAAI;OAAO,SAAS,IAAI;MAAQ;KAClE;IACF;IAEA,MAAM,YAAY,UAA2B;KAC3C,WAAW;KACX,IAAI,SAAS,WAAW,GAAG;KAC3B,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,QAAQ;MAC9C,KAAK,MAAM,CAAC,GAAG,OAAO,IAAI,QAAQ,GAAG;OACnC,MAAM,QAAQ,SAAS;OAEvB,MAAM,GAAG,MAAM,eAAe,MAAM,mBAAmB,CAAC,EAAE,CAAC;OAC3D,MAAM,GAAG,MAAM,eAAe,MAAM,+CAA+C;QACjF;QACA,MAAM;QACN,MAAM,WAAW,KAAK,UAAU,MAAM,QAAQ,IAAI;OACpD,CAAC;MACH;KACF,CAAC;IACH;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAO,MAAM,iBAAiB,SAAS;KAC7C,MAAM,cAAc,IAAI,OAAO,OAAO;MACpC,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;MAC1C,MAAM,GAAG,MAAM,eAAe,MAAM,mDAAmD,CACrF,KAAK,UAAU,GAAG,CACpB,CAAC;KACH,CAAC;IACH;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAAiD;KACzE,WAAW;KACX,MAAM,QAAQ,cAAc,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;KAClD,MAAM,OAAO,MAAM,GAAG,MAMpB,yDAAyD,MAAM,UAAU,OAAO,6BAA6B,MAAM,IAAI,iBACvH,MAAM,MACR;KACA,KAAK,MAAM,KAAK,MACd,MAAM;MACJ,MAAM,EAAE;MACR,SAAS,EAAE;MACX,SAAS,EAAE;MACX,GAAI,EAAE,WAAW,EAAE,UAAU,KAAK,MAAM,EAAE,QAAQ,EAAc,IAAI,CAAC;KACvE;IAEJ;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,GAAG,KAAK,wBAAwB,OAAO;IAC/C;GAES;EACb;CACF;AACF"}

package/dist/ngram-provider.js

@@ -49,7 +49,7 @@ function newSqliteNgramProvider() {
 				store: storeKind
 			});
 			await store.init();
-			const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`);
+			const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () => ensureOpen());
 			let closed = false;
 			const ensureOpen = () => {
 				if (closed) throw new Error("NgramIndex is closed");
@@ -113,6 +113,7 @@ function newSqliteNgramProvider() {
 				getDocumentPaths: (p) => list.paths(p),
 				getDocumentBlocksRefs: (p) => list.refs(p),
 				async *getDocumentsBlocks(p) {
+					ensureOpen();
 					const entries = await list.entries(p);
 					const texts = await store.getTexts(entries.map((e) => e.id));
 					for (const e of entries) {

package/dist/ngram-provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  NGRAM_TYPE,\n  type NgramBlock,\n  type NgramConfig,\n  type NgramIndex,\n  type NgramMatch,\n  type NgramQuery,\n  type NgramResult,\n  type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n  const breaks: number[] = [];\n  for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n  let line = 1;\n  let b = 0;\n  return [...matches]\n    .sort((x, y) => x.start - y.start)\n    .map(({ start, end }) => {\n      while (b < breaks.length && (breaks[b] as number) < start) {\n        b++;\n        line++;\n      }\n      return { start, end, line };\n    });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n  return {\n    type: NGRAM_TYPE,\n\n    async drop({ db, prefix }) {\n      await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const storeKind = config.store ?? \"fts5\";\n      const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n      await store.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"NgramIndex is closed\");\n      };\n\n      const index: NgramIndex = {\n        async getIndexInfo() {\n          return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n        },\n\n        async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n          ensureOpen();\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const result = await store.search(params.pattern, {\n            regex: params.regex ?? false,\n            caseSensitive: params.caseSensitive ?? true,\n            limit: params.topK ?? 100,\n            ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n          });\n          const hits = result.hits.filter((h) => h.matches.length > 0);\n          const ids = hits.map((h) => h.id);\n          const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n          // Array sort is stable: equal counts keep the store's ascending-id order.\n          const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n          for (const h of ranked) {\n            const ref = refs.get(h.id);\n            if (!ref) continue;\n            const count = h.matches.length;\n            yield {\n              ...ref,\n              score: count,\n              count,\n              matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n            };\n          }\n          return { truncated: result.truncated };\n        },\n\n        async addDocument(blocks: NgramBlock[]) {\n          ensureOpen();\n          for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n            const batch = blocks.slice(i, i + UPSERT_BATCH);\n            const ids = await blockIds.ensure(db, batch);\n            await store.upsert(\n              ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n            );\n          }\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await store.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n          const entries = await list.entries(p);\n          const texts = await store.getTexts(entries.map((e) => e.id));\n          for (const e of entries) {\n            const content = texts.get(e.id);\n            if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await store.drop();\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,SAAS;GAC3E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAwFA,OAAO;IArFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}
+{"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n  NGRAM_TYPE,\n  type NgramBlock,\n  type NgramConfig,\n  type NgramIndex,\n  type NgramMatch,\n  type NgramQuery,\n  type NgramResult,\n  type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n  const breaks: number[] = [];\n  for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n  let line = 1;\n  let b = 0;\n  return [...matches]\n    .sort((x, y) => x.start - y.start)\n    .map(({ start, end }) => {\n      while (b < breaks.length && (breaks[b] as number) < start) {\n        b++;\n        line++;\n      }\n      return { start, end, line };\n    });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n  return {\n    type: NGRAM_TYPE,\n\n    async drop({ db, prefix }) {\n      await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const storeKind = config.store ?? \"fts5\";\n      const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n      await store.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>\n        ensureOpen(),\n      );\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"NgramIndex is closed\");\n      };\n\n      const index: NgramIndex = {\n        async getIndexInfo() {\n          return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n        },\n\n        async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n          ensureOpen();\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const result = await store.search(params.pattern, {\n            regex: params.regex ?? false,\n            caseSensitive: params.caseSensitive ?? true,\n            limit: params.topK ?? 100,\n            ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n          });\n          const hits = result.hits.filter((h) => h.matches.length > 0);\n          const ids = hits.map((h) => h.id);\n          const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n          // Array sort is stable: equal counts keep the store's ascending-id order.\n          const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n          for (const h of ranked) {\n            const ref = refs.get(h.id);\n            if (!ref) continue;\n            const count = h.matches.length;\n            yield {\n              ...ref,\n              score: count,\n              count,\n              matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n            };\n          }\n          return { truncated: result.truncated };\n        },\n\n        async addDocument(blocks: NgramBlock[]) {\n          ensureOpen();\n          for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n            const batch = blocks.slice(i, i + UPSERT_BATCH);\n            const ids = await blockIds.ensure(db, batch);\n            await store.upsert(\n              ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n            );\n          }\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await store.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n          ensureOpen();\n          const entries = await list.entries(p);\n          const texts = await store.getTexts(entries.map((e) => e.id));\n          for (const e of entries) {\n            const content = texts.get(e.id);\n            if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await store.drop();\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,iBAChE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAyFA,OAAO;IAtFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}

package/dist/vector-provider.js

@@ -23,7 +23,7 @@ function newSqliteVectorProvider() {
 				metric: "cosine"
 			});
 			await engine.init();
-			const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`);
+			const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`, () => ensureOpen());
 			let closed = false;
 			const ensureOpen = () => {
 				if (closed) throw new Error("VectorIndex is closed");
@@ -39,6 +39,7 @@ function newSqliteVectorProvider() {
 				},
 				async *search(params) {
 					ensureOpen();
+					for (const q of params.embeddings ?? []) validateDimensionality(info, q);
 					const k = params.topK ?? 100;
 					const allowed = params.paths?.length ? new Set(await blockIds.select(db, params.paths.map((path) => ({ path })))) : void 0;
 					const filter = allowed ? (id) => allowed.has(id) : void 0;
@@ -85,6 +86,7 @@ function newSqliteVectorProvider() {
 				getDocumentPaths: (p) => list.paths(p),
 				getDocumentBlocksRefs: (p) => list.refs(p),
 				async *getDocumentsBlocks(p) {
+					ensureOpen();
 					const entries = await list.entries(p);
 					const vectors = await engine.getVectors(entries.map((e) => e.id));
 					for (const e of entries) {

package/dist/vector-provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"vector-provider.js","names":["VectorEngine"],"sources":["../src/vector-provider.ts"],"sourcesContent":["import { dropVectorIndex, VectorIndex as VectorEngine } from \"@statewalker/db-sqlite-index/vector\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable, validateDimensionality } from \"@statewalker/indexer-core\";\nimport {\n  VECTOR_TYPE,\n  type VectorBlock,\n  type VectorConfig,\n  type VectorIndex,\n  type VectorQuery,\n  type VectorResult,\n} from \"@statewalker/indexer-vector\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Wraps the id-based `VectorIndex` engine. The engine owns its tables under\n * `{prefix}vec_` and its own transactions (never call it inside `inTransaction`);\n * this provider adds only the `{path, blockId}` mapping. Size and enumeration\n * join the engine's `idTable`, so they list exactly what the engine holds.\n */\nexport function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig, VectorIndex> {\n  return {\n    type: VECTOR_TYPE,\n\n    async drop({ db, prefix }) {\n      await dropVectorIndex(db, `${prefix}vec_`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const engine = new VectorEngine({\n        db,\n        prefix: `${prefix}vec_`,\n        dims: config.dimensionality,\n        metric: \"cosine\",\n      });\n      await engine.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`);\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"VectorIndex is closed\");\n      };\n      const info = {\n        dimensionality: config.dimensionality,\n        model: config.model,\n        metadata: config.metadata,\n      };\n\n      const index: VectorIndex = {\n        async getIndexInfo() {\n          return { ...info };\n        },\n\n        async *search(params: VectorQuery): AsyncGenerator<VectorResult> {\n          ensureOpen();\n          const k = params.topK ?? 100;\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const filter = allowed ? (id: number) => allowed.has(id) : undefined;\n          // Several embeddings merge by max score, like every other backend.\n          const best = new Map<number, number>();\n          for (const q of params.embeddings ?? []) {\n            for (const hit of await engine.search(q, { k, ...(filter ? { filter } : {}) })) {\n              const score = 1 - hit.distance;\n              if (score > (best.get(hit.id) ?? Number.NEGATIVE_INFINITY)) best.set(hit.id, score);\n            }\n          }\n          const top = [...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, k);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, score] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score };\n          }\n        },\n\n        async addDocument(blocks: VectorBlock[]) {\n          ensureOpen();\n          if (blocks.length === 0) return;\n          for (const b of blocks) validateDimensionality(info, b.embedding);\n          const ids = await blockIds.ensure(db, blocks);\n          await engine.upsert(\n            ids.map((id, i) => ({ id, vector: (blocks[i] as VectorBlock).embedding })),\n          );\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await engine.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<VectorBlock> {\n          const entries = await list.entries(p);\n          const vectors = await engine.getVectors(entries.map((e) => e.id));\n          for (const e of entries) {\n            const embedding = vectors.get(e.id);\n            if (embedding) yield { path: e.path, blockId: e.blockId, embedding };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await dropVectorIndex(db, `${prefix}vec_`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;AAoBA,SAAgB,0BAA6E;CAC3F,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;EAC3C;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,SAAS,IAAIA,YAAa;IAC9B;IACA,QAAQ,GAAG,OAAO;IAClB,MAAM,OAAO;IACb,QAAQ;GACV,CAAC;GACD,MAAM,OAAO,KAAK;GAClB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,OAAO,SAAS;GAC5E,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;GACrD;GACA,MAAM,OAAO;IACX,gBAAgB,OAAO;IACvB,OAAO,OAAO;IACd,UAAU,OAAO;GACnB;GAmFA,OAAO;IAhFL,MAAM,eAAe;KACnB,OAAO,EAAE,GAAG,KAAK;IACnB;IAEA,OAAO,OAAO,QAAmD;KAC/D,WAAW;KACX,MAAM,IAAI,OAAO,QAAQ;KACzB,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,WAAW,OAAe,QAAQ,IAAI,EAAE,IAAI,KAAA;KAE3D,MAAM,uBAAO,IAAI,IAAoB;KACrC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC,GACpC,KAAK,MAAM,OAAO,MAAM,OAAO,OAAO,GAAG;MAAE;MAAG,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;KAAG,CAAC,GAAG;MAC9E,MAAM,QAAQ,IAAI,IAAI;MACtB,IAAI,SAAS,KAAK,IAAI,IAAI,EAAE,KAAK,OAAO,oBAAoB,KAAK,IAAI,IAAI,IAAI,KAAK;KACpF;KAEF,MAAM,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;KACtE,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,UAAU,KAAK;MAC7B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK;MAAM;KACjC;IACF;IAEA,MAAM,YAAY,QAAuB;KACvC,WAAW;KACX,IAAI,OAAO,WAAW,GAAG;KACzB,KAAK,MAAM,KAAK,QAAQ,uBAAuB,MAAM,EAAE,SAAS;KAChE,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM;KAC5C,MAAM,OAAO,OACX,IAAI,KAAK,IAAI,OAAO;MAAE;MAAI,QAAS,OAAO,EAAE,CAAiB;KAAU,EAAE,CAC3E;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,OAAO,OAAO,GAAG;IACzB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA+C;KACvE,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,UAAU,MAAM,OAAO,WAAW,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAChE,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,YAAY,QAAQ,IAAI,EAAE,EAAE;MAClC,IAAI,WAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAU;KACrE;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;IAC3C;GAES;EACb;CACF;AACF"}
+{"version":3,"file":"vector-provider.js","names":["VectorEngine"],"sources":["../src/vector-provider.ts"],"sourcesContent":["import { dropVectorIndex, VectorIndex as VectorEngine } from \"@statewalker/db-sqlite-index/vector\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable, validateDimensionality } from \"@statewalker/indexer-core\";\nimport {\n  VECTOR_TYPE,\n  type VectorBlock,\n  type VectorConfig,\n  type VectorIndex,\n  type VectorQuery,\n  type VectorResult,\n} from \"@statewalker/indexer-vector\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Wraps the id-based `VectorIndex` engine. The engine owns its tables under\n * `{prefix}vec_` and its own transactions (never call it inside `inTransaction`);\n * this provider adds only the `{path, blockId}` mapping. Size and enumeration\n * join the engine's `idTable`, so they list exactly what the engine holds.\n */\nexport function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig, VectorIndex> {\n  return {\n    type: VECTOR_TYPE,\n\n    async drop({ db, prefix }) {\n      await dropVectorIndex(db, `${prefix}vec_`);\n    },\n\n    async open({ db, prefix, blockIds }, config) {\n      const engine = new VectorEngine({\n        db,\n        prefix: `${prefix}vec_`,\n        dims: config.dimensionality,\n        metric: \"cosine\",\n      });\n      await engine.init();\n      const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`, () =>\n        ensureOpen(),\n      );\n      let closed = false;\n      const ensureOpen = () => {\n        if (closed) throw new Error(\"VectorIndex is closed\");\n      };\n      const info = {\n        dimensionality: config.dimensionality,\n        model: config.model,\n        metadata: config.metadata,\n      };\n\n      const index: VectorIndex = {\n        async getIndexInfo() {\n          return { ...info };\n        },\n\n        async *search(params: VectorQuery): AsyncGenerator<VectorResult> {\n          ensureOpen();\n          for (const q of params.embeddings ?? []) validateDimensionality(info, q);\n          const k = params.topK ?? 100;\n          const allowed = params.paths?.length\n            ? new Set(\n                await blockIds.select(\n                  db,\n                  params.paths.map((path) => ({ path })),\n                ),\n              )\n            : undefined;\n          const filter = allowed ? (id: number) => allowed.has(id) : undefined;\n          // Several embeddings merge by max score, like every other backend.\n          const best = new Map<number, number>();\n          for (const q of params.embeddings ?? []) {\n            for (const hit of await engine.search(q, { k, ...(filter ? { filter } : {}) })) {\n              const score = 1 - hit.distance;\n              if (score > (best.get(hit.id) ?? Number.NEGATIVE_INFINITY)) best.set(hit.id, score);\n            }\n          }\n          const top = [...best.entries()].sort((a, b) => b[1] - a[1]).slice(0, k);\n          const refs = await blockIds.refs(\n            db,\n            top.map(([id]) => id),\n          );\n          for (const [id, score] of top) {\n            const ref = refs.get(id);\n            if (ref) yield { ...ref, score };\n          }\n        },\n\n        async addDocument(blocks: VectorBlock[]) {\n          ensureOpen();\n          if (blocks.length === 0) return;\n          for (const b of blocks) validateDimensionality(info, b.embedding);\n          const ids = await blockIds.ensure(db, blocks);\n          await engine.upsert(\n            ids.map((id, i) => ({ id, vector: (blocks[i] as VectorBlock).embedding })),\n          );\n        },\n\n        async addDocuments(batches) {\n          for await (const batch of batches) await this.addDocument(batch);\n        },\n\n        async deleteDocuments(selectors) {\n          ensureOpen();\n          const sels: PathSelector[] = [];\n          for await (const s of toAsyncIterable(selectors)) sels.push(s);\n          const ids = await blockIds.select(db, sels);\n          if (ids.length === 0) return;\n          await engine.delete(ids);\n        },\n\n        getSize: (p?: DocumentPath) => list.size(p),\n        getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n        getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n        async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<VectorBlock> {\n          ensureOpen();\n          const entries = await list.entries(p);\n          const vectors = await engine.getVectors(entries.map((e) => e.id));\n          for (const e of entries) {\n            const embedding = vectors.get(e.id);\n            if (embedding) yield { path: e.path, blockId: e.blockId, embedding };\n          }\n        },\n\n        async close() {\n          closed = true;\n        },\n        async flush() {},\n        async deleteIndex() {\n          closed = true;\n          await dropVectorIndex(db, `${prefix}vec_`);\n        },\n      };\n      return index;\n    },\n  };\n}\n"],"mappings":";;;;;;;;;;;AAoBA,SAAgB,0BAA6E;CAC3F,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;EAC3C;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,SAAS,IAAIA,YAAa;IAC9B;IACA,QAAQ,GAAG,OAAO;IAClB,MAAM,OAAO;IACb,QAAQ;GACV,CAAC;GACD,MAAM,OAAO,KAAK;GAClB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,OAAO,iBACjE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,uBAAuB;GACrD;GACA,MAAM,OAAO;IACX,gBAAgB,OAAO;IACvB,OAAO,OAAO;IACd,UAAU,OAAO;GACnB;GAqFA,OAAO;IAlFL,MAAM,eAAe;KACnB,OAAO,EAAE,GAAG,KAAK;IACnB;IAEA,OAAO,OAAO,QAAmD;KAC/D,WAAW;KACX,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC,GAAG,uBAAuB,MAAM,CAAC;KACvE,MAAM,IAAI,OAAO,QAAQ;KACzB,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,WAAW,OAAe,QAAQ,IAAI,EAAE,IAAI,KAAA;KAE3D,MAAM,uBAAO,IAAI,IAAoB;KACrC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC,GACpC,KAAK,MAAM,OAAO,MAAM,OAAO,OAAO,GAAG;MAAE;MAAG,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;KAAG,CAAC,GAAG;MAC9E,MAAM,QAAQ,IAAI,IAAI;MACtB,IAAI,SAAS,KAAK,IAAI,IAAI,EAAE,KAAK,OAAO,oBAAoB,KAAK,IAAI,IAAI,IAAI,KAAK;KACpF;KAEF,MAAM,MAAM,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;KACtE,MAAM,OAAO,MAAM,SAAS,KAC1B,IACA,IAAI,KAAK,CAAC,QAAQ,EAAE,CACtB;KACA,KAAK,MAAM,CAAC,IAAI,UAAU,KAAK;MAC7B,MAAM,MAAM,KAAK,IAAI,EAAE;MACvB,IAAI,KAAK,MAAM;OAAE,GAAG;OAAK;MAAM;KACjC;IACF;IAEA,MAAM,YAAY,QAAuB;KACvC,WAAW;KACX,IAAI,OAAO,WAAW,GAAG;KACzB,KAAK,MAAM,KAAK,QAAQ,uBAAuB,MAAM,EAAE,SAAS;KAChE,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,MAAM;KAC5C,MAAM,OAAO,OACX,IAAI,KAAK,IAAI,OAAO;MAAE;MAAI,QAAS,OAAO,EAAE,CAAiB;KAAU,EAAE,CAC3E;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,OAAO,OAAO,GAAG;IACzB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA+C;KACvE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,UAAU,MAAM,OAAO,WAAW,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAChE,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,YAAY,QAAQ,IAAI,EAAE,EAAE;MAClC,IAAI,WAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAU;KACrE;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,gBAAgB,IAAI,GAAG,OAAO,KAAK;IAC3C;GAES;EACb;CACF;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statewalker/indexer-sqlite",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "private": false,
   "type": "module",
   "description": "SQLite-backed indexer: FTS5, vector and n-gram/grep sub-indexes in one @statewalker/db-api Db (Node, browser worker, Durable Objects).",
@@ -34,11 +34,11 @@
     "@statewalker/db-api": "^0.1.0",
     "@statewalker/db-sqlite": "^0.1.0",
     "@statewalker/db-sqlite-index": "^0.1.0",
-    "@statewalker/indexer-api": "^0.1.3",
-    "@statewalker/indexer-core": "^0.1.2",
-    "@statewalker/indexer-fulltext": "^0.1.3",
-    "@statewalker/indexer-ngram": "^0.1.0",
-    "@statewalker/indexer-vector": "^0.1.3"
+    "@statewalker/indexer-api": "^0.1.4",
+    "@statewalker/indexer-core": "^0.1.4",
+    "@statewalker/indexer-fulltext": "^0.1.5",
+    "@statewalker/indexer-ngram": "^0.1.1",
+    "@statewalker/indexer-vector": "^0.1.5"
   },
   "devDependencies": {
     "@statewalker/db-sqlite-do": "^0.1.0",

package/src/create-sqlite-indexer.ts

@@ -6,6 +6,7 @@ import type {
   Index,
   Indexer,
   IndexInfo,
+  Metadata,
 } from "@statewalker/indexer-api";
 import { createCompositeIndex, createSerialiser } from "@statewalker/indexer-core";
 import { type BlockIds, openBlockIds } from "./block-ids.js";
@@ -21,6 +22,8 @@ export interface SqliteIndexerOptions {
 type SubSpec = { type: string } & Record<string, unknown>;
 interface IndexManifest {
   subIndexes: Record<string, SubSpec>;
+  /** Index-level metadata from `createIndex` (absent in manifests written before it was saved). */
+  metadata?: Metadata;
 }
 
 const MANIFEST_TABLE = "__indexer_manifest";
@@ -85,6 +88,7 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
     name: string,
     subIndexes: Record<string, SubSpec>,
     options?: GetIndexOptions,
+    metadata?: Metadata,
   ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {
     const onMissing = options?.onMissingProvider ?? "throw";
     const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
@@ -121,9 +125,14 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
       effective[sub] = spec;
     }
 
-    const index = createCompositeIndex({
+    const index: Index = createCompositeIndex({
       name,
+      metadata,
       bindings,
+      // A closed instance leaves the cache, so getIndex opens a new one.
+      onClose: async () => {
+        if (indexes.get(name) === index) indexes.delete(name);
+      },
       onDeleteIndex: async () => {
         // Bound sub-indexes dropped themselves; skipped ones still own tables.
         for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {
@@ -141,7 +150,9 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
   return {
     async getIndexNames(): Promise<IndexInfo[]> {
       ensureOpen();
-      return [...manifest.keys()].map((name) => ({ name }));
+      return [...manifest.entries()].map(([name, entry]) =>
+        entry.metadata !== undefined ? { name, metadata: entry.metadata } : { name },
+      );
     },
 
     createIndex(params: CreateIndexParams): Promise<Index> {
@@ -161,9 +172,11 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
           indexes.delete(name);
           await dropIndexTables(name, existing?.subIndexes ?? {});
         }
-        const { index, effective } = await openIndex(name, subIndexes);
-        await writeManifest(name, { subIndexes: effective });
-        if (metadata !== undefined) (index as { metadata?: unknown }).metadata = metadata;
+        const { index, effective } = await openIndex(name, subIndexes, undefined, metadata);
+        await writeManifest(name, {
+          subIndexes: effective,
+          ...(metadata !== undefined ? { metadata } : {}),
+        });
         indexes.set(name, index);
         return index;
       });
@@ -178,9 +191,14 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
         if (cached) return cached;
         const entry = manifest.get(name);
         if (!entry) return null;
-        const { index, effective } = await openIndex(name, entry.subIndexes, options);
+        const { index, effective } = await openIndex(
+          name,
+          entry.subIndexes,
+          options,
+          entry.metadata,
+        );
         if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {
-          await writeManifest(name, { subIndexes: effective });
+          await writeManifest(name, { ...entry, subIndexes: effective });
         }
         indexes.set(name, index);
         return index;
@@ -192,27 +210,33 @@ export async function createSqliteIndexer(options: SqliteIndexerOptions): Promis
       return manifest.has(name);
     },
 
-    async deleteIndex(name: string): Promise<void> {
-      ensureOpen();
-      const live = indexes.get(name);
-      if (live) {
-        await live.deleteIndex();
-        return;
-      }
-      const entry = manifest.get(name);
-      if (entry) await dropIndexTables(name, entry.subIndexes);
+    deleteIndex(name: string): Promise<void> {
+      return runExclusive(async () => {
+        ensureOpen();
+        const live = indexes.get(name);
+        if (live) {
+          await live.deleteIndex();
+          return;
+        }
+        const entry = manifest.get(name);
+        if (entry) await dropIndexTables(name, entry.subIndexes);
+      });
     },
 
-    async flush(): Promise<void> {
-      ensureOpen();
-      for (const index of indexes.values()) await index.flush();
+    flush(): Promise<void> {
+      return runExclusive(async () => {
+        ensureOpen();
+        for (const index of indexes.values()) await index.flush();
+      });
     },
 
-    async close(): Promise<void> {
-      if (closed) return;
-      closed = true;
-      for (const index of indexes.values()) await index.close().catch(() => undefined);
-      indexes.clear();
+    close(): Promise<void> {
+      return runExclusive(async () => {
+        if (closed) return;
+        closed = true;
+        for (const index of [...indexes.values()]) await index.close().catch(() => undefined);
+        indexes.clear();
+      });
     },
   } satisfies Indexer;
 }

package/src/enumeration.ts

@@ -15,8 +15,16 @@ export interface Enumeration {
   refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;
 }
 
-/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */
-export function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string): Enumeration {
+/**
+ * `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`.
+ * `ensureOpen` runs before every call, so a closed sub-index throws like the other backends.
+ */
+export function newEnumeration(
+  db: Db,
+  blockIds: BlockIds,
+  memberIdsSql: string,
+  ensureOpen: () => void = () => {},
+): Enumeration {
   const from = (prefix?: DocumentPath) => {
     const scope = pathPrefixSql("b.path", prefix ? [prefix] : []);
     return {
@@ -26,11 +34,13 @@ export function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string)
   };
   return {
     async size(prefix) {
+      ensureOpen();
       const f = from(prefix);
       const rows = await db.query<{ n: number }>(`SELECT COUNT(*) AS n ${f.sql}`, f.params);
       return Number(rows[0]?.n ?? 0);
     },
     async entries(prefix) {
+      ensureOpen();
       const f = from(prefix);
       const rows = await db.query<{ id: number; path: string; block_id: string }>(
         `SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`,
@@ -43,6 +53,7 @@ export function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string)
       }));
     },
     async *paths(prefix) {
+      ensureOpen();
       const f = from(prefix);
       const rows = await db.query<{ path: string }>(`SELECT DISTINCT b.path ${f.sql}`, f.params);
       for (const r of rows) yield r.path as DocumentPath;

package/src/fts-provider.ts

@@ -47,7 +47,9 @@ export function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConf
       await db.exec(
         `CREATE VIRTUAL TABLE IF NOT EXISTS ${table} USING fts5(content, metadata UNINDEXED, tokenize='${tokenizerFor(config.language)}')`,
       );
-      const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`);
+      const list = newEnumeration(db, blockIds, `SELECT rowid AS id FROM ${table}`, () =>
+        ensureOpen(),
+      );
       let closed = false;
       const ensureOpen = () => {
         if (closed) throw new Error("FullTextIndex is closed");
@@ -135,6 +137,7 @@ export function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConf
         getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),
 
         async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<FullTextBlock> {
+          ensureOpen();
           const scope = pathPrefixSql("b.path", p ? [p] : []);
           const rows = await db.query<{
             path: string;

package/src/ngram-provider.ts

@@ -55,7 +55,9 @@ export function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, Ng
       const storeKind = config.store ?? "fts5";
       const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });
       await store.init();
-      const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`);
+      const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>
+        ensureOpen(),
+      );
       let closed = false;
       const ensureOpen = () => {
         if (closed) throw new Error("NgramIndex is closed");
@@ -130,6 +132,7 @@ export function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, Ng
         getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),
 
         async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {
+          ensureOpen();
           const entries = await list.entries(p);
           const texts = await store.getTexts(entries.map((e) => e.id));
           for (const e of entries) {

package/src/vector-provider.ts

@@ -34,7 +34,9 @@ export function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig,
         metric: "cosine",
       });
       await engine.init();
-      const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`);
+      const list = newEnumeration(db, blockIds, `SELECT id FROM ${engine.idTable}`, () =>
+        ensureOpen(),
+      );
       let closed = false;
       const ensureOpen = () => {
         if (closed) throw new Error("VectorIndex is closed");
@@ -52,6 +54,7 @@ export function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig,
 
         async *search(params: VectorQuery): AsyncGenerator<VectorResult> {
           ensureOpen();
+          for (const q of params.embeddings ?? []) validateDimensionality(info, q);
           const k = params.topK ?? 100;
           const allowed = params.paths?.length
             ? new Set(
@@ -109,6 +112,7 @@ export function newSqliteVectorProvider(): SqliteModalityProvider<VectorConfig,
         getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),
 
         async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<VectorBlock> {
+          ensureOpen();
           const entries = await list.entries(p);
           const vectors = await engine.getVectors(entries.map((e) => e.id));
           for (const e of entries) {