@statewalker/indexer-sqlite 0.0.0-stage → 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Kotelnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,240 @@
-# Temporary Holding Version
+# @statewalker/indexer-sqlite
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+## What it is
+
+An `Indexer` whose full-text (FTS5), vector and n-gram/grep sub-indexes all live in one SQLite
+`Db` from `@statewalker/db-api`: `node:sqlite` on Node.js, sqlite-wasm in a browser worker, or a
+Durable Object's `ctx.storage.sql`. Each modality is a provider you register; an Index combines
+any number of named sub-indexes and fuses their results by reciprocal rank fusion (RRF).
+
+## Why it exists
+
+One SQLite file holds every modality of every Index, so there is nothing to save or load: a new
+indexer over the same database lists the indexes and searches the old data. Inside a Durable
+Object it runs on the object's own SQLite storage, so a cold start opens the indexes where they
+are, with nothing to load. The n-gram sub-index answers exact substring and regex queries (`fooBar(`, `\bTODO\b`)
+that word-based full-text search cannot.
+
+## How to use
+
+```sh
+pnpm add @statewalker/indexer-sqlite
+```
+
+- `createSqliteIndexer({ db, providers }): Promise<Indexer>` — `db` is a `Db` from
+  `@statewalker/db-sqlite-node`, `@statewalker/db-sqlite-do` or another `@statewalker/db-sqlite*`
+  driver. Pass one provider per modality your indexes use; for all three:
+  `[newSqliteFullTextProvider(), newSqliteVectorProvider(), newSqliteNgramProvider()]`. A
+  sub-index type with no provider fails `createIndex` and `getIndex` (see below).
+- The indexer never closes `db`. `indexer.close()` closes the open indexes; close the `Db`
+  yourself.
+- Sub-indexes are configured, queried and read with the modality packages:
+  `@statewalker/indexer-fulltext`, `@statewalker/indexer-vector`, `@statewalker/indexer-ngram`
+  (including its `grep` helper).
+- `SqliteModalityProvider`, `openBlockIds`, `newEnumeration` and `pathPrefixSql` are exported for
+  writing another provider.
+
+## Examples
+
+Node.js, with a file database and all three modalities:
+
+```ts
+import { newNodeSqliteDb } from "@statewalker/db-sqlite-node";
+import { newFullTextAccess, setFullTextConfig } from "@statewalker/indexer-fulltext";
+import { grep, newNgramAccess, setNgramConfig } from "@statewalker/indexer-ngram";
+import { newVectorAccess, setVectorConfig } from "@statewalker/indexer-vector";
+import {
+  createSqliteIndexer,
+  newSqliteFullTextProvider,
+  newSqliteNgramProvider,
+  newSqliteVectorProvider,
+} from "@statewalker/indexer-sqlite";
+
+const db = await newNodeSqliteDb({ path: "./search.sqlite" });
+const indexer = await createSqliteIndexer({
+  db,
+  providers: [newSqliteFullTextProvider(), newSqliteVectorProvider(), newSqliteNgramProvider()],
+});
+
+const params = { name: "docs" };
+setFullTextConfig(params, "text", { language: "en" });
+setVectorConfig(params, "emb", { dimensionality: 3, model: "my-model" });
+setNgramConfig(params, "code", { store: "fts5" });
+const index = await indexer.createIndex(params);
+
+const text = newFullTextAccess("text");
+const emb = newVectorAccess("emb");
+const code = newNgramAccess("code");
+const block = { path: "/src/a.ts" as const, blockId: "1" };
+const content = "function fooBar() {\n  return running(fooBar);\n}\n";
+await text.get(index).addDocument([{ ...block, content }]);
+await emb.get(index).addDocument([{ ...block, embedding: Float32Array.of(0.1, 0.9, 0) }]);
+await code.get(index).addDocument([{ ...block, content }]);
+
+// One fused search over all three sub-indexes.
+const request = { topK: 10 };
+text.setQuery(request, { queries: ["run"] }); // Porter stemming: matches "running"
+emb.setQuery(request, { embeddings: [Float32Array.of(0, 1, 0)] });
+code.setQuery(request, { pattern: "fooBar" });
+for await (const r of index.search(request)) {
+  console.log(r.path, r.blockId, r.score, code.getResult(r)?.count); // RRF score; 2 matches
+}
+
+// grep: every match with offsets and lines, plus whether the search was cut short.
+const { hits, truncated } = await grep(index, "code", "fooBar\\(", { regex: true });
+console.log(hits[0]?.matches, truncated); // [{ start: 9, end: 16, line: 1 }] false
+
+await indexer.close();
+await db.close();
+```
+
+Inside a Durable Object (`newDoSqliteDb` is synchronous; the indexer is created on first use):
+
+```ts
+import { DurableObject } from "cloudflare:workers";
+import { newDoSqliteDb } from "@statewalker/db-sqlite-do";
+import type { Indexer } from "@statewalker/indexer-api";
+import {
+  createSqliteIndexer,
+  newSqliteFullTextProvider,
+  newSqliteNgramProvider,
+  newSqliteVectorProvider,
+} from "@statewalker/indexer-sqlite";
+
+export class Search extends DurableObject {
+  db = newDoSqliteDb(this.ctx.storage);
+  #indexer?: Promise<Indexer>;
+
+  indexer(): Promise<Indexer> {
+    this.#indexer ??= createSqliteIndexer({
+      db: this.db,
+      providers: [newSqliteFullTextProvider(), newSqliteVectorProvider(), newSqliteNgramProvider()],
+    });
+    return this.#indexer;
+  }
+}
+```
+
+In Node.js tests, `newDoSqliteDb(newFakeDoStorage())` (from `@statewalker/db-sqlite-do/fake`)
+gives a `Db` with the Durable Object's limits.
+
+## Internals
+
+### Tables are prefixed per index and sub-index, and names are case-distinct
+
+```
+__indexer_manifest(name, config)                      one row per Index: its sub-index configs
+idx_<index>__blocks(id INTEGER PRIMARY KEY, path, block_id)    shared by every sub-index
+idx_<index>__<sub>_fts                                FTS5 table, rowid = block id
+idx_<index>__<sub>_vec_*                              the vector engine's tables
+idx_<index>__<sub>_ng_*                               the n-gram engine's tables
+```
+
+Index and sub-index names are escaped into the prefixes, upper-case letters included: SQLite table names ignore case, so `Docs` and `docs` become distinct prefixes and are independent indexes. The vector and n-gram sub-indexes
+compute size and listings by joining the engine's own id table with the blocks table, so nothing
+lists a block the engine does not hold, and there is no second write that a crash could separate
+from the engine's write.
+
+### Why blocks get integer ids
+
+The engines (`@statewalker/db-sqlite-index`) index integer ids, not `{path, blockId}` pairs. The
+blocks table maps one to the other, and its rows are kept until `deleteIndex`, even after a
+block is deleted from every sub-index: an id stays stable for its block, so an engine never sees
+one id reused for different content. Re-adding a `{path, blockId}` replaces its content; it never
+creates a second entry.
+
+### Why each modality uses its own table prefix
+
+The vector and n-gram engines both create a `{prefix}meta` table with different columns. Each
+provider therefore appends its own suffix (`fts`, `vec_`, `ng_`) to the sub-index prefix; a new
+provider must do the same.
+
+### Path scope uses `substr`, not `LIKE`, and one JSON parameter
+
+A path prefix matches on component boundaries (`/foo` matches `/foo` and `/foo/x`, never
+`/foobar`) and case-sensitively (`/Docs` does not match `/docs/a`). SQLite's `LIKE` ignores ASCII
+case, and `PRAGMA case_sensitive_like` is not authorised in a Durable Object, so the condition is
+built with `substr`. All prefixes are passed as a single JSON parameter (`json_each`), because
+a Durable Object rejects statements with more than 100 bound variables; any number of `paths` works.
+
+### Full text runs on FTS5 with stemming for exactly "en"
+
+- Only the exact language `"en"` gets the Porter stemmer (`porter unicode61 remove_diacritics
+  2`). Every other value, including `"en-US"` and unknown codes, gets `unicode61
+  remove_diacritics 2` without stemming; no language throws.
+- Each query string is split into words; every word with a letter or a digit is double-quoted
+  and the words are OR-joined, so FTS5 syntax in user text (`"`, `*`, `AND`, `NEAR(`, `-`) is
+  searched as plain words and never raises `fts5: syntax error`. Text with no letters or digits
+  finds nothing.
+- `score = -bm25(...)`; the snippet is `snippet(<table>, 0, '', '', '…', 16)`.
+- Several `queries` in one sub-query **add up** (SUM): a block matching more of them ranks
+  higher. The PGlite and DuckDB indexers keep each block's best single query instead, so the same
+  request can rank differently there. Each query is cut to `topK` before summing, so a block just
+  below the cut-off in one query loses that part of its score.
+
+### Vector sub-indexes use cosine scores and do not store block metadata
+
+Cosine metric; `score = 1 − distance`. Several embeddings in one query keep each block's best
+score. The engine commits an `upsert` in transactions of 200 vectors: if a call fails part-way,
+the earlier batches stay written.
+`VectorBlock.metadata` is not stored. `getDocumentsBlocks` returns the cosine-normalised vectors, not the embeddings that were added.
+
+### N-gram sub-indexes answer grep, with a bounded candidate budget
+
+- `store` is `"fts5"` (default) or `"postings"`; both give the same results.
+- `caseSensitive` defaults to `true`, as in grep. A literal pattern is matched exactly; with
+  `regex: true` it is a JavaScript regex without the `u` flag, and `\p{…}` / `\P{…}` are rejected
+  with a `SyntaxError`.
+- Matches stay inside one block. Offsets are UTF-16 code units of the stored text (lone
+  surrogates are stored as U+FFFD), `end` exclusive; `line` is 1-based, counted from `\n`.
+- `score = count`, the number of matches in the block. The engine returns its first `topK`
+  matching blocks (default 100) in ascending id order; the provider then ranks those by count.
+  When `topK` or the engine's candidate budget (10,000 candidates) cut the search short, `grep`
+  returns `truncated: true` and the hits are the ones found so far, not necessarily the blocks with
+  the most matches.
+- `NgramBlock.metadata` is not stored (the full-text sub-index does store it).
+- A `paths`-scoped search spends the candidate budget on out-of-scope blocks too: the engine
+  filters by path after it cuts to the budget, so a narrow scope in a large index can return few
+  hits together with `truncated: true` although more matches exist in scope.
+- `addDocument` hands the engine at most 500 blocks per write transaction; each batch commits on
+  its own, so if a later batch fails the earlier ones stay written.
+
+### Dependencies: the Db, the engines and the modality contracts
+
+- `@statewalker/db-api` — the `Db` it runs on.
+- `@statewalker/db-sqlite` — `inTransaction` (FTS writes).
+- `@statewalker/db-sqlite-index` — the vector (`./vector`) and n-gram (`./ngram`) engines.
+- `@statewalker/indexer-api`, `@statewalker/indexer-core` — the `Indexer` contract, RRF
+  composition, shared helpers.
+- `@statewalker/indexer-fulltext`, `@statewalker/indexer-vector`, `@statewalker/indexer-ngram` —
+  the modality contracts the providers implement.
+
+## What will surprise you
+
+- **A wrapped `Db` loses the transaction hook.** Pass the driver's `Db` itself, not an object
+  that forwards to it. With a wrapper, creating an Index with a vector or n-gram sub-index, or
+  the first write to a full-text sub-index, rejects with:
+  `inTransaction: this Db has no SQLite transaction hook. It is not a @statewalker/db-sqlite* Db,
+  or it wraps one (a wrapper object loses the hook - pass the driver's Db itself), or it is the
+  handle of a running transaction (nesting is not supported).`
+- **Never call an indexer, an Index or a sub-index from inside your own
+  `inTransaction(db, …)` callback.** The sub-indexes run their own transactions and queries on
+  `db`; on Node.js and in the browser those wait for your transaction to end, and it waits for
+  them. The call hangs, with no error.
+- **A provider's `open` and `drop` must not call back into the indexer** (`getIndex`,
+  `createIndex`): the indexer runs them inside its own one-at-a-time queue, and a nested call waits
+  for itself forever.
+- **Durable Object limits apply to every write.** A single string or BLOB value over about 2 MB
+  rejects with `string or blob too big`, so very large blocks must be split before indexing. An
+  operation that runs past the platform's deadline (15 s on celld) is retried rather than failed;
+  keep each `addDocument` call modest.
+- **`truncated: true` is easy to miss.** `for await (const hit of sub.search(q))` drops the
+  summary the generator returns; use `grep` to read it.
+- **Re-opening needs the same providers.** `getIndex` on an Index with a sub-index whose type has
+  no registered provider throws `Cannot load index: sub-index "<name>" requires modality
+  "<type>" but only [...] are registered.`; pass `{ onMissingProvider: "warn" }` to open it
+  without that sub-index.
+
+## License
+
+MIT

package/dist/block-ids.d.ts

@@ -0,0 +1,20 @@
+import { Db } from "@statewalker/db-api";
+import { BlockReference, PathSelector } from "@statewalker/indexer-api";
+//#region src/block-ids.d.ts
+/**
+ * The Index-wide `{path, blockId} ↔ integer id` table shared by every sub-index.
+ * Every method takes the `Db` to run on, so it works on `tx` inside `inTransaction`.
+ * Rows are never deleted before `deleteIndex`: an id stays stable for its block.
+ */
+export interface BlockIds {
+  readonly table: string;
+  /** Ids for `refs`, in order; missing rows are created. */
+  ensure(db: Db, refs: readonly BlockReference[]): Promise<number[]>;
+  /** Ids of every known block matched by any selector (path prefix, or exact block). */
+  select(db: Db, selectors: readonly PathSelector[]): Promise<number[]>;
+  /** References of `ids`; unknown ids are absent. */
+  refs(db: Db, ids: readonly number[]): Promise<Map<number, BlockReference>>;
+}
+export declare function openBlockIds(db: Db, table: string): Promise<BlockIds>;
+//#endregion
+//# sourceMappingURL=block-ids.d.ts.map

package/dist/block-ids.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-ids.d.ts","names":[],"sources":["../src/block-ids.ts"],"mappings":";;;;;;;;iBAUiB;WACN;;EAET,OAAO,IAAI,IAAI,eAAe,mBAAmB;;EAEjD,OAAO,IAAI,IAAI,oBAAoB,iBAAiB;;EAEpD,KAAK,IAAI,IAAI,yBAAyB,QAAQ,YAAY;;wBAGtC,aAAa,IAAI,IAAI,gBAAgB,QAAQ"}

package/dist/block-ids.js

@@ -0,0 +1,47 @@
+import { pathPrefixSql } from "./path-sql.js";
+import { compositeKey } from "@statewalker/indexer-core";
+//#region src/block-ids.ts
+async function openBlockIds(db, table) {
+	await db.exec(`CREATE TABLE IF NOT EXISTS ${table} (id INTEGER PRIMARY KEY, path TEXT NOT NULL, block_id TEXT NOT NULL, UNIQUE (path, block_id))`);
+	return {
+		table,
+		async ensure(tx, refs) {
+			if (refs.length === 0) return [];
+			const json = JSON.stringify(refs.map((r) => [r.path, r.blockId]));
+			await tx.query(`INSERT INTO ${table} (path, block_id) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?) WHERE true ON CONFLICT (path, block_id) DO NOTHING`, [json]);
+			const rows = await tx.query(`SELECT b.id, b.path, b.block_id FROM ${table} b JOIN json_each(?) j ON b.path = json_extract(j.value, '$[0]') AND b.block_id = json_extract(j.value, '$[1]')`, [json]);
+			const byKey = new Map(rows.map((r) => [compositeKey(r.path, r.block_id), Number(r.id)]));
+			return refs.map((r) => {
+				const id = byKey.get(compositeKey(r.path, r.blockId));
+				if (id === void 0) throw new Error(`block id missing for ${r.path}#${r.blockId}`);
+				return id;
+			});
+		},
+		async select(tx, selectors) {
+			const ids = /* @__PURE__ */ new Set();
+			for (const sel of selectors) {
+				const where = sel.blockId !== void 0 ? {
+					sql: "b.path = ? AND b.block_id = ?",
+					params: [sel.path, sel.blockId]
+				} : pathPrefixSql("b.path", [sel.path]);
+				const rows = await tx.query(`SELECT b.id FROM ${table} b WHERE ${where.sql}`, where.params);
+				for (const r of rows) ids.add(Number(r.id));
+			}
+			return [...ids];
+		},
+		async refs(tx, ids) {
+			const out = /* @__PURE__ */ new Map();
+			if (ids.length === 0) return out;
+			const rows = await tx.query(`SELECT id, path, block_id FROM ${table} WHERE id IN (SELECT value FROM json_each(?))`, [JSON.stringify(ids)]);
+			for (const r of rows) out.set(Number(r.id), {
+				path: r.path,
+				blockId: r.block_id
+			});
+			return out;
+		}
+	};
+}
+//#endregion
+export { openBlockIds };
+
+//# sourceMappingURL=block-ids.js.map

package/dist/block-ids.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-ids.js","names":[],"sources":["../src/block-ids.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { BlockReference, DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { compositeKey } from \"@statewalker/indexer-core\";\nimport { pathPrefixSql } from \"./path-sql.js\";\n\n/**\n * The Index-wide `{path, blockId} ↔ integer id` table shared by every sub-index.\n * Every method takes the `Db` to run on, so it works on `tx` inside `inTransaction`.\n * Rows are never deleted before `deleteIndex`: an id stays stable for its block.\n */\nexport interface BlockIds {\n  readonly table: string;\n  /** Ids for `refs`, in order; missing rows are created. */\n  ensure(db: Db, refs: readonly BlockReference[]): Promise<number[]>;\n  /** Ids of every known block matched by any selector (path prefix, or exact block). */\n  select(db: Db, selectors: readonly PathSelector[]): Promise<number[]>;\n  /** References of `ids`; unknown ids are absent. */\n  refs(db: Db, ids: readonly number[]): Promise<Map<number, BlockReference>>;\n}\n\nexport async function openBlockIds(db: Db, table: string): Promise<BlockIds> {\n  await db.exec(\n    `CREATE TABLE IF NOT EXISTS ${table} (id INTEGER PRIMARY KEY, path TEXT NOT NULL, block_id TEXT NOT NULL, UNIQUE (path, block_id))`,\n  );\n  return {\n    table,\n    async ensure(tx, refs) {\n      if (refs.length === 0) return [];\n      const json = JSON.stringify(refs.map((r) => [r.path, r.blockId]));\n      // `WHERE true` disambiguates `SELECT … ON CONFLICT` for the SQLite parser.\n      await tx.query(\n        `INSERT INTO ${table} (path, block_id) SELECT json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?) WHERE true ON CONFLICT (path, block_id) DO NOTHING`,\n        [json],\n      );\n      const rows = await tx.query<{ id: number; path: string; block_id: string }>(\n        `SELECT b.id, b.path, b.block_id FROM ${table} b JOIN json_each(?) j ON b.path = json_extract(j.value, '$[0]') AND b.block_id = json_extract(j.value, '$[1]')`,\n        [json],\n      );\n      const byKey = new Map(\n        rows.map((r) => [compositeKey(r.path as DocumentPath, r.block_id), Number(r.id)]),\n      );\n      return refs.map((r) => {\n        const id = byKey.get(compositeKey(r.path, r.blockId));\n        if (id === undefined) throw new Error(`block id missing for ${r.path}#${r.blockId}`);\n        return id;\n      });\n    },\n    async select(tx, selectors) {\n      const ids = new Set<number>();\n      for (const sel of selectors) {\n        const where =\n          sel.blockId !== undefined\n            ? { sql: \"b.path = ? AND b.block_id = ?\", params: [sel.path, sel.blockId] }\n            : pathPrefixSql(\"b.path\", [sel.path]);\n        const rows = await tx.query<{ id: number }>(\n          `SELECT b.id FROM ${table} b WHERE ${where.sql}`,\n          where.params,\n        );\n        for (const r of rows) ids.add(Number(r.id));\n      }\n      return [...ids];\n    },\n    async refs(tx, ids) {\n      const out = new Map<number, BlockReference>();\n      if (ids.length === 0) return out;\n      const rows = await tx.query<{ id: number; path: string; block_id: string }>(\n        `SELECT id, path, block_id FROM ${table} WHERE id IN (SELECT value FROM json_each(?))`,\n        [JSON.stringify(ids)],\n      );\n      for (const r of rows)\n        out.set(Number(r.id), { path: r.path as DocumentPath, blockId: r.block_id });\n      return out;\n    },\n  };\n}\n"],"mappings":";;;AAoBA,eAAsB,aAAa,IAAQ,OAAkC;CAC3E,MAAM,GAAG,KACP,8BAA8B,MAAM,+FACtC;CACA,OAAO;EACL;EACA,MAAM,OAAO,IAAI,MAAM;GACrB,IAAI,KAAK,WAAW,GAAG,OAAO,CAAC;GAC/B,MAAM,OAAO,KAAK,UAAU,KAAK,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;GAEhE,MAAM,GAAG,MACP,eAAe,MAAM,yJACrB,CAAC,IAAI,CACP;GACA,MAAM,OAAO,MAAM,GAAG,MACpB,wCAAwC,MAAM,kHAC9C,CAAC,IAAI,CACP;GACA,MAAM,QAAQ,IAAI,IAChB,KAAK,KAAK,MAAM,CAAC,aAAa,EAAE,MAAsB,EAAE,QAAQ,GAAG,OAAO,EAAE,EAAE,CAAC,CAAC,CAClF;GACA,OAAO,KAAK,KAAK,MAAM;IACrB,MAAM,KAAK,MAAM,IAAI,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC;IACpD,IAAI,OAAO,KAAA,GAAW,MAAM,IAAI,MAAM,wBAAwB,EAAE,KAAK,GAAG,EAAE,SAAS;IACnF,OAAO;GACT,CAAC;EACH;EACA,MAAM,OAAO,IAAI,WAAW;GAC1B,MAAM,sBAAM,IAAI,IAAY;GAC5B,KAAK,MAAM,OAAO,WAAW;IAC3B,MAAM,QACJ,IAAI,YAAY,KAAA,IACZ;KAAE,KAAK;KAAiC,QAAQ,CAAC,IAAI,MAAM,IAAI,OAAO;IAAE,IACxE,cAAc,UAAU,CAAC,IAAI,IAAI,CAAC;IACxC,MAAM,OAAO,MAAM,GAAG,MACpB,oBAAoB,MAAM,WAAW,MAAM,OAC3C,MAAM,MACR;IACA,KAAK,MAAM,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,EAAE,CAAC;GAC5C;GACA,OAAO,CAAC,GAAG,GAAG;EAChB;EACA,MAAM,KAAK,IAAI,KAAK;GAClB,MAAM,sBAAM,IAAI,IAA4B;GAC5C,IAAI,IAAI,WAAW,GAAG,OAAO;GAC7B,MAAM,OAAO,MAAM,GAAG,MACpB,kCAAkC,MAAM,gDACxC,CAAC,KAAK,UAAU,GAAG,CAAC,CACtB;GACA,KAAK,MAAM,KAAK,MACd,IAAI,IAAI,OAAO,EAAE,EAAE,GAAG;IAAE,MAAM,EAAE;IAAsB,SAAS,EAAE;GAAS,CAAC;GAC7E,OAAO;EACT;CACF;AACF"}

package/dist/create-sqlite-indexer.d.ts

@@ -0,0 +1,18 @@
+import { SqliteModalityProvider } from "./types.js";
+import { Db } from "@statewalker/db-api";
+import { Indexer } from "@statewalker/indexer-api";
+//#region src/create-sqlite-indexer.d.ts
+export interface SqliteIndexerOptions {
+  /** The database. The indexer never closes it. */
+  db: Db;
+  /** One provider per modality type, e.g. FTS5, vector, n-gram. */
+  providers: SqliteModalityProvider[];
+}
+/**
+ * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared
+ * `{indexPrefix}blocks` id table per Index, and opens each sub-index through
+ * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).
+ */
+export declare function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer>;
+//#endregion
+//# sourceMappingURL=create-sqlite-indexer.d.ts.map

package/dist/create-sqlite-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-sqlite-indexer.d.ts","names":[],"sources":["../src/create-sqlite-indexer.ts"],"mappings":";;;;iBAaiB;;EAEf,IAAI;;EAEJ,WAAW;;;;;;;wBAqBS,oBAAoB,SAAS,uBAAuB,QAAQ"}

package/dist/create-sqlite-indexer.js

@@ -0,0 +1,152 @@
+import { openBlockIds } from "./block-ids.js";
+import { createCompositeIndex, createSerialiser } from "@statewalker/indexer-core";
+//#region src/create-sqlite-indexer.ts
+const MANIFEST_TABLE = "__indexer_manifest";
+/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so "Docs" and "docs" must not collide. */
+const sanitizePrefix = (name) => name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);
+const indexPrefix = (name) => `idx_${sanitizePrefix(name)}__`;
+/**
+* `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared
+* `{indexPrefix}blocks` id table per Index, and opens each sub-index through
+* its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).
+*/
+async function createSqliteIndexer(options) {
+	const { db } = options;
+	const providers = new Map(options.providers.map((p) => [p.type, p]));
+	const indexes = /* @__PURE__ */ new Map();
+	const manifest = /* @__PURE__ */ new Map();
+	const runExclusive = createSerialiser();
+	let closed = false;
+	await db.exec(`CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`);
+	for (const row of await db.query(`SELECT name, config FROM ${MANIFEST_TABLE}`)) manifest.set(row.name, JSON.parse(row.config));
+	const ensureOpen = () => {
+		if (closed) throw new Error("Indexer is closed");
+	};
+	const contextFor = (name, sub, blockIds) => ({
+		db,
+		prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,
+		blockIds
+	});
+	async function writeManifest(name, entry) {
+		await db.query(`INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`, [name, JSON.stringify(entry)]);
+		manifest.set(name, entry);
+	}
+	async function dropIndexTables(name, subIndexes) {
+		const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
+		for (const [sub, spec] of Object.entries(subIndexes)) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
+		await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
+		await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
+		manifest.delete(name);
+	}
+	async function openIndex(name, subIndexes, options) {
+		const onMissing = options?.onMissingProvider ?? "throw";
+		const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
+		const bindings = [];
+		const effective = {};
+		for (const [sub, saved] of Object.entries(subIndexes)) {
+			const action = options?.subIndexes?.[sub];
+			if (action && "skip" in action && action.skip === true) {
+				effective[sub] = saved;
+				continue;
+			}
+			const ctx = contextFor(name, sub, blockIds);
+			const spec = action && !("skip" in action) ? action : saved;
+			const provider = providers.get(spec.type);
+			if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);
+			if (!provider) {
+				if (onMissing === "throw") throw new Error(`Cannot load index: sub-index "${sub}" requires modality "${spec.type}" but only [${[...providers.keys()].map((t) => `"${t}"`).join(", ")}] are registered.`);
+				globalThis.console?.warn(`Sub-index "${sub}" (type "${spec.type}") has no provider; skipping.`);
+				effective[sub] = saved;
+				continue;
+			}
+			const { type, ...config } = spec;
+			bindings.push({
+				name: sub,
+				type,
+				config,
+				index: await provider.open(ctx, config)
+			});
+			effective[sub] = spec;
+		}
+		return {
+			index: createCompositeIndex({
+				name,
+				bindings,
+				onDeleteIndex: async () => {
+					for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
+					await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
+					await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
+					manifest.delete(name);
+					indexes.delete(name);
+				}
+			}),
+			effective
+		};
+	}
+	return {
+		async getIndexNames() {
+			ensureOpen();
+			return [...manifest.keys()].map((name) => ({ name }));
+		},
+		createIndex(params) {
+			return runExclusive(async () => {
+				ensureOpen();
+				const { name, subIndexes, overwrite, metadata } = params;
+				if (!subIndexes || Object.keys(subIndexes).length === 0) throw new Error("Cannot create index without any subIndexes");
+				const existing = manifest.get(name);
+				if (existing || indexes.has(name)) {
+					if (!overwrite) throw new Error(`Index "${name}" already exists`);
+					await indexes.get(name)?.close().catch(() => void 0);
+					indexes.delete(name);
+					await dropIndexTables(name, existing?.subIndexes ?? {});
+				}
+				const { index, effective } = await openIndex(name, subIndexes);
+				await writeManifest(name, { subIndexes: effective });
+				if (metadata !== void 0) index.metadata = metadata;
+				indexes.set(name, index);
+				return index;
+			});
+		},
+		getIndex(name, options) {
+			return runExclusive(async () => {
+				ensureOpen();
+				const cached = indexes.get(name);
+				if (cached) return cached;
+				const entry = manifest.get(name);
+				if (!entry) return null;
+				const { index, effective } = await openIndex(name, entry.subIndexes, options);
+				if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) await writeManifest(name, { subIndexes: effective });
+				indexes.set(name, index);
+				return index;
+			});
+		},
+		async hasIndex(name) {
+			ensureOpen();
+			return manifest.has(name);
+		},
+		async deleteIndex(name) {
+			ensureOpen();
+			const live = indexes.get(name);
+			if (live) {
+				await live.deleteIndex();
+				return;
+			}
+			const entry = manifest.get(name);
+			if (entry) await dropIndexTables(name, entry.subIndexes);
+		},
+		async flush() {
+			ensureOpen();
+			for (const index of indexes.values()) await index.flush();
+		},
+		async close() {
+			if (closed) return;
+			closed = true;
+			for (const index of indexes.values()) await index.close().catch(() => void 0);
+			indexes.clear();
+		}
+	};
+}
+//#endregion
+export { createSqliteIndexer };
+
+//# sourceMappingURL=create-sqlite-indexer.js.map

package/dist/create-sqlite-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-sqlite-indexer.js","names":[],"sources":["../src/create-sqlite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type {\n  AnySubIndexBinding,\n  CreateIndexParams,\n  GetIndexOptions,\n  Index,\n  Indexer,\n  IndexInfo,\n} from \"@statewalker/indexer-api\";\nimport { createCompositeIndex, createSerialiser } from \"@statewalker/indexer-core\";\nimport { type BlockIds, openBlockIds } from \"./block-ids.js\";\nimport type { SqliteModalityContext, SqliteModalityProvider } from \"./types.js\";\n\nexport interface SqliteIndexerOptions {\n  /** The database. The indexer never closes it. */\n  db: Db;\n  /** One provider per modality type, e.g. FTS5, vector, n-gram. */\n  providers: SqliteModalityProvider[];\n}\n\ntype SubSpec = { type: string } & Record<string, unknown>;\ninterface IndexManifest {\n  subIndexes: Record<string, SubSpec>;\n}\n\nconst MANIFEST_TABLE = \"__indexer_manifest\";\n\n/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so \"Docs\" and \"docs\" must not collide. */\nconst sanitizePrefix = (name: string) =>\n  name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);\n\nconst indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;\n\n/**\n * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared\n * `{indexPrefix}blocks` id table per Index, and opens each sub-index through\n * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).\n */\nexport async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {\n  const { db } = options;\n  const providers = new Map(options.providers.map((p) => [p.type, p]));\n  const indexes = new Map<string, Index>();\n  const manifest = new Map<string, IndexManifest>();\n  const runExclusive = createSerialiser();\n  let closed = false;\n\n  await db.exec(\n    `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,\n  );\n  for (const row of await db.query<{ name: string; config: string }>(\n    `SELECT name, config FROM ${MANIFEST_TABLE}`,\n  )) {\n    manifest.set(row.name, JSON.parse(row.config) as IndexManifest);\n  }\n\n  const ensureOpen = () => {\n    if (closed) throw new Error(\"Indexer is closed\");\n  };\n\n  const contextFor = (name: string, sub: string, blockIds: BlockIds): SqliteModalityContext => ({\n    db,\n    prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,\n    blockIds,\n  });\n\n  async function writeManifest(name: string, entry: IndexManifest): Promise<void> {\n    await db.query(\n      `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,\n      [name, JSON.stringify(entry)],\n    );\n    manifest.set(name, entry);\n  }\n\n  async function dropIndexTables(name: string, subIndexes: Record<string, SubSpec>): Promise<void> {\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    for (const [sub, spec] of Object.entries(subIndexes)) {\n      await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n    }\n    await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n    await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n    manifest.delete(name);\n  }\n\n  async function openIndex(\n    name: string,\n    subIndexes: Record<string, SubSpec>,\n    options?: GetIndexOptions,\n  ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {\n    const onMissing = options?.onMissingProvider ?? \"throw\";\n    const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n    const bindings: AnySubIndexBinding[] = [];\n    const effective: Record<string, SubSpec> = {};\n\n    for (const [sub, saved] of Object.entries(subIndexes)) {\n      const action = options?.subIndexes?.[sub];\n      if (action && \"skip\" in action && action.skip === true) {\n        effective[sub] = saved;\n        continue;\n      }\n      const ctx = contextFor(name, sub, blockIds);\n      const spec = action && !(\"skip\" in action) ? (action as SubSpec) : saved;\n      const provider = providers.get(spec.type);\n      if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);\n      if (!provider) {\n        if (onMissing === \"throw\") {\n          throw new Error(\n            `Cannot load index: sub-index \"${sub}\" requires modality \"${spec.type}\" but only [${[\n              ...providers.keys(),\n            ]\n              .map((t) => `\"${t}\"`)\n              .join(\", \")}] are registered.`,\n          );\n        }\n        const g = globalThis as { console?: { warn(...args: unknown[]): void } };\n        g.console?.warn(`Sub-index \"${sub}\" (type \"${spec.type}\") has no provider; skipping.`);\n        effective[sub] = saved;\n        continue;\n      }\n      const { type, ...config } = spec;\n      bindings.push({ name: sub, type, config, index: await provider.open(ctx, config) });\n      effective[sub] = spec;\n    }\n\n    const index = createCompositeIndex({\n      name,\n      bindings,\n      onDeleteIndex: async () => {\n        // Bound sub-indexes dropped themselves; skipped ones still own tables.\n        for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {\n          await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n        }\n        await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n        await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n        manifest.delete(name);\n        indexes.delete(name);\n      },\n    });\n    return { index, effective };\n  }\n\n  return {\n    async getIndexNames(): Promise<IndexInfo[]> {\n      ensureOpen();\n      return [...manifest.keys()].map((name) => ({ name }));\n    },\n\n    createIndex(params: CreateIndexParams): Promise<Index> {\n      return runExclusive(async () => {\n        ensureOpen();\n        const { name, subIndexes, overwrite, metadata } = params;\n        if (!subIndexes || Object.keys(subIndexes).length === 0) {\n          throw new Error(\"Cannot create index without any subIndexes\");\n        }\n        const existing = manifest.get(name);\n        if (existing || indexes.has(name)) {\n          if (!overwrite) throw new Error(`Index \"${name}\" already exists`);\n          await indexes\n            .get(name)\n            ?.close()\n            .catch(() => undefined);\n          indexes.delete(name);\n          await dropIndexTables(name, existing?.subIndexes ?? {});\n        }\n        const { index, effective } = await openIndex(name, subIndexes);\n        await writeManifest(name, { subIndexes: effective });\n        if (metadata !== undefined) (index as { metadata?: unknown }).metadata = metadata;\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    getIndex(name: string, options?: GetIndexOptions): Promise<Index | null> {\n      // Serialised with createIndex so concurrent first opens open once\n      // (a second reinit would otherwise drop tables the first just wrote).\n      return runExclusive(async () => {\n        ensureOpen();\n        const cached = indexes.get(name);\n        if (cached) return cached;\n        const entry = manifest.get(name);\n        if (!entry) return null;\n        const { index, effective } = await openIndex(name, entry.subIndexes, options);\n        if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {\n          await writeManifest(name, { subIndexes: effective });\n        }\n        indexes.set(name, index);\n        return index;\n      });\n    },\n\n    async hasIndex(name: string): Promise<boolean> {\n      ensureOpen();\n      return manifest.has(name);\n    },\n\n    async deleteIndex(name: string): Promise<void> {\n      ensureOpen();\n      const live = indexes.get(name);\n      if (live) {\n        await live.deleteIndex();\n        return;\n      }\n      const entry = manifest.get(name);\n      if (entry) await dropIndexTables(name, entry.subIndexes);\n    },\n\n    async flush(): Promise<void> {\n      ensureOpen();\n      for (const index of indexes.values()) await index.flush();\n    },\n\n    async close(): Promise<void> {\n      if (closed) return;\n      closed = true;\n      for (const index of indexes.values()) await index.close().catch(() => undefined);\n      indexes.clear();\n    },\n  } satisfies Indexer;\n}\n"],"mappings":";;;AAyBA,MAAM,iBAAiB;;AAGvB,MAAM,kBAAkB,SACtB,KAAK,QAAQ,eAAe,OAAO,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;AAE5D,MAAM,eAAe,SAAiB,OAAO,eAAe,IAAI,EAAE;;;;;;AAOlE,eAAsB,oBAAoB,SAAiD;CACzF,MAAM,EAAE,OAAO;CACf,MAAM,YAAY,IAAI,IAAI,QAAQ,UAAU,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;CACnE,MAAM,0BAAU,IAAI,IAAmB;CACvC,MAAM,2BAAW,IAAI,IAA2B;CAChD,MAAM,eAAe,iBAAiB;CACtC,IAAI,SAAS;CAEb,MAAM,GAAG,KACP,8BAA8B,eAAe,+CAC/C;CACA,KAAK,MAAM,OAAO,MAAM,GAAG,MACzB,4BAA4B,gBAC9B,GACE,SAAS,IAAI,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAkB;CAGhE,MAAM,mBAAmB;EACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,mBAAmB;CACjD;CAEA,MAAM,cAAc,MAAc,KAAa,cAA+C;EAC5F;EACA,QAAQ,GAAG,YAAY,IAAI,IAAI,eAAe,GAAG,EAAE;EACnD;CACF;CAEA,eAAe,cAAc,MAAc,OAAqC;EAC9E,MAAM,GAAG,MACP,eAAe,eAAe,0FAC9B,CAAC,MAAM,KAAK,UAAU,KAAK,CAAC,CAC9B;EACA,SAAS,IAAI,MAAM,KAAK;CAC1B;CAEA,eAAe,gBAAgB,MAAc,YAAoD;EAC/F,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAU,GACjD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;EAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;EACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;EACrE,SAAS,OAAO,IAAI;CACtB;CAEA,eAAe,UACb,MACA,YACA,SAC+D;EAC/D,MAAM,YAAY,SAAS,qBAAqB;EAChD,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,MAAM,WAAiC,CAAC;EACxC,MAAM,YAAqC,CAAC;EAE5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,UAAU,GAAG;GACrD,MAAM,SAAS,SAAS,aAAa;GACrC,IAAI,UAAU,UAAU,UAAU,OAAO,SAAS,MAAM;IACtD,UAAU,OAAO;IACjB;GACF;GACA,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ;GAC1C,MAAM,OAAO,UAAU,EAAE,UAAU,UAAW,SAAqB;GACnE,MAAM,WAAW,UAAU,IAAI,KAAK,IAAI;GACxC,IAAI,YAAY,SAAS,OAAO,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,EAAE,KAAK,GAAG;GACzE,IAAI,CAAC,UAAU;IACb,IAAI,cAAc,SAChB,MAAM,IAAI,MACR,iCAAiC,IAAI,uBAAuB,KAAK,KAAK,cAAc,CAClF,GAAG,UAAU,KAAK,CACpB,CAAC,CACE,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CACpB,KAAK,IAAI,EAAE,kBAChB;IAGF,WAAE,SAAS,KAAK,cAAc,IAAI,WAAW,KAAK,KAAK,8BAA8B;IACrF,UAAU,OAAO;IACjB;GACF;GACA,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,SAAS,KAAK;IAAE,MAAM;IAAK;IAAM;IAAQ,OAAO,MAAM,SAAS,KAAK,KAAK,MAAM;GAAE,CAAC;GAClF,UAAU,OAAO;EACnB;EAgBA,OAAO;GAAE,OAdK,qBAAqB;IACjC;IACA;IACA,eAAe,YAAY;KAEzB,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,IAAI,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC,GAC3E,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;KAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;KACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;KACrE,SAAS,OAAO,IAAI;KACpB,QAAQ,OAAO,IAAI;IACrB;GACF,CACa;GAAG;EAAU;CAC5B;CAEA,OAAO;EACL,MAAM,gBAAsC;GAC1C,WAAW;GACX,OAAO,CAAC,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,UAAU,EAAE,KAAK,EAAE;EACtD;EAEA,YAAY,QAA2C;GACrD,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,EAAE,MAAM,YAAY,WAAW,aAAa;IAClD,IAAI,CAAC,cAAc,OAAO,KAAK,UAAU,CAAC,CAAC,WAAW,GACpD,MAAM,IAAI,MAAM,4CAA4C;IAE9D,MAAM,WAAW,SAAS,IAAI,IAAI;IAClC,IAAI,YAAY,QAAQ,IAAI,IAAI,GAAG;KACjC,IAAI,CAAC,WAAW,MAAM,IAAI,MAAM,UAAU,KAAK,iBAAiB;KAChE,MAAM,QACH,IAAI,IAAI,CAAC,EACR,MAAM,CAAC,CACR,YAAY,KAAA,CAAS;KACxB,QAAQ,OAAO,IAAI;KACnB,MAAM,gBAAgB,MAAM,UAAU,cAAc,CAAC,CAAC;IACxD;IACA,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,UAAU;IAC7D,MAAM,cAAc,MAAM,EAAE,YAAY,UAAU,CAAC;IACnD,IAAI,aAAa,KAAA,GAAW,MAAkC,WAAW;IACzE,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,SAAS,MAAc,SAAkD;GAGvE,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,SAAS,QAAQ,IAAI,IAAI;IAC/B,IAAI,QAAQ,OAAO;IACnB,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,CAAC,OAAO,OAAO;IACnB,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,MAAM,YAAY,OAAO;IAC5E,IAAI,KAAK,UAAU,SAAS,MAAM,KAAK,UAAU,MAAM,UAAU,GAC/D,MAAM,cAAc,MAAM,EAAE,YAAY,UAAU,CAAC;IAErD,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,MAAM,SAAS,MAAgC;GAC7C,WAAW;GACX,OAAO,SAAS,IAAI,IAAI;EAC1B;EAEA,MAAM,YAAY,MAA6B;GAC7C,WAAW;GACX,MAAM,OAAO,QAAQ,IAAI,IAAI;GAC7B,IAAI,MAAM;IACR,MAAM,KAAK,YAAY;IACvB;GACF;GACA,MAAM,QAAQ,SAAS,IAAI,IAAI;GAC/B,IAAI,OAAO,MAAM,gBAAgB,MAAM,MAAM,UAAU;EACzD;EAEA,MAAM,QAAuB;GAC3B,WAAW;GACX,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM;EAC1D;EAEA,MAAM,QAAuB;GAC3B,IAAI,QAAQ;GACZ,SAAS;GACT,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;GAC/E,QAAQ,MAAM;EAChB;CACF;AACF"}

package/dist/enumeration.d.ts

@@ -0,0 +1,18 @@
+import { BlockIds } from "./block-ids.js";
+import { Db } from "@statewalker/db-api";
+import { BlockReference, DocumentPath } from "@statewalker/indexer-api";
+//#region src/enumeration.d.ts
+export interface BlockEntry extends BlockReference {
+  id: number;
+}
+/** `SearchIndex` enumeration over a sub-index's member ids joined with the blocks table. */
+export interface Enumeration {
+  size(prefix?: DocumentPath): Promise<number>;
+  entries(prefix?: DocumentPath): Promise<BlockEntry[]>;
+  paths(prefix?: DocumentPath): AsyncGenerator<DocumentPath>;
+  refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;
+}
+/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */
+export declare function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string): Enumeration;
+//#endregion
+//# sourceMappingURL=enumeration.d.ts.map

package/dist/enumeration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enumeration.d.ts","names":[],"sources":["../src/enumeration.ts"],"mappings":";;;;iBAKiB,mBAAmB;EAClC;;;iBAIe;EACf,KAAK,SAAS,eAAe;EAC7B,QAAQ,SAAS,eAAe,QAAQ;EACxC,MAAM,SAAS,eAAe,eAAe;EAC7C,KAAK,SAAS,eAAe,eAAe;;;wBAI9B,eAAe,IAAI,IAAI,UAAU,UAAU,uBAAuB"}

package/dist/enumeration.js

@@ -0,0 +1,42 @@
+import { pathPrefixSql } from "./path-sql.js";
+//#region src/enumeration.ts
+/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */
+function newEnumeration(db, blockIds, memberIdsSql) {
+	const from = (prefix) => {
+		const scope = pathPrefixSql("b.path", prefix ? [prefix] : []);
+		return {
+			sql: `FROM (${memberIdsSql}) m JOIN ${blockIds.table} b ON b.id = m.id WHERE ${scope.sql}`,
+			params: scope.params
+		};
+	};
+	return {
+		async size(prefix) {
+			const f = from(prefix);
+			const rows = await db.query(`SELECT COUNT(*) AS n ${f.sql}`, f.params);
+			return Number(rows[0]?.n ?? 0);
+		},
+		async entries(prefix) {
+			const f = from(prefix);
+			return (await db.query(`SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`, f.params)).map((r) => ({
+				id: Number(r.id),
+				path: r.path,
+				blockId: r.block_id
+			}));
+		},
+		async *paths(prefix) {
+			const f = from(prefix);
+			const rows = await db.query(`SELECT DISTINCT b.path ${f.sql}`, f.params);
+			for (const r of rows) yield r.path;
+		},
+		async *refs(prefix) {
+			for (const e of await this.entries(prefix)) yield {
+				path: e.path,
+				blockId: e.blockId
+			};
+		}
+	};
+}
+//#endregion
+export { newEnumeration };
+
+//# sourceMappingURL=enumeration.js.map

package/dist/enumeration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"enumeration.js","names":[],"sources":["../src/enumeration.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { BlockReference, DocumentPath } from \"@statewalker/indexer-api\";\nimport type { BlockIds } from \"./block-ids.js\";\nimport { pathPrefixSql } from \"./path-sql.js\";\n\nexport interface BlockEntry extends BlockReference {\n  id: number;\n}\n\n/** `SearchIndex` enumeration over a sub-index's member ids joined with the blocks table. */\nexport interface Enumeration {\n  size(prefix?: DocumentPath): Promise<number>;\n  entries(prefix?: DocumentPath): Promise<BlockEntry[]>;\n  paths(prefix?: DocumentPath): AsyncGenerator<DocumentPath>;\n  refs(prefix?: DocumentPath): AsyncGenerator<BlockReference>;\n}\n\n/** `memberIdsSql` is a `SELECT … AS id` listing the sub-index's ids, e.g. `SELECT rowid AS id FROM t`. */\nexport function newEnumeration(db: Db, blockIds: BlockIds, memberIdsSql: string): Enumeration {\n  const from = (prefix?: DocumentPath) => {\n    const scope = pathPrefixSql(\"b.path\", prefix ? [prefix] : []);\n    return {\n      sql: `FROM (${memberIdsSql}) m JOIN ${blockIds.table} b ON b.id = m.id WHERE ${scope.sql}`,\n      params: scope.params,\n    };\n  };\n  return {\n    async size(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ n: number }>(`SELECT COUNT(*) AS n ${f.sql}`, f.params);\n      return Number(rows[0]?.n ?? 0);\n    },\n    async entries(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ id: number; path: string; block_id: string }>(\n        `SELECT b.id, b.path, b.block_id ${f.sql} ORDER BY b.id`,\n        f.params,\n      );\n      return rows.map((r) => ({\n        id: Number(r.id),\n        path: r.path as DocumentPath,\n        blockId: r.block_id,\n      }));\n    },\n    async *paths(prefix) {\n      const f = from(prefix);\n      const rows = await db.query<{ path: string }>(`SELECT DISTINCT b.path ${f.sql}`, f.params);\n      for (const r of rows) yield r.path as DocumentPath;\n    },\n    async *refs(prefix) {\n      for (const e of await this.entries(prefix)) yield { path: e.path, blockId: e.blockId };\n    },\n  };\n}\n"],"mappings":";;;AAkBA,SAAgB,eAAe,IAAQ,UAAoB,cAAmC;CAC5F,MAAM,QAAQ,WAA0B;EACtC,MAAM,QAAQ,cAAc,UAAU,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;EAC5D,OAAO;GACL,KAAK,SAAS,aAAa,WAAW,SAAS,MAAM,0BAA0B,MAAM;GACrF,QAAQ,MAAM;EAChB;CACF;CACA,OAAO;EACL,MAAM,KAAK,QAAQ;GACjB,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAqB,wBAAwB,EAAE,OAAO,EAAE,MAAM;GACpF,OAAO,OAAO,KAAK,EAAE,EAAE,KAAK,CAAC;EAC/B;EACA,MAAM,QAAQ,QAAQ;GACpB,MAAM,IAAI,KAAK,MAAM;GAKrB,QAAO,MAJY,GAAG,MACpB,mCAAmC,EAAE,IAAI,iBACzC,EAAE,MACJ,EAAA,CACY,KAAK,OAAO;IACtB,IAAI,OAAO,EAAE,EAAE;IACf,MAAM,EAAE;IACR,SAAS,EAAE;GACb,EAAE;EACJ;EACA,OAAO,MAAM,QAAQ;GACnB,MAAM,IAAI,KAAK,MAAM;GACrB,MAAM,OAAO,MAAM,GAAG,MAAwB,0BAA0B,EAAE,OAAO,EAAE,MAAM;GACzF,KAAK,MAAM,KAAK,MAAM,MAAM,EAAE;EAChC;EACA,OAAO,KAAK,QAAQ;GAClB,KAAK,MAAM,KAAK,MAAM,KAAK,QAAQ,MAAM,GAAG,MAAM;IAAE,MAAM,EAAE;IAAM,SAAS,EAAE;GAAQ;EACvF;CACF;AACF"}

package/dist/fts-provider.d.ts

@@ -0,0 +1,6 @@
+import { SqliteModalityProvider } from "./types.js";
+import { FullTextConfig, FullTextIndex } from "@statewalker/indexer-fulltext";
+//#region src/fts-provider.d.ts
+export declare function newSqliteFullTextProvider(): SqliteModalityProvider<FullTextConfig, FullTextIndex>;
+//#endregion
+//# sourceMappingURL=fts-provider.d.ts.map

package/dist/fts-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts-provider.d.ts","names":[],"sources":["../src/fts-provider.ts"],"mappings":";;;wBAmCgB,6BAA6B,uBAAuB,gBAAgB"}