@statewalker/indexer-sqlite 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/README.md +5 -1
  2. package/dist/create-sqlite-indexer.d.ts +3 -3
  3. package/dist/create-sqlite-indexer.d.ts.map +1 -1
  4. package/dist/create-sqlite-indexer.js +23 -151
  5. package/dist/create-sqlite-indexer.js.map +1 -1
  6. package/dist/ngram-provider.js +1 -0
  7. package/dist/ngram-provider.js.map +1 -1
  8. package/dist/types.d.ts +5 -4
  9. package/dist/types.d.ts.map +1 -1
  10. package/package.json +6 -6
  11. package/src/create-sqlite-indexer.ts +23 -208
  12. package/src/ngram-provider.ts +5 -0
  13. package/src/types.ts +5 -4
package/README.md CHANGED
@@ -120,6 +120,10 @@ gives a `Db` with the Durable Object's limits.
120
120
 
121
121
  ## Internals
122
122
 
123
+ ### The lifecycle is the shared SQL kernel
124
+
125
+ `createSqliteIndexer` is `createSqlKernelIndexer` from `@statewalker/indexer-core` over a SQLite storage (the manifest table and the per-index blocks table) and your providers. DuckDB and PGlite run the same kernel, so the load actions (adopt, skip, reinit, `onMissingProvider`) and the delete and close rules behave the same on all three. `SqliteModalityProvider` is the kernel's `SqlModalityProvider` over a SQLite context.
126
+
123
127
  ### Tables are prefixed per index and sub-index, and names are case-distinct
124
128
 
125
129
  ```
@@ -231,7 +235,7 @@ the earlier batches stay written.
231
235
  - **`truncated: true` is easy to miss.** `for await (const hit of sub.search(q))` drops the
232
236
  summary the generator returns; use `grep` to read it.
233
237
  - **Re-opening needs the same providers.** `getIndex` on an Index with a sub-index whose type has
234
- no registered provider throws `Cannot load index: sub-index "<name>" requires modality
238
+ no registered provider throws `Cannot load index "<index>": sub-index "<name>" requires modality
235
239
  "<type>" but only [...] are registered.`; pass `{ onMissingProvider: "warn" }` to open it
236
240
  without that sub-index.
237
241
 
package/dist/create-sqlite-indexer.d.ts CHANGED
@@ -9,9 +9,9 @@ export interface SqliteIndexerOptions {
9
9
  providers: SqliteModalityProvider[];
10
10
  }
11
11
  /**
12
- * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared
13
- * `{indexPrefix}blocks` id table per Index, and opens each sub-index through
14
- * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).
12
+ * `Indexer` over one SQLite `Db`: the shared SQL kernel (`createSqlKernelIndexer`) over SQLite
13
+ * storage. Keeps a manifest row per Index, one shared `{indexPrefix}blocks` id table per Index,
14
+ * and opens each sub-index through its provider. Search fuses sub-indexes by RRF.
15
15
  */
16
16
  export declare function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer>;
17
17
  //#endregion
package/dist/create-sqlite-indexer.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"create-sqlite-indexer.d.ts","names":[],"sources":["../src/create-sqlite-indexer.ts"],"mappings":";;;;iBAciB;;EAEf,IAAI;;EAEJ,WAAW;;;;;;;wBAuBS,oBAAoB,SAAS,uBAAuB,QAAQ"}
1
+ {"version":3,"file":"create-sqlite-indexer.d.ts","names":[],"sources":["../src/create-sqlite-indexer.ts"],"mappings":";;;;iBAMiB;;EAEf,IAAI;;EAEJ,WAAW;;;;;;;wBAgBS,oBAAoB,SAAS,uBAAuB,QAAQ"}
package/dist/create-sqlite-indexer.js CHANGED
@@ -1,169 +1,41 @@
1
1
  import { openBlockIds } from "./block-ids.js";
2
- import { createCompositeIndex, createSerialiser } from "@statewalker/indexer-core";
2
+ import { createSqlKernelIndexer } from "@statewalker/indexer-core";
3
3
  //#region src/create-sqlite-indexer.ts
4
4
  const MANIFEST_TABLE = "__indexer_manifest";
5
5
  /** Escapes everything but lower-case letters and digits: SQLite table names fold case, so "Docs" and "docs" must not collide. */
6
6
  const sanitizePrefix = (name) => name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);
7
7
  const indexPrefix = (name) => `idx_${sanitizePrefix(name)}__`;
8
8
  /**
9
- * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared
10
- * `{indexPrefix}blocks` id table per Index, and opens each sub-index through
11
- * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).
9
+ * `Indexer` over one SQLite `Db`: the shared SQL kernel (`createSqlKernelIndexer`) over SQLite
10
+ * storage. Keeps a manifest row per Index, one shared `{indexPrefix}blocks` id table per Index,
11
+ * and opens each sub-index through its provider. Search fuses sub-indexes by RRF.
12
12
  */
13
13
  async function createSqliteIndexer(options) {
14
14
  const { db } = options;
15
- const providers = new Map(options.providers.map((p) => [p.type, p]));
16
- const indexes = /* @__PURE__ */ new Map();
17
- const manifest = /* @__PURE__ */ new Map();
18
- const runExclusive = createSerialiser();
19
- let closed = false;
20
15
  await db.exec(`CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`);
21
- for (const row of await db.query(`SELECT name, config FROM ${MANIFEST_TABLE}`)) manifest.set(row.name, JSON.parse(row.config));
22
- const ensureOpen = () => {
23
- if (closed) throw new Error("Indexer is closed");
24
- };
25
- const contextFor = (name, sub, blockIds) => ({
26
- db,
27
- prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,
28
- blockIds
29
- });
30
- async function writeManifest(name, entry) {
31
- await db.query(`INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`, [name, JSON.stringify(entry)]);
32
- manifest.set(name, entry);
33
- }
34
- async function dropIndexTables(name, subIndexes) {
35
- const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
36
- for (const [sub, spec] of Object.entries(subIndexes)) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
37
- await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
38
- await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
39
- manifest.delete(name);
40
- }
41
- async function openIndex(name, subIndexes, options, metadata) {
42
- const onMissing = options?.onMissingProvider ?? "throw";
43
- const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
44
- const bindings = [];
45
- const effective = {};
46
- for (const [sub, saved] of Object.entries(subIndexes)) {
47
- const action = options?.subIndexes?.[sub];
48
- if (action && "skip" in action && action.skip === true) {
49
- effective[sub] = saved;
50
- continue;
51
- }
52
- const ctx = contextFor(name, sub, blockIds);
53
- const spec = action && !("skip" in action) ? action : saved;
54
- const provider = providers.get(spec.type);
55
- if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);
56
- if (!provider) {
57
- if (onMissing === "throw") throw new Error(`Cannot load index: sub-index "${sub}" requires modality "${spec.type}" but only [${[...providers.keys()].map((t) => `"${t}"`).join(", ")}] are registered.`);
58
- globalThis.console?.warn(`Sub-index "${sub}" (type "${spec.type}") has no provider; skipping.`);
59
- effective[sub] = saved;
60
- continue;
61
- }
62
- const { type, ...config } = spec;
63
- bindings.push({
64
- name: sub,
65
- type,
66
- config,
67
- index: await provider.open(ctx, config)
68
- });
69
- effective[sub] = spec;
70
- }
71
- const index = createCompositeIndex({
72
- name,
73
- metadata,
74
- bindings,
75
- onClose: async () => {
76
- if (indexes.get(name) === index) indexes.delete(name);
16
+ return createSqlKernelIndexer({
17
+ storage: {
18
+ readManifest: () => db.query(`SELECT name, config FROM ${MANIFEST_TABLE}`),
19
+ async writeManifest(name, config) {
20
+ await db.query(`INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`, [name, config]);
77
21
  },
78
- onDeleteIndex: async () => {
79
- for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
80
- await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
22
+ async deleteManifest(name) {
81
23
  await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
82
- manifest.delete(name);
83
- indexes.delete(name);
24
+ },
25
+ openShared: (name) => openBlockIds(db, `${indexPrefix(name)}blocks`),
26
+ async dropShared(_name, blockIds) {
27
+ await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
28
+ },
29
+ async subContext(name, subName, _type, blockIds) {
30
+ return {
31
+ db,
32
+ prefix: `${indexPrefix(name)}${sanitizePrefix(subName)}_`,
33
+ blockIds
34
+ };
84
35
  }
85
- });
86
- return {
87
- index,
88
- effective
89
- };
90
- }
91
- return {
92
- async getIndexNames() {
93
- ensureOpen();
94
- return [...manifest.entries()].map(([name, entry]) => entry.metadata !== void 0 ? {
95
- name,
96
- metadata: entry.metadata
97
- } : { name });
98
36
  },
99
- createIndex(params) {
100
- return runExclusive(async () => {
101
- ensureOpen();
102
- const { name, subIndexes, overwrite, metadata } = params;
103
- if (!subIndexes || Object.keys(subIndexes).length === 0) throw new Error("Cannot create index without any subIndexes");
104
- const existing = manifest.get(name);
105
- if (existing || indexes.has(name)) {
106
- if (!overwrite) throw new Error(`Index "${name}" already exists`);
107
- await indexes.get(name)?.close().catch(() => void 0);
108
- indexes.delete(name);
109
- await dropIndexTables(name, existing?.subIndexes ?? {});
110
- }
111
- const { index, effective } = await openIndex(name, subIndexes, void 0, metadata);
112
- await writeManifest(name, {
113
- subIndexes: effective,
114
- ...metadata !== void 0 ? { metadata } : {}
115
- });
116
- indexes.set(name, index);
117
- return index;
118
- });
119
- },
120
- getIndex(name, options) {
121
- return runExclusive(async () => {
122
- ensureOpen();
123
- const cached = indexes.get(name);
124
- if (cached) return cached;
125
- const entry = manifest.get(name);
126
- if (!entry) return null;
127
- const { index, effective } = await openIndex(name, entry.subIndexes, options, entry.metadata);
128
- if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) await writeManifest(name, {
129
- ...entry,
130
- subIndexes: effective
131
- });
132
- indexes.set(name, index);
133
- return index;
134
- });
135
- },
136
- async hasIndex(name) {
137
- ensureOpen();
138
- return manifest.has(name);
139
- },
140
- deleteIndex(name) {
141
- return runExclusive(async () => {
142
- ensureOpen();
143
- const live = indexes.get(name);
144
- if (live) {
145
- await live.deleteIndex();
146
- return;
147
- }
148
- const entry = manifest.get(name);
149
- if (entry) await dropIndexTables(name, entry.subIndexes);
150
- });
151
- },
152
- flush() {
153
- return runExclusive(async () => {
154
- ensureOpen();
155
- for (const index of indexes.values()) await index.flush();
156
- });
157
- },
158
- close() {
159
- return runExclusive(async () => {
160
- if (closed) return;
161
- closed = true;
162
- for (const index of [...indexes.values()]) await index.close().catch(() => void 0);
163
- indexes.clear();
164
- });
165
- }
166
- };
37
+ providers: options.providers
38
+ });
167
39
  }
168
40
  //#endregion
169
41
  export { createSqliteIndexer };
package/dist/create-sqlite-indexer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"create-sqlite-indexer.js","names":[],"sources":["../src/create-sqlite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type {\n AnySubIndexBinding,\n CreateIndexParams,\n GetIndexOptions,\n Index,\n Indexer,\n IndexInfo,\n Metadata,\n} from \"@statewalker/indexer-api\";\nimport { createCompositeIndex, createSerialiser } from \"@statewalker/indexer-core\";\nimport { type BlockIds, openBlockIds } from \"./block-ids.js\";\nimport type { SqliteModalityContext, SqliteModalityProvider } from \"./types.js\";\n\nexport interface SqliteIndexerOptions {\n /** The database. The indexer never closes it. */\n db: Db;\n /** One provider per modality type, e.g. FTS5, vector, n-gram. */\n providers: SqliteModalityProvider[];\n}\n\ntype SubSpec = { type: string } & Record<string, unknown>;\ninterface IndexManifest {\n subIndexes: Record<string, SubSpec>;\n /** Index-level metadata from `createIndex` (absent in manifests written before it was saved). */\n metadata?: Metadata;\n}\n\nconst MANIFEST_TABLE = \"__indexer_manifest\";\n\n/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so \"Docs\" and \"docs\" must not collide. */\nconst sanitizePrefix = (name: string) =>\n name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);\n\nconst indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;\n\n/**\n * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared\n * `{indexPrefix}blocks` id table per Index, and opens each sub-index through\n * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).\n */\nexport async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {\n const { db } = options;\n const providers = new Map(options.providers.map((p) => [p.type, p]));\n const indexes = new Map<string, Index>();\n const manifest = new Map<string, IndexManifest>();\n const runExclusive = createSerialiser();\n let closed = false;\n\n await db.exec(\n `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,\n );\n for (const row of await db.query<{ name: string; config: string }>(\n `SELECT name, config FROM ${MANIFEST_TABLE}`,\n )) {\n manifest.set(row.name, JSON.parse(row.config) as IndexManifest);\n }\n\n const ensureOpen = () => {\n if (closed) throw new Error(\"Indexer is closed\");\n };\n\n const contextFor = (name: string, sub: string, blockIds: BlockIds): SqliteModalityContext => ({\n db,\n prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,\n blockIds,\n });\n\n async function writeManifest(name: string, entry: IndexManifest): Promise<void> {\n await db.query(\n `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,\n [name, JSON.stringify(entry)],\n );\n manifest.set(name, entry);\n }\n\n async function dropIndexTables(name: string, subIndexes: Record<string, SubSpec>): Promise<void> {\n const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n for (const [sub, spec] of Object.entries(subIndexes)) {\n await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n }\n await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n manifest.delete(name);\n }\n\n async function openIndex(\n name: string,\n subIndexes: Record<string, SubSpec>,\n options?: GetIndexOptions,\n metadata?: Metadata,\n ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {\n const onMissing = options?.onMissingProvider ?? \"throw\";\n const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);\n const bindings: AnySubIndexBinding[] = [];\n const effective: Record<string, SubSpec> = {};\n\n for (const [sub, saved] of Object.entries(subIndexes)) {\n const action = options?.subIndexes?.[sub];\n if (action && \"skip\" in action && action.skip === true) {\n effective[sub] = saved;\n continue;\n }\n const ctx = contextFor(name, sub, blockIds);\n const spec = action && !(\"skip\" in action) ? (action as SubSpec) : saved;\n const provider = providers.get(spec.type);\n if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);\n if (!provider) {\n if (onMissing === \"throw\") {\n throw new Error(\n `Cannot load index: sub-index \"${sub}\" requires modality \"${spec.type}\" but only [${[\n ...providers.keys(),\n ]\n .map((t) => `\"${t}\"`)\n .join(\", \")}] are registered.`,\n );\n }\n const g = globalThis as { console?: { warn(...args: unknown[]): void } };\n g.console?.warn(`Sub-index \"${sub}\" (type \"${spec.type}\") has no provider; skipping.`);\n effective[sub] = saved;\n continue;\n }\n const { type, ...config } = spec;\n bindings.push({ name: sub, type, config, index: await provider.open(ctx, config) });\n effective[sub] = spec;\n }\n\n const index: Index = createCompositeIndex({\n name,\n metadata,\n bindings,\n // A closed instance leaves the cache, so getIndex opens a new one.\n onClose: async () => {\n if (indexes.get(name) === index) indexes.delete(name);\n },\n onDeleteIndex: async () => {\n // Bound sub-indexes dropped themselves; skipped ones still own tables.\n for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {\n await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));\n }\n await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n manifest.delete(name);\n indexes.delete(name);\n },\n });\n return { index, effective };\n }\n\n return {\n async getIndexNames(): Promise<IndexInfo[]> {\n ensureOpen();\n return [...manifest.entries()].map(([name, entry]) =>\n entry.metadata !== undefined ? { name, metadata: entry.metadata } : { name },\n );\n },\n\n createIndex(params: CreateIndexParams): Promise<Index> {\n return runExclusive(async () => {\n ensureOpen();\n const { name, subIndexes, overwrite, metadata } = params;\n if (!subIndexes || Object.keys(subIndexes).length === 0) {\n throw new Error(\"Cannot create index without any subIndexes\");\n }\n const existing = manifest.get(name);\n if (existing || indexes.has(name)) {\n if (!overwrite) throw new Error(`Index \"${name}\" already exists`);\n await indexes\n .get(name)\n ?.close()\n .catch(() => undefined);\n indexes.delete(name);\n await dropIndexTables(name, existing?.subIndexes ?? {});\n }\n const { index, effective } = await openIndex(name, subIndexes, undefined, metadata);\n await writeManifest(name, {\n subIndexes: effective,\n ...(metadata !== undefined ? { metadata } : {}),\n });\n indexes.set(name, index);\n return index;\n });\n },\n\n getIndex(name: string, options?: GetIndexOptions): Promise<Index | null> {\n // Serialised with createIndex so concurrent first opens open once\n // (a second reinit would otherwise drop tables the first just wrote).\n return runExclusive(async () => {\n ensureOpen();\n const cached = indexes.get(name);\n if (cached) return cached;\n const entry = manifest.get(name);\n if (!entry) return null;\n const { index, effective } = await openIndex(\n name,\n entry.subIndexes,\n options,\n entry.metadata,\n );\n if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {\n await writeManifest(name, { ...entry, subIndexes: effective });\n }\n indexes.set(name, index);\n return index;\n });\n },\n\n async hasIndex(name: string): Promise<boolean> {\n ensureOpen();\n return manifest.has(name);\n },\n\n deleteIndex(name: string): Promise<void> {\n return runExclusive(async () => {\n ensureOpen();\n const live = indexes.get(name);\n if (live) {\n await live.deleteIndex();\n return;\n }\n const entry = manifest.get(name);\n if (entry) await dropIndexTables(name, entry.subIndexes);\n });\n },\n\n flush(): Promise<void> {\n return runExclusive(async () => {\n ensureOpen();\n for (const index of indexes.values()) await index.flush();\n });\n },\n\n close(): Promise<void> {\n return runExclusive(async () => {\n if (closed) return;\n closed = true;\n for (const index of [...indexes.values()]) await index.close().catch(() => undefined);\n indexes.clear();\n });\n },\n } satisfies Indexer;\n}\n"],"mappings":";;;AA4BA,MAAM,iBAAiB;;AAGvB,MAAM,kBAAkB,SACtB,KAAK,QAAQ,eAAe,OAAO,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;AAE5D,MAAM,eAAe,SAAiB,OAAO,eAAe,IAAI,EAAE;;;;;;AAOlE,eAAsB,oBAAoB,SAAiD;CACzF,MAAM,EAAE,OAAO;CACf,MAAM,YAAY,IAAI,IAAI,QAAQ,UAAU,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;CACnE,MAAM,0BAAU,IAAI,IAAmB;CACvC,MAAM,2BAAW,IAAI,IAA2B;CAChD,MAAM,eAAe,iBAAiB;CACtC,IAAI,SAAS;CAEb,MAAM,GAAG,KACP,8BAA8B,eAAe,+CAC/C;CACA,KAAK,MAAM,OAAO,MAAM,GAAG,MACzB,4BAA4B,gBAC9B,GACE,SAAS,IAAI,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,CAAkB;CAGhE,MAAM,mBAAmB;EACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,mBAAmB;CACjD;CAEA,MAAM,cAAc,MAAc,KAAa,cAA+C;EAC5F;EACA,QAAQ,GAAG,YAAY,IAAI,IAAI,eAAe,GAAG,EAAE;EACnD;CACF;CAEA,eAAe,cAAc,MAAc,OAAqC;EAC9E,MAAM,GAAG,MACP,eAAe,eAAe,0FAC9B,CAAC,MAAM,KAAK,UAAU,KAAK,CAAC,CAC9B;EACA,SAAS,IAAI,MAAM,KAAK;CAC1B;CAEA,eAAe,gBAAgB,MAAc,YAAoD;EAC/F,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,UAAU,GACjD,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;EAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;EACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;EACrE,SAAS,OAAO,IAAI;CACtB;CAEA,eAAe,UACb,MACA,YACA,SACA,UAC+D;EAC/D,MAAM,YAAY,SAAS,qBAAqB;EAChD,MAAM,WAAW,MAAM,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;EACpE,MAAM,WAAiC,CAAC;EACxC,MAAM,YAAqC,CAAC;EAE5C,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,UAAU,GAAG;GACrD,MAAM,SAAS,SAAS,aAAa;GACrC,IAAI,UAAU,UAAU,UAAU,OAAO,SAAS,MAAM;IACtD,UAAU,OAAO;IACjB;GACF;GACA,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ;GAC1C,MAAM,OAAO,UAAU,EAAE,UAAU,UAAW,SAAqB;GACnE,MAAM,WAAW,UAAU,IAAI,KAAK,IAAI;GACxC,IAAI,YAAY,SAAS,OAAO,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,EAAE,KAAK,GAAG;GACzE,IAAI,CAAC,UAAU;IACb,IAAI,cAAc,SAChB,MAAM,IAAI,MACR,iCAAiC,IAAI,uBAAuB,KAAK,KAAK,cAAc,CAClF,GAAG,UAAU,KAAK,CACpB,CAAC,CACE,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CACpB,KAAK,IAAI,EAAE,kBAChB;IAGF,WAAE,SAAS,KAAK,cAAc,IAAI,WAAW,KAAK,KAAK,8BAA8B;IACrF,UAAU,OAAO;IACjB;GACF;GACA,MAAM,EAAE,MAAM,GAAG,WAAW;GAC5B,SAAS,KAAK;IAAE,MAAM;IAAK;IAAM;IAAQ,OAAO,MAAM,SAAS,KAAK,KAAK,MAAM;GAAE,CAAC;GAClF,UAAU,OAAO;EACnB;EAEA,MAAM,QAAe,qBAAqB;GACxC;GACA;GACA;GAEA,SAAS,YAAY;IACnB,IAAI,QAAQ,IAAI,IAAI,MAAM,OAAO,QAAQ,OAAO,IAAI;GACtD;GACA,eAAe,YAAY;IAEzB,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,SAAS,IAAI,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC,GAC3E,MAAM,UAAU,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,WAAW,MAAM,KAAK,QAAQ,CAAC;IAEtE,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;IACtD,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;IACrE,SAAS,OAAO,IAAI;IACpB,QAAQ,OAAO,IAAI;GACrB;EACF,CAAC;EACD,OAAO;GAAE;GAAO;EAAU;CAC5B;CAEA,OAAO;EACL,MAAM,gBAAsC;GAC1C,WAAW;GACX,OAAO,CAAC,GAAG,SAAS,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,WACzC,MAAM,aAAa,KAAA,IAAY;IAAE;IAAM,UAAU,MAAM;GAAS,IAAI,EAAE,KAAK,CAC7E;EACF;EAEA,YAAY,QAA2C;GACrD,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,EAAE,MAAM,YAAY,WAAW,aAAa;IAClD,IAAI,CAAC,cAAc,OAAO,KAAK,UAAU,CAAC,CAAC,WAAW,GACpD,MAAM,IAAI,MAAM,4CAA4C;IAE9D,MAAM,WAAW,SAAS,IAAI,IAAI;IAClC,IAAI,YAAY,QAAQ,IAAI,IAAI,GAAG;KACjC,IAAI,CAAC,WAAW,MAAM,IAAI,MAAM,UAAU,KAAK,iBAAiB;KAChE,MAAM,QACH,IAAI,IAAI,CAAC,EACR,MAAM,CAAC,CACR,YAAY,KAAA,CAAS;KACxB,QAAQ,OAAO,IAAI;KACnB,MAAM,gBAAgB,MAAM,UAAU,cAAc,CAAC,CAAC;IACxD;IACA,MAAM,EAAE,OAAO,cAAc,MAAM,UAAU,MAAM,YAAY,KAAA,GAAW,QAAQ;IAClF,MAAM,cAAc,MAAM;KACxB,YAAY;KACZ,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;IAC/C,CAAC;IACD,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,SAAS,MAAc,SAAkD;GAGvE,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,SAAS,QAAQ,IAAI,IAAI;IAC/B,IAAI,QAAQ,OAAO;IACnB,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,CAAC,OAAO,OAAO;IACnB,MAAM,EAAE,OAAO,cAAc,MAAM,UACjC,MACA,MAAM,YACN,SACA,MAAM,QACR;IACA,IAAI,KAAK,UAAU,SAAS,MAAM,KAAK,UAAU,MAAM,UAAU,GAC/D,MAAM,cAAc,MAAM;KAAE,GAAG;KAAO,YAAY;IAAU,CAAC;IAE/D,QAAQ,IAAI,MAAM,KAAK;IACvB,OAAO;GACT,CAAC;EACH;EAEA,MAAM,SAAS,MAAgC;GAC7C,WAAW;GACX,OAAO,SAAS,IAAI,IAAI;EAC1B;EAEA,YAAY,MAA6B;GACvC,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,MAAM,OAAO,QAAQ,IAAI,IAAI;IAC7B,IAAI,MAAM;KACR,MAAM,KAAK,YAAY;KACvB;IACF;IACA,MAAM,QAAQ,SAAS,IAAI,IAAI;IAC/B,IAAI,OAAO,MAAM,gBAAgB,MAAM,MAAM,UAAU;GACzD,CAAC;EACH;EAEA,QAAuB;GACrB,OAAO,aAAa,YAAY;IAC9B,WAAW;IACX,KAAK,MAAM,SAAS,QAAQ,OAAO,GAAG,MAAM,MAAM,MAAM;GAC1D,CAAC;EACH;EAEA,QAAuB;GACrB,OAAO,aAAa,YAAY;IAC9B,IAAI,QAAQ;IACZ,SAAS;IACT,KAAK,MAAM,SAAS,CAAC,GAAG,QAAQ,OAAO,CAAC,GAAG,MAAM,MAAM,MAAM,CAAC,CAAC,YAAY,KAAA,CAAS;IACpF,QAAQ,MAAM;GAChB,CAAC;EACH;CACF;AACF"}
1
+ {"version":3,"file":"create-sqlite-indexer.js","names":[],"sources":["../src/create-sqlite-indexer.ts"],"sourcesContent":["import type { Db } from \"@statewalker/db-api\";\nimport type { Indexer } from \"@statewalker/indexer-api\";\nimport { createSqlKernelIndexer, type SqlKernelStorage } from \"@statewalker/indexer-core\";\nimport { type BlockIds, openBlockIds } from \"./block-ids.js\";\nimport type { SqliteModalityContext, SqliteModalityProvider } from \"./types.js\";\n\nexport interface SqliteIndexerOptions {\n /** The database. The indexer never closes it. */\n db: Db;\n /** One provider per modality type, e.g. FTS5, vector, n-gram. */\n providers: SqliteModalityProvider[];\n}\n\nconst MANIFEST_TABLE = \"__indexer_manifest\";\n\n/** Escapes everything but lower-case letters and digits: SQLite table names fold case, so \"Docs\" and \"docs\" must not collide. */\nconst sanitizePrefix = (name: string) =>\n name.replace(/[^a-z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`);\n\nconst indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;\n\n/**\n * `Indexer` over one SQLite `Db`: the shared SQL kernel (`createSqlKernelIndexer`) over SQLite\n * storage. Keeps a manifest row per Index, one shared `{indexPrefix}blocks` id table per Index,\n * and opens each sub-index through its provider. Search fuses sub-indexes by RRF.\n */\nexport async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {\n const { db } = options;\n await db.exec(\n `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,\n );\n\n const storage: SqlKernelStorage<BlockIds, SqliteModalityContext> = {\n readManifest: () =>\n db.query<{ name: string; config: string }>(`SELECT name, config FROM ${MANIFEST_TABLE}`),\n async writeManifest(name, config) {\n await db.query(\n `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,\n [name, config],\n );\n },\n async deleteManifest(name) {\n await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);\n },\n openShared: (name) => openBlockIds(db, `${indexPrefix(name)}blocks`),\n async dropShared(_name, blockIds) {\n await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);\n },\n // SQLite table names have always escaped upper case and have no length limit, so there is\n // no earlier naming scheme to fall back to.\n async subContext(name, subName, _type, blockIds) {\n return { db, prefix: `${indexPrefix(name)}${sanitizePrefix(subName)}_`, blockIds };\n },\n };\n\n return createSqlKernelIndexer({ storage, providers: options.providers });\n}\n"],"mappings":";;;AAaA,MAAM,iBAAiB;;AAGvB,MAAM,kBAAkB,SACtB,KAAK,QAAQ,eAAe,OAAO,IAAI,GAAG,WAAW,CAAC,EAAE,EAAE;AAE5D,MAAM,eAAe,SAAiB,OAAO,eAAe,IAAI,EAAE;;;;;;AAOlE,eAAsB,oBAAoB,SAAiD;CACzF,MAAM,EAAE,OAAO;CACf,MAAM,GAAG,KACP,8BAA8B,eAAe,+CAC/C;CAyBA,OAAO,uBAAuB;EAAE,SAAA;GAtB9B,oBACE,GAAG,MAAwC,4BAA4B,gBAAgB;GACzF,MAAM,cAAc,MAAM,QAAQ;IAChC,MAAM,GAAG,MACP,eAAe,eAAe,0FAC9B,CAAC,MAAM,MAAM,CACf;GACF;GACA,MAAM,eAAe,MAAM;IACzB,MAAM,GAAG,MAAM,eAAe,eAAe,kBAAkB,CAAC,IAAI,CAAC;GACvE;GACA,aAAa,SAAS,aAAa,IAAI,GAAG,YAAY,IAAI,EAAE,OAAO;GACnE,MAAM,WAAW,OAAO,UAAU;IAChC,MAAM,GAAG,KAAK,wBAAwB,SAAS,OAAO;GACxD;GAGA,MAAM,WAAW,MAAM,SAAS,OAAO,UAAU;IAC/C,OAAO;KAAE;KAAI,QAAQ,GAAG,YAAY,IAAI,IAAI,eAAe,OAAO,EAAE;KAAI;IAAS;GACnF;EAGoC;EAAG,WAAW,QAAQ;CAAU,CAAC;AACzE"}
package/dist/ngram-provider.js CHANGED
@@ -43,6 +43,7 @@ function newSqliteNgramProvider() {
43
43
  },
44
44
  async open({ db, prefix, blockIds }, config) {
45
45
  const storeKind = config.store ?? "fts5";
46
+ if (storeKind !== "fts5" && storeKind !== "postings") throw new Error(`ngram: store "${storeKind}" is not available on this backend (it has "fts5" and "postings")`);
46
47
  const store = new NgramStore({
47
48
  db,
48
49
  prefix: `${prefix}ng_`,
package/dist/ngram-provider.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n NGRAM_TYPE,\n type NgramBlock,\n type NgramConfig,\n type NgramIndex,\n type NgramMatch,\n type NgramQuery,\n type NgramResult,\n type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n const breaks: number[] = [];\n for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n let line = 1;\n let b = 0;\n return [...matches]\n .sort((x, y) => x.start - y.start)\n .map(({ start, end }) => {\n while (b < breaks.length && (breaks[b] as number) < start) {\n b++;\n line++;\n }\n return { start, end, line };\n });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n return {\n type: NGRAM_TYPE,\n\n async drop({ db, prefix }) {\n await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n },\n\n async open({ db, prefix, blockIds }, config) {\n const storeKind = config.store ?? \"fts5\";\n const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n await store.init();\n const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>\n ensureOpen(),\n );\n let closed = false;\n const ensureOpen = () => {\n if (closed) throw new Error(\"NgramIndex is closed\");\n };\n\n const index: NgramIndex = {\n async getIndexInfo() {\n return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n },\n\n async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n ensureOpen();\n const allowed = params.paths?.length\n ? new Set(\n await blockIds.select(\n db,\n params.paths.map((path) => ({ path })),\n ),\n )\n : undefined;\n const result = await store.search(params.pattern, {\n regex: params.regex ?? false,\n caseSensitive: params.caseSensitive ?? true,\n limit: params.topK ?? 100,\n ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n });\n const hits = result.hits.filter((h) => h.matches.length > 0);\n const ids = hits.map((h) => h.id);\n const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n // Array sort is stable: equal counts keep the store's ascending-id order.\n const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n for (const h of ranked) {\n const ref = refs.get(h.id);\n if (!ref) continue;\n const count = h.matches.length;\n yield {\n ...ref,\n score: count,\n count,\n matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n };\n }\n return { truncated: result.truncated };\n },\n\n async addDocument(blocks: NgramBlock[]) {\n ensureOpen();\n for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n const batch = blocks.slice(i, i + UPSERT_BATCH);\n const ids = await blockIds.ensure(db, batch);\n await store.upsert(\n ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n );\n }\n },\n\n async addDocuments(batches) {\n for await (const batch of batches) await this.addDocument(batch);\n },\n\n async deleteDocuments(selectors) {\n ensureOpen();\n const sels: PathSelector[] = [];\n for await (const s of toAsyncIterable(selectors)) sels.push(s);\n const ids = await blockIds.select(db, sels);\n if (ids.length === 0) return;\n await store.delete(ids);\n },\n\n getSize: (p?: DocumentPath) => list.size(p),\n getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n ensureOpen();\n const entries = await list.entries(p);\n const texts = await store.getTexts(entries.map((e) => e.id));\n for (const e of entries) {\n const content = texts.get(e.id);\n if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n }\n },\n\n async close() {\n closed = true;\n },\n async flush() {},\n async deleteIndex() {\n closed = true;\n await store.drop();\n },\n };\n return index;\n },\n };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,iBAChE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAyFA,OAAO;IAtFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}
1
+ {"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n NGRAM_TYPE,\n type NgramBlock,\n type NgramConfig,\n type NgramIndex,\n type NgramMatch,\n type NgramQuery,\n type NgramResult,\n type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n const breaks: number[] = [];\n for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n let line = 1;\n let b = 0;\n return [...matches]\n .sort((x, y) => x.start - y.start)\n .map(({ start, end }) => {\n while (b < breaks.length && (breaks[b] as number) < start) {\n b++;\n line++;\n }\n return { start, end, line };\n });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n return {\n type: NGRAM_TYPE,\n\n async drop({ db, prefix }) {\n await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n },\n\n async open({ db, prefix, blockIds }, config) {\n const storeKind = config.store ?? \"fts5\";\n if (storeKind !== \"fts5\" && storeKind !== \"postings\") {\n throw new Error(\n `ngram: store \"${storeKind}\" is not available on this backend (it has \"fts5\" and \"postings\")`,\n );\n }\n const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n await store.init();\n const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>\n ensureOpen(),\n );\n let closed = false;\n const ensureOpen = () => {\n if (closed) throw new Error(\"NgramIndex is closed\");\n };\n\n const index: NgramIndex = {\n async getIndexInfo() {\n return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n },\n\n async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n ensureOpen();\n const allowed = params.paths?.length\n ? new Set(\n await blockIds.select(\n db,\n params.paths.map((path) => ({ path })),\n ),\n )\n : undefined;\n const result = await store.search(params.pattern, {\n regex: params.regex ?? false,\n caseSensitive: params.caseSensitive ?? true,\n limit: params.topK ?? 100,\n ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n });\n const hits = result.hits.filter((h) => h.matches.length > 0);\n const ids = hits.map((h) => h.id);\n const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n // Array sort is stable: equal counts keep the store's ascending-id order.\n const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n for (const h of ranked) {\n const ref = refs.get(h.id);\n if (!ref) continue;\n const count = h.matches.length;\n yield {\n ...ref,\n score: count,\n count,\n matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n };\n }\n return { truncated: result.truncated };\n },\n\n async addDocument(blocks: NgramBlock[]) {\n ensureOpen();\n for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n const batch = blocks.slice(i, i + UPSERT_BATCH);\n const ids = await blockIds.ensure(db, batch);\n await store.upsert(\n ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n );\n }\n },\n\n async addDocuments(batches) {\n for await (const batch of batches) await this.addDocument(batch);\n },\n\n async deleteDocuments(selectors) {\n ensureOpen();\n const sels: PathSelector[] = [];\n for await (const s of toAsyncIterable(selectors)) sels.push(s);\n const ids = await blockIds.select(db, sels);\n if (ids.length === 0) return;\n await store.delete(ids);\n },\n\n getSize: (p?: DocumentPath) => list.size(p),\n getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n ensureOpen();\n const entries = await list.entries(p);\n const texts = await store.getTexts(entries.map((e) => e.id));\n for (const e of entries) {\n const content = texts.get(e.id);\n if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n }\n },\n\n async close() {\n closed = true;\n },\n async flush() {},\n async deleteIndex() {\n closed = true;\n await store.drop();\n },\n };\n return index;\n },\n };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,IAAI,cAAc,UAAU,cAAc,YACxC,MAAM,IAAI,MACR,iBAAiB,UAAU,kEAC7B;GAEF,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,iBAChE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAyFA,OAAO;IAtFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}
package/dist/types.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { BlockIds } from "./block-ids.js";
2
+ import { SqlModalityProvider } from "@statewalker/indexer-core";
2
3
  import { Db } from "@statewalker/db-api";
3
4
  import { ScoredHit, SearchIndex } from "@statewalker/indexer-api";
4
5
  //#region src/types.d.ts
@@ -11,11 +12,11 @@ export interface SqliteModalityContext {
11
12
  blockIds: BlockIds;
12
13
  }
13
14
  /**
14
- * A modality backed by tables in one SQLite `Db`. Unlike `ModalityProvider`,
15
- * opening is async (tables are created, existing data is adopted) and the
16
- * provider can drop its tables without an open instance.
15
+ * A modality backed by tables in one SQLite `Db`: the SQL kernel's `SqlModalityProvider` over a
16
+ * SQLite context. Unlike `ModalityProvider`, opening is async (tables are created, existing data
17
+ * is adopted) and the provider can drop its tables without an open instance.
17
18
  */
18
- export interface SqliteModalityProvider<C = unknown, I extends SearchIndex<unknown, unknown, ScoredHit> = SearchIndex<unknown, unknown, ScoredHit>> {
19
+ export interface SqliteModalityProvider<C = unknown, I extends SearchIndex<unknown, unknown, ScoredHit> = SearchIndex<unknown, unknown, ScoredHit>> extends SqlModalityProvider<SqliteModalityContext, C, I> {
19
20
  readonly type: string;
20
21
  open(ctx: SqliteModalityContext, config: C): Promise<I>;
21
22
  drop(ctx: SqliteModalityContext): Promise<void>;
package/dist/types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;iBAIiB;;EAEf,IAAI;;EAEJ;;EAEA,UAAU;;;;;;;iBAQK,uBACf,aACA,UAAU,8BAA8B,aAAa,8BAA8B;WAE1E;EACT,KAAK,KAAK,uBAAuB,QAAQ,IAAI,QAAQ;EACrD,KAAK,KAAK,wBAAwB"}
1
+ {"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;;;iBAKiB;;EAEf,IAAI;;EAEJ;;EAEA,UAAU;;;;;;;iBAQK,uBACf,aACA,UAAU,8BAA8B,aAAa,8BAA8B,oBAC3E,oBAAoB,uBAAuB,GAAG;WAC7C;EACT,KAAK,KAAK,uBAAuB,QAAQ,IAAI,QAAQ;EACrD,KAAK,KAAK,wBAAwB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/indexer-sqlite",
3
- "version": "0.1.2",
3
+ "version": "0.3.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "SQLite-backed indexer: FTS5, vector and n-gram/grep sub-indexes in one @statewalker/db-api Db (Node, browser worker, Durable Objects).",
@@ -34,11 +34,11 @@
34
34
  "@statewalker/db-api": "^0.1.0",
35
35
  "@statewalker/db-sqlite": "^0.1.0",
36
36
  "@statewalker/db-sqlite-index": "^0.1.0",
37
- "@statewalker/indexer-api": "^0.1.4",
38
- "@statewalker/indexer-core": "^0.1.4",
39
- "@statewalker/indexer-fulltext": "^0.1.5",
40
- "@statewalker/indexer-ngram": "^0.1.1",
41
- "@statewalker/indexer-vector": "^0.1.5"
37
+ "@statewalker/indexer-api": "^0.1.5",
38
+ "@statewalker/indexer-core": "^0.3.0",
39
+ "@statewalker/indexer-fulltext": "^0.1.6",
40
+ "@statewalker/indexer-ngram": "^0.2.0",
41
+ "@statewalker/indexer-vector": "^0.1.6"
42
42
  },
43
43
  "devDependencies": {
44
44
  "@statewalker/db-sqlite-do": "^0.1.0",
package/src/create-sqlite-indexer.ts CHANGED
@@ -1,14 +1,6 @@
1
1
  import type { Db } from "@statewalker/db-api";
2
- import type {
3
- AnySubIndexBinding,
4
- CreateIndexParams,
5
- GetIndexOptions,
6
- Index,
7
- Indexer,
8
- IndexInfo,
9
- Metadata,
10
- } from "@statewalker/indexer-api";
11
- import { createCompositeIndex, createSerialiser } from "@statewalker/indexer-core";
2
+ import type { Indexer } from "@statewalker/indexer-api";
3
+ import { createSqlKernelIndexer, type SqlKernelStorage } from "@statewalker/indexer-core";
12
4
  import { type BlockIds, openBlockIds } from "./block-ids.js";
13
5
  import type { SqliteModalityContext, SqliteModalityProvider } from "./types.js";
14
6
 
@@ -19,13 +11,6 @@ export interface SqliteIndexerOptions {
19
11
  providers: SqliteModalityProvider[];
20
12
  }
21
13
 
22
- type SubSpec = { type: string } & Record<string, unknown>;
23
- interface IndexManifest {
24
- subIndexes: Record<string, SubSpec>;
25
- /** Index-level metadata from `createIndex` (absent in manifests written before it was saved). */
26
- metadata?: Metadata;
27
- }
28
-
29
14
  const MANIFEST_TABLE = "__indexer_manifest";
30
15
 
31
16
  /** Escapes everything but lower-case letters and digits: SQLite table names fold case, so "Docs" and "docs" must not collide. */
@@ -35,208 +20,38 @@ const sanitizePrefix = (name: string) =>
35
20
  const indexPrefix = (name: string) => `idx_${sanitizePrefix(name)}__`;
36
21
 
37
22
  /**
38
- * `Indexer` over one SQLite `Db`. Keeps a manifest row per Index, one shared
39
- * `{indexPrefix}blocks` id table per Index, and opens each sub-index through
40
- * its provider. Search fuses sub-indexes by RRF (`createCompositeIndex`).
23
+ * `Indexer` over one SQLite `Db`: the shared SQL kernel (`createSqlKernelIndexer`) over SQLite
24
+ * storage. Keeps a manifest row per Index, one shared `{indexPrefix}blocks` id table per Index,
25
+ * and opens each sub-index through its provider. Search fuses sub-indexes by RRF.
41
26
  */
42
27
  export async function createSqliteIndexer(options: SqliteIndexerOptions): Promise<Indexer> {
43
28
  const { db } = options;
44
- const providers = new Map(options.providers.map((p) => [p.type, p]));
45
- const indexes = new Map<string, Index>();
46
- const manifest = new Map<string, IndexManifest>();
47
- const runExclusive = createSerialiser();
48
- let closed = false;
49
-
50
29
  await db.exec(
51
30
  `CREATE TABLE IF NOT EXISTS ${MANIFEST_TABLE} (name TEXT PRIMARY KEY, config TEXT NOT NULL)`,
52
31
  );
53
- for (const row of await db.query<{ name: string; config: string }>(
54
- `SELECT name, config FROM ${MANIFEST_TABLE}`,
55
- )) {
56
- manifest.set(row.name, JSON.parse(row.config) as IndexManifest);
57
- }
58
-
59
- const ensureOpen = () => {
60
- if (closed) throw new Error("Indexer is closed");
61
- };
62
-
63
- const contextFor = (name: string, sub: string, blockIds: BlockIds): SqliteModalityContext => ({
64
- db,
65
- prefix: `${indexPrefix(name)}${sanitizePrefix(sub)}_`,
66
- blockIds,
67
- });
68
-
69
- async function writeManifest(name: string, entry: IndexManifest): Promise<void> {
70
- await db.query(
71
- `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,
72
- [name, JSON.stringify(entry)],
73
- );
74
- manifest.set(name, entry);
75
- }
76
-
77
- async function dropIndexTables(name: string, subIndexes: Record<string, SubSpec>): Promise<void> {
78
- const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
79
- for (const [sub, spec] of Object.entries(subIndexes)) {
80
- await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
81
- }
82
- await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
83
- await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
84
- manifest.delete(name);
85
- }
86
-
87
- async function openIndex(
88
- name: string,
89
- subIndexes: Record<string, SubSpec>,
90
- options?: GetIndexOptions,
91
- metadata?: Metadata,
92
- ): Promise<{ index: Index; effective: Record<string, SubSpec> }> {
93
- const onMissing = options?.onMissingProvider ?? "throw";
94
- const blockIds = await openBlockIds(db, `${indexPrefix(name)}blocks`);
95
- const bindings: AnySubIndexBinding[] = [];
96
- const effective: Record<string, SubSpec> = {};
97
-
98
- for (const [sub, saved] of Object.entries(subIndexes)) {
99
- const action = options?.subIndexes?.[sub];
100
- if (action && "skip" in action && action.skip === true) {
101
- effective[sub] = saved;
102
- continue;
103
- }
104
- const ctx = contextFor(name, sub, blockIds);
105
- const spec = action && !("skip" in action) ? (action as SubSpec) : saved;
106
- const provider = providers.get(spec.type);
107
- if (provider && spec !== saved) await providers.get(saved.type)?.drop(ctx);
108
- if (!provider) {
109
- if (onMissing === "throw") {
110
- throw new Error(
111
- `Cannot load index: sub-index "${sub}" requires modality "${spec.type}" but only [${[
112
- ...providers.keys(),
113
- ]
114
- .map((t) => `"${t}"`)
115
- .join(", ")}] are registered.`,
116
- );
117
- }
118
- const g = globalThis as { console?: { warn(...args: unknown[]): void } };
119
- g.console?.warn(`Sub-index "${sub}" (type "${spec.type}") has no provider; skipping.`);
120
- effective[sub] = saved;
121
- continue;
122
- }
123
- const { type, ...config } = spec;
124
- bindings.push({ name: sub, type, config, index: await provider.open(ctx, config) });
125
- effective[sub] = spec;
126
- }
127
32
 
128
- const index: Index = createCompositeIndex({
129
- name,
130
- metadata,
131
- bindings,
132
- // A closed instance leaves the cache, so getIndex opens a new one.
133
- onClose: async () => {
134
- if (indexes.get(name) === index) indexes.delete(name);
135
- },
136
- onDeleteIndex: async () => {
137
- // Bound sub-indexes dropped themselves; skipped ones still own tables.
138
- for (const [sub, spec] of Object.entries(manifest.get(name)?.subIndexes ?? {})) {
139
- await providers.get(spec.type)?.drop(contextFor(name, sub, blockIds));
140
- }
141
- await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
142
- await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
143
- manifest.delete(name);
144
- indexes.delete(name);
145
- },
146
- });
147
- return { index, effective };
148
- }
149
-
150
- return {
151
- async getIndexNames(): Promise<IndexInfo[]> {
152
- ensureOpen();
153
- return [...manifest.entries()].map(([name, entry]) =>
154
- entry.metadata !== undefined ? { name, metadata: entry.metadata } : { name },
33
+ const storage: SqlKernelStorage<BlockIds, SqliteModalityContext> = {
34
+ readManifest: () =>
35
+ db.query<{ name: string; config: string }>(`SELECT name, config FROM ${MANIFEST_TABLE}`),
36
+ async writeManifest(name, config) {
37
+ await db.query(
38
+ `INSERT INTO ${MANIFEST_TABLE} (name, config) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET config = excluded.config`,
39
+ [name, config],
155
40
  );
156
41
  },
157
-
158
- createIndex(params: CreateIndexParams): Promise<Index> {
159
- return runExclusive(async () => {
160
- ensureOpen();
161
- const { name, subIndexes, overwrite, metadata } = params;
162
- if (!subIndexes || Object.keys(subIndexes).length === 0) {
163
- throw new Error("Cannot create index without any subIndexes");
164
- }
165
- const existing = manifest.get(name);
166
- if (existing || indexes.has(name)) {
167
- if (!overwrite) throw new Error(`Index "${name}" already exists`);
168
- await indexes
169
- .get(name)
170
- ?.close()
171
- .catch(() => undefined);
172
- indexes.delete(name);
173
- await dropIndexTables(name, existing?.subIndexes ?? {});
174
- }
175
- const { index, effective } = await openIndex(name, subIndexes, undefined, metadata);
176
- await writeManifest(name, {
177
- subIndexes: effective,
178
- ...(metadata !== undefined ? { metadata } : {}),
179
- });
180
- indexes.set(name, index);
181
- return index;
182
- });
183
- },
184
-
185
- getIndex(name: string, options?: GetIndexOptions): Promise<Index | null> {
186
- // Serialised with createIndex so concurrent first opens open once
187
- // (a second reinit would otherwise drop tables the first just wrote).
188
- return runExclusive(async () => {
189
- ensureOpen();
190
- const cached = indexes.get(name);
191
- if (cached) return cached;
192
- const entry = manifest.get(name);
193
- if (!entry) return null;
194
- const { index, effective } = await openIndex(
195
- name,
196
- entry.subIndexes,
197
- options,
198
- entry.metadata,
199
- );
200
- if (JSON.stringify(effective) !== JSON.stringify(entry.subIndexes)) {
201
- await writeManifest(name, { ...entry, subIndexes: effective });
202
- }
203
- indexes.set(name, index);
204
- return index;
205
- });
206
- },
207
-
208
- async hasIndex(name: string): Promise<boolean> {
209
- ensureOpen();
210
- return manifest.has(name);
42
+ async deleteManifest(name) {
43
+ await db.query(`DELETE FROM ${MANIFEST_TABLE} WHERE name = ?`, [name]);
211
44
  },
212
-
213
- deleteIndex(name: string): Promise<void> {
214
- return runExclusive(async () => {
215
- ensureOpen();
216
- const live = indexes.get(name);
217
- if (live) {
218
- await live.deleteIndex();
219
- return;
220
- }
221
- const entry = manifest.get(name);
222
- if (entry) await dropIndexTables(name, entry.subIndexes);
223
- });
45
+ openShared: (name) => openBlockIds(db, `${indexPrefix(name)}blocks`),
46
+ async dropShared(_name, blockIds) {
47
+ await db.exec(`DROP TABLE IF EXISTS ${blockIds.table}`);
224
48
  },
225
-
226
- flush(): Promise<void> {
227
- return runExclusive(async () => {
228
- ensureOpen();
229
- for (const index of indexes.values()) await index.flush();
230
- });
49
+ // SQLite table names have always escaped upper case and have no length limit, so there is
50
+ // no earlier naming scheme to fall back to.
51
+ async subContext(name, subName, _type, blockIds) {
52
+ return { db, prefix: `${indexPrefix(name)}${sanitizePrefix(subName)}_`, blockIds };
231
53
  },
54
+ };
232
55
 
233
- close(): Promise<void> {
234
- return runExclusive(async () => {
235
- if (closed) return;
236
- closed = true;
237
- for (const index of [...indexes.values()]) await index.close().catch(() => undefined);
238
- indexes.clear();
239
- });
240
- },
241
- } satisfies Indexer;
56
+ return createSqlKernelIndexer({ storage, providers: options.providers });
242
57
  }
package/src/ngram-provider.ts CHANGED
@@ -53,6 +53,11 @@ export function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, Ng
53
53
 
54
54
  async open({ db, prefix, blockIds }, config) {
55
55
  const storeKind = config.store ?? "fts5";
56
+ if (storeKind !== "fts5" && storeKind !== "postings") {
57
+ throw new Error(
58
+ `ngram: store "${storeKind}" is not available on this backend (it has "fts5" and "postings")`,
59
+ );
60
+ }
56
61
  const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });
57
62
  await store.init();
58
63
  const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>
package/src/types.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import type { Db } from "@statewalker/db-api";
2
2
  import type { ScoredHit, SearchIndex } from "@statewalker/indexer-api";
3
+ import type { SqlModalityProvider } from "@statewalker/indexer-core";
3
4
  import type { BlockIds } from "./block-ids.js";
4
5
 
5
6
  export interface SqliteModalityContext {
@@ -12,14 +13,14 @@ export interface SqliteModalityContext {
12
13
  }
13
14
 
14
15
  /**
15
- * A modality backed by tables in one SQLite `Db`. Unlike `ModalityProvider`,
16
- * opening is async (tables are created, existing data is adopted) and the
17
- * provider can drop its tables without an open instance.
16
+ * A modality backed by tables in one SQLite `Db`: the SQL kernel's `SqlModalityProvider` over a
17
+ * SQLite context. Unlike `ModalityProvider`, opening is async (tables are created, existing data
18
+ * is adopted) and the provider can drop its tables without an open instance.
18
19
  */
19
20
  export interface SqliteModalityProvider<
20
21
  C = unknown,
21
22
  I extends SearchIndex<unknown, unknown, ScoredHit> = SearchIndex<unknown, unknown, ScoredHit>,
22
- > {
23
+ > extends SqlModalityProvider<SqliteModalityContext, C, I> {
23
24
  readonly type: string;
24
25
  open(ctx: SqliteModalityContext, config: C): Promise<I>;
25
26
  drop(ctx: SqliteModalityContext): Promise<void>;