@statewalker/indexer-sqlite 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/ngram-provider.js
CHANGED
|
@@ -43,6 +43,7 @@ function newSqliteNgramProvider() {
|
|
|
43
43
|
},
|
|
44
44
|
async open({ db, prefix, blockIds }, config) {
|
|
45
45
|
const storeKind = config.store ?? "fts5";
|
|
46
|
+
if (storeKind !== "fts5" && storeKind !== "postings") throw new Error(`ngram: store "${storeKind}" is not available on this backend (it has "fts5" and "postings")`);
|
|
46
47
|
const store = new NgramStore({
|
|
47
48
|
db,
|
|
48
49
|
prefix: `${prefix}ng_`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n NGRAM_TYPE,\n type NgramBlock,\n type NgramConfig,\n type NgramIndex,\n type NgramMatch,\n type NgramQuery,\n type NgramResult,\n type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n const breaks: number[] = [];\n for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n let line = 1;\n let b = 0;\n return [...matches]\n .sort((x, y) => x.start - y.start)\n .map(({ start, end }) => {\n while (b < breaks.length && (breaks[b] as number) < start) {\n b++;\n line++;\n }\n return { start, end, line };\n });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n return {\n type: NGRAM_TYPE,\n\n async drop({ db, prefix }) {\n await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n },\n\n async open({ db, prefix, blockIds }, config) {\n const storeKind = config.store ?? \"fts5\";\n const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n await store.init();\n const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>\n ensureOpen(),\n );\n let closed = false;\n const ensureOpen = () => {\n if (closed) throw new Error(\"NgramIndex is closed\");\n };\n\n const index: NgramIndex = {\n async getIndexInfo() {\n return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n },\n\n async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n ensureOpen();\n const allowed = params.paths?.length\n ? new Set(\n await blockIds.select(\n db,\n params.paths.map((path) => ({ path })),\n ),\n )\n : undefined;\n const result = await store.search(params.pattern, {\n regex: params.regex ?? false,\n caseSensitive: params.caseSensitive ?? true,\n limit: params.topK ?? 100,\n ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n });\n const hits = result.hits.filter((h) => h.matches.length > 0);\n const ids = hits.map((h) => h.id);\n const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n // Array sort is stable: equal counts keep the store's ascending-id order.\n const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n for (const h of ranked) {\n const ref = refs.get(h.id);\n if (!ref) continue;\n const count = h.matches.length;\n yield {\n ...ref,\n score: count,\n count,\n matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n };\n }\n return { truncated: result.truncated };\n },\n\n async addDocument(blocks: NgramBlock[]) {\n ensureOpen();\n for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n const batch = blocks.slice(i, i + UPSERT_BATCH);\n const ids = await blockIds.ensure(db, batch);\n await store.upsert(\n ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n );\n }\n },\n\n async addDocuments(batches) {\n for await (const batch of batches) await this.addDocument(batch);\n },\n\n async deleteDocuments(selectors) {\n ensureOpen();\n const sels: PathSelector[] = [];\n for await (const s of toAsyncIterable(selectors)) sels.push(s);\n const ids = await blockIds.select(db, sels);\n if (ids.length === 0) return;\n await store.delete(ids);\n },\n\n getSize: (p?: DocumentPath) => list.size(p),\n getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n ensureOpen();\n const entries = await list.entries(p);\n const texts = await store.getTexts(entries.map((e) => e.id));\n for (const e of entries) {\n const content = texts.get(e.id);\n if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n }\n },\n\n async close() {\n closed = true;\n },\n async flush() {},\n async deleteIndex() {\n closed = true;\n await store.drop();\n },\n };\n return index;\n },\n };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,iBAChE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAyFA,OAAO;IAtFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"ngram-provider.js","names":[],"sources":["../src/ngram-provider.ts"],"sourcesContent":["import { NgramStore } from \"@statewalker/db-sqlite-index/ngram\";\nimport type { DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { toAsyncIterable } from \"@statewalker/indexer-core\";\nimport {\n NGRAM_TYPE,\n type NgramBlock,\n type NgramConfig,\n type NgramIndex,\n type NgramMatch,\n type NgramQuery,\n type NgramResult,\n type NgramSearchSummary,\n} from \"@statewalker/indexer-ngram\";\nimport { newEnumeration } from \"./enumeration.js\";\nimport type { SqliteModalityProvider } from \"./types.js\";\n\n/**\n * Blocks per `NgramStore.upsert`: each upsert is one transaction, and a\n * Durable Object retries a write that runs past its ~15 s deadline.\n */\nconst UPSERT_BATCH = 500;\n\n/** 1-based line of each offset, from the positions of `\\n` in `text`. */\nfunction withLines(text: string, matches: { start: number; end: number }[]): NgramMatch[] {\n const breaks: number[] = [];\n for (let i = text.indexOf(\"\\n\"); i !== -1; i = text.indexOf(\"\\n\", i + 1)) breaks.push(i);\n let line = 1;\n let b = 0;\n return [...matches]\n .sort((x, y) => x.start - y.start)\n .map(({ start, end }) => {\n while (b < breaks.length && (breaks[b] as number) < start) {\n b++;\n line++;\n }\n return { start, end, line };\n });\n}\n\n/**\n * Wraps the id-based `NgramStore` (own transactions; never call it inside\n * `inTransaction`). Tables live under `{prefix}ng_`. Size and enumeration join\n * the store's `idTable`. Score = match count; ties keep the store's\n * ascending-id order.\n */\nexport function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, NgramIndex> {\n return {\n type: NGRAM_TYPE,\n\n async drop({ db, prefix }) {\n await new NgramStore({ db, prefix: `${prefix}ng_` }).drop(); // drops either store's tables\n },\n\n async open({ db, prefix, blockIds }, config) {\n const storeKind = config.store ?? \"fts5\";\n if (storeKind !== \"fts5\" && storeKind !== \"postings\") {\n throw new Error(\n `ngram: store \"${storeKind}\" is not available on this backend (it has \"fts5\" and \"postings\")`,\n );\n }\n const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });\n await store.init();\n const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>\n ensureOpen(),\n );\n let closed = false;\n const ensureOpen = () => {\n if (closed) throw new Error(\"NgramIndex is closed\");\n };\n\n const index: NgramIndex = {\n async getIndexInfo() {\n return { store: storeKind, ...(config.metadata ? { metadata: config.metadata } : {}) };\n },\n\n async *search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary> {\n ensureOpen();\n const allowed = params.paths?.length\n ? new Set(\n await blockIds.select(\n db,\n params.paths.map((path) => ({ path })),\n ),\n )\n : undefined;\n const result = await store.search(params.pattern, {\n regex: params.regex ?? false,\n caseSensitive: params.caseSensitive ?? true,\n limit: params.topK ?? 100,\n ...(allowed ? { filter: (id: number) => allowed.has(id) } : {}),\n });\n const hits = result.hits.filter((h) => h.matches.length > 0);\n const ids = hits.map((h) => h.id);\n const [texts, refs] = await Promise.all([store.getTexts(ids), blockIds.refs(db, ids)]);\n // Array sort is stable: equal counts keep the store's ascending-id order.\n const ranked = hits.sort((x, y) => y.matches.length - x.matches.length);\n for (const h of ranked) {\n const ref = refs.get(h.id);\n if (!ref) continue;\n const count = h.matches.length;\n yield {\n ...ref,\n score: count,\n count,\n matches: withLines(texts.get(h.id) ?? \"\", h.matches),\n };\n }\n return { truncated: result.truncated };\n },\n\n async addDocument(blocks: NgramBlock[]) {\n ensureOpen();\n for (let i = 0; i < blocks.length; i += UPSERT_BATCH) {\n const batch = blocks.slice(i, i + UPSERT_BATCH);\n const ids = await blockIds.ensure(db, batch);\n await store.upsert(\n ids.map((id, j) => ({ id, text: (batch[j] as NgramBlock).content })),\n );\n }\n },\n\n async addDocuments(batches) {\n for await (const batch of batches) await this.addDocument(batch);\n },\n\n async deleteDocuments(selectors) {\n ensureOpen();\n const sels: PathSelector[] = [];\n for await (const s of toAsyncIterable(selectors)) sels.push(s);\n const ids = await blockIds.select(db, sels);\n if (ids.length === 0) return;\n await store.delete(ids);\n },\n\n getSize: (p?: DocumentPath) => list.size(p),\n getDocumentPaths: (p?: DocumentPath) => list.paths(p),\n getDocumentBlocksRefs: (p?: DocumentPath) => list.refs(p),\n\n async *getDocumentsBlocks(p?: DocumentPath): AsyncGenerator<NgramBlock> {\n ensureOpen();\n const entries = await list.entries(p);\n const texts = await store.getTexts(entries.map((e) => e.id));\n for (const e of entries) {\n const content = texts.get(e.id);\n if (content !== undefined) yield { path: e.path, blockId: e.blockId, content };\n }\n },\n\n async close() {\n closed = true;\n },\n async flush() {},\n async deleteIndex() {\n closed = true;\n await store.drop();\n },\n };\n return index;\n },\n };\n}\n"],"mappings":";;;;;;;;;AAoBA,MAAM,eAAe;;AAGrB,SAAS,UAAU,MAAc,SAAyD;CACxF,MAAM,SAAmB,CAAC;CAC1B,KAAK,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,MAAM,IAAI,IAAI,KAAK,QAAQ,MAAM,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC;CACvF,IAAI,OAAO;CACX,IAAI,IAAI;CACR,OAAO,CAAC,GAAG,OAAO,CAAC,CAChB,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,KAAK,EAAE,OAAO,UAAU;EACvB,OAAO,IAAI,OAAO,UAAW,OAAO,KAAgB,OAAO;GACzD;GACA;EACF;EACA,OAAO;GAAE;GAAO;GAAK;EAAK;CAC5B,CAAC;AACL;;;;;;;AAQA,SAAgB,yBAA0E;CACxF,OAAO;EACL,MAAM;EAEN,MAAM,KAAK,EAAE,IAAI,UAAU;GACzB,MAAM,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;GAAK,CAAC,CAAC,CAAC,KAAK;EAC5D;EAEA,MAAM,KAAK,EAAE,IAAI,QAAQ,YAAY,QAAQ;GAC3C,MAAM,YAAY,OAAO,SAAS;GAClC,IAAI,cAAc,UAAU,cAAc,YACxC,MAAM,IAAI,MACR,iBAAiB,UAAU,kEAC7B;GAEF,MAAM,QAAQ,IAAI,WAAW;IAAE;IAAI,QAAQ,GAAG,OAAO;IAAM,OAAO;GAAU,CAAC;GAC7E,MAAM,MAAM,KAAK;GACjB,MAAM,OAAO,eAAe,IAAI,UAAU,kBAAkB,MAAM,iBAChE,WAAW,CACb;GACA,IAAI,SAAS;GACb,MAAM,mBAAmB;IACvB,IAAI,QAAQ,MAAM,IAAI,MAAM,sBAAsB;GACpD;GAyFA,OAAO;IAtFL,MAAM,eAAe;KACnB,OAAO;MAAE,OAAO;MAAW,GAAI,OAAO,WAAW,EAAE,UAAU,OAAO,SAAS,IAAI,CAAC;KAAG;IACvF;IAEA,OAAO,OAAO,QAAqE;KACjF,WAAW;KACX,MAAM,UAAU,OAAO,OAAO,SAC1B,IAAI,IACF,MAAM,SAAS,OACb,IACA,OAAO,MAAM,KAAK,UAAU,EAAE,KAAK,EAAE,CACvC,CACF,IACA,KAAA;KACJ,MAAM,SAAS,MAAM,MAAM,OAAO,OAAO,SAAS;MAChD,OAAO,OAAO,SAAS;MACvB,eAAe,OAAO,iBAAiB;MACvC,OAAO,OAAO,QAAQ;MACtB,GAAI,UAAU,EAAE,SAAS,OAAe,QAAQ,IAAI,EAAE,EAAE,IAAI,CAAC;KAC/D,CAAC;KACD,MAAM,OAAO,OAAO,KAAK,QAAQ,MAAM,EAAE,QAAQ,SAAS,CAAC;KAC3D,MAAM,MAAM,KAAK,KAAK,MAAM,EAAE,EAAE;KAChC,MAAM,CAAC,OAAO,QAAQ,MAAM,QAAQ,IAAI,CAAC,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,IAAI,GAAG,CAAC,CAAC;KAErF,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM,EAAE,QAAQ,SAAS,EAAE,QAAQ,MAAM;KACtE,KAAK,MAAM,KAAK,QAAQ;MACtB,MAAM,MAAM,KAAK,IAAI,EAAE,EAAE;MACzB,IAAI,CAAC,KAAK;MACV,MAAM,QAAQ,EAAE,QAAQ;MACxB,MAAM;OACJ,GAAG;OACH,OAAO;OACP;OACA,SAAS,UAAU,MAAM,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE,OAAO;MACrD;KACF;KACA,OAAO,EAAE,WAAW,OAAO,UAAU;IACvC;IAEA,MAAM,YAAY,QAAsB;KACtC,WAAW;KACX,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,cAAc;MACpD,MAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,YAAY;MAC9C,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;MAC3C,MAAM,MAAM,OACV,IAAI,KAAK,IAAI,OAAO;OAAE;OAAI,MAAO,MAAM,EAAE,CAAgB;MAAQ,EAAE,CACrE;KACF;IACF;IAEA,MAAM,aAAa,SAAS;KAC1B,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,YAAY,KAAK;IACjE;IAEA,MAAM,gBAAgB,WAAW;KAC/B,WAAW;KACX,MAAM,OAAuB,CAAC;KAC9B,WAAW,MAAM,KAAK,gBAAgB,SAAS,GAAG,KAAK,KAAK,CAAC;KAC7D,MAAM,MAAM,MAAM,SAAS,OAAO,IAAI,IAAI;KAC1C,IAAI,IAAI,WAAW,GAAG;KACtB,MAAM,MAAM,OAAO,GAAG;IACxB;IAEA,UAAU,MAAqB,KAAK,KAAK,CAAC;IAC1C,mBAAmB,MAAqB,KAAK,MAAM,CAAC;IACpD,wBAAwB,MAAqB,KAAK,KAAK,CAAC;IAExD,OAAO,mBAAmB,GAA8C;KACtE,WAAW;KACX,MAAM,UAAU,MAAM,KAAK,QAAQ,CAAC;KACpC,MAAM,QAAQ,MAAM,MAAM,SAAS,QAAQ,KAAK,MAAM,EAAE,EAAE,CAAC;KAC3D,KAAK,MAAM,KAAK,SAAS;MACvB,MAAM,UAAU,MAAM,IAAI,EAAE,EAAE;MAC9B,IAAI,YAAY,KAAA,GAAW,MAAM;OAAE,MAAM,EAAE;OAAM,SAAS,EAAE;OAAS;MAAQ;KAC/E;IACF;IAEA,MAAM,QAAQ;KACZ,SAAS;IACX;IACA,MAAM,QAAQ,CAAC;IACf,MAAM,cAAc;KAClB,SAAS;KACT,MAAM,MAAM,KAAK;IACnB;GAES;EACb;CACF;AACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/indexer-sqlite",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "SQLite-backed indexer: FTS5, vector and n-gram/grep sub-indexes in one @statewalker/db-api Db (Node, browser worker, Durable Objects).",
|
|
@@ -35,10 +35,10 @@
|
|
|
35
35
|
"@statewalker/db-sqlite": "^0.1.0",
|
|
36
36
|
"@statewalker/db-sqlite-index": "^0.1.0",
|
|
37
37
|
"@statewalker/indexer-api": "^0.1.5",
|
|
38
|
+
"@statewalker/indexer-core": "^0.3.0",
|
|
38
39
|
"@statewalker/indexer-fulltext": "^0.1.6",
|
|
39
|
-
"@statewalker/indexer-ngram": "^0.
|
|
40
|
-
"@statewalker/indexer-vector": "^0.1.6"
|
|
41
|
-
"@statewalker/indexer-core": "^0.2.0"
|
|
40
|
+
"@statewalker/indexer-ngram": "^0.2.0",
|
|
41
|
+
"@statewalker/indexer-vector": "^0.1.6"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"@statewalker/db-sqlite-do": "^0.1.0",
|
package/src/ngram-provider.ts
CHANGED
|
@@ -53,6 +53,11 @@ export function newSqliteNgramProvider(): SqliteModalityProvider<NgramConfig, Ng
|
|
|
53
53
|
|
|
54
54
|
async open({ db, prefix, blockIds }, config) {
|
|
55
55
|
const storeKind = config.store ?? "fts5";
|
|
56
|
+
if (storeKind !== "fts5" && storeKind !== "postings") {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`ngram: store "${storeKind}" is not available on this backend (it has "fts5" and "postings")`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
56
61
|
const store = new NgramStore({ db, prefix: `${prefix}ng_`, store: storeKind });
|
|
57
62
|
await store.init();
|
|
58
63
|
const list = newEnumeration(db, blockIds, `SELECT id FROM ${store.idTable}`, () =>
|