@statewalker/indexer-mem-flexsearch 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/flexsearch-full-text-index.d.ts +11 -58
  2. package/dist/flexsearch-full-text-index.d.ts.map +1 -1
  3. package/dist/flexsearch-full-text-index.js +69 -253
  4. package/dist/flexsearch-full-text-index.js.map +1 -1
  5. package/dist/flexsearch-indexer.js +1 -1
  6. package/package.json +4 -4
  7. package/src/flexsearch-full-text-index.ts +61 -380
package/dist/flexsearch-full-text-index.d.ts CHANGED
@@ -1,66 +1,19 @@
1
- import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
2
- import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
1
+ import { MemFullTextEngine, MemFullTextIndex } from "@statewalker/indexer-mem";
2
+ import { FullTextIndexInfo } from "@statewalker/indexer-fulltext";
3
3
  //#region src/flexsearch-full-text-index.d.ts
4
- export declare class FlexSearchFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
5
- private readonly info;
6
- private flexIndex;
7
- private keyToNum;
8
- private numToKey;
9
- private blocks;
10
- private nextNum;
11
- private closed;
12
- /**
13
- * Dirty bit. `true` whenever the FTS index has been mutated since the
14
- * last `serialize()`. The serializer caches the last produced JSON and
15
- * returns it directly when `!dirty`, so a no-op re-sync skips the
16
- * expensive FlexSearch export entirely. Starts `true` so the first
17
- * serialize after construction always runs.
18
- */
19
- private dirty;
20
- private cachedSerialized?;
4
+ /**
5
+ * In-memory full-text sub-index backed by FlexSearch: a FlexSearch adapter
6
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
7
+ */
8
+ export declare class FlexSearchFullTextIndex extends MemFullTextIndex {
21
9
  constructor(info: FullTextIndexInfo);
10
+ protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine;
22
11
  /**
23
- * `true` when the in-memory state has changed since the last serialize.
24
- * Callers (e.g. the persistence layer) may use it to skip serialization
25
- * and writes when the index has not been touched.
26
- */
27
- isDirty(): boolean;
28
- private ensureOpen;
29
- private getOrAssignNum;
30
- private searchInternal;
31
- getIndexInfo(): Promise<FullTextIndexInfo>;
32
- search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
33
- addDocument(blocks: FullTextBlock[]): Promise<void>;
34
- addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
35
- deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
36
- /** Returns true when the key was present and got removed. */
37
- private removeByKey;
38
- getSize(pathPrefix?: DocumentPath): Promise<number>;
39
- getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
40
- getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
41
- getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
42
- close(_options?: {
43
- force?: boolean;
44
- }): Promise<void>;
45
- flush(): Promise<void>;
46
- deleteIndex(): Promise<void>;
47
- /**
48
- * Stream the FTS sub-index's state as a single named entry. The composite
49
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
50
- */
51
- serialise(): AsyncIterable<PersistenceEntry>;
52
- /** Restore the FTS sub-index from a previously-produced entry set. */
53
- loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
54
- /**
55
- * Serialize to JSON string (v3 format with path data).
56
- *
57
- * Caches the result; while `dirty` is false (no mutations since the
58
- * last call) returns the same string, skipping the FlexSearch export
59
- * entirely. The persistence layer still calls into us each flush, but
60
- * the heavy work only runs when the index actually changed.
12
+ * Serialize to a JSON string (format version 3). Cached: while the index is
13
+ * unchanged, returns the same string without exporting FlexSearch again.
61
14
  */
62
15
  serialize(): Promise<string>;
63
- /** Deserialize from JSON — handles v3 format */
16
+ /** Deserialize from JSON (format version 3); throws on any other version. */
64
17
  static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex;
65
18
  }
66
19
  //#endregion
package/dist/flexsearch-full-text-index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"flexsearch-full-text-index.d.ts","names":["FullTextSearchParams","FullTextSearchResult"],"sources":["../src/flexsearch-full-text-index.ts"],"mappings":";;;qBAgCa,mCAET,eACA,uBAAuB,eAAeA,eAAsBC;mBAE7C;UACT;UACA;UACA;UACA;UACA;UACA;;;;;;;;UAQA;UACA;EAER,YAAY,MAAM;;;;;;EAclB;UAIQ;UAMA;UAUA;EAwCF,gBAAgB,QAAQ;EAKvB,OAAO,QAAQD,gBAAuB,eAAeC;EAqCtD,YAAY,QAAQ,kBAAkB;EAqBtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;;UAuBK;EAYF,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAe;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;EAgBd,aAAa,cAAc;;EAM5B,SAAS,SAAS,SAAS,oBAAoB;;;;;;;;;EA2B/C,aAAa;;SAqCZ,YAAY,MAAM,mBAAmB,eAAe"}
1
+ {"version":3,"file":"flexsearch-full-text-index.d.ts","names":[],"sources":["../src/flexsearch-full-text-index.ts"],"mappings":";;;;;;;qBA8Ea,gCAAgC;EAC3C,YAAY,MAAM;YAIR,cAAc,OAAO,0BAA0B;;;;;EAQnD,aAAa;;SAKZ,YAAY,MAAM,mBAAmB,eAAe"}
package/dist/flexsearch-full-text-index.js CHANGED
@@ -1,281 +1,97 @@
1
- import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, toBytes } from "@statewalker/indexer-core";
1
+ import { MemFullTextIndex } from "@statewalker/indexer-mem";
2
2
  import FlexSearch from "flexsearch";
3
3
  //#region src/flexsearch-full-text-index.ts
4
- var FlexSearchFullTextIndex = class FlexSearchFullTextIndex {
5
- info;
6
- flexIndex;
4
+ function newFlexIndex() {
5
+ return new FlexSearch.Index({
6
+ tokenize: "forward",
7
+ resolution: 9,
8
+ cache: true
9
+ });
10
+ }
11
+ /**
12
+ * FlexSearch adapter for the in-memory block store. FlexSearch ids are
13
+ * numbers, so the adapter keeps a store-key ↔ number map; it is part of the
14
+ * exported state (`chunks`, `keyToNum`, `nextNum`).
15
+ */
16
+ var FlexSearchEngine = class FlexSearchEngine {
17
+ flexIndex = newFlexIndex();
7
18
  keyToNum = /* @__PURE__ */ new Map();
8
19
  numToKey = /* @__PURE__ */ new Map();
9
- blocks = /* @__PURE__ */ new Map();
10
20
  nextNum = 1;
11
- closed = false;
12
- /**
13
- * Dirty bit. `true` whenever the FTS index has been mutated since the
14
- * last `serialize()`. The serializer caches the last produced JSON and
15
- * returns it directly when `!dirty`, so a no-op re-sync skips the
16
- * expensive FlexSearch export entirely. Starts `true` so the first
17
- * serialize after construction always runs.
18
- */
19
- dirty = true;
20
- cachedSerialized;
21
- constructor(info) {
22
- this.info = info;
23
- this.flexIndex = new FlexSearch.Index({
24
- tokenize: "forward",
25
- resolution: 9,
26
- cache: true
27
- });
28
- }
29
- /**
30
- * `true` when the in-memory state has changed since the last serialize.
31
- * Callers (e.g. the persistence layer) may use it to skip serialization
32
- * and writes when the index has not been touched.
33
- */
34
- isDirty() {
35
- return this.dirty;
36
- }
37
- ensureOpen() {
38
- if (this.closed) throw new Error("FullTextIndex is closed");
39
- }
40
- getOrAssignNum(key) {
21
+ add(key, block) {
41
22
  let num = this.keyToNum.get(key);
42
23
  if (num === void 0) {
43
24
  num = this.nextNum++;
44
25
  this.keyToNum.set(key, num);
45
26
  this.numToKey.set(num, key);
46
27
  }
47
- return num;
28
+ this.flexIndex.add(num, block.content);
48
29
  }
49
- searchInternal(query, topK, pathPrefixes) {
50
- const ids = this.flexIndex.search(query, topK * 3, { suggest: true });
51
- let results = ids.map((numId, rank) => ({
52
- key: this.numToKey.get(numId) ?? "",
53
- score: 1 - rank / ids.length
54
- })).filter((r) => r.key !== "");
55
- if (pathPrefixes && pathPrefixes.length > 0) results = results.filter((r) => {
56
- const block = this.blocks.get(r.key);
57
- return block !== void 0 && pathPrefixes.some((p) => matchesPrefix(block.path, p));
58
- });
59
- results.sort((a, b) => b.score - a.score);
60
- return results.slice(0, topK).map((r) => {
61
- const block = this.blocks.get(r.key);
62
- return {
63
- path: block?.path ?? "/",
64
- blockId: block?.blockId ?? "",
65
- snippet: block?.content ?? "",
66
- score: r.score
67
- };
30
+ remove(key) {
31
+ const num = this.keyToNum.get(key);
32
+ if (num === void 0) return;
33
+ this.flexIndex.remove(num);
34
+ this.keyToNum.delete(key);
35
+ this.numToKey.delete(num);
36
+ }
37
+ search(query, limit) {
38
+ const ids = this.flexIndex.search(query, limit * 3, { suggest: true });
39
+ const hits = [];
40
+ ids.forEach((num, rank) => {
41
+ const key = this.numToKey.get(num);
42
+ if (key !== void 0) hits.push({
43
+ key,
44
+ score: 1 - rank / ids.length
45
+ });
68
46
  });
47
+ return hits;
69
48
  }
70
- async getIndexInfo() {
71
- this.ensureOpen();
72
- return { ...this.info };
73
- }
74
- async *search(params) {
75
- this.ensureOpen();
76
- const { queries, paths } = params;
77
- const topK = params.topK ?? 100;
78
- if (!queries || queries.length === 0) return;
79
- const merged = /* @__PURE__ */ new Map();
80
- for (const query of queries) {
81
- const results = this.searchInternal(query, topK, paths);
82
- for (const r of results) {
83
- const key = compositeKey(r.path, r.blockId);
84
- const existing = merged.get(key);
85
- if (!existing) merged.set(key, {
86
- result: r,
87
- best: r.score,
88
- total: r.score
89
- });
90
- else {
91
- existing.total += r.score;
92
- if (r.score > existing.best) {
93
- existing.best = r.score;
94
- existing.result = r;
95
- }
96
- }
97
- }
98
- }
99
- const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
100
- for (const { result, total } of sorted.slice(0, topK)) yield {
101
- ...result,
102
- score: total
49
+ exportState() {
50
+ const chunks = {};
51
+ this.flexIndex.export((key, data) => {
52
+ chunks[String(key)] = data;
53
+ });
54
+ return {
55
+ chunks,
56
+ keyToNum: [...this.keyToNum.entries()],
57
+ nextNum: this.nextNum
103
58
  };
104
59
  }
105
- async addDocument(blocks) {
106
- this.ensureOpen();
107
- if (blocks.length === 0) return;
108
- for (const block of blocks) {
109
- const key = compositeKey(block.path, block.blockId);
110
- const num = this.getOrAssignNum(key);
111
- if (this.blocks.has(key)) this.flexIndex.remove(num);
112
- this.flexIndex.add(num, block.content);
113
- this.blocks.set(key, {
114
- path: block.path,
115
- blockId: block.blockId,
116
- content: block.content,
117
- metadata: block.metadata
118
- });
119
- }
120
- this.dirty = true;
121
- }
122
- async addDocuments(blocks) {
123
- this.ensureOpen();
124
- for await (const batch of blocks) await this.addDocument(batch);
125
- }
126
- async deleteDocuments(pathSelectors) {
127
- this.ensureOpen();
128
- let removed = 0;
129
- for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
130
- const key = compositeKey(sel.path, sel.blockId);
131
- if (this.removeByKey(key)) removed += 1;
132
- } else {
133
- const keysToDelete = [];
134
- for (const [key, block] of this.blocks) if (matchesPrefix(block.path, sel.path)) keysToDelete.push(key);
135
- for (const key of keysToDelete) if (this.removeByKey(key)) removed += 1;
136
- }
137
- if (removed > 0) this.dirty = true;
138
- }
139
- /** Returns true when the key was present and got removed. */
140
- removeByKey(key) {
141
- const num = this.keyToNum.get(key);
142
- const had = this.blocks.has(key);
143
- if (num !== void 0) {
144
- this.flexIndex.remove(num);
145
- this.keyToNum.delete(key);
146
- this.numToKey.delete(num);
60
+ static restore(state) {
61
+ const engine = new FlexSearchEngine();
62
+ const chunks = state.chunks ?? {};
63
+ for (const [key, data] of Object.entries(chunks)) engine.flexIndex.import(key, data);
64
+ for (const [key, num] of state.keyToNum ?? []) {
65
+ engine.keyToNum.set(key, num);
66
+ engine.numToKey.set(num, key);
147
67
  }
148
- this.blocks.delete(key);
149
- return had;
68
+ engine.nextNum = typeof state.nextNum === "number" ? state.nextNum : engine.keyToNum.size + 1;
69
+ return engine;
150
70
  }
151
- async getSize(pathPrefix) {
152
- this.ensureOpen();
153
- if (pathPrefix === void 0) return this.blocks.size;
154
- let count = 0;
155
- for (const block of this.blocks.values()) if (matchesPrefix(block.path, pathPrefix)) count++;
156
- return count;
157
- }
158
- async *getDocumentPaths(pathPrefix) {
159
- this.ensureOpen();
160
- const paths = /* @__PURE__ */ new Set();
161
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) paths.add(block.path);
162
- for (const p of paths) yield p;
163
- }
164
- async *getDocumentBlocksRefs(pathPrefix) {
165
- this.ensureOpen();
166
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
167
- path: block.path,
168
- blockId: block.blockId
169
- };
170
- }
171
- async *getDocumentsBlocks(pathPrefix) {
172
- this.ensureOpen();
173
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
174
- path: block.path,
175
- blockId: block.blockId,
176
- content: block.content,
177
- metadata: block.metadata
178
- };
179
- }
180
- async close(_options) {
181
- this.closed = true;
182
- }
183
- async flush() {
184
- this.ensureOpen();
185
- }
186
- async deleteIndex() {
187
- this.ensureOpen();
188
- this.blocks.clear();
189
- this.keyToNum.clear();
190
- this.numToKey.clear();
191
- this.cachedSerialized = void 0;
192
- this.dirty = true;
193
- this.closed = true;
194
- }
195
- /**
196
- * Stream the FTS sub-index's state as a single named entry. The composite
197
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
198
- */
199
- async *serialise() {
200
- const json = await this.serialize();
201
- yield {
202
- name: "json",
203
- content: singleChunk(toBytes(json))
204
- };
71
+ };
72
+ /**
73
+ * In-memory full-text sub-index backed by FlexSearch: a FlexSearch adapter
74
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
75
+ */
76
+ var FlexSearchFullTextIndex = class FlexSearchFullTextIndex extends MemFullTextIndex {
77
+ constructor(info) {
78
+ super(info, "FlexSearchFullTextIndex", new FlexSearchEngine());
205
79
  }
206
- /** Restore the FTS sub-index from a previously-produced entry set. */
207
- async loadFrom(entries) {
208
- this.ensureOpen();
209
- for (const e of entries) {
210
- if (e.name !== "json") continue;
211
- const bytes = await readEntryBytes(e);
212
- const json = new TextDecoder().decode(bytes);
213
- const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);
214
- this.flexIndex = rebuilt.flexIndex;
215
- this.keyToNum = rebuilt.keyToNum;
216
- this.numToKey = rebuilt.numToKey;
217
- this.blocks = rebuilt.blocks;
218
- this.nextNum = rebuilt.nextNum;
219
- this.cachedSerialized = json;
220
- this.dirty = false;
221
- return;
222
- }
80
+ restoreEngine(state) {
81
+ return FlexSearchEngine.restore(state);
223
82
  }
224
83
  /**
225
- * Serialize to JSON string (v3 format with path data).
226
- *
227
- * Caches the result; while `dirty` is false (no mutations since the
228
- * last call) returns the same string, skipping the FlexSearch export
229
- * entirely. The persistence layer still calls into us each flush, but
230
- * the heavy work only runs when the index actually changed.
84
+ * Serialize to a JSON string (format version 3). Cached: while the index is
85
+ * unchanged, returns the same string without exporting FlexSearch again.
231
86
  */
232
87
  async serialize() {
233
- if (!this.dirty && this.cachedSerialized !== void 0) return this.cachedSerialized;
234
- const chunks = /* @__PURE__ */ new Map();
235
- await this.flexIndex.export((key, data) => {
236
- chunks.set(String(key), data);
237
- });
238
- const blocksData = [];
239
- for (const block of this.blocks.values()) blocksData.push({
240
- path: block.path,
241
- blockId: block.blockId,
242
- content: block.content,
243
- metadata: block.metadata
244
- });
245
- const out = JSON.stringify({
246
- version: 3,
247
- chunks: Object.fromEntries(chunks),
248
- blocks: blocksData,
249
- keyToNum: [...this.keyToNum.entries()],
250
- nextNum: this.nextNum
251
- });
252
- this.cachedSerialized = out;
253
- this.dirty = false;
254
- return out;
88
+ return this.serializeState();
255
89
  }
256
- /** Deserialize from JSON — handles v3 format */
90
+ /** Deserialize from JSON (format version 3); throws on any other version. */
257
91
  static deserialize(info, json) {
258
- const parsed = JSON.parse(json);
259
- if (parsed.version !== 3) throw new Error(`FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`);
260
- const fts = new FlexSearchFullTextIndex(info);
261
- for (const [key, data] of Object.entries(parsed.chunks)) fts.flexIndex.import(key, data);
262
- for (const [key, num] of parsed.keyToNum) {
263
- fts.keyToNum.set(key, num);
264
- fts.numToKey.set(num, key);
265
- }
266
- fts.nextNum = parsed.nextNum;
267
- for (const block of parsed.blocks) {
268
- const key = compositeKey(block.path, block.blockId);
269
- fts.blocks.set(key, {
270
- path: block.path,
271
- blockId: block.blockId,
272
- content: block.content,
273
- metadata: block.metadata
274
- });
275
- }
276
- fts.cachedSerialized = json;
277
- fts.dirty = false;
278
- return fts;
92
+ const index = new FlexSearchFullTextIndex(info);
93
+ index.restoreState(json);
94
+ return index;
279
95
  }
280
96
  };
281
97
  //#endregion
package/dist/flexsearch-full-text-index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"flexsearch-full-text-index.js","names":[],"sources":["../src/flexsearch-full-text-index.ts"],"sourcesContent":["import type {\n BlockReference,\n DocumentPath,\n Metadata,\n PathSelector,\n PersistableSearchIndex,\n PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n compositeKey,\n matchesPrefix,\n readEntryBytes,\n singleChunk,\n toAsyncIterable,\n toBytes,\n} from \"@statewalker/indexer-core\";\nimport type {\n FullTextBlock,\n FullTextIndex,\n FullTextIndexInfo,\n FulltextQuery as FullTextSearchParams,\n FulltextResult as FullTextSearchResult,\n} from \"@statewalker/indexer-fulltext\";\nimport FlexSearch, { type Index as FlexIndex } from \"flexsearch\";\n\ninterface StoredBlock {\n path: DocumentPath;\n blockId: string;\n content: string;\n metadata?: Metadata;\n}\n\nexport class FlexSearchFullTextIndex\n implements\n FullTextIndex,\n PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>\n{\n private readonly info: FullTextIndexInfo;\n private flexIndex: FlexIndex;\n private keyToNum = new Map<string, number>();\n private numToKey = new Map<number, string>();\n private blocks = new Map<string, StoredBlock>();\n private nextNum = 1;\n private closed = false;\n /**\n * Dirty bit. `true` whenever the FTS index has been mutated since the\n * last `serialize()`. The serializer caches the last produced JSON and\n * returns it directly when `!dirty`, so a no-op re-sync skips the\n * expensive FlexSearch export entirely. Starts `true` so the first\n * serialize after construction always runs.\n */\n private dirty = true;\n private cachedSerialized?: string;\n\n constructor(info: FullTextIndexInfo) {\n this.info = info;\n this.flexIndex = new FlexSearch.Index({\n tokenize: \"forward\",\n resolution: 9,\n cache: true,\n });\n }\n\n /**\n * `true` when the in-memory state has changed since the last serialize.\n * Callers (e.g. the persistence layer) may use it to skip serialization\n * and writes when the index has not been touched.\n */\n isDirty(): boolean {\n return this.dirty;\n }\n\n private ensureOpen(): void {\n if (this.closed) {\n throw new Error(\"FullTextIndex is closed\");\n }\n }\n\n private getOrAssignNum(key: string): number {\n let num = this.keyToNum.get(key);\n if (num === undefined) {\n num = this.nextNum++;\n this.keyToNum.set(key, num);\n this.numToKey.set(num, key);\n }\n return num;\n }\n\n private searchInternal(\n query: string,\n topK: number,\n pathPrefixes?: DocumentPath[],\n ): FullTextSearchResult[] {\n // `suggest: true` relaxes the default all-terms-AND for a multi-word query into a\n // ranked partial match — sections are ordered by how much of the term set they match\n // (FlexSearch's own resolution scoring), instead of returning nothing unless one\n // section contains every term. For single-word or fully-intersecting queries it is\n // identical to the default, so it only ever broadens recall, never narrows it.\n const ids = this.flexIndex.search(query, topK * 3, { suggest: true }) as number[];\n\n let results = ids\n .map((numId, rank) => ({\n key: this.numToKey.get(numId) ?? \"\",\n score: 1 - rank / ids.length,\n }))\n .filter((r) => r.key !== \"\");\n\n // Filter by path prefixes\n if (pathPrefixes && pathPrefixes.length > 0) {\n results = results.filter((r) => {\n const block = this.blocks.get(r.key);\n return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));\n });\n }\n\n results.sort((a, b) => b.score - a.score);\n\n return results.slice(0, topK).map((r) => {\n const block = this.blocks.get(r.key);\n return {\n path: block?.path ?? (\"/\" as DocumentPath),\n blockId: block?.blockId ?? \"\",\n snippet: block?.content ?? \"\",\n score: r.score,\n };\n });\n }\n\n async getIndexInfo(): Promise<FullTextIndexInfo> {\n this.ensureOpen();\n return { ...this.info };\n }\n\n async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {\n this.ensureOpen();\n const { queries, paths } = params;\n const topK = params.topK ?? 100;\n\n if (!queries || queries.length === 0) return;\n\n // Multi-query: search for each and ACCUMULATE per-block scores, so a block\n // matching more queries ranks higher (the FulltextQuery contract). Taking the\n // max instead would let a block that is the top hit for one query outrank a\n // block matching several queries. The snippet is kept from the strongest\n // single-query match.\n const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();\n\n for (const query of queries) {\n const results = this.searchInternal(query, topK, paths);\n for (const r of results) {\n const key = compositeKey(r.path, r.blockId);\n const existing = merged.get(key);\n if (!existing) {\n merged.set(key, { result: r, best: r.score, total: r.score });\n } else {\n existing.total += r.score;\n if (r.score > existing.best) {\n existing.best = r.score;\n existing.result = r;\n }\n }\n }\n }\n\n const sorted = [...merged.values()].sort((a, b) => b.total - a.total);\n for (const { result, total } of sorted.slice(0, topK)) {\n yield { ...result, score: total };\n }\n }\n\n async addDocument(blocks: FullTextBlock[]): Promise<void> {\n this.ensureOpen();\n if (blocks.length === 0) return;\n for (const block of blocks) {\n const key = compositeKey(block.path, block.blockId);\n const num = this.getOrAssignNum(key);\n // Remove old content if exists\n if (this.blocks.has(key)) {\n this.flexIndex.remove(num);\n }\n this.flexIndex.add(num, block.content);\n this.blocks.set(key, {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n this.dirty = true;\n }\n\n async addDocuments(\n blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const batch of blocks) {\n await this.addDocument(batch);\n }\n }\n\n async deleteDocuments(\n pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n ): Promise<void> {\n this.ensureOpen();\n let removed = 0;\n for await (const sel of toAsyncIterable(pathSelectors)) {\n if (sel.blockId !== undefined) {\n const key = compositeKey(sel.path, sel.blockId);\n if (this.removeByKey(key)) removed += 1;\n } else {\n const keysToDelete: string[] = [];\n for (const [key, block] of this.blocks) {\n if (matchesPrefix(block.path, sel.path)) {\n keysToDelete.push(key);\n }\n }\n for (const key of keysToDelete) {\n if (this.removeByKey(key)) removed += 1;\n }\n }\n }\n if (removed > 0) this.dirty = true;\n }\n\n /** Returns true when the key was present and got removed. */\n private removeByKey(key: string): boolean {\n const num = this.keyToNum.get(key);\n const had = this.blocks.has(key);\n if (num !== undefined) {\n this.flexIndex.remove(num);\n this.keyToNum.delete(key);\n this.numToKey.delete(num);\n }\n this.blocks.delete(key);\n return had;\n }\n\n async getSize(pathPrefix?: DocumentPath): Promise<number> {\n this.ensureOpen();\n if (pathPrefix === undefined) return this.blocks.size;\n let count = 0;\n for (const block of this.blocks.values()) {\n if (matchesPrefix(block.path, pathPrefix)) count++;\n }\n return count;\n }\n\n async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n this.ensureOpen();\n const paths = new Set<DocumentPath>();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n paths.add(block.path);\n }\n }\n for (const p of paths) yield p;\n }\n\n async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield { path: block.path, blockId: block.blockId };\n }\n }\n }\n\n async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n };\n }\n }\n }\n\n async close(_options?: { force?: boolean }): Promise<void> {\n this.closed = true;\n }\n\n async flush(): Promise<void> {\n this.ensureOpen();\n // no-op for in-memory\n }\n\n async deleteIndex(): Promise<void> {\n this.ensureOpen();\n this.blocks.clear();\n this.keyToNum.clear();\n this.numToKey.clear();\n this.cachedSerialized = undefined;\n this.dirty = true;\n this.closed = true;\n }\n\n // --- PersistableSearchIndex --------------------------------------------------\n\n /**\n * Stream the FTS sub-index's state as a single named entry. The composite\n * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n */\n async *serialise(): AsyncIterable<PersistenceEntry> {\n const json = await this.serialize();\n yield { name: \"json\", content: singleChunk(toBytes(json)) };\n }\n\n /** Restore the FTS sub-index from a previously-produced entry set. */\n async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n this.ensureOpen();\n for (const e of entries) {\n if (e.name !== \"json\") continue;\n const bytes = await readEntryBytes(e);\n const json = new TextDecoder().decode(bytes);\n const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);\n // Adopt the rebuilt instance's internal state.\n this.flexIndex = rebuilt.flexIndex;\n this.keyToNum = rebuilt.keyToNum;\n this.numToKey = rebuilt.numToKey;\n this.blocks = rebuilt.blocks;\n this.nextNum = rebuilt.nextNum;\n this.cachedSerialized = json;\n this.dirty = false;\n return;\n }\n }\n\n /**\n * Serialize to JSON string (v3 format with path data).\n *\n * Caches the result; while `dirty` is false (no mutations since the\n * last call) returns the same string, skipping the FlexSearch export\n * entirely. The persistence layer still calls into us each flush, but\n * the heavy work only runs when the index actually changed.\n */\n async serialize(): Promise<string> {\n if (!this.dirty && this.cachedSerialized !== undefined) {\n return this.cachedSerialized;\n }\n const chunks = new Map<string, string>();\n await this.flexIndex.export((key: string | number, data: string) => {\n chunks.set(String(key), data);\n });\n\n const blocksData: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }> = [];\n for (const block of this.blocks.values()) {\n blocksData.push({\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n const out = JSON.stringify({\n version: 3,\n chunks: Object.fromEntries(chunks),\n blocks: blocksData,\n keyToNum: [...this.keyToNum.entries()],\n nextNum: this.nextNum,\n });\n this.cachedSerialized = out;\n this.dirty = false;\n return out;\n }\n\n /** Deserialize from JSON — handles v3 format */\n static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex {\n const parsed = JSON.parse(json) as {\n version?: number;\n chunks: Record<string, string>;\n blocks: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }>;\n keyToNum: Array<[string, number]>;\n nextNum: number;\n };\n\n if (parsed.version !== 3) {\n throw new Error(\n `FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,\n );\n }\n\n const fts = new FlexSearchFullTextIndex(info);\n for (const [key, data] of Object.entries(parsed.chunks)) {\n fts.flexIndex.import(key, data);\n }\n for (const [key, num] of parsed.keyToNum) {\n fts.keyToNum.set(key, num);\n fts.numToKey.set(num, key);\n }\n fts.nextNum = parsed.nextNum;\n for (const block of parsed.blocks) {\n const key = compositeKey(block.path as DocumentPath, block.blockId);\n fts.blocks.set(key, {\n path: block.path as DocumentPath,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n // Prime the cache: a deserialized index that's never mutated should\n // return the same JSON it was loaded from when re-serialized.\n fts.cachedSerialized = json;\n fts.dirty = false;\n return fts;\n }\n}\n"],"mappings":";;;AAgCA,IAAa,0BAAb,MAAa,wBAIb;CACE;CACA;CACA,2BAAmB,IAAI,IAAoB;CAC3C,2BAAmB,IAAI,IAAoB;CAC3C,yBAAiB,IAAI,IAAyB;CAC9C,UAAkB;CAClB,SAAiB;;;;;;;;CAQjB,QAAgB;CAChB;CAEA,YAAY,MAAyB;EACnC,KAAK,OAAO;EACZ,KAAK,YAAY,IAAI,WAAW,MAAM;GACpC,UAAU;GACV,YAAY;GACZ,OAAO;EACT,CAAC;CACH;;;;;;CAOA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,yBAAyB;CAE7C;CAEA,eAAuB,KAAqB;EAC1C,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG;EAC/B,IAAI,QAAQ,KAAA,GAAW;GACrB,MAAM,KAAK;GACX,KAAK,SAAS,IAAI,KAAK,GAAG;GAC1B,KAAK,SAAS,IAAI,KAAK,GAAG;EAC5B;EACA,OAAO;CACT;CAEA,eACE,OACA,MACA,cACwB;EAMxB,MAAM,MAAM,KAAK,UAAU,OAAO,OAAO,OAAO,GAAG,EAAE,SAAS,KAAK,CAAC;EAEpE,IAAI,UAAU,IACX,KAAK,OAAO,UAAU;GACrB,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK;GACjC,OAAO,IAAI,OAAO,IAAI;EACxB,EAAE,CAAC,CACF,QAAQ,MAAM,EAAE,QAAQ,EAAE;EAG7B,IAAI,gBAAgB,aAAa,SAAS,GACxC,UAAU,QAAQ,QAAQ,MAAM;GAC9B,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO,UAAU,KAAA,KAAa,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC;EACrF,CAAC;EAGH,QAAQ,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EAExC,OAAO,QAAQ,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,MAAM;GACvC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO;IACL,MAAM,OAAO,QAAS;IACtB,SAAS,OAAO,WAAW;IAC3B,SAAS,OAAO,WAAW;IAC3B,OAAO,EAAE;GACX;EACF,CAAC;CACH;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAoE;EAChF,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EAOtC,MAAM,yBAAS,IAAI,IAA2E;EAE9F,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,UAAU,KAAK,eAAe,OAAO,MAAM,KAAK;GACtD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,OAAO,IAAI,GAAG;IAC/B,IAAI,CAAC,UACH,OAAO,IAAI,KAAK;KAAE,QAAQ;KAAG,MAAM,EAAE;KAAO,OAAO,EAAE;IAAM,CAAC;SACvD;KACL,SAAS,SAAS,EAAE;KACpB,IAAI,EAAE,QAAQ,SAAS,MAAM;MAC3B,SAAS,OAAO,EAAE;MAClB,SAAS,SAAS;KACpB;IACF;GACF;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACpE,KAAK,MAAM,EAAE,QAAQ,WAAW,OAAO,MAAM,GAAG,IAAI,GAClD,MAAM;GAAE,GAAG;GAAQ,OAAO;EAAM;CAEpC;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,IAAI,OAAO,WAAW,GAAG;EACzB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,MAAM,MAAM,KAAK,eAAe,GAAG;GAEnC,IAAI,KAAK,OAAO,IAAI,GAAG,GACrB,KAAK,UAAU,OAAO,GAAG;GAE3B,KAAK,UAAU,IAAI,KAAK,MAAM,OAAO;GACrC,KAAK,OAAO,IAAI,KAAK;IACnB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EACA,KAAK,QAAQ;CACf;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GAAW;GAC7B,MAAM,MAAM,aAAa,IAAI,MAAM,IAAI,OAAO;GAC9C,IAAI,KAAK,YAAY,GAAG,GAAG,WAAW;EACxC,OAAO;GACL,MAAM,eAAyB,CAAC;GAChC,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GACpC,aAAa,KAAK,GAAG;GAGzB,KAAK,MAAM,OAAO,cAChB,IAAI,KAAK,YAAY,GAAG,GAAG,WAAW;EAE1C;EAEF,IAAI,UAAU,GAAG,KAAK,QAAQ;CAChC;;CAGA,YAAoB,KAAsB;EACxC,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG;EACjC,MAAM,MAAM,KAAK,OAAO,IAAI,GAAG;EAC/B,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,UAAU,OAAO,GAAG;GACzB,KAAK,SAAS,OAAO,GAAG;GACxB,KAAK,SAAS,OAAO,GAAG;EAC1B;EACA,KAAK,OAAO,OAAO,GAAG;EACtB,OAAO;CACT;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,OAAO,MAAM;EAClB,KAAK,SAAS,MAAM;EACpB,KAAK,SAAS,MAAM;EACpB,KAAK,mBAAmB,KAAA;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM,OAAO,MAAM,KAAK,UAAU;EAClC,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,IAAI,CAAC;EAAE;CAC5D;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;GAC3C,MAAM,UAAU,wBAAwB,YAAY,KAAK,MAAM,IAAI;GAEnE,KAAK,YAAY,QAAQ;GACzB,KAAK,WAAW,QAAQ;GACxB,KAAK,WAAW,QAAQ;GACxB,KAAK,SAAS,QAAQ;GACtB,KAAK,UAAU,QAAQ;GACvB,KAAK,mBAAmB;GACxB,KAAK,QAAQ;GACb;EACF;CACF;;;;;;;;;CAUA,MAAM,YAA6B;EACjC,IAAI,CAAC,KAAK,SAAS,KAAK,qBAAqB,KAAA,GAC3C,OAAO,KAAK;EAEd,MAAM,yBAAS,IAAI,IAAoB;EACvC,MAAM,KAAK,UAAU,QAAQ,KAAsB,SAAiB;GAClE,OAAO,IAAI,OAAO,GAAG,GAAG,IAAI;EAC9B,CAAC;EAED,MAAM,aAKD,CAAC;EACN,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,WAAW,KAAK;GACd,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB,CAAC;EAGH,MAAM,MAAM,KAAK,UAAU;GACzB,SAAS;GACT,QAAQ,OAAO,YAAY,MAAM;GACjC,QAAQ;GACR,UAAU,CAAC,GAAG,KAAK,SAAS,QAAQ,CAAC;GACrC,SAAS,KAAK;EAChB,CAAC;EACD,KAAK,mBAAmB;EACxB,KAAK,QAAQ;EACb,OAAO;CACT;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,SAAS,KAAK,MAAM,IAAI;EAa9B,IAAI,OAAO,YAAY,GACrB,MAAM,IAAI,MACR,2DAA2D,OAAO,OAAO,OAAO,EAAE,cACpF;EAGF,MAAM,MAAM,IAAI,wBAAwB,IAAI;EAC5C,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,OAAO,MAAM,GACpD,IAAI,UAAU,OAAO,KAAK,IAAI;EAEhC,KAAK,MAAM,CAAC,KAAK,QAAQ,OAAO,UAAU;GACxC,IAAI,SAAS,IAAI,KAAK,GAAG;GACzB,IAAI,SAAS,IAAI,KAAK,GAAG;EAC3B;EACA,IAAI,UAAU,OAAO;EACrB,KAAK,MAAM,SAAS,OAAO,QAAQ;GACjC,MAAM,MAAM,aAAa,MAAM,MAAsB,MAAM,OAAO;GAClE,IAAI,OAAO,IAAI,KAAK;IAClB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EAGA,IAAI,mBAAmB;EACvB,IAAI,QAAQ;EACZ,OAAO;CACT;AACF"}
1
+ {"version":3,"file":"flexsearch-full-text-index.js","names":[],"sources":["../src/flexsearch-full-text-index.ts"],"sourcesContent":["import type { FullTextBlock, FullTextIndexInfo } from \"@statewalker/indexer-fulltext\";\nimport { type MemFullTextEngine, MemFullTextIndex, type RankedKey } from \"@statewalker/indexer-mem\";\nimport FlexSearch, { type Index as FlexIndex } from \"flexsearch\";\n\nfunction newFlexIndex(): FlexIndex {\n return new FlexSearch.Index({ tokenize: \"forward\", resolution: 9, cache: true });\n}\n\n/**\n * FlexSearch adapter for the in-memory block store. FlexSearch ids are\n * numbers, so the adapter keeps a store-key ↔ number map; it is part of the\n * exported state (`chunks`, `keyToNum`, `nextNum`).\n */\nclass FlexSearchEngine implements MemFullTextEngine {\n private flexIndex = newFlexIndex();\n private keyToNum = new Map<string, number>();\n private numToKey = new Map<number, string>();\n private nextNum = 1;\n\n add(key: string, block: FullTextBlock): void {\n let num = this.keyToNum.get(key);\n if (num === undefined) {\n num = this.nextNum++;\n this.keyToNum.set(key, num);\n this.numToKey.set(num, key);\n }\n this.flexIndex.add(num, block.content);\n }\n\n remove(key: string): void {\n const num = this.keyToNum.get(key);\n if (num === undefined) return;\n this.flexIndex.remove(num);\n this.keyToNum.delete(key);\n this.numToKey.delete(num);\n }\n\n search(query: string, limit: number): RankedKey[] {\n // `suggest: true` relaxes the default all-terms-AND for a multi-word query into a\n // ranked partial match — sections are ordered by how much of the term set they match\n // (FlexSearch's own resolution scoring), instead of returning nothing unless one\n // section contains every term. For single-word or fully-intersecting queries it is\n // identical to the default, so it only ever broadens recall, never narrows it.\n // Over-fetch (×3) so path filtering in the store still leaves `limit` hits.\n const ids = this.flexIndex.search(query, limit * 3, { suggest: true }) as number[];\n const hits: RankedKey[] = [];\n ids.forEach((num, rank) => {\n const key = this.numToKey.get(num);\n if (key !== undefined) hits.push({ key, score: 1 - rank / ids.length });\n });\n return hits;\n }\n\n exportState(): Record<string, unknown> {\n const chunks: Record<string, string> = {};\n this.flexIndex.export((key: string | number, data: string) => {\n chunks[String(key)] = data;\n });\n return { chunks, keyToNum: [...this.keyToNum.entries()], nextNum: this.nextNum };\n }\n\n static restore(state: Record<string, unknown>): FlexSearchEngine {\n const engine = new FlexSearchEngine();\n const chunks = (state.chunks ?? {}) as Record<string, string>;\n for (const [key, data] of Object.entries(chunks)) engine.flexIndex.import(key, data);\n for (const [key, num] of (state.keyToNum ?? []) as Array<[string, number]>) {\n engine.keyToNum.set(key, num);\n engine.numToKey.set(num, key);\n }\n engine.nextNum = typeof state.nextNum === \"number\" ? state.nextNum : engine.keyToNum.size + 1;\n return engine;\n }\n}\n\n/**\n * In-memory full-text sub-index backed by FlexSearch: a FlexSearch adapter\n * over the shared in-memory block store ({@link MemFullTextIndex}).\n */\nexport class FlexSearchFullTextIndex extends MemFullTextIndex {\n constructor(info: FullTextIndexInfo) {\n super(info, \"FlexSearchFullTextIndex\", new FlexSearchEngine());\n }\n\n protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine {\n return FlexSearchEngine.restore(state);\n }\n\n /**\n * Serialize to a JSON string (format version 3). Cached: while the index is\n * unchanged, returns the same string without exporting FlexSearch again.\n */\n async serialize(): Promise<string> {\n return this.serializeState();\n }\n\n /** Deserialize from JSON (format version 3); throws on any other version. */\n static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex {\n const index = new FlexSearchFullTextIndex(info);\n index.restoreState(json);\n return index;\n }\n}\n"],"mappings":";;;AAIA,SAAS,eAA0B;CACjC,OAAO,IAAI,WAAW,MAAM;EAAE,UAAU;EAAW,YAAY;EAAG,OAAO;CAAK,CAAC;AACjF;;;;;;AAOA,IAAM,mBAAN,MAAM,iBAA8C;CAClD,YAAoB,aAAa;CACjC,2BAAmB,IAAI,IAAoB;CAC3C,2BAAmB,IAAI,IAAoB;CAC3C,UAAkB;CAElB,IAAI,KAAa,OAA4B;EAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG;EAC/B,IAAI,QAAQ,KAAA,GAAW;GACrB,MAAM,KAAK;GACX,KAAK,SAAS,IAAI,KAAK,GAAG;GAC1B,KAAK,SAAS,IAAI,KAAK,GAAG;EAC5B;EACA,KAAK,UAAU,IAAI,KAAK,MAAM,OAAO;CACvC;CAEA,OAAO,KAAmB;EACxB,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG;EACjC,IAAI,QAAQ,KAAA,GAAW;EACvB,KAAK,UAAU,OAAO,GAAG;EACzB,KAAK,SAAS,OAAO,GAAG;EACxB,KAAK,SAAS,OAAO,GAAG;CAC1B;CAEA,OAAO,OAAe,OAA4B;EAOhD,MAAM,MAAM,KAAK,UAAU,OAAO,OAAO,QAAQ,GAAG,EAAE,SAAS,KAAK,CAAC;EACrE,MAAM,OAAoB,CAAC;EAC3B,IAAI,SAAS,KAAK,SAAS;GACzB,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG;GACjC,IAAI,QAAQ,KAAA,GAAW,KAAK,KAAK;IAAE;IAAK,OAAO,IAAI,OAAO,IAAI;GAAO,CAAC;EACxE,CAAC;EACD,OAAO;CACT;CAEA,cAAuC;EACrC,MAAM,SAAiC,CAAC;EACxC,KAAK,UAAU,QAAQ,KAAsB,SAAiB;GAC5D,OAAO,OAAO,GAAG,KAAK;EACxB,CAAC;EACD,OAAO;GAAE;GAAQ,UAAU,CAAC,GAAG,KAAK,SAAS,QAAQ,CAAC;GAAG,SAAS,KAAK;EAAQ;CACjF;CAEA,OAAO,QAAQ,OAAkD;EAC/D,MAAM,SAAS,IAAI,iBAAiB;EACpC,MAAM,SAAU,MAAM,UAAU,CAAC;EACjC,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,GAAG,OAAO,UAAU,OAAO,KAAK,IAAI;EACnF,KAAK,MAAM,CAAC,KAAK,QAAS,MAAM,YAAY,CAAC,GAA+B;GAC1E,OAAO,SAAS,IAAI,KAAK,GAAG;GAC5B,OAAO,SAAS,IAAI,KAAK,GAAG;EAC9B;EACA,OAAO,UAAU,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU,OAAO,SAAS,OAAO;EAC5F,OAAO;CACT;AACF;;;;;AAMA,IAAa,0BAAb,MAAa,gCAAgC,iBAAiB;CAC5D,YAAY,MAAyB;EACnC,MAAM,MAAM,2BAA2B,IAAI,iBAAiB,CAAC;CAC/D;CAEA,cAAwB,OAAmD;EACzE,OAAO,iBAAiB,QAAQ,KAAK;CACvC;;;;;CAMA,MAAM,YAA6B;EACjC,OAAO,KAAK,eAAe;CAC7B;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,QAAQ,IAAI,wBAAwB,IAAI;EAC9C,MAAM,aAAa,IAAI;EACvB,OAAO;CACT;AACF"}
package/dist/flexsearch-indexer.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { flexSearchFullTextProvider } from "./flexsearch-provider.js";
2
- import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
3
2
  import { memVectorProvider } from "@statewalker/indexer-mem";
3
+ import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
4
4
  //#region src/flexsearch-indexer.ts
5
5
  /**
6
6
  * In-memory Indexer combining FlexSearch full-text and brute-force vector
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/indexer-mem-flexsearch",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "In-memory indexer implementation backed by FlexSearch for full-text search.",
@@ -33,17 +33,17 @@
33
33
  "dependencies": {
34
34
  "flexsearch": "^0.8.212",
35
35
  "@statewalker/indexer-api": "^0.1.5",
36
- "@statewalker/indexer-core": "^0.2.0",
37
36
  "@statewalker/indexer-fulltext": "^0.1.6",
38
- "@statewalker/indexer-mem": "^0.2.0"
37
+ "@statewalker/indexer-core": "^0.3.0",
38
+ "@statewalker/indexer-mem": "^0.4.0"
39
39
  },
40
40
  "devDependencies": {
41
41
  "rimraf": "^6.1.3",
42
42
  "tsdown": "^0.23.0",
43
43
  "typescript": "^7.0.2",
44
44
  "vitest": "^5.0.3",
45
- "@statewalker/indexer-tests": "^0.0.0",
46
45
  "@statewalker/indexer-search": "^0.1.0",
46
+ "@statewalker/indexer-tests": "^0.0.0",
47
47
  "@statewalker/indexer-vector": "^0.1.6"
48
48
  },
49
49
  "sideEffects": false,
package/src/flexsearch-full-text-index.ts CHANGED
@@ -1,421 +1,102 @@
1
- import type {
2
- BlockReference,
3
- DocumentPath,
4
- Metadata,
5
- PathSelector,
6
- PersistableSearchIndex,
7
- PersistenceEntry,
8
- } from "@statewalker/indexer-api";
9
- import {
10
- compositeKey,
11
- matchesPrefix,
12
- readEntryBytes,
13
- singleChunk,
14
- toAsyncIterable,
15
- toBytes,
16
- } from "@statewalker/indexer-core";
17
- import type {
18
- FullTextBlock,
19
- FullTextIndex,
20
- FullTextIndexInfo,
21
- FulltextQuery as FullTextSearchParams,
22
- FulltextResult as FullTextSearchResult,
23
- } from "@statewalker/indexer-fulltext";
1
+ import type { FullTextBlock, FullTextIndexInfo } from "@statewalker/indexer-fulltext";
2
+ import { type MemFullTextEngine, MemFullTextIndex, type RankedKey } from "@statewalker/indexer-mem";
24
3
  import FlexSearch, { type Index as FlexIndex } from "flexsearch";
25
4
 
26
- interface StoredBlock {
27
- path: DocumentPath;
28
- blockId: string;
29
- content: string;
30
- metadata?: Metadata;
5
+ function newFlexIndex(): FlexIndex {
6
+ return new FlexSearch.Index({ tokenize: "forward", resolution: 9, cache: true });
31
7
  }
32
8
 
33
- export class FlexSearchFullTextIndex
34
- implements
35
- FullTextIndex,
36
- PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>
37
- {
38
- private readonly info: FullTextIndexInfo;
39
- private flexIndex: FlexIndex;
9
+ /**
10
+ * FlexSearch adapter for the in-memory block store. FlexSearch ids are
11
+ * numbers, so the adapter keeps a store-key ↔ number map; it is part of the
12
+ * exported state (`chunks`, `keyToNum`, `nextNum`).
13
+ */
14
+ class FlexSearchEngine implements MemFullTextEngine {
15
+ private flexIndex = newFlexIndex();
40
16
  private keyToNum = new Map<string, number>();
41
17
  private numToKey = new Map<number, string>();
42
- private blocks = new Map<string, StoredBlock>();
43
18
  private nextNum = 1;
44
- private closed = false;
45
- /**
46
- * Dirty bit. `true` whenever the FTS index has been mutated since the
47
- * last `serialize()`. The serializer caches the last produced JSON and
48
- * returns it directly when `!dirty`, so a no-op re-sync skips the
49
- * expensive FlexSearch export entirely. Starts `true` so the first
50
- * serialize after construction always runs.
51
- */
52
- private dirty = true;
53
- private cachedSerialized?: string;
54
-
55
- constructor(info: FullTextIndexInfo) {
56
- this.info = info;
57
- this.flexIndex = new FlexSearch.Index({
58
- tokenize: "forward",
59
- resolution: 9,
60
- cache: true,
61
- });
62
- }
63
-
64
- /**
65
- * `true` when the in-memory state has changed since the last serialize.
66
- * Callers (e.g. the persistence layer) may use it to skip serialization
67
- * and writes when the index has not been touched.
68
- */
69
- isDirty(): boolean {
70
- return this.dirty;
71
- }
72
19
 
73
- private ensureOpen(): void {
74
- if (this.closed) {
75
- throw new Error("FullTextIndex is closed");
76
- }
77
- }
78
-
79
- private getOrAssignNum(key: string): number {
20
+ add(key: string, block: FullTextBlock): void {
80
21
  let num = this.keyToNum.get(key);
81
22
  if (num === undefined) {
82
23
  num = this.nextNum++;
83
24
  this.keyToNum.set(key, num);
84
25
  this.numToKey.set(num, key);
85
26
  }
86
- return num;
27
+ this.flexIndex.add(num, block.content);
87
28
  }
88
29
 
89
- private searchInternal(
90
- query: string,
91
- topK: number,
92
- pathPrefixes?: DocumentPath[],
93
- ): FullTextSearchResult[] {
30
+ remove(key: string): void {
31
+ const num = this.keyToNum.get(key);
32
+ if (num === undefined) return;
33
+ this.flexIndex.remove(num);
34
+ this.keyToNum.delete(key);
35
+ this.numToKey.delete(num);
36
+ }
37
+
38
+ search(query: string, limit: number): RankedKey[] {
94
39
  // `suggest: true` relaxes the default all-terms-AND for a multi-word query into a
95
40
  // ranked partial match — sections are ordered by how much of the term set they match
96
41
  // (FlexSearch's own resolution scoring), instead of returning nothing unless one
97
42
  // section contains every term. For single-word or fully-intersecting queries it is
98
43
  // identical to the default, so it only ever broadens recall, never narrows it.
99
- const ids = this.flexIndex.search(query, topK * 3, { suggest: true }) as number[];
100
-
101
- let results = ids
102
- .map((numId, rank) => ({
103
- key: this.numToKey.get(numId) ?? "",
104
- score: 1 - rank / ids.length,
105
- }))
106
- .filter((r) => r.key !== "");
107
-
108
- // Filter by path prefixes
109
- if (pathPrefixes && pathPrefixes.length > 0) {
110
- results = results.filter((r) => {
111
- const block = this.blocks.get(r.key);
112
- return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));
113
- });
114
- }
115
-
116
- results.sort((a, b) => b.score - a.score);
117
-
118
- return results.slice(0, topK).map((r) => {
119
- const block = this.blocks.get(r.key);
120
- return {
121
- path: block?.path ?? ("/" as DocumentPath),
122
- blockId: block?.blockId ?? "",
123
- snippet: block?.content ?? "",
124
- score: r.score,
125
- };
44
+ // Over-fetch (×3) so path filtering in the store still leaves `limit` hits.
45
+ const ids = this.flexIndex.search(query, limit * 3, { suggest: true }) as number[];
46
+ const hits: RankedKey[] = [];
47
+ ids.forEach((num, rank) => {
48
+ const key = this.numToKey.get(num);
49
+ if (key !== undefined) hits.push({ key, score: 1 - rank / ids.length });
126
50
  });
51
+ return hits;
127
52
  }
128
53
 
129
- async getIndexInfo(): Promise<FullTextIndexInfo> {
130
- this.ensureOpen();
131
- return { ...this.info };
132
- }
133
-
134
- async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {
135
- this.ensureOpen();
136
- const { queries, paths } = params;
137
- const topK = params.topK ?? 100;
138
-
139
- if (!queries || queries.length === 0) return;
140
-
141
- // Multi-query: search for each and ACCUMULATE per-block scores, so a block
142
- // matching more queries ranks higher (the FulltextQuery contract). Taking the
143
- // max instead would let a block that is the top hit for one query outrank a
144
- // block matching several queries. The snippet is kept from the strongest
145
- // single-query match.
146
- const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();
147
-
148
- for (const query of queries) {
149
- const results = this.searchInternal(query, topK, paths);
150
- for (const r of results) {
151
- const key = compositeKey(r.path, r.blockId);
152
- const existing = merged.get(key);
153
- if (!existing) {
154
- merged.set(key, { result: r, best: r.score, total: r.score });
155
- } else {
156
- existing.total += r.score;
157
- if (r.score > existing.best) {
158
- existing.best = r.score;
159
- existing.result = r;
160
- }
161
- }
162
- }
163
- }
164
-
165
- const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
166
- for (const { result, total } of sorted.slice(0, topK)) {
167
- yield { ...result, score: total };
168
- }
169
- }
170
-
171
- async addDocument(blocks: FullTextBlock[]): Promise<void> {
172
- this.ensureOpen();
173
- if (blocks.length === 0) return;
174
- for (const block of blocks) {
175
- const key = compositeKey(block.path, block.blockId);
176
- const num = this.getOrAssignNum(key);
177
- // Remove old content if exists
178
- if (this.blocks.has(key)) {
179
- this.flexIndex.remove(num);
180
- }
181
- this.flexIndex.add(num, block.content);
182
- this.blocks.set(key, {
183
- path: block.path,
184
- blockId: block.blockId,
185
- content: block.content,
186
- metadata: block.metadata,
187
- });
188
- }
189
- this.dirty = true;
190
- }
191
-
192
- async addDocuments(
193
- blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
194
- ): Promise<void> {
195
- this.ensureOpen();
196
- for await (const batch of blocks) {
197
- await this.addDocument(batch);
198
- }
199
- }
200
-
201
- async deleteDocuments(
202
- pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
203
- ): Promise<void> {
204
- this.ensureOpen();
205
- let removed = 0;
206
- for await (const sel of toAsyncIterable(pathSelectors)) {
207
- if (sel.blockId !== undefined) {
208
- const key = compositeKey(sel.path, sel.blockId);
209
- if (this.removeByKey(key)) removed += 1;
210
- } else {
211
- const keysToDelete: string[] = [];
212
- for (const [key, block] of this.blocks) {
213
- if (matchesPrefix(block.path, sel.path)) {
214
- keysToDelete.push(key);
215
- }
216
- }
217
- for (const key of keysToDelete) {
218
- if (this.removeByKey(key)) removed += 1;
219
- }
220
- }
221
- }
222
- if (removed > 0) this.dirty = true;
223
- }
224
-
225
- /** Returns true when the key was present and got removed. */
226
- private removeByKey(key: string): boolean {
227
- const num = this.keyToNum.get(key);
228
- const had = this.blocks.has(key);
229
- if (num !== undefined) {
230
- this.flexIndex.remove(num);
231
- this.keyToNum.delete(key);
232
- this.numToKey.delete(num);
233
- }
234
- this.blocks.delete(key);
235
- return had;
236
- }
237
-
238
- async getSize(pathPrefix?: DocumentPath): Promise<number> {
239
- this.ensureOpen();
240
- if (pathPrefix === undefined) return this.blocks.size;
241
- let count = 0;
242
- for (const block of this.blocks.values()) {
243
- if (matchesPrefix(block.path, pathPrefix)) count++;
244
- }
245
- return count;
246
- }
247
-
248
- async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
249
- this.ensureOpen();
250
- const paths = new Set<DocumentPath>();
251
- for (const block of this.blocks.values()) {
252
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
253
- paths.add(block.path);
254
- }
255
- }
256
- for (const p of paths) yield p;
257
- }
258
-
259
- async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
260
- this.ensureOpen();
261
- for (const block of this.blocks.values()) {
262
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
263
- yield { path: block.path, blockId: block.blockId };
264
- }
265
- }
54
+ exportState(): Record<string, unknown> {
55
+ const chunks: Record<string, string> = {};
56
+ this.flexIndex.export((key: string | number, data: string) => {
57
+ chunks[String(key)] = data;
58
+ });
59
+ return { chunks, keyToNum: [...this.keyToNum.entries()], nextNum: this.nextNum };
266
60
  }
267
61
 
268
- async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
269
- this.ensureOpen();
270
- for (const block of this.blocks.values()) {
271
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
272
- yield {
273
- path: block.path,
274
- blockId: block.blockId,
275
- content: block.content,
276
- metadata: block.metadata,
277
- };
278
- }
62
+ static restore(state: Record<string, unknown>): FlexSearchEngine {
63
+ const engine = new FlexSearchEngine();
64
+ const chunks = (state.chunks ?? {}) as Record<string, string>;
65
+ for (const [key, data] of Object.entries(chunks)) engine.flexIndex.import(key, data);
66
+ for (const [key, num] of (state.keyToNum ?? []) as Array<[string, number]>) {
67
+ engine.keyToNum.set(key, num);
68
+ engine.numToKey.set(num, key);
279
69
  }
70
+ engine.nextNum = typeof state.nextNum === "number" ? state.nextNum : engine.keyToNum.size + 1;
71
+ return engine;
280
72
  }
73
+ }
281
74
 
282
- async close(_options?: { force?: boolean }): Promise<void> {
283
- this.closed = true;
284
- }
285
-
286
- async flush(): Promise<void> {
287
- this.ensureOpen();
288
- // no-op for in-memory
289
- }
290
-
291
- async deleteIndex(): Promise<void> {
292
- this.ensureOpen();
293
- this.blocks.clear();
294
- this.keyToNum.clear();
295
- this.numToKey.clear();
296
- this.cachedSerialized = undefined;
297
- this.dirty = true;
298
- this.closed = true;
299
- }
300
-
301
- // --- PersistableSearchIndex --------------------------------------------------
302
-
303
- /**
304
- * Stream the FTS sub-index's state as a single named entry. The composite
305
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
306
- */
307
- async *serialise(): AsyncIterable<PersistenceEntry> {
308
- const json = await this.serialize();
309
- yield { name: "json", content: singleChunk(toBytes(json)) };
75
+ /**
76
+ * In-memory full-text sub-index backed by FlexSearch: a FlexSearch adapter
77
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
78
+ */
79
+ export class FlexSearchFullTextIndex extends MemFullTextIndex {
80
+ constructor(info: FullTextIndexInfo) {
81
+ super(info, "FlexSearchFullTextIndex", new FlexSearchEngine());
310
82
  }
311
83
 
312
- /** Restore the FTS sub-index from a previously-produced entry set. */
313
- async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
314
- this.ensureOpen();
315
- for (const e of entries) {
316
- if (e.name !== "json") continue;
317
- const bytes = await readEntryBytes(e);
318
- const json = new TextDecoder().decode(bytes);
319
- const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);
320
- // Adopt the rebuilt instance's internal state.
321
- this.flexIndex = rebuilt.flexIndex;
322
- this.keyToNum = rebuilt.keyToNum;
323
- this.numToKey = rebuilt.numToKey;
324
- this.blocks = rebuilt.blocks;
325
- this.nextNum = rebuilt.nextNum;
326
- this.cachedSerialized = json;
327
- this.dirty = false;
328
- return;
329
- }
84
+ protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine {
85
+ return FlexSearchEngine.restore(state);
330
86
  }
331
87
 
332
88
  /**
333
- * Serialize to JSON string (v3 format with path data).
334
- *
335
- * Caches the result; while `dirty` is false (no mutations since the
336
- * last call) returns the same string, skipping the FlexSearch export
337
- * entirely. The persistence layer still calls into us each flush, but
338
- * the heavy work only runs when the index actually changed.
89
+ * Serialize to a JSON string (format version 3). Cached: while the index is
90
+ * unchanged, returns the same string without exporting FlexSearch again.
339
91
  */
340
92
  async serialize(): Promise<string> {
341
- if (!this.dirty && this.cachedSerialized !== undefined) {
342
- return this.cachedSerialized;
343
- }
344
- const chunks = new Map<string, string>();
345
- await this.flexIndex.export((key: string | number, data: string) => {
346
- chunks.set(String(key), data);
347
- });
348
-
349
- const blocksData: Array<{
350
- path: string;
351
- blockId: string;
352
- content: string;
353
- metadata?: Metadata;
354
- }> = [];
355
- for (const block of this.blocks.values()) {
356
- blocksData.push({
357
- path: block.path,
358
- blockId: block.blockId,
359
- content: block.content,
360
- metadata: block.metadata,
361
- });
362
- }
363
-
364
- const out = JSON.stringify({
365
- version: 3,
366
- chunks: Object.fromEntries(chunks),
367
- blocks: blocksData,
368
- keyToNum: [...this.keyToNum.entries()],
369
- nextNum: this.nextNum,
370
- });
371
- this.cachedSerialized = out;
372
- this.dirty = false;
373
- return out;
93
+ return this.serializeState();
374
94
  }
375
95
 
376
- /** Deserialize from JSON — handles v3 format */
96
+ /** Deserialize from JSON (format version 3); throws on any other version. */
377
97
  static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex {
378
- const parsed = JSON.parse(json) as {
379
- version?: number;
380
- chunks: Record<string, string>;
381
- blocks: Array<{
382
- path: string;
383
- blockId: string;
384
- content: string;
385
- metadata?: Metadata;
386
- }>;
387
- keyToNum: Array<[string, number]>;
388
- nextNum: number;
389
- };
390
-
391
- if (parsed.version !== 3) {
392
- throw new Error(
393
- `FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,
394
- );
395
- }
396
-
397
- const fts = new FlexSearchFullTextIndex(info);
398
- for (const [key, data] of Object.entries(parsed.chunks)) {
399
- fts.flexIndex.import(key, data);
400
- }
401
- for (const [key, num] of parsed.keyToNum) {
402
- fts.keyToNum.set(key, num);
403
- fts.numToKey.set(num, key);
404
- }
405
- fts.nextNum = parsed.nextNum;
406
- for (const block of parsed.blocks) {
407
- const key = compositeKey(block.path as DocumentPath, block.blockId);
408
- fts.blocks.set(key, {
409
- path: block.path as DocumentPath,
410
- blockId: block.blockId,
411
- content: block.content,
412
- metadata: block.metadata,
413
- });
414
- }
415
- // Prime the cache: a deserialized index that's never mutated should
416
- // return the same JSON it was loaded from when re-serialized.
417
- fts.cachedSerialized = json;
418
- fts.dirty = false;
419
- return fts;
98
+ const index = new FlexSearchFullTextIndex(info);
99
+ index.restoreState(json);
100
+ return index;
420
101
  }
421
102
  }