@statewalker/indexer-mem-minisearch 0.1.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/minisearch-full-text-index.d.ts +11 -32
  2. package/dist/minisearch-full-text-index.d.ts.map +1 -1
  3. package/dist/minisearch-full-text-index.js +48 -192
  4. package/dist/minisearch-full-text-index.js.map +1 -1
  5. package/dist/minisearch-indexer.js +1 -1
  6. package/package.json +6 -6
  7. package/src/minisearch-full-text-index.ts +46 -315
package/dist/minisearch-full-text-index.d.ts CHANGED
@@ -1,40 +1,19 @@
1
- import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
2
- import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
1
+ import { MemFullTextEngine, MemFullTextIndex } from "@statewalker/indexer-mem";
2
+ import { FullTextIndexInfo } from "@statewalker/indexer-fulltext";
3
3
  //#region src/minisearch-full-text-index.d.ts
4
- export declare class MiniSearchFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
5
- private readonly info;
6
- private miniSearch;
7
- private blocks;
8
- private keySet;
9
- private closed;
4
+ /**
5
+ * In-memory full-text sub-index backed by MiniSearch: a MiniSearch adapter
6
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
7
+ */
8
+ export declare class MiniSearchFullTextIndex extends MemFullTextIndex {
10
9
  constructor(info: FullTextIndexInfo);
11
- private ensureOpen;
12
- private searchInternal;
13
- getIndexInfo(): Promise<FullTextIndexInfo>;
14
- search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
15
- addDocument(blocks: FullTextBlock[]): Promise<void>;
16
- addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
17
- deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
18
- private removeByKey;
19
- getSize(pathPrefix?: DocumentPath): Promise<number>;
20
- getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
21
- getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
22
- getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
23
- close(_options?: {
24
- force?: boolean;
25
- }): Promise<void>;
26
- flush(): Promise<void>;
27
- deleteIndex(): Promise<void>;
10
+ protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine;
28
11
  /**
29
- * Stream the FTS sub-index's state as a single named entry. The composite
30
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
12
+ * Serialize to a JSON string (format version 3). Cached: while the index is
13
+ * unchanged, returns the same string without exporting MiniSearch again.
31
14
  */
32
- serialise(): AsyncIterable<PersistenceEntry>;
33
- /** Restore the FTS sub-index from a previously-produced entry set. */
34
- loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
35
- /** Serialize to JSON (v3 format with path data) */
36
15
  serialize(): string;
37
- /** Deserialize from JSON — handles v3 format */
16
+ /** Deserialize from JSON (format version 3); throws on any other version. */
38
17
  static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex;
39
18
  }
40
19
  //#endregion
package/dist/minisearch-full-text-index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"minisearch-full-text-index.d.ts","names":["FullTextSearchParams","FullTextSearchResult"],"sources":["../src/minisearch-full-text-index.ts"],"mappings":";;;qBAgCa,mCAET,eACA,uBAAuB,eAAeA,eAAsBC;mBAE7C;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;UAQV;UAMA;EAiDF,gBAAgB,QAAQ;EAKvB,OAAO,QAAQD,gBAAuB,eAAeC;EAqCtD,YAAY,QAAQ,kBAAkB;EAkBtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;UAoBK;EAQF,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAe;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAIT,eAAe;;;;;EAad,aAAa,cAAc;;EAK5B,SAAS,SAAS,SAAS,oBAAoB;;EAerD;;SAyBO,YAAY,MAAM,mBAAmB,eAAe"}
1
+ {"version":3,"file":"minisearch-full-text-index.d.ts","names":[],"sources":["../src/minisearch-full-text-index.ts"],"mappings":";;;;;;;qBA4Da,gCAAgC;EAC3C,YAAY,MAAM;YAIR,cAAc,OAAO,0BAA0B;;;;;EAQzD;;SAKO,YAAY,MAAM,mBAAmB,eAAe"}
package/dist/minisearch-full-text-index.js CHANGED
@@ -1,23 +1,33 @@
1
- import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, toBytes } from "@statewalker/indexer-core";
1
+ import { MemFullTextIndex } from "@statewalker/indexer-mem";
2
2
  import MiniSearch from "minisearch";
3
3
  //#region src/minisearch-full-text-index.ts
4
- var MiniSearchFullTextIndex = class MiniSearchFullTextIndex {
5
- info;
4
+ const MINISEARCH_OPTIONS = {
5
+ fields: ["content"],
6
+ idField: "key"
7
+ };
8
+ /**
9
+ * MiniSearch adapter for the in-memory block store. Exported state:
10
+ * `miniSearch` (MiniSearch's own JSON) and `keys` (the indexed keys, kept for
11
+ * readers of format 3 that rebuild their key set from it).
12
+ */
13
+ var MiniSearchEngine = class MiniSearchEngine {
6
14
  miniSearch;
7
- blocks = /* @__PURE__ */ new Map();
8
- keySet = /* @__PURE__ */ new Set();
9
- closed = false;
10
- constructor(info) {
11
- this.info = info;
12
- this.miniSearch = new MiniSearch({
13
- fields: ["content"],
14
- idField: "key"
15
+ keys = /* @__PURE__ */ new Set();
16
+ constructor(miniSearch = new MiniSearch(MINISEARCH_OPTIONS)) {
17
+ this.miniSearch = miniSearch;
18
+ }
19
+ add(key, block) {
20
+ this.miniSearch.add({
21
+ key,
22
+ content: block.content
15
23
  });
24
+ this.keys.add(key);
16
25
  }
17
- ensureOpen() {
18
- if (this.closed) throw new Error("FullTextIndex is closed");
26
+ remove(key) {
27
+ if (!this.keys.delete(key)) return;
28
+ this.miniSearch.discard(key);
19
29
  }
20
- searchInternal(query, topK, pathPrefixes) {
30
+ search(query, _limit) {
21
31
  let results = this.miniSearch.search(query, {
22
32
  prefix: true,
23
33
  fuzzy: .2
@@ -30,199 +40,45 @@ var MiniSearchFullTextIndex = class MiniSearchFullTextIndex {
30
40
  combineWith: "OR"
31
41
  });
32
42
  }
33
- let filtered = results.map((r) => ({
43
+ return results.map((r) => ({
34
44
  key: r.id,
35
45
  score: r.score
36
46
  }));
37
- if (pathPrefixes && pathPrefixes.length > 0) filtered = filtered.filter((r) => {
38
- const block = this.blocks.get(r.key);
39
- return block !== void 0 && pathPrefixes.some((p) => matchesPrefix(block.path, p));
40
- });
41
- return filtered.slice(0, topK).map((r) => {
42
- const block = this.blocks.get(r.key);
43
- return {
44
- path: block?.path ?? "/",
45
- blockId: block?.blockId ?? "",
46
- snippet: block?.content ?? "",
47
- score: r.score
48
- };
49
- });
50
- }
51
- async getIndexInfo() {
52
- this.ensureOpen();
53
- return { ...this.info };
54
- }
55
- async *search(params) {
56
- this.ensureOpen();
57
- const { queries, paths } = params;
58
- const topK = params.topK ?? 100;
59
- if (!queries || queries.length === 0) return;
60
- const merged = /* @__PURE__ */ new Map();
61
- for (const query of queries) {
62
- const results = this.searchInternal(query, topK, paths);
63
- for (const r of results) {
64
- const key = compositeKey(r.path, r.blockId);
65
- const existing = merged.get(key);
66
- if (!existing) merged.set(key, {
67
- result: r,
68
- best: r.score,
69
- total: r.score
70
- });
71
- else {
72
- existing.total += r.score;
73
- if (r.score > existing.best) {
74
- existing.best = r.score;
75
- existing.result = r;
76
- }
77
- }
78
- }
79
- }
80
- const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
81
- for (const { result, total } of sorted.slice(0, topK)) yield {
82
- ...result,
83
- score: total
84
- };
85
- }
86
- async addDocument(blocks) {
87
- this.ensureOpen();
88
- for (const block of blocks) {
89
- const key = compositeKey(block.path, block.blockId);
90
- if (this.keySet.has(key)) this.miniSearch.discard(key);
91
- this.miniSearch.add({
92
- key,
93
- content: block.content
94
- });
95
- this.keySet.add(key);
96
- this.blocks.set(key, {
97
- path: block.path,
98
- blockId: block.blockId,
99
- content: block.content,
100
- metadata: block.metadata
101
- });
102
- }
103
- }
104
- async addDocuments(blocks) {
105
- this.ensureOpen();
106
- for await (const batch of blocks) await this.addDocument(batch);
107
- }
108
- async deleteDocuments(pathSelectors) {
109
- this.ensureOpen();
110
- for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
111
- const key = compositeKey(sel.path, sel.blockId);
112
- this.removeByKey(key);
113
- } else {
114
- const keysToDelete = [];
115
- for (const [key, block] of this.blocks) if (matchesPrefix(block.path, sel.path)) keysToDelete.push(key);
116
- for (const key of keysToDelete) this.removeByKey(key);
117
- }
118
- }
119
- removeByKey(key) {
120
- if (this.keySet.has(key)) {
121
- this.miniSearch.discard(key);
122
- this.keySet.delete(key);
123
- }
124
- this.blocks.delete(key);
125
47
  }
126
- async getSize(pathPrefix) {
127
- this.ensureOpen();
128
- if (pathPrefix === void 0) return this.blocks.size;
129
- let count = 0;
130
- for (const block of this.blocks.values()) if (matchesPrefix(block.path, pathPrefix)) count++;
131
- return count;
132
- }
133
- async *getDocumentPaths(pathPrefix) {
134
- this.ensureOpen();
135
- const paths = /* @__PURE__ */ new Set();
136
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) paths.add(block.path);
137
- for (const p of paths) yield p;
138
- }
139
- async *getDocumentBlocksRefs(pathPrefix) {
140
- this.ensureOpen();
141
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
142
- path: block.path,
143
- blockId: block.blockId
144
- };
145
- }
146
- async *getDocumentsBlocks(pathPrefix) {
147
- this.ensureOpen();
148
- for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
149
- path: block.path,
150
- blockId: block.blockId,
151
- content: block.content,
152
- metadata: block.metadata
48
+ exportState() {
49
+ return {
50
+ miniSearch: this.miniSearch.toJSON(),
51
+ keys: [...this.keys]
153
52
  };
154
53
  }
155
- async close(_options) {
156
- this.closed = true;
54
+ static restore(state) {
55
+ const engine = new MiniSearchEngine(MiniSearch.loadJSON(JSON.stringify(state.miniSearch), MINISEARCH_OPTIONS));
56
+ engine.keys = new Set(state.keys ?? []);
57
+ return engine;
157
58
  }
158
- async flush() {
159
- this.ensureOpen();
59
+ };
60
+ /**
61
+ * In-memory full-text sub-index backed by MiniSearch: a MiniSearch adapter
62
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
63
+ */
64
+ var MiniSearchFullTextIndex = class MiniSearchFullTextIndex extends MemFullTextIndex {
65
+ constructor(info) {
66
+ super(info, "MiniSearchFullTextIndex", new MiniSearchEngine());
160
67
  }
161
- async deleteIndex() {
162
- this.ensureOpen();
163
- this.blocks.clear();
164
- this.keySet.clear();
165
- this.closed = true;
68
+ restoreEngine(state) {
69
+ return MiniSearchEngine.restore(state);
166
70
  }
167
71
  /**
168
- * Stream the FTS sub-index's state as a single named entry. The composite
169
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
72
+ * Serialize to a JSON string (format version 3). Cached: while the index is
73
+ * unchanged, returns the same string without exporting MiniSearch again.
170
74
  */
171
- async *serialise() {
172
- yield {
173
- name: "json",
174
- content: singleChunk(toBytes(this.serialize()))
175
- };
176
- }
177
- /** Restore the FTS sub-index from a previously-produced entry set. */
178
- async loadFrom(entries) {
179
- this.ensureOpen();
180
- for (const e of entries) {
181
- if (e.name !== "json") continue;
182
- const bytes = await readEntryBytes(e);
183
- const json = new TextDecoder().decode(bytes);
184
- const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
185
- this.miniSearch = rebuilt.miniSearch;
186
- this.blocks = rebuilt.blocks;
187
- this.keySet = rebuilt.keySet;
188
- return;
189
- }
190
- }
191
- /** Serialize to JSON (v3 format with path data) */
192
75
  serialize() {
193
- const blocksData = [];
194
- for (const block of this.blocks.values()) blocksData.push({
195
- path: block.path,
196
- blockId: block.blockId,
197
- content: block.content,
198
- metadata: block.metadata
199
- });
200
- return JSON.stringify({
201
- version: 3,
202
- miniSearch: this.miniSearch.toJSON(),
203
- blocks: blocksData,
204
- keys: [...this.keySet]
205
- });
76
+ return this.serializeState();
206
77
  }
207
- /** Deserialize from JSON — handles v3 format */
78
+ /** Deserialize from JSON (format version 3); throws on any other version. */
208
79
  static deserialize(info, json) {
209
- const parsed = JSON.parse(json);
210
- if (parsed.version !== 3) throw new Error(`MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`);
211
80
  const index = new MiniSearchFullTextIndex(info);
212
- index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
213
- fields: ["content"],
214
- idField: "key"
215
- });
216
- index.keySet = new Set(parsed.keys);
217
- for (const block of parsed.blocks) {
218
- const key = compositeKey(block.path, block.blockId);
219
- index.blocks.set(key, {
220
- path: block.path,
221
- blockId: block.blockId,
222
- content: block.content,
223
- metadata: block.metadata
224
- });
225
- }
81
+ index.restoreState(json);
226
82
  return index;
227
83
  }
228
84
  };
package/dist/minisearch-full-text-index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"minisearch-full-text-index.js","names":[],"sources":["../src/minisearch-full-text-index.ts"],"sourcesContent":["import type {\n BlockReference,\n DocumentPath,\n Metadata,\n PathSelector,\n PersistableSearchIndex,\n PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n compositeKey,\n matchesPrefix,\n readEntryBytes,\n singleChunk,\n toAsyncIterable,\n toBytes,\n} from \"@statewalker/indexer-core\";\nimport type {\n FullTextBlock,\n FullTextIndex,\n FullTextIndexInfo,\n FulltextQuery as FullTextSearchParams,\n FulltextResult as FullTextSearchResult,\n} from \"@statewalker/indexer-fulltext\";\nimport MiniSearch from \"minisearch\";\n\ninterface StoredBlock {\n path: DocumentPath;\n blockId: string;\n content: string;\n metadata?: Metadata;\n}\n\nexport class MiniSearchFullTextIndex\n implements\n FullTextIndex,\n PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>\n{\n private readonly info: FullTextIndexInfo;\n private miniSearch: MiniSearch;\n private blocks = new Map<string, StoredBlock>();\n private keySet = new Set<string>();\n private closed = false;\n\n constructor(info: FullTextIndexInfo) {\n this.info = info;\n this.miniSearch = new MiniSearch({\n fields: [\"content\"],\n idField: \"key\",\n });\n }\n\n private ensureOpen(): void {\n if (this.closed) {\n throw new Error(\"FullTextIndex is closed\");\n }\n }\n\n private searchInternal(\n query: string,\n topK: number,\n pathPrefixes?: DocumentPath[],\n ): FullTextSearchResult[] {\n let results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n });\n\n if (results.length === 0) {\n results = this.miniSearch.search(query, { prefix: true });\n }\n\n if (results.length === 0) {\n const words = query.split(/\\s+/).filter((w) => w.length > 1);\n if (words.length > 1) {\n results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n combineWith: \"OR\",\n });\n }\n }\n\n // Filter by path prefixes\n let filtered = results.map((r) => ({\n key: r.id as string,\n score: r.score,\n }));\n\n if (pathPrefixes && pathPrefixes.length > 0) {\n filtered = filtered.filter((r) => {\n const block = this.blocks.get(r.key);\n return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));\n });\n }\n\n return filtered.slice(0, topK).map((r) => {\n const block = this.blocks.get(r.key);\n return {\n path: block?.path ?? (\"/\" as DocumentPath),\n blockId: block?.blockId ?? \"\",\n snippet: block?.content ?? \"\",\n score: r.score,\n };\n });\n }\n\n async getIndexInfo(): Promise<FullTextIndexInfo> {\n this.ensureOpen();\n return { ...this.info };\n }\n\n async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {\n this.ensureOpen();\n const { queries, paths } = params;\n const topK = params.topK ?? 100;\n\n if (!queries || queries.length === 0) return;\n\n // Multi-query: search for each and ACCUMULATE per-block scores, so a block\n // matching more queries ranks higher (the FulltextQuery contract). Taking the\n // max instead would let a block that is the top hit for one query outrank a\n // block matching several queries. The snippet is kept from the strongest\n // single-query match.\n const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();\n\n for (const query of queries) {\n const results = this.searchInternal(query, topK, paths);\n for (const r of results) {\n const key = compositeKey(r.path, r.blockId);\n const existing = merged.get(key);\n if (!existing) {\n merged.set(key, { result: r, best: r.score, total: r.score });\n } else {\n existing.total += r.score;\n if (r.score > existing.best) {\n existing.best = r.score;\n existing.result = r;\n }\n }\n }\n }\n\n const sorted = [...merged.values()].sort((a, b) => b.total - a.total);\n for (const { result, total } of sorted.slice(0, topK)) {\n yield { ...result, score: total };\n }\n }\n\n async addDocument(blocks: FullTextBlock[]): Promise<void> {\n this.ensureOpen();\n for (const block of blocks) {\n const key = compositeKey(block.path, block.blockId);\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n }\n this.miniSearch.add({ key, content: block.content });\n this.keySet.add(key);\n this.blocks.set(key, {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n }\n\n async addDocuments(\n blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const batch of blocks) {\n await this.addDocument(batch);\n }\n }\n\n async deleteDocuments(\n pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const sel of toAsyncIterable(pathSelectors)) {\n if (sel.blockId !== undefined) {\n const key = compositeKey(sel.path, sel.blockId);\n this.removeByKey(key);\n } else {\n const keysToDelete: string[] = [];\n for (const [key, block] of this.blocks) {\n if (matchesPrefix(block.path, sel.path)) {\n keysToDelete.push(key);\n }\n }\n for (const key of keysToDelete) {\n this.removeByKey(key);\n }\n }\n }\n }\n\n private removeByKey(key: string): void {\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n this.keySet.delete(key);\n }\n this.blocks.delete(key);\n }\n\n async getSize(pathPrefix?: DocumentPath): Promise<number> {\n this.ensureOpen();\n if (pathPrefix === undefined) return this.blocks.size;\n let count = 0;\n for (const block of this.blocks.values()) {\n if (matchesPrefix(block.path, pathPrefix)) count++;\n }\n return count;\n }\n\n async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n this.ensureOpen();\n const paths = new Set<DocumentPath>();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n paths.add(block.path);\n }\n }\n for (const p of paths) yield p;\n }\n\n async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield { path: block.path, blockId: block.blockId };\n }\n }\n }\n\n async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n };\n }\n }\n }\n\n async close(_options?: { force?: boolean }): Promise<void> {\n this.closed = true;\n }\n\n async flush(): Promise<void> {\n this.ensureOpen();\n }\n\n async deleteIndex(): Promise<void> {\n this.ensureOpen();\n this.blocks.clear();\n this.keySet.clear();\n this.closed = true;\n }\n\n // --- PersistableSearchIndex --------------------------------------------------\n\n /**\n * Stream the FTS sub-index's state as a single named entry. The composite\n * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n */\n async *serialise(): AsyncIterable<PersistenceEntry> {\n yield { name: \"json\", content: singleChunk(toBytes(this.serialize())) };\n }\n\n /** Restore the FTS sub-index from a previously-produced entry set. */\n async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n this.ensureOpen();\n for (const e of entries) {\n if (e.name !== \"json\") continue;\n const bytes = await readEntryBytes(e);\n const json = new TextDecoder().decode(bytes);\n const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);\n this.miniSearch = rebuilt.miniSearch;\n this.blocks = rebuilt.blocks;\n this.keySet = rebuilt.keySet;\n return;\n }\n }\n\n /** Serialize to JSON (v3 format with path data) */\n serialize(): string {\n const blocksData: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }> = [];\n for (const block of this.blocks.values()) {\n blocksData.push({\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return JSON.stringify({\n version: 3,\n miniSearch: this.miniSearch.toJSON(),\n blocks: blocksData,\n keys: [...this.keySet],\n });\n }\n\n /** Deserialize from JSON — handles v3 format */\n static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {\n const parsed = JSON.parse(json) as {\n version?: number;\n miniSearch: unknown;\n blocks: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }>;\n keys: string[];\n };\n\n if (parsed.version !== 3) {\n throw new Error(\n `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,\n );\n }\n\n const index = new MiniSearchFullTextIndex(info);\n index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {\n fields: [\"content\"],\n idField: \"key\",\n });\n index.keySet = new Set(parsed.keys);\n for (const block of parsed.blocks) {\n const key = compositeKey(block.path as DocumentPath, block.blockId);\n index.blocks.set(key, {\n path: block.path as DocumentPath,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return index;\n }\n}\n"],"mappings":";;;AAgCA,IAAa,0BAAb,MAAa,wBAIb;CACE;CACA;CACA,yBAAiB,IAAI,IAAyB;CAC9C,yBAAiB,IAAI,IAAY;CACjC,SAAiB;CAEjB,YAAY,MAAyB;EACnC,KAAK,OAAO;EACZ,KAAK,aAAa,IAAI,WAAW;GAC/B,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;CACH;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,yBAAyB;CAE7C;CAEA,eACE,OACA,MACA,cACwB;EACxB,IAAI,UAAU,KAAK,WAAW,OAAO,OAAO;GAC1C,QAAQ;GACR,OAAO;EACT,CAAC;EAED,IAAI,QAAQ,WAAW,GACrB,UAAU,KAAK,WAAW,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;EAG1D,IAAI,QAAQ,WAAW,GACP;OAAA,MAAM,MAAM,KAAK,CAAC,CAAC,QAAQ,MAAM,EAAE,SAAS,CAClD,CAAC,CAAC,SAAS,GACjB,UAAU,KAAK,WAAW,OAAO,OAAO;IACtC,QAAQ;IACR,OAAO;IACP,aAAa;GACf,CAAC;EAAA;EAKL,IAAI,WAAW,QAAQ,KAAK,OAAO;GACjC,KAAK,EAAE;GACP,OAAO,EAAE;EACX,EAAE;EAEF,IAAI,gBAAgB,aAAa,SAAS,GACxC,WAAW,SAAS,QAAQ,MAAM;GAChC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO,UAAU,KAAA,KAAa,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC;EACrF,CAAC;EAGH,OAAO,SAAS,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,MAAM;GACxC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO;IACL,MAAM,OAAO,QAAS;IACtB,SAAS,OAAO,WAAW;IAC3B,SAAS,OAAO,WAAW;IAC3B,OAAO,EAAE;GACX;EACF,CAAC;CACH;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAoE;EAChF,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EAOtC,MAAM,yBAAS,IAAI,IAA2E;EAE9F,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,UAAU,KAAK,eAAe,OAAO,MAAM,KAAK;GACtD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,OAAO,IAAI,GAAG;IAC/B,IAAI,CAAC,UACH,OAAO,IAAI,KAAK;KAAE,QAAQ;KAAG,MAAM,EAAE;KAAO,OAAO,EAAE;IAAM,CAAC;SACvD;KACL,SAAS,SAAS,EAAE;KACpB,IAAI,EAAE,QAAQ,SAAS,MAAM;MAC3B,SAAS,OAAO,EAAE;MAClB,SAAS,SAAS;KACpB;IACF;GACF;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACpE,KAAK,MAAM,EAAE,QAAQ,WAAW,OAAO,MAAM,GAAG,IAAI,GAClD,MAAM;GAAE,GAAG;GAAQ,OAAO;EAAM;CAEpC;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,IAAI,KAAK,OAAO,IAAI,GAAG,GACrB,KAAK,WAAW,QAAQ,GAAG;GAE7B,KAAK,WAAW,IAAI;IAAE;IAAK,SAAS,MAAM;GAAQ,CAAC;GACnD,KAAK,OAAO,IAAI,GAAG;GACnB,KAAK,OAAO,IAAI,KAAK;IACnB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;CACF;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GAAW;GAC7B,MAAM,MAAM,aAAa,IAAI,MAAM,IAAI,OAAO;GAC9C,KAAK,YAAY,GAAG;EACtB,OAAO;GACL,MAAM,eAAyB,CAAC;GAChC,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GACpC,aAAa,KAAK,GAAG;GAGzB,KAAK,MAAM,OAAO,cAChB,KAAK,YAAY,GAAG;EAExB;CAEJ;CAEA,YAAoB,KAAmB;EACrC,IAAI,KAAK,OAAO,IAAI,GAAG,GAAG;GACxB,KAAK,WAAW,QAAQ,GAAG;GAC3B,KAAK,OAAO,OAAO,GAAG;EACxB;EACA,KAAK,OAAO,OAAO,GAAG;CACxB;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAClB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,OAAO,MAAM;EAClB,KAAK,OAAO,MAAM;EAClB,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,KAAK,UAAU,CAAC,CAAC;EAAE;CACxE;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;GAC3C,MAAM,UAAU,wBAAwB,YAAY,KAAK,MAAM,IAAI;GACnE,KAAK,aAAa,QAAQ;GAC1B,KAAK,SAAS,QAAQ;GACtB,KAAK,SAAS,QAAQ;GACtB;EACF;CACF;;CAGA,YAAoB;EAClB,MAAM,aAKD,CAAC;EACN,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,WAAW,KAAK;GACd,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB,CAAC;EAGH,OAAO,KAAK,UAAU;GACpB,SAAS;GACT,YAAY,KAAK,WAAW,OAAO;GACnC,QAAQ;GACR,MAAM,CAAC,GAAG,KAAK,MAAM;EACvB,CAAC;CACH;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,SAAS,KAAK,MAAM,IAAI;EAY9B,IAAI,OAAO,YAAY,GACrB,MAAM,IAAI,MACR,2DAA2D,OAAO,OAAO,OAAO,EAAE,cACpF;EAGF,MAAM,QAAQ,IAAI,wBAAwB,IAAI;EAC9C,MAAM,aAAa,WAAW,SAAS,KAAK,UAAU,OAAO,UAAU,GAAG;GACxE,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;EACD,MAAM,SAAS,IAAI,IAAI,OAAO,IAAI;EAClC,KAAK,MAAM,SAAS,OAAO,QAAQ;GACjC,MAAM,MAAM,aAAa,MAAM,MAAsB,MAAM,OAAO;GAClE,MAAM,OAAO,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EAEA,OAAO;CACT;AACF"}
1
+ {"version":3,"file":"minisearch-full-text-index.js","names":[],"sources":["../src/minisearch-full-text-index.ts"],"sourcesContent":["import type { FullTextBlock, FullTextIndexInfo } from \"@statewalker/indexer-fulltext\";\nimport { type MemFullTextEngine, MemFullTextIndex, type RankedKey } from \"@statewalker/indexer-mem\";\nimport MiniSearch from \"minisearch\";\n\nconst MINISEARCH_OPTIONS = { fields: [\"content\"], idField: \"key\" };\n\n/**\n * MiniSearch adapter for the in-memory block store. Exported state:\n * `miniSearch` (MiniSearch's own JSON) and `keys` (the indexed keys, kept for\n * readers of format 3 that rebuild their key set from it).\n */\nclass MiniSearchEngine implements MemFullTextEngine {\n private miniSearch: MiniSearch;\n private keys = new Set<string>();\n\n constructor(miniSearch = new MiniSearch(MINISEARCH_OPTIONS)) {\n this.miniSearch = miniSearch;\n }\n\n add(key: string, block: FullTextBlock): void {\n this.miniSearch.add({ key, content: block.content });\n this.keys.add(key);\n }\n\n remove(key: string): void {\n if (!this.keys.delete(key)) return;\n this.miniSearch.discard(key);\n }\n\n search(query: string, _limit: number): RankedKey[] {\n let results = this.miniSearch.search(query, { prefix: true, fuzzy: 0.2 });\n if (results.length === 0) {\n results = this.miniSearch.search(query, { prefix: true });\n }\n if (results.length === 0) {\n const words = query.split(/\\s+/).filter((w) => w.length > 1);\n if (words.length > 1) {\n results = this.miniSearch.search(query, { prefix: true, fuzzy: 0.2, combineWith: \"OR\" });\n }\n }\n return results.map((r) => ({ key: r.id as string, score: r.score }));\n }\n\n exportState(): Record<string, unknown> {\n return { miniSearch: this.miniSearch.toJSON(), keys: [...this.keys] };\n }\n\n static restore(state: Record<string, unknown>): MiniSearchEngine {\n const engine = new MiniSearchEngine(\n MiniSearch.loadJSON(JSON.stringify(state.miniSearch), MINISEARCH_OPTIONS),\n );\n engine.keys = new Set((state.keys ?? []) as string[]);\n return engine;\n }\n}\n\n/**\n * In-memory full-text sub-index backed by MiniSearch: a MiniSearch adapter\n * over the shared in-memory block store ({@link MemFullTextIndex}).\n */\nexport class MiniSearchFullTextIndex extends MemFullTextIndex {\n constructor(info: FullTextIndexInfo) {\n super(info, \"MiniSearchFullTextIndex\", new MiniSearchEngine());\n }\n\n protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine {\n return MiniSearchEngine.restore(state);\n }\n\n /**\n * Serialize to a JSON string (format version 3). Cached: while the index is\n * unchanged, returns the same string without exporting MiniSearch again.\n */\n serialize(): string {\n return this.serializeState();\n }\n\n /** Deserialize from JSON (format version 3); throws on any other version. */\n static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {\n const index = new MiniSearchFullTextIndex(info);\n index.restoreState(json);\n return index;\n }\n}\n"],"mappings":";;;AAIA,MAAM,qBAAqB;CAAE,QAAQ,CAAC,SAAS;CAAG,SAAS;AAAM;;;;;;AAOjE,IAAM,mBAAN,MAAM,iBAA8C;CAClD;CACA,uBAAe,IAAI,IAAY;CAE/B,YAAY,aAAa,IAAI,WAAW,kBAAkB,GAAG;EAC3D,KAAK,aAAa;CACpB;CAEA,IAAI,KAAa,OAA4B;EAC3C,KAAK,WAAW,IAAI;GAAE;GAAK,SAAS,MAAM;EAAQ,CAAC;EACnD,KAAK,KAAK,IAAI,GAAG;CACnB;CAEA,OAAO,KAAmB;EACxB,IAAI,CAAC,KAAK,KAAK,OAAO,GAAG,GAAG;EAC5B,KAAK,WAAW,QAAQ,GAAG;CAC7B;CAEA,OAAO,OAAe,QAA6B;EACjD,IAAI,UAAU,KAAK,WAAW,OAAO,OAAO;GAAE,QAAQ;GAAM,OAAO;EAAI,CAAC;EACxE,IAAI,QAAQ,WAAW,GACrB,UAAU,KAAK,WAAW,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;EAE1D,IAAI,QAAQ,WAAW,GACP;OAAA,MAAM,MAAM,KAAK,CAAC,CAAC,QAAQ,MAAM,EAAE,SAAS,CAClD,CAAC,CAAC,SAAS,GACjB,UAAU,KAAK,WAAW,OAAO,OAAO;IAAE,QAAQ;IAAM,OAAO;IAAK,aAAa;GAAK,CAAC;EAAA;EAG3F,OAAO,QAAQ,KAAK,OAAO;GAAE,KAAK,EAAE;GAAc,OAAO,EAAE;EAAM,EAAE;CACrE;CAEA,cAAuC;EACrC,OAAO;GAAE,YAAY,KAAK,WAAW,OAAO;GAAG,MAAM,CAAC,GAAG,KAAK,IAAI;EAAE;CACtE;CAEA,OAAO,QAAQ,OAAkD;EAC/D,MAAM,SAAS,IAAI,iBACjB,WAAW,SAAS,KAAK,UAAU,MAAM,UAAU,GAAG,kBAAkB,CAC1E;EACA,OAAO,OAAO,IAAI,IAAK,MAAM,QAAQ,CAAC,CAAc;EACpD,OAAO;CACT;AACF;;;;;AAMA,IAAa,0BAAb,MAAa,gCAAgC,iBAAiB;CAC5D,YAAY,MAAyB;EACnC,MAAM,MAAM,2BAA2B,IAAI,iBAAiB,CAAC;CAC/D;CAEA,cAAwB,OAAmD;EACzE,OAAO,iBAAiB,QAAQ,KAAK;CACvC;;;;;CAMA,YAAoB;EAClB,OAAO,KAAK,eAAe;CAC7B;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,QAAQ,IAAI,wBAAwB,IAAI;EAC9C,MAAM,aAAa,IAAI;EACvB,OAAO;CACT;AACF"}
package/dist/minisearch-indexer.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { miniSearchFullTextProvider } from "./minisearch-provider.js";
2
- import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
3
2
  import { memVectorProvider } from "@statewalker/indexer-mem";
3
+ import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
4
4
  //#region src/minisearch-indexer.ts
5
5
  /**
6
6
  * In-memory Indexer combining MiniSearch full-text and brute-force vector
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/indexer-mem-minisearch",
3
- "version": "0.1.5",
3
+ "version": "0.3.0",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "In-memory indexer implementation backed by MiniSearch for full-text search.",
@@ -32,10 +32,10 @@
32
32
  ],
33
33
  "dependencies": {
34
34
  "minisearch": "^7.2.0",
35
- "@statewalker/indexer-mem": "^0.1.5",
36
- "@statewalker/indexer-fulltext": "^0.1.5",
37
- "@statewalker/indexer-api": "^0.1.4",
38
- "@statewalker/indexer-core": "^0.1.4"
35
+ "@statewalker/indexer-api": "^0.1.5",
36
+ "@statewalker/indexer-fulltext": "^0.1.6",
37
+ "@statewalker/indexer-mem": "^0.3.0",
38
+ "@statewalker/indexer-core": "^0.2.0"
39
39
  },
40
40
  "devDependencies": {
41
41
  "rimraf": "^6.1.3",
@@ -43,7 +43,7 @@
43
43
  "typescript": "^7.0.2",
44
44
  "vitest": "^5.0.3",
45
45
  "@statewalker/indexer-tests": "^0.0.0",
46
- "@statewalker/indexer-vector": "^0.1.5"
46
+ "@statewalker/indexer-vector": "^0.1.6"
47
47
  },
48
48
  "sideEffects": false,
49
49
  "publishConfig": {
package/src/minisearch-full-text-index.ts CHANGED
@@ -1,353 +1,84 @@
1
- import type {
2
- BlockReference,
3
- DocumentPath,
4
- Metadata,
5
- PathSelector,
6
- PersistableSearchIndex,
7
- PersistenceEntry,
8
- } from "@statewalker/indexer-api";
9
- import {
10
- compositeKey,
11
- matchesPrefix,
12
- readEntryBytes,
13
- singleChunk,
14
- toAsyncIterable,
15
- toBytes,
16
- } from "@statewalker/indexer-core";
17
- import type {
18
- FullTextBlock,
19
- FullTextIndex,
20
- FullTextIndexInfo,
21
- FulltextQuery as FullTextSearchParams,
22
- FulltextResult as FullTextSearchResult,
23
- } from "@statewalker/indexer-fulltext";
1
+ import type { FullTextBlock, FullTextIndexInfo } from "@statewalker/indexer-fulltext";
2
+ import { type MemFullTextEngine, MemFullTextIndex, type RankedKey } from "@statewalker/indexer-mem";
24
3
  import MiniSearch from "minisearch";
25
4
 
26
- interface StoredBlock {
27
- path: DocumentPath;
28
- blockId: string;
29
- content: string;
30
- metadata?: Metadata;
31
- }
5
+ const MINISEARCH_OPTIONS = { fields: ["content"], idField: "key" };
32
6
 
33
- export class MiniSearchFullTextIndex
34
- implements
35
- FullTextIndex,
36
- PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>
37
- {
38
- private readonly info: FullTextIndexInfo;
7
+ /**
8
+ * MiniSearch adapter for the in-memory block store. Exported state:
9
+ * `miniSearch` (MiniSearch's own JSON) and `keys` (the indexed keys, kept for
10
+ * readers of format 3 that rebuild their key set from it).
11
+ */
12
+ class MiniSearchEngine implements MemFullTextEngine {
39
13
  private miniSearch: MiniSearch;
40
- private blocks = new Map<string, StoredBlock>();
41
- private keySet = new Set<string>();
42
- private closed = false;
14
+ private keys = new Set<string>();
43
15
 
44
- constructor(info: FullTextIndexInfo) {
45
- this.info = info;
46
- this.miniSearch = new MiniSearch({
47
- fields: ["content"],
48
- idField: "key",
49
- });
16
+ constructor(miniSearch = new MiniSearch(MINISEARCH_OPTIONS)) {
17
+ this.miniSearch = miniSearch;
50
18
  }
51
19
 
52
- private ensureOpen(): void {
53
- if (this.closed) {
54
- throw new Error("FullTextIndex is closed");
55
- }
20
+ add(key: string, block: FullTextBlock): void {
21
+ this.miniSearch.add({ key, content: block.content });
22
+ this.keys.add(key);
56
23
  }
57
24
 
58
- private searchInternal(
59
- query: string,
60
- topK: number,
61
- pathPrefixes?: DocumentPath[],
62
- ): FullTextSearchResult[] {
63
- let results = this.miniSearch.search(query, {
64
- prefix: true,
65
- fuzzy: 0.2,
66
- });
25
+ remove(key: string): void {
26
+ if (!this.keys.delete(key)) return;
27
+ this.miniSearch.discard(key);
28
+ }
67
29
 
30
+ search(query: string, _limit: number): RankedKey[] {
31
+ let results = this.miniSearch.search(query, { prefix: true, fuzzy: 0.2 });
68
32
  if (results.length === 0) {
69
33
  results = this.miniSearch.search(query, { prefix: true });
70
34
  }
71
-
72
35
  if (results.length === 0) {
73
36
  const words = query.split(/\s+/).filter((w) => w.length > 1);
74
37
  if (words.length > 1) {
75
- results = this.miniSearch.search(query, {
76
- prefix: true,
77
- fuzzy: 0.2,
78
- combineWith: "OR",
79
- });
80
- }
81
- }
82
-
83
- // Filter by path prefixes
84
- let filtered = results.map((r) => ({
85
- key: r.id as string,
86
- score: r.score,
87
- }));
88
-
89
- if (pathPrefixes && pathPrefixes.length > 0) {
90
- filtered = filtered.filter((r) => {
91
- const block = this.blocks.get(r.key);
92
- return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));
93
- });
94
- }
95
-
96
- return filtered.slice(0, topK).map((r) => {
97
- const block = this.blocks.get(r.key);
98
- return {
99
- path: block?.path ?? ("/" as DocumentPath),
100
- blockId: block?.blockId ?? "",
101
- snippet: block?.content ?? "",
102
- score: r.score,
103
- };
104
- });
105
- }
106
-
107
- async getIndexInfo(): Promise<FullTextIndexInfo> {
108
- this.ensureOpen();
109
- return { ...this.info };
110
- }
111
-
112
- async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {
113
- this.ensureOpen();
114
- const { queries, paths } = params;
115
- const topK = params.topK ?? 100;
116
-
117
- if (!queries || queries.length === 0) return;
118
-
119
- // Multi-query: search for each and ACCUMULATE per-block scores, so a block
120
- // matching more queries ranks higher (the FulltextQuery contract). Taking the
121
- // max instead would let a block that is the top hit for one query outrank a
122
- // block matching several queries. The snippet is kept from the strongest
123
- // single-query match.
124
- const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();
125
-
126
- for (const query of queries) {
127
- const results = this.searchInternal(query, topK, paths);
128
- for (const r of results) {
129
- const key = compositeKey(r.path, r.blockId);
130
- const existing = merged.get(key);
131
- if (!existing) {
132
- merged.set(key, { result: r, best: r.score, total: r.score });
133
- } else {
134
- existing.total += r.score;
135
- if (r.score > existing.best) {
136
- existing.best = r.score;
137
- existing.result = r;
138
- }
139
- }
140
- }
141
- }
142
-
143
- const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
144
- for (const { result, total } of sorted.slice(0, topK)) {
145
- yield { ...result, score: total };
146
- }
147
- }
148
-
149
- async addDocument(blocks: FullTextBlock[]): Promise<void> {
150
- this.ensureOpen();
151
- for (const block of blocks) {
152
- const key = compositeKey(block.path, block.blockId);
153
- if (this.keySet.has(key)) {
154
- this.miniSearch.discard(key);
155
- }
156
- this.miniSearch.add({ key, content: block.content });
157
- this.keySet.add(key);
158
- this.blocks.set(key, {
159
- path: block.path,
160
- blockId: block.blockId,
161
- content: block.content,
162
- metadata: block.metadata,
163
- });
164
- }
165
- }
166
-
167
- async addDocuments(
168
- blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
169
- ): Promise<void> {
170
- this.ensureOpen();
171
- for await (const batch of blocks) {
172
- await this.addDocument(batch);
173
- }
174
- }
175
-
176
- async deleteDocuments(
177
- pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
178
- ): Promise<void> {
179
- this.ensureOpen();
180
- for await (const sel of toAsyncIterable(pathSelectors)) {
181
- if (sel.blockId !== undefined) {
182
- const key = compositeKey(sel.path, sel.blockId);
183
- this.removeByKey(key);
184
- } else {
185
- const keysToDelete: string[] = [];
186
- for (const [key, block] of this.blocks) {
187
- if (matchesPrefix(block.path, sel.path)) {
188
- keysToDelete.push(key);
189
- }
190
- }
191
- for (const key of keysToDelete) {
192
- this.removeByKey(key);
193
- }
38
+ results = this.miniSearch.search(query, { prefix: true, fuzzy: 0.2, combineWith: "OR" });
194
39
  }
195
40
  }
41
+ return results.map((r) => ({ key: r.id as string, score: r.score }));
196
42
  }
197
43
 
198
- private removeByKey(key: string): void {
199
- if (this.keySet.has(key)) {
200
- this.miniSearch.discard(key);
201
- this.keySet.delete(key);
202
- }
203
- this.blocks.delete(key);
204
- }
205
-
206
- async getSize(pathPrefix?: DocumentPath): Promise<number> {
207
- this.ensureOpen();
208
- if (pathPrefix === undefined) return this.blocks.size;
209
- let count = 0;
210
- for (const block of this.blocks.values()) {
211
- if (matchesPrefix(block.path, pathPrefix)) count++;
212
- }
213
- return count;
214
- }
215
-
216
- async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
217
- this.ensureOpen();
218
- const paths = new Set<DocumentPath>();
219
- for (const block of this.blocks.values()) {
220
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
221
- paths.add(block.path);
222
- }
223
- }
224
- for (const p of paths) yield p;
225
- }
226
-
227
- async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
228
- this.ensureOpen();
229
- for (const block of this.blocks.values()) {
230
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
231
- yield { path: block.path, blockId: block.blockId };
232
- }
233
- }
234
- }
235
-
236
- async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
237
- this.ensureOpen();
238
- for (const block of this.blocks.values()) {
239
- if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
240
- yield {
241
- path: block.path,
242
- blockId: block.blockId,
243
- content: block.content,
244
- metadata: block.metadata,
245
- };
246
- }
247
- }
44
+ exportState(): Record<string, unknown> {
45
+ return { miniSearch: this.miniSearch.toJSON(), keys: [...this.keys] };
248
46
  }
249
47
 
250
- async close(_options?: { force?: boolean }): Promise<void> {
251
- this.closed = true;
48
+ static restore(state: Record<string, unknown>): MiniSearchEngine {
49
+ const engine = new MiniSearchEngine(
50
+ MiniSearch.loadJSON(JSON.stringify(state.miniSearch), MINISEARCH_OPTIONS),
51
+ );
52
+ engine.keys = new Set((state.keys ?? []) as string[]);
53
+ return engine;
252
54
  }
55
+ }
253
56
 
254
- async flush(): Promise<void> {
255
- this.ensureOpen();
57
+ /**
58
+ * In-memory full-text sub-index backed by MiniSearch: a MiniSearch adapter
59
+ * over the shared in-memory block store ({@link MemFullTextIndex}).
60
+ */
61
+ export class MiniSearchFullTextIndex extends MemFullTextIndex {
62
+ constructor(info: FullTextIndexInfo) {
63
+ super(info, "MiniSearchFullTextIndex", new MiniSearchEngine());
256
64
  }
257
65
 
258
- async deleteIndex(): Promise<void> {
259
- this.ensureOpen();
260
- this.blocks.clear();
261
- this.keySet.clear();
262
- this.closed = true;
66
+ protected restoreEngine(state: Record<string, unknown>): MemFullTextEngine {
67
+ return MiniSearchEngine.restore(state);
263
68
  }
264
69
 
265
- // --- PersistableSearchIndex --------------------------------------------------
266
-
267
70
  /**
268
- * Stream the FTS sub-index's state as a single named entry. The composite
269
- * prefixes this with `${indexName}/${subName}/` before it hits persistence.
71
+ * Serialize to a JSON string (format version 3). Cached: while the index is
72
+ * unchanged, returns the same string without exporting MiniSearch again.
270
73
  */
271
- async *serialise(): AsyncIterable<PersistenceEntry> {
272
- yield { name: "json", content: singleChunk(toBytes(this.serialize())) };
273
- }
274
-
275
- /** Restore the FTS sub-index from a previously-produced entry set. */
276
- async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
277
- this.ensureOpen();
278
- for (const e of entries) {
279
- if (e.name !== "json") continue;
280
- const bytes = await readEntryBytes(e);
281
- const json = new TextDecoder().decode(bytes);
282
- const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
283
- this.miniSearch = rebuilt.miniSearch;
284
- this.blocks = rebuilt.blocks;
285
- this.keySet = rebuilt.keySet;
286
- return;
287
- }
288
- }
289
-
290
- /** Serialize to JSON (v3 format with path data) */
291
74
  serialize(): string {
292
- const blocksData: Array<{
293
- path: string;
294
- blockId: string;
295
- content: string;
296
- metadata?: Metadata;
297
- }> = [];
298
- for (const block of this.blocks.values()) {
299
- blocksData.push({
300
- path: block.path,
301
- blockId: block.blockId,
302
- content: block.content,
303
- metadata: block.metadata,
304
- });
305
- }
306
-
307
- return JSON.stringify({
308
- version: 3,
309
- miniSearch: this.miniSearch.toJSON(),
310
- blocks: blocksData,
311
- keys: [...this.keySet],
312
- });
75
+ return this.serializeState();
313
76
  }
314
77
 
315
- /** Deserialize from JSON — handles v3 format */
78
+ /** Deserialize from JSON (format version 3); throws on any other version. */
316
79
  static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {
317
- const parsed = JSON.parse(json) as {
318
- version?: number;
319
- miniSearch: unknown;
320
- blocks: Array<{
321
- path: string;
322
- blockId: string;
323
- content: string;
324
- metadata?: Metadata;
325
- }>;
326
- keys: string[];
327
- };
328
-
329
- if (parsed.version !== 3) {
330
- throw new Error(
331
- `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,
332
- );
333
- }
334
-
335
80
  const index = new MiniSearchFullTextIndex(info);
336
- index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
337
- fields: ["content"],
338
- idField: "key",
339
- });
340
- index.keySet = new Set(parsed.keys);
341
- for (const block of parsed.blocks) {
342
- const key = compositeKey(block.path as DocumentPath, block.blockId);
343
- index.blocks.set(key, {
344
- path: block.path as DocumentPath,
345
- blockId: block.blockId,
346
- content: block.content,
347
- metadata: block.metadata,
348
- });
349
- }
350
-
81
+ index.restoreState(json);
351
82
  return index;
352
83
  }
353
84
  }