@statewalker/indexer-mem-minisearch 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Mikhail Kotelnikov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,53 @@
1
+ # @statewalker/indexer-mem-minisearch
2
+
3
+ In-memory implementation of `@statewalker/indexer-api` combining [MiniSearch](https://github.com/lucaong/minisearch) for full-text search with `@statewalker/indexer-mem`'s `MemVectorIndex` for embeddings. Optional streaming persistence via `IndexerPersistence`.
4
+
5
+ ## Installation
6
+
7
+ ```sh
8
+ pnpm add @statewalker/indexer-mem-minisearch
9
+ ```
10
+
11
+ ## Usage
12
+
13
+ ```ts
14
+ import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch";
15
+
16
+ const indexer = createMiniSearchIndexer();
17
+ const index = await indexer.createIndex({
18
+ name: "docs",
19
+ fulltext: { language: "en" },
20
+ vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" },
21
+ });
22
+
23
+ await index.addDocument([
24
+ { path: "/docs/a", blockId: "1", content: "hello world" },
25
+ ]);
26
+
27
+ for await (const hit of index.search({ queries: ["hello"], topK: 10 })) {
28
+ console.log(hit.path, hit.blockId, hit.score);
29
+ }
30
+ ```
31
+
32
+ ### With persistence
33
+
34
+ ```ts
35
+ import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch";
36
+ import type { IndexerPersistence } from "@statewalker/indexer-api";
37
+
38
+ const persistence: IndexerPersistence = /* … your save/load adapter … */;
39
+ const indexer = createMiniSearchIndexer({ persistence });
40
+ ```
41
+
42
+ Wire format (stable across releases): `__manifest__` (JSON array of index names), `${name}/__config__`, `${name}/fts` (MiniSearch `toJSON()`), `${name}/vec` (Arrow IPC from `MemVectorIndex`).
43
+
44
+ ## API
45
+
46
+ - `createMiniSearchIndexer(options?)` — returns an `Indexer`. Options: `persistence?: IndexerPersistence`.
47
+ - `MiniSearchIndexerOptions` — option type.
48
+
49
+ ## Related
50
+
51
+ - `@statewalker/indexer-api` — the pluggable contract.
52
+ - `@statewalker/indexer-mem-flexsearch` — drop-in alternative using FlexSearch.
53
+ - `@statewalker/indexer-mem` — provides the vector sub-index (`MemVectorIndex`).
@@ -0,0 +1,4 @@
1
+ import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
2
+ import { MiniSearchIndexerOptions, createMiniSearchIndexer } from "./minisearch-indexer.js";
3
+ import { miniSearchFullTextProvider } from "./minisearch-provider.js";
4
+ export { MiniSearchFullTextIndex, type MiniSearchIndexerOptions, createMiniSearchIndexer, miniSearchFullTextProvider };
package/dist/index.js ADDED
@@ -0,0 +1,4 @@
1
+ import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
2
+ import { miniSearchFullTextProvider } from "./minisearch-provider.js";
3
+ import { createMiniSearchIndexer } from "./minisearch-indexer.js";
4
+ export { MiniSearchFullTextIndex, createMiniSearchIndexer, miniSearchFullTextProvider };
@@ -0,0 +1,41 @@
1
+ import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
2
+ import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
3
+ //#region src/minisearch-full-text-index.d.ts
4
+ export declare class MiniSearchFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
5
+ private readonly info;
6
+ private miniSearch;
7
+ private blocks;
8
+ private keySet;
9
+ private closed;
10
+ constructor(info: FullTextIndexInfo);
11
+ private ensureOpen;
12
+ private searchInternal;
13
+ getIndexInfo(): Promise<FullTextIndexInfo>;
14
+ search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
15
+ addDocument(blocks: FullTextBlock[]): Promise<void>;
16
+ addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
17
+ deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
18
+ private removeByKey;
19
+ getSize(pathPrefix?: DocumentPath): Promise<number>;
20
+ getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
21
+ getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
22
+ getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
23
+ close(_options?: {
24
+ force?: boolean;
25
+ }): Promise<void>;
26
+ flush(): Promise<void>;
27
+ deleteIndex(): Promise<void>;
28
+ /**
29
+ * Stream the FTS sub-index's state as a single named entry. The composite
30
+ * prefixes this with `${indexName}/${subName}/` before it hits persistence.
31
+ */
32
+ serialise(): AsyncIterable<PersistenceEntry>;
33
+ /** Restore the FTS sub-index from a previously-produced entry set. */
34
+ loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
35
+ /** Serialize to JSON (v3 format with path data) */
36
+ serialize(): string;
37
+ /** Deserialize from JSON — handles v3 format */
38
+ static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex;
39
+ }
40
+ //#endregion
41
+ //# sourceMappingURL=minisearch-full-text-index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-full-text-index.d.ts","names":["FullTextSearchParams","FullTextSearchResult"],"sources":["../src/minisearch-full-text-index.ts"],"mappings":";;;qBAgCa,mCAET,eACA,uBAAuB,eAAeA,eAAsBC;mBAE7C;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;UAQV;UAMA;EAiDF,gBAAgB,QAAQ;EAKvB,OAAO,QAAQD,gBAAuB,eAAeC;EAqCtD,YAAY,QAAQ,kBAAkB;EAkBtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;UAoBK;EAQF,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAe;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAIT,eAAe;;;;;EAad,aAAa,cAAc;;EAK5B,SAAS,SAAS,SAAS,oBAAoB;;EAerD;;SAyBO,YAAY,MAAM,mBAAmB,eAAe"}
@@ -0,0 +1,232 @@
1
+ import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, toBytes } from "@statewalker/indexer-core";
2
+ import MiniSearch from "minisearch";
3
+ //#region src/minisearch-full-text-index.ts
4
+ var MiniSearchFullTextIndex = class MiniSearchFullTextIndex {
5
+ info;
6
+ miniSearch;
7
+ blocks = /* @__PURE__ */ new Map();
8
+ keySet = /* @__PURE__ */ new Set();
9
+ closed = false;
10
+ constructor(info) {
11
+ this.info = info;
12
+ this.miniSearch = new MiniSearch({
13
+ fields: ["content"],
14
+ idField: "key"
15
+ });
16
+ }
17
+ ensureOpen() {
18
+ if (this.closed) throw new Error("FullTextIndex is closed");
19
+ }
20
+ searchInternal(query, topK, pathPrefixes) {
21
+ let results = this.miniSearch.search(query, {
22
+ prefix: true,
23
+ fuzzy: .2
24
+ });
25
+ if (results.length === 0) results = this.miniSearch.search(query, { prefix: true });
26
+ if (results.length === 0) {
27
+ if (query.split(/\s+/).filter((w) => w.length > 1).length > 1) results = this.miniSearch.search(query, {
28
+ prefix: true,
29
+ fuzzy: .2,
30
+ combineWith: "OR"
31
+ });
32
+ }
33
+ let filtered = results.map((r) => ({
34
+ key: r.id,
35
+ score: r.score
36
+ }));
37
+ if (pathPrefixes && pathPrefixes.length > 0) filtered = filtered.filter((r) => {
38
+ const block = this.blocks.get(r.key);
39
+ return block !== void 0 && pathPrefixes.some((p) => matchesPrefix(block.path, p));
40
+ });
41
+ return filtered.slice(0, topK).map((r) => {
42
+ const block = this.blocks.get(r.key);
43
+ return {
44
+ path: block?.path ?? "/",
45
+ blockId: block?.blockId ?? "",
46
+ snippet: block?.content ?? "",
47
+ score: r.score
48
+ };
49
+ });
50
+ }
51
+ async getIndexInfo() {
52
+ this.ensureOpen();
53
+ return { ...this.info };
54
+ }
55
+ async *search(params) {
56
+ this.ensureOpen();
57
+ const { queries, paths } = params;
58
+ const topK = params.topK ?? 100;
59
+ if (!queries || queries.length === 0) return;
60
+ const merged = /* @__PURE__ */ new Map();
61
+ for (const query of queries) {
62
+ const results = this.searchInternal(query, topK, paths);
63
+ for (const r of results) {
64
+ const key = compositeKey(r.path, r.blockId);
65
+ const existing = merged.get(key);
66
+ if (!existing) merged.set(key, {
67
+ result: r,
68
+ best: r.score,
69
+ total: r.score
70
+ });
71
+ else {
72
+ existing.total += r.score;
73
+ if (r.score > existing.best) {
74
+ existing.best = r.score;
75
+ existing.result = r;
76
+ }
77
+ }
78
+ }
79
+ }
80
+ const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
81
+ for (const { result, total } of sorted.slice(0, topK)) yield {
82
+ ...result,
83
+ score: total
84
+ };
85
+ }
86
+ async addDocument(blocks) {
87
+ this.ensureOpen();
88
+ for (const block of blocks) {
89
+ const key = compositeKey(block.path, block.blockId);
90
+ if (this.keySet.has(key)) this.miniSearch.discard(key);
91
+ this.miniSearch.add({
92
+ key,
93
+ content: block.content
94
+ });
95
+ this.keySet.add(key);
96
+ this.blocks.set(key, {
97
+ path: block.path,
98
+ blockId: block.blockId,
99
+ content: block.content,
100
+ metadata: block.metadata
101
+ });
102
+ }
103
+ }
104
+ async addDocuments(blocks) {
105
+ this.ensureOpen();
106
+ for await (const batch of blocks) await this.addDocument(batch);
107
+ }
108
+ async deleteDocuments(pathSelectors) {
109
+ this.ensureOpen();
110
+ for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
111
+ const key = compositeKey(sel.path, sel.blockId);
112
+ this.removeByKey(key);
113
+ } else {
114
+ const keysToDelete = [];
115
+ for (const [key, block] of this.blocks) if (matchesPrefix(block.path, sel.path)) keysToDelete.push(key);
116
+ for (const key of keysToDelete) this.removeByKey(key);
117
+ }
118
+ }
119
+ removeByKey(key) {
120
+ if (this.keySet.has(key)) {
121
+ this.miniSearch.discard(key);
122
+ this.keySet.delete(key);
123
+ }
124
+ this.blocks.delete(key);
125
+ }
126
+ async getSize(pathPrefix) {
127
+ this.ensureOpen();
128
+ if (pathPrefix === void 0) return this.blocks.size;
129
+ let count = 0;
130
+ for (const block of this.blocks.values()) if (matchesPrefix(block.path, pathPrefix)) count++;
131
+ return count;
132
+ }
133
+ async *getDocumentPaths(pathPrefix) {
134
+ this.ensureOpen();
135
+ const paths = /* @__PURE__ */ new Set();
136
+ for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) paths.add(block.path);
137
+ for (const p of paths) yield p;
138
+ }
139
+ async *getDocumentBlocksRefs(pathPrefix) {
140
+ this.ensureOpen();
141
+ for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
142
+ path: block.path,
143
+ blockId: block.blockId
144
+ };
145
+ }
146
+ async *getDocumentsBlocks(pathPrefix) {
147
+ this.ensureOpen();
148
+ for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
149
+ path: block.path,
150
+ blockId: block.blockId,
151
+ content: block.content,
152
+ metadata: block.metadata
153
+ };
154
+ }
155
+ async close(_options) {
156
+ this.closed = true;
157
+ }
158
+ async flush() {
159
+ this.ensureOpen();
160
+ }
161
+ async deleteIndex() {
162
+ this.ensureOpen();
163
+ this.blocks.clear();
164
+ this.keySet.clear();
165
+ this.closed = true;
166
+ }
167
+ /**
168
+ * Stream the FTS sub-index's state as a single named entry. The composite
169
+ * prefixes this with `${indexName}/${subName}/` before it hits persistence.
170
+ */
171
+ async *serialise() {
172
+ yield {
173
+ name: "json",
174
+ content: singleChunk(toBytes(this.serialize()))
175
+ };
176
+ }
177
+ /** Restore the FTS sub-index from a previously-produced entry set. */
178
+ async loadFrom(entries) {
179
+ this.ensureOpen();
180
+ for (const e of entries) {
181
+ if (e.name !== "json") continue;
182
+ const bytes = await readEntryBytes(e);
183
+ const json = new TextDecoder().decode(bytes);
184
+ const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
185
+ this.miniSearch = rebuilt.miniSearch;
186
+ this.blocks = rebuilt.blocks;
187
+ this.keySet = rebuilt.keySet;
188
+ return;
189
+ }
190
+ }
191
+ /** Serialize to JSON (v3 format with path data) */
192
+ serialize() {
193
+ const blocksData = [];
194
+ for (const block of this.blocks.values()) blocksData.push({
195
+ path: block.path,
196
+ blockId: block.blockId,
197
+ content: block.content,
198
+ metadata: block.metadata
199
+ });
200
+ return JSON.stringify({
201
+ version: 3,
202
+ miniSearch: this.miniSearch.toJSON(),
203
+ blocks: blocksData,
204
+ keys: [...this.keySet]
205
+ });
206
+ }
207
+ /** Deserialize from JSON — handles v3 format */
208
+ static deserialize(info, json) {
209
+ const parsed = JSON.parse(json);
210
+ if (parsed.version !== 3) throw new Error(`MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`);
211
+ const index = new MiniSearchFullTextIndex(info);
212
+ index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
213
+ fields: ["content"],
214
+ idField: "key"
215
+ });
216
+ index.keySet = new Set(parsed.keys);
217
+ for (const block of parsed.blocks) {
218
+ const key = compositeKey(block.path, block.blockId);
219
+ index.blocks.set(key, {
220
+ path: block.path,
221
+ blockId: block.blockId,
222
+ content: block.content,
223
+ metadata: block.metadata
224
+ });
225
+ }
226
+ return index;
227
+ }
228
+ };
229
+ //#endregion
230
+ export { MiniSearchFullTextIndex };
231
+
232
+ //# sourceMappingURL=minisearch-full-text-index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-full-text-index.js","names":[],"sources":["../src/minisearch-full-text-index.ts"],"sourcesContent":["import type {\n BlockReference,\n DocumentPath,\n Metadata,\n PathSelector,\n PersistableSearchIndex,\n PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n compositeKey,\n matchesPrefix,\n readEntryBytes,\n singleChunk,\n toAsyncIterable,\n toBytes,\n} from \"@statewalker/indexer-core\";\nimport type {\n FullTextBlock,\n FullTextIndex,\n FullTextIndexInfo,\n FulltextQuery as FullTextSearchParams,\n FulltextResult as FullTextSearchResult,\n} from \"@statewalker/indexer-fulltext\";\nimport MiniSearch from \"minisearch\";\n\ninterface StoredBlock {\n path: DocumentPath;\n blockId: string;\n content: string;\n metadata?: Metadata;\n}\n\nexport class MiniSearchFullTextIndex\n implements\n FullTextIndex,\n PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>\n{\n private readonly info: FullTextIndexInfo;\n private miniSearch: MiniSearch;\n private blocks = new Map<string, StoredBlock>();\n private keySet = new Set<string>();\n private closed = false;\n\n constructor(info: FullTextIndexInfo) {\n this.info = info;\n this.miniSearch = new MiniSearch({\n fields: [\"content\"],\n idField: \"key\",\n });\n }\n\n private ensureOpen(): void {\n if (this.closed) {\n throw new Error(\"FullTextIndex is closed\");\n }\n }\n\n private searchInternal(\n query: string,\n topK: number,\n pathPrefixes?: DocumentPath[],\n ): FullTextSearchResult[] {\n let results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n });\n\n if (results.length === 0) {\n results = this.miniSearch.search(query, { prefix: true });\n }\n\n if (results.length === 0) {\n const words = query.split(/\\s+/).filter((w) => w.length > 1);\n if (words.length > 1) {\n results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n combineWith: \"OR\",\n });\n }\n }\n\n // Filter by path prefixes\n let filtered = results.map((r) => ({\n key: r.id as string,\n score: r.score,\n }));\n\n if (pathPrefixes && pathPrefixes.length > 0) {\n filtered = filtered.filter((r) => {\n const block = this.blocks.get(r.key);\n return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));\n });\n }\n\n return filtered.slice(0, topK).map((r) => {\n const block = this.blocks.get(r.key);\n return {\n path: block?.path ?? (\"/\" as DocumentPath),\n blockId: block?.blockId ?? \"\",\n snippet: block?.content ?? \"\",\n score: r.score,\n };\n });\n }\n\n async getIndexInfo(): Promise<FullTextIndexInfo> {\n this.ensureOpen();\n return { ...this.info };\n }\n\n async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {\n this.ensureOpen();\n const { queries, paths } = params;\n const topK = params.topK ?? 100;\n\n if (!queries || queries.length === 0) return;\n\n // Multi-query: search for each and ACCUMULATE per-block scores, so a block\n // matching more queries ranks higher (the FulltextQuery contract). Taking the\n // max instead would let a block that is the top hit for one query outrank a\n // block matching several queries. The snippet is kept from the strongest\n // single-query match.\n const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();\n\n for (const query of queries) {\n const results = this.searchInternal(query, topK, paths);\n for (const r of results) {\n const key = compositeKey(r.path, r.blockId);\n const existing = merged.get(key);\n if (!existing) {\n merged.set(key, { result: r, best: r.score, total: r.score });\n } else {\n existing.total += r.score;\n if (r.score > existing.best) {\n existing.best = r.score;\n existing.result = r;\n }\n }\n }\n }\n\n const sorted = [...merged.values()].sort((a, b) => b.total - a.total);\n for (const { result, total } of sorted.slice(0, topK)) {\n yield { ...result, score: total };\n }\n }\n\n async addDocument(blocks: FullTextBlock[]): Promise<void> {\n this.ensureOpen();\n for (const block of blocks) {\n const key = compositeKey(block.path, block.blockId);\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n }\n this.miniSearch.add({ key, content: block.content });\n this.keySet.add(key);\n this.blocks.set(key, {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n }\n\n async addDocuments(\n blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const batch of blocks) {\n await this.addDocument(batch);\n }\n }\n\n async deleteDocuments(\n pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const sel of toAsyncIterable(pathSelectors)) {\n if (sel.blockId !== undefined) {\n const key = compositeKey(sel.path, sel.blockId);\n this.removeByKey(key);\n } else {\n const keysToDelete: string[] = [];\n for (const [key, block] of this.blocks) {\n if (matchesPrefix(block.path, sel.path)) {\n keysToDelete.push(key);\n }\n }\n for (const key of keysToDelete) {\n this.removeByKey(key);\n }\n }\n }\n }\n\n private removeByKey(key: string): void {\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n this.keySet.delete(key);\n }\n this.blocks.delete(key);\n }\n\n async getSize(pathPrefix?: DocumentPath): Promise<number> {\n this.ensureOpen();\n if (pathPrefix === undefined) return this.blocks.size;\n let count = 0;\n for (const block of this.blocks.values()) {\n if (matchesPrefix(block.path, pathPrefix)) count++;\n }\n return count;\n }\n\n async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n this.ensureOpen();\n const paths = new Set<DocumentPath>();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n paths.add(block.path);\n }\n }\n for (const p of paths) yield p;\n }\n\n async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield { path: block.path, blockId: block.blockId };\n }\n }\n }\n\n async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n };\n }\n }\n }\n\n async close(_options?: { force?: boolean }): Promise<void> {\n this.closed = true;\n }\n\n async flush(): Promise<void> {\n this.ensureOpen();\n }\n\n async deleteIndex(): Promise<void> {\n this.ensureOpen();\n this.blocks.clear();\n this.keySet.clear();\n this.closed = true;\n }\n\n // --- PersistableSearchIndex --------------------------------------------------\n\n /**\n * Stream the FTS sub-index's state as a single named entry. The composite\n * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n */\n async *serialise(): AsyncIterable<PersistenceEntry> {\n yield { name: \"json\", content: singleChunk(toBytes(this.serialize())) };\n }\n\n /** Restore the FTS sub-index from a previously-produced entry set. */\n async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n this.ensureOpen();\n for (const e of entries) {\n if (e.name !== \"json\") continue;\n const bytes = await readEntryBytes(e);\n const json = new TextDecoder().decode(bytes);\n const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);\n this.miniSearch = rebuilt.miniSearch;\n this.blocks = rebuilt.blocks;\n this.keySet = rebuilt.keySet;\n return;\n }\n }\n\n /** Serialize to JSON (v3 format with path data) */\n serialize(): string {\n const blocksData: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }> = [];\n for (const block of this.blocks.values()) {\n blocksData.push({\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return JSON.stringify({\n version: 3,\n miniSearch: this.miniSearch.toJSON(),\n blocks: blocksData,\n keys: [...this.keySet],\n });\n }\n\n /** Deserialize from JSON — handles v3 format */\n static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {\n const parsed = JSON.parse(json) as {\n version?: number;\n miniSearch: unknown;\n blocks: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }>;\n keys: string[];\n };\n\n if (parsed.version !== 3) {\n throw new Error(\n `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,\n );\n }\n\n const index = new MiniSearchFullTextIndex(info);\n index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {\n fields: [\"content\"],\n idField: \"key\",\n });\n index.keySet = new Set(parsed.keys);\n for (const block of parsed.blocks) {\n const key = compositeKey(block.path as DocumentPath, block.blockId);\n index.blocks.set(key, {\n path: block.path as DocumentPath,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return index;\n }\n}\n"],"mappings":";;;AAgCA,IAAa,0BAAb,MAAa,wBAIb;CACE;CACA;CACA,yBAAiB,IAAI,IAAyB;CAC9C,yBAAiB,IAAI,IAAY;CACjC,SAAiB;CAEjB,YAAY,MAAyB;EACnC,KAAK,OAAO;EACZ,KAAK,aAAa,IAAI,WAAW;GAC/B,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;CACH;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,yBAAyB;CAE7C;CAEA,eACE,OACA,MACA,cACwB;EACxB,IAAI,UAAU,KAAK,WAAW,OAAO,OAAO;GAC1C,QAAQ;GACR,OAAO;EACT,CAAC;EAED,IAAI,QAAQ,WAAW,GACrB,UAAU,KAAK,WAAW,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;EAG1D,IAAI,QAAQ,WAAW,GACP;OAAA,MAAM,MAAM,KAAK,CAAC,CAAC,QAAQ,MAAM,EAAE,SAAS,CAClD,CAAC,CAAC,SAAS,GACjB,UAAU,KAAK,WAAW,OAAO,OAAO;IACtC,QAAQ;IACR,OAAO;IACP,aAAa;GACf,CAAC;EAAA;EAKL,IAAI,WAAW,QAAQ,KAAK,OAAO;GACjC,KAAK,EAAE;GACP,OAAO,EAAE;EACX,EAAE;EAEF,IAAI,gBAAgB,aAAa,SAAS,GACxC,WAAW,SAAS,QAAQ,MAAM;GAChC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO,UAAU,KAAA,KAAa,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC;EACrF,CAAC;EAGH,OAAO,SAAS,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,MAAM;GACxC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO;IACL,MAAM,OAAO,QAAS;IACtB,SAAS,OAAO,WAAW;IAC3B,SAAS,OAAO,WAAW;IAC3B,OAAO,EAAE;GACX;EACF,CAAC;CACH;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAoE;EAChF,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EAOtC,MAAM,yBAAS,IAAI,IAA2E;EAE9F,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,UAAU,KAAK,eAAe,OAAO,MAAM,KAAK;GACtD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,OAAO,IAAI,GAAG;IAC/B,IAAI,CAAC,UACH,OAAO,IAAI,KAAK;KAAE,QAAQ;KAAG,MAAM,EAAE;KAAO,OAAO,EAAE;IAAM,CAAC;SACvD;KACL,SAAS,SAAS,EAAE;KACpB,IAAI,EAAE,QAAQ,SAAS,MAAM;MAC3B,SAAS,OAAO,EAAE;MAClB,SAAS,SAAS;KACpB;IACF;GACF;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACpE,KAAK,MAAM,EAAE,QAAQ,WAAW,OAAO,MAAM,GAAG,IAAI,GAClD,MAAM;GAAE,GAAG;GAAQ,OAAO;EAAM;CAEpC;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,IAAI,KAAK,OAAO,IAAI,GAAG,GACrB,KAAK,WAAW,QAAQ,GAAG;GAE7B,KAAK,WAAW,IAAI;IAAE;IAAK,SAAS,MAAM;GAAQ,CAAC;GACnD,KAAK,OAAO,IAAI,GAAG;GACnB,KAAK,OAAO,IAAI,KAAK;IACnB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;CACF;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GAAW;GAC7B,MAAM,MAAM,aAAa,IAAI,MAAM,IAAI,OAAO;GAC9C,KAAK,YAAY,GAAG;EACtB,OAAO;GACL,MAAM,eAAyB,CAAC;GAChC,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GACpC,aAAa,KAAK,GAAG;GAGzB,KAAK,MAAM,OAAO,cAChB,KAAK,YAAY,GAAG;EAExB;CAEJ;CAEA,YAAoB,KAAmB;EACrC,IAAI,KAAK,OAAO,IAAI,GAAG,GAAG;GACxB,KAAK,WAAW,QAAQ,GAAG;GAC3B,KAAK,OAAO,OAAO,GAAG;EACxB;EACA,KAAK,OAAO,OAAO,GAAG;CACxB;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAClB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,OAAO,MAAM;EAClB,KAAK,OAAO,MAAM;EAClB,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,KAAK,UAAU,CAAC,CAAC;EAAE;CACxE;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;GAC3C,MAAM,UAAU,wBAAwB,YAAY,KAAK,MAAM,IAAI;GACnE,KAAK,aAAa,QAAQ;GAC1B,KAAK,SAAS,QAAQ;GACtB,KAAK,SAAS,QAAQ;GACtB;EACF;CACF;;CAGA,YAAoB;EAClB,MAAM,aAKD,CAAC;EACN,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,WAAW,KAAK;GACd,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB,CAAC;EAGH,OAAO,KAAK,UAAU;GACpB,SAAS;GACT,YAAY,KAAK,WAAW,OAAO;GACnC,QAAQ;GACR,MAAM,CAAC,GAAG,KAAK,MAAM;EACvB,CAAC;CACH;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,SAAS,KAAK,MAAM,IAAI;EAY9B,IAAI,OAAO,YAAY,GACrB,MAAM,IAAI,MACR,2DAA2D,OAAO,OAAO,OAAO,EAAE,cACpF;EAGF,MAAM,QAAQ,IAAI,wBAAwB,IAAI;EAC9C,MAAM,aAAa,WAAW,SAAS,KAAK,UAAU,OAAO,UAAU,GAAG;GACxE,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;EACD,MAAM,SAAS,IAAI,IAAI,OAAO,IAAI;EAClC,KAAK,MAAM,SAAS,OAAO,QAAQ;GACjC,MAAM,MAAM,aAAa,MAAM,MAAsB,MAAM,OAAO;GAClE,MAAM,OAAO,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EAEA,OAAO;CACT;AACF"}
@@ -0,0 +1,13 @@
1
+ import { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
2
+ //#region src/minisearch-indexer.d.ts
3
+ export interface MiniSearchIndexerOptions {
4
+ persistence?: IndexerPersistence;
5
+ }
6
+ /**
7
+ * In-memory Indexer combining MiniSearch full-text and brute-force vector
8
+ * sub-indexes via the Provider model. Multiple sub-indexes of either modality
9
+ * can be created on one Index (e.g. English and French FTS sub-indexes).
10
+ */
11
+ export declare function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer;
12
+ //#endregion
13
+ //# sourceMappingURL=minisearch-indexer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-indexer.d.ts","names":[],"sources":["../src/minisearch-indexer.ts"],"mappings":";;iBAKiB;EACf,cAAc;;;;;;;wBAQA,wBAAwB,UAAU,2BAA2B"}
@@ -0,0 +1,19 @@
1
+ import { miniSearchFullTextProvider } from "./minisearch-provider.js";
2
+ import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
3
+ import { memVectorProvider } from "@statewalker/indexer-mem";
4
+ //#region src/minisearch-indexer.ts
5
+ /**
6
+ * In-memory Indexer combining MiniSearch full-text and brute-force vector
7
+ * sub-indexes via the Provider model. Multiple sub-indexes of either modality
8
+ * can be created on one Index (e.g. English and French FTS sub-indexes).
9
+ */
10
+ function createMiniSearchIndexer(options) {
11
+ return createPersistenceBackedIndexer({
12
+ persistence: options?.persistence,
13
+ providers: [miniSearchFullTextProvider, memVectorProvider]
14
+ });
15
+ }
16
+ //#endregion
17
+ export { createMiniSearchIndexer };
18
+
19
+ //# sourceMappingURL=minisearch-indexer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-indexer.js","names":[],"sources":["../src/minisearch-indexer.ts"],"sourcesContent":["import type { Indexer, IndexerPersistence } from \"@statewalker/indexer-api\";\nimport { createPersistenceBackedIndexer } from \"@statewalker/indexer-core\";\nimport { memVectorProvider } from \"@statewalker/indexer-mem\";\nimport { miniSearchFullTextProvider } from \"./minisearch-provider.js\";\n\nexport interface MiniSearchIndexerOptions {\n persistence?: IndexerPersistence;\n}\n\n/**\n * In-memory Indexer combining MiniSearch full-text and brute-force vector\n * sub-indexes via the Provider model. Multiple sub-indexes of either modality\n * can be created on one Index (e.g. English and French FTS sub-indexes).\n */\nexport function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer {\n return createPersistenceBackedIndexer({\n persistence: options?.persistence,\n providers: [miniSearchFullTextProvider, memVectorProvider],\n });\n}\n"],"mappings":";;;;;;;;;AAcA,SAAgB,wBAAwB,SAA6C;CACnF,OAAO,+BAA+B;EACpC,aAAa,SAAS;EACtB,WAAW,CAAC,4BAA4B,iBAAiB;CAC3D,CAAC;AACH"}
@@ -0,0 +1,10 @@
1
+ import { FullTextProvider } from "@statewalker/indexer-fulltext";
2
+ //#region src/minisearch-provider.d.ts
3
+ /**
4
+ * `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
5
+ * `Indexer` alongside a `VectorProvider` to compose a full-text + vector
6
+ * Indexer.
7
+ */
8
+ export declare const miniSearchFullTextProvider: FullTextProvider;
9
+ //#endregion
10
+ //# sourceMappingURL=minisearch-provider.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-provider.d.ts","names":[],"sources":["../src/minisearch-provider.ts"],"mappings":";;;;;;;qBAYa,4BAA4B"}
@@ -0,0 +1,21 @@
1
+ import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
2
+ import { FULL_TEXT_TYPE } from "@statewalker/indexer-fulltext";
3
+ //#region src/minisearch-provider.ts
4
+ /**
5
+ * `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
6
+ * `Indexer` alongside a `VectorProvider` to compose a full-text + vector
7
+ * Indexer.
8
+ */
9
+ const miniSearchFullTextProvider = {
10
+ type: FULL_TEXT_TYPE,
11
+ create(config) {
12
+ return new MiniSearchFullTextIndex({
13
+ language: config.language,
14
+ metadata: config.metadata
15
+ });
16
+ }
17
+ };
18
+ //#endregion
19
+ export { miniSearchFullTextProvider };
20
+
21
+ //# sourceMappingURL=minisearch-provider.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"minisearch-provider.js","names":[],"sources":["../src/minisearch-provider.ts"],"sourcesContent":["import {\n FULL_TEXT_TYPE,\n type FullTextConfig,\n type FullTextProvider,\n} from \"@statewalker/indexer-fulltext\";\nimport { MiniSearchFullTextIndex } from \"./minisearch-full-text-index.js\";\n\n/**\n * `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an\n * `Indexer` alongside a `VectorProvider` to compose a full-text + vector\n * Indexer.\n */\nexport const miniSearchFullTextProvider: FullTextProvider = {\n type: FULL_TEXT_TYPE,\n create(config: FullTextConfig): MiniSearchFullTextIndex {\n return new MiniSearchFullTextIndex({\n language: config.language,\n metadata: config.metadata,\n });\n },\n};\n"],"mappings":";;;;;;;;AAYA,MAAa,6BAA+C;CAC1D,MAAM;CACN,OAAO,QAAiD;EACtD,OAAO,IAAI,wBAAwB;GACjC,UAAU,OAAO;GACjB,UAAU,OAAO;EACnB,CAAC;CACH;AACF"}
package/package.json ADDED
@@ -0,0 +1,58 @@
1
+ {
2
+ "name": "@statewalker/indexer-mem-minisearch",
3
+ "version": "0.1.2",
4
+ "private": false,
5
+ "type": "module",
6
+ "description": "In-memory indexer implementation backed by MiniSearch for full-text search.",
7
+ "homepage": "https://github.com/statewalker/statewalker-indexer",
8
+ "author": {
9
+ "name": "Mikhail Kotelnikov",
10
+ "email": "mikhail.kotelnikov@gmail.com"
11
+ },
12
+ "license": "MIT",
13
+ "repository": {
14
+ "type": "git",
15
+ "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git"
16
+ },
17
+ "exports": {
18
+ ".": {
19
+ "source": "./src/index.ts",
20
+ "types": "./dist/index.d.ts",
21
+ "import": "./dist/index.js",
22
+ "default": "./dist/index.js"
23
+ }
24
+ },
25
+ "files": [
26
+ "dist",
27
+ "src"
28
+ ],
29
+ "dependencies": {
30
+ "minisearch": "^7.2.0",
31
+ "@statewalker/indexer-core": "^0.1.1",
32
+ "@statewalker/indexer-api": "^0.1.1",
33
+ "@statewalker/indexer-fulltext": "^0.1.1",
34
+ "@statewalker/indexer-mem": "^0.1.2"
35
+ },
36
+ "devDependencies": {
37
+ "rimraf": "^6.1.3",
38
+ "tsdown": "^0.23.0",
39
+ "typescript": "^7.0.2",
40
+ "vitest": "^5.0.3",
41
+ "@statewalker/indexer-tests": "^0.0.0",
42
+ "@statewalker/indexer-vector": "^0.1.1"
43
+ },
44
+ "sideEffects": false,
45
+ "publishConfig": {
46
+ "access": "public"
47
+ },
48
+ "scripts": {
49
+ "build": "tsdown",
50
+ "dev": "tsdown --watch",
51
+ "test": "vitest run --passWithNoTests",
52
+ "test:watch": "vitest",
53
+ "typecheck": "tsc --noEmit",
54
+ "clean": "rimraf dist",
55
+ "lint": "biome check --write .",
56
+ "format": "biome format --write ."
57
+ }
58
+ }
package/src/index.ts ADDED
@@ -0,0 +1,3 @@
1
+ export { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
2
+ export { createMiniSearchIndexer, type MiniSearchIndexerOptions } from "./minisearch-indexer.js";
3
+ export { miniSearchFullTextProvider } from "./minisearch-provider.js";
@@ -0,0 +1,353 @@
1
+ import type {
2
+ BlockReference,
3
+ DocumentPath,
4
+ Metadata,
5
+ PathSelector,
6
+ PersistableSearchIndex,
7
+ PersistenceEntry,
8
+ } from "@statewalker/indexer-api";
9
+ import {
10
+ compositeKey,
11
+ matchesPrefix,
12
+ readEntryBytes,
13
+ singleChunk,
14
+ toAsyncIterable,
15
+ toBytes,
16
+ } from "@statewalker/indexer-core";
17
+ import type {
18
+ FullTextBlock,
19
+ FullTextIndex,
20
+ FullTextIndexInfo,
21
+ FulltextQuery as FullTextSearchParams,
22
+ FulltextResult as FullTextSearchResult,
23
+ } from "@statewalker/indexer-fulltext";
24
+ import MiniSearch from "minisearch";
25
+
26
+ interface StoredBlock {
27
+ path: DocumentPath;
28
+ blockId: string;
29
+ content: string;
30
+ metadata?: Metadata;
31
+ }
32
+
33
+ export class MiniSearchFullTextIndex
34
+ implements
35
+ FullTextIndex,
36
+ PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>
37
+ {
38
+ private readonly info: FullTextIndexInfo;
39
+ private miniSearch: MiniSearch;
40
+ private blocks = new Map<string, StoredBlock>();
41
+ private keySet = new Set<string>();
42
+ private closed = false;
43
+
44
+ constructor(info: FullTextIndexInfo) {
45
+ this.info = info;
46
+ this.miniSearch = new MiniSearch({
47
+ fields: ["content"],
48
+ idField: "key",
49
+ });
50
+ }
51
+
52
+ private ensureOpen(): void {
53
+ if (this.closed) {
54
+ throw new Error("FullTextIndex is closed");
55
+ }
56
+ }
57
+
58
+ private searchInternal(
59
+ query: string,
60
+ topK: number,
61
+ pathPrefixes?: DocumentPath[],
62
+ ): FullTextSearchResult[] {
63
+ let results = this.miniSearch.search(query, {
64
+ prefix: true,
65
+ fuzzy: 0.2,
66
+ });
67
+
68
+ if (results.length === 0) {
69
+ results = this.miniSearch.search(query, { prefix: true });
70
+ }
71
+
72
+ if (results.length === 0) {
73
+ const words = query.split(/\s+/).filter((w) => w.length > 1);
74
+ if (words.length > 1) {
75
+ results = this.miniSearch.search(query, {
76
+ prefix: true,
77
+ fuzzy: 0.2,
78
+ combineWith: "OR",
79
+ });
80
+ }
81
+ }
82
+
83
+ // Filter by path prefixes
84
+ let filtered = results.map((r) => ({
85
+ key: r.id as string,
86
+ score: r.score,
87
+ }));
88
+
89
+ if (pathPrefixes && pathPrefixes.length > 0) {
90
+ filtered = filtered.filter((r) => {
91
+ const block = this.blocks.get(r.key);
92
+ return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));
93
+ });
94
+ }
95
+
96
+ return filtered.slice(0, topK).map((r) => {
97
+ const block = this.blocks.get(r.key);
98
+ return {
99
+ path: block?.path ?? ("/" as DocumentPath),
100
+ blockId: block?.blockId ?? "",
101
+ snippet: block?.content ?? "",
102
+ score: r.score,
103
+ };
104
+ });
105
+ }
106
+
107
+ async getIndexInfo(): Promise<FullTextIndexInfo> {
108
+ this.ensureOpen();
109
+ return { ...this.info };
110
+ }
111
+
112
+ async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {
113
+ this.ensureOpen();
114
+ const { queries, paths } = params;
115
+ const topK = params.topK ?? 100;
116
+
117
+ if (!queries || queries.length === 0) return;
118
+
119
+ // Multi-query: search for each and ACCUMULATE per-block scores, so a block
120
+ // matching more queries ranks higher (the FulltextQuery contract). Taking the
121
+ // max instead would let a block that is the top hit for one query outrank a
122
+ // block matching several queries. The snippet is kept from the strongest
123
+ // single-query match.
124
+ const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();
125
+
126
+ for (const query of queries) {
127
+ const results = this.searchInternal(query, topK, paths);
128
+ for (const r of results) {
129
+ const key = compositeKey(r.path, r.blockId);
130
+ const existing = merged.get(key);
131
+ if (!existing) {
132
+ merged.set(key, { result: r, best: r.score, total: r.score });
133
+ } else {
134
+ existing.total += r.score;
135
+ if (r.score > existing.best) {
136
+ existing.best = r.score;
137
+ existing.result = r;
138
+ }
139
+ }
140
+ }
141
+ }
142
+
143
+ const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
144
+ for (const { result, total } of sorted.slice(0, topK)) {
145
+ yield { ...result, score: total };
146
+ }
147
+ }
148
+
149
+ async addDocument(blocks: FullTextBlock[]): Promise<void> {
150
+ this.ensureOpen();
151
+ for (const block of blocks) {
152
+ const key = compositeKey(block.path, block.blockId);
153
+ if (this.keySet.has(key)) {
154
+ this.miniSearch.discard(key);
155
+ }
156
+ this.miniSearch.add({ key, content: block.content });
157
+ this.keySet.add(key);
158
+ this.blocks.set(key, {
159
+ path: block.path,
160
+ blockId: block.blockId,
161
+ content: block.content,
162
+ metadata: block.metadata,
163
+ });
164
+ }
165
+ }
166
+
167
+ async addDocuments(
168
+ blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
169
+ ): Promise<void> {
170
+ this.ensureOpen();
171
+ for await (const batch of blocks) {
172
+ await this.addDocument(batch);
173
+ }
174
+ }
175
+
176
+ async deleteDocuments(
177
+ pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
178
+ ): Promise<void> {
179
+ this.ensureOpen();
180
+ for await (const sel of toAsyncIterable(pathSelectors)) {
181
+ if (sel.blockId !== undefined) {
182
+ const key = compositeKey(sel.path, sel.blockId);
183
+ this.removeByKey(key);
184
+ } else {
185
+ const keysToDelete: string[] = [];
186
+ for (const [key, block] of this.blocks) {
187
+ if (matchesPrefix(block.path, sel.path)) {
188
+ keysToDelete.push(key);
189
+ }
190
+ }
191
+ for (const key of keysToDelete) {
192
+ this.removeByKey(key);
193
+ }
194
+ }
195
+ }
196
+ }
197
+
198
+ private removeByKey(key: string): void {
199
+ if (this.keySet.has(key)) {
200
+ this.miniSearch.discard(key);
201
+ this.keySet.delete(key);
202
+ }
203
+ this.blocks.delete(key);
204
+ }
205
+
206
+ async getSize(pathPrefix?: DocumentPath): Promise<number> {
207
+ this.ensureOpen();
208
+ if (pathPrefix === undefined) return this.blocks.size;
209
+ let count = 0;
210
+ for (const block of this.blocks.values()) {
211
+ if (matchesPrefix(block.path, pathPrefix)) count++;
212
+ }
213
+ return count;
214
+ }
215
+
216
+ async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
217
+ this.ensureOpen();
218
+ const paths = new Set<DocumentPath>();
219
+ for (const block of this.blocks.values()) {
220
+ if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
221
+ paths.add(block.path);
222
+ }
223
+ }
224
+ for (const p of paths) yield p;
225
+ }
226
+
227
+ async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
228
+ this.ensureOpen();
229
+ for (const block of this.blocks.values()) {
230
+ if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
231
+ yield { path: block.path, blockId: block.blockId };
232
+ }
233
+ }
234
+ }
235
+
236
+ async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
237
+ this.ensureOpen();
238
+ for (const block of this.blocks.values()) {
239
+ if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
240
+ yield {
241
+ path: block.path,
242
+ blockId: block.blockId,
243
+ content: block.content,
244
+ metadata: block.metadata,
245
+ };
246
+ }
247
+ }
248
+ }
249
+
250
+ async close(_options?: { force?: boolean }): Promise<void> {
251
+ this.closed = true;
252
+ }
253
+
254
+ async flush(): Promise<void> {
255
+ this.ensureOpen();
256
+ }
257
+
258
+ async deleteIndex(): Promise<void> {
259
+ this.ensureOpen();
260
+ this.blocks.clear();
261
+ this.keySet.clear();
262
+ this.closed = true;
263
+ }
264
+
265
+ // --- PersistableSearchIndex --------------------------------------------------
266
+
267
+ /**
268
+ * Stream the FTS sub-index's state as a single named entry. The composite
269
+ * prefixes this with `${indexName}/${subName}/` before it hits persistence.
270
+ */
271
+ async *serialise(): AsyncIterable<PersistenceEntry> {
272
+ yield { name: "json", content: singleChunk(toBytes(this.serialize())) };
273
+ }
274
+
275
+ /** Restore the FTS sub-index from a previously-produced entry set. */
276
+ async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
277
+ this.ensureOpen();
278
+ for (const e of entries) {
279
+ if (e.name !== "json") continue;
280
+ const bytes = await readEntryBytes(e);
281
+ const json = new TextDecoder().decode(bytes);
282
+ const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
283
+ this.miniSearch = rebuilt.miniSearch;
284
+ this.blocks = rebuilt.blocks;
285
+ this.keySet = rebuilt.keySet;
286
+ return;
287
+ }
288
+ }
289
+
290
+ /** Serialize to JSON (v3 format with path data) */
291
+ serialize(): string {
292
+ const blocksData: Array<{
293
+ path: string;
294
+ blockId: string;
295
+ content: string;
296
+ metadata?: Metadata;
297
+ }> = [];
298
+ for (const block of this.blocks.values()) {
299
+ blocksData.push({
300
+ path: block.path,
301
+ blockId: block.blockId,
302
+ content: block.content,
303
+ metadata: block.metadata,
304
+ });
305
+ }
306
+
307
+ return JSON.stringify({
308
+ version: 3,
309
+ miniSearch: this.miniSearch.toJSON(),
310
+ blocks: blocksData,
311
+ keys: [...this.keySet],
312
+ });
313
+ }
314
+
315
+ /** Deserialize from JSON — handles v3 format */
316
+ static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {
317
+ const parsed = JSON.parse(json) as {
318
+ version?: number;
319
+ miniSearch: unknown;
320
+ blocks: Array<{
321
+ path: string;
322
+ blockId: string;
323
+ content: string;
324
+ metadata?: Metadata;
325
+ }>;
326
+ keys: string[];
327
+ };
328
+
329
+ if (parsed.version !== 3) {
330
+ throw new Error(
331
+ `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,
332
+ );
333
+ }
334
+
335
+ const index = new MiniSearchFullTextIndex(info);
336
+ index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
337
+ fields: ["content"],
338
+ idField: "key",
339
+ });
340
+ index.keySet = new Set(parsed.keys);
341
+ for (const block of parsed.blocks) {
342
+ const key = compositeKey(block.path as DocumentPath, block.blockId);
343
+ index.blocks.set(key, {
344
+ path: block.path as DocumentPath,
345
+ blockId: block.blockId,
346
+ content: block.content,
347
+ metadata: block.metadata,
348
+ });
349
+ }
350
+
351
+ return index;
352
+ }
353
+ }
@@ -0,0 +1,20 @@
1
+ import type { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
2
+ import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
3
+ import { memVectorProvider } from "@statewalker/indexer-mem";
4
+ import { miniSearchFullTextProvider } from "./minisearch-provider.js";
5
+
6
+ export interface MiniSearchIndexerOptions {
7
+ persistence?: IndexerPersistence;
8
+ }
9
+
10
+ /**
11
+ * In-memory Indexer combining MiniSearch full-text and brute-force vector
12
+ * sub-indexes via the Provider model. Multiple sub-indexes of either modality
13
+ * can be created on one Index (e.g. English and French FTS sub-indexes).
14
+ */
15
+ export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer {
16
+ return createPersistenceBackedIndexer({
17
+ persistence: options?.persistence,
18
+ providers: [miniSearchFullTextProvider, memVectorProvider],
19
+ });
20
+ }
@@ -0,0 +1,21 @@
1
+ import {
2
+ FULL_TEXT_TYPE,
3
+ type FullTextConfig,
4
+ type FullTextProvider,
5
+ } from "@statewalker/indexer-fulltext";
6
+ import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
7
+
8
+ /**
9
+ * `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
10
+ * `Indexer` alongside a `VectorProvider` to compose a full-text + vector
11
+ * Indexer.
12
+ */
13
+ export const miniSearchFullTextProvider: FullTextProvider = {
14
+ type: FULL_TEXT_TYPE,
15
+ create(config: FullTextConfig): MiniSearchFullTextIndex {
16
+ return new MiniSearchFullTextIndex({
17
+ language: config.language,
18
+ metadata: config.metadata,
19
+ });
20
+ },
21
+ };
@@ -0,0 +1,3 @@
1
+ // See ../../indexer-core/src/text-codec.d.ts for rationale.
2
+ declare const TextEncoder: { new (): { encode(input: string): Uint8Array } };
3
+ declare const TextDecoder: { new (label?: string): { decode(input?: Uint8Array): string } };