@statewalker/indexer-mem-minisearch 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +53 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/minisearch-full-text-index.d.ts +41 -0
- package/dist/minisearch-full-text-index.d.ts.map +1 -0
- package/dist/minisearch-full-text-index.js +232 -0
- package/dist/minisearch-full-text-index.js.map +1 -0
- package/dist/minisearch-indexer.d.ts +13 -0
- package/dist/minisearch-indexer.d.ts.map +1 -0
- package/dist/minisearch-indexer.js +19 -0
- package/dist/minisearch-indexer.js.map +1 -0
- package/dist/minisearch-provider.d.ts +10 -0
- package/dist/minisearch-provider.d.ts.map +1 -0
- package/dist/minisearch-provider.js +21 -0
- package/dist/minisearch-provider.js.map +1 -0
- package/package.json +58 -0
- package/src/index.ts +3 -0
- package/src/minisearch-full-text-index.ts +353 -0
- package/src/minisearch-indexer.ts +20 -0
- package/src/minisearch-provider.ts +21 -0
- package/src/text-codec.d.ts +3 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mikhail Kotelnikov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# @statewalker/indexer-mem-minisearch
|
|
2
|
+
|
|
3
|
+
In-memory implementation of `@statewalker/indexer-api` combining [MiniSearch](https://github.com/lucaong/minisearch) for full-text search with `@statewalker/indexer-mem`'s `MemVectorIndex` for embeddings. Optional streaming persistence via `IndexerPersistence`.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pnpm add @statewalker/indexer-mem-minisearch
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch";
|
|
15
|
+
|
|
16
|
+
const indexer = createMiniSearchIndexer();
|
|
17
|
+
const index = await indexer.createIndex({
|
|
18
|
+
name: "docs",
|
|
19
|
+
fulltext: { language: "en" },
|
|
20
|
+
vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" },
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
await index.addDocument([
|
|
24
|
+
{ path: "/docs/a", blockId: "1", content: "hello world" },
|
|
25
|
+
]);
|
|
26
|
+
|
|
27
|
+
for await (const hit of index.search({ queries: ["hello"], topK: 10 })) {
|
|
28
|
+
console.log(hit.path, hit.blockId, hit.score);
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### With persistence
|
|
33
|
+
|
|
34
|
+
```ts
|
|
35
|
+
import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch";
|
|
36
|
+
import type { IndexerPersistence } from "@statewalker/indexer-api";
|
|
37
|
+
|
|
38
|
+
const persistence: IndexerPersistence = /* … your save/load adapter … */;
|
|
39
|
+
const indexer = createMiniSearchIndexer({ persistence });
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Wire format (stable across releases): `__manifest__` (JSON array of index names), `${name}/__config__`, `${name}/fts` (MiniSearch `toJSON()`), `${name}/vec` (Arrow IPC from `MemVectorIndex`).
|
|
43
|
+
|
|
44
|
+
## API
|
|
45
|
+
|
|
46
|
+
- `createMiniSearchIndexer(options?)` — returns an `Indexer`. Options: `persistence?: IndexerPersistence`.
|
|
47
|
+
- `MiniSearchIndexerOptions` — option type.
|
|
48
|
+
|
|
49
|
+
## Related
|
|
50
|
+
|
|
51
|
+
- `@statewalker/indexer-api` — the pluggable contract.
|
|
52
|
+
- `@statewalker/indexer-mem-flexsearch` — drop-in alternative using FlexSearch.
|
|
53
|
+
- `@statewalker/indexer-mem` — provides the vector sub-index (`MemVectorIndex`).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
|
|
2
|
+
import { MiniSearchIndexerOptions, createMiniSearchIndexer } from "./minisearch-indexer.js";
|
|
3
|
+
import { miniSearchFullTextProvider } from "./minisearch-provider.js";
|
|
4
|
+
export { MiniSearchFullTextIndex, type MiniSearchIndexerOptions, createMiniSearchIndexer, miniSearchFullTextProvider };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
|
|
2
|
+
import { miniSearchFullTextProvider } from "./minisearch-provider.js";
|
|
3
|
+
import { createMiniSearchIndexer } from "./minisearch-indexer.js";
|
|
4
|
+
export { MiniSearchFullTextIndex, createMiniSearchIndexer, miniSearchFullTextProvider };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
|
|
2
|
+
import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
|
|
3
|
+
//#region src/minisearch-full-text-index.d.ts
|
|
4
|
+
export declare class MiniSearchFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
|
|
5
|
+
private readonly info;
|
|
6
|
+
private miniSearch;
|
|
7
|
+
private blocks;
|
|
8
|
+
private keySet;
|
|
9
|
+
private closed;
|
|
10
|
+
constructor(info: FullTextIndexInfo);
|
|
11
|
+
private ensureOpen;
|
|
12
|
+
private searchInternal;
|
|
13
|
+
getIndexInfo(): Promise<FullTextIndexInfo>;
|
|
14
|
+
search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
|
|
15
|
+
addDocument(blocks: FullTextBlock[]): Promise<void>;
|
|
16
|
+
addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
|
|
17
|
+
deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
|
|
18
|
+
private removeByKey;
|
|
19
|
+
getSize(pathPrefix?: DocumentPath): Promise<number>;
|
|
20
|
+
getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
|
|
21
|
+
getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
|
|
22
|
+
getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
|
|
23
|
+
close(_options?: {
|
|
24
|
+
force?: boolean;
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
flush(): Promise<void>;
|
|
27
|
+
deleteIndex(): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Stream the FTS sub-index's state as a single named entry. The composite
|
|
30
|
+
* prefixes this with `${indexName}/${subName}/` before it hits persistence.
|
|
31
|
+
*/
|
|
32
|
+
serialise(): AsyncIterable<PersistenceEntry>;
|
|
33
|
+
/** Restore the FTS sub-index from a previously-produced entry set. */
|
|
34
|
+
loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
|
|
35
|
+
/** Serialize to JSON (v3 format with path data) */
|
|
36
|
+
serialize(): string;
|
|
37
|
+
/** Deserialize from JSON — handles v3 format */
|
|
38
|
+
static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex;
|
|
39
|
+
}
|
|
40
|
+
//#endregion
|
|
41
|
+
//# sourceMappingURL=minisearch-full-text-index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-full-text-index.d.ts","names":["FullTextSearchParams","FullTextSearchResult"],"sources":["../src/minisearch-full-text-index.ts"],"mappings":";;;qBAgCa,mCAET,eACA,uBAAuB,eAAeA,eAAsBC;mBAE7C;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;UAQV;UAMA;EAiDF,gBAAgB,QAAQ;EAKvB,OAAO,QAAQD,gBAAuB,eAAeC;EAqCtD,YAAY,QAAQ,kBAAkB;EAkBtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;UAoBK;EAQF,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAe;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAIT,eAAe;;;;;EAad,aAAa,cAAc;;EAK5B,SAAS,SAAS,SAAS,oBAAoB;;EAerD;;SAyBO,YAAY,MAAM,mBAAmB,eAAe"}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, toBytes } from "@statewalker/indexer-core";
|
|
2
|
+
import MiniSearch from "minisearch";
|
|
3
|
+
//#region src/minisearch-full-text-index.ts
|
|
4
|
+
var MiniSearchFullTextIndex = class MiniSearchFullTextIndex {
|
|
5
|
+
info;
|
|
6
|
+
miniSearch;
|
|
7
|
+
blocks = /* @__PURE__ */ new Map();
|
|
8
|
+
keySet = /* @__PURE__ */ new Set();
|
|
9
|
+
closed = false;
|
|
10
|
+
constructor(info) {
|
|
11
|
+
this.info = info;
|
|
12
|
+
this.miniSearch = new MiniSearch({
|
|
13
|
+
fields: ["content"],
|
|
14
|
+
idField: "key"
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
ensureOpen() {
|
|
18
|
+
if (this.closed) throw new Error("FullTextIndex is closed");
|
|
19
|
+
}
|
|
20
|
+
searchInternal(query, topK, pathPrefixes) {
|
|
21
|
+
let results = this.miniSearch.search(query, {
|
|
22
|
+
prefix: true,
|
|
23
|
+
fuzzy: .2
|
|
24
|
+
});
|
|
25
|
+
if (results.length === 0) results = this.miniSearch.search(query, { prefix: true });
|
|
26
|
+
if (results.length === 0) {
|
|
27
|
+
if (query.split(/\s+/).filter((w) => w.length > 1).length > 1) results = this.miniSearch.search(query, {
|
|
28
|
+
prefix: true,
|
|
29
|
+
fuzzy: .2,
|
|
30
|
+
combineWith: "OR"
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
let filtered = results.map((r) => ({
|
|
34
|
+
key: r.id,
|
|
35
|
+
score: r.score
|
|
36
|
+
}));
|
|
37
|
+
if (pathPrefixes && pathPrefixes.length > 0) filtered = filtered.filter((r) => {
|
|
38
|
+
const block = this.blocks.get(r.key);
|
|
39
|
+
return block !== void 0 && pathPrefixes.some((p) => matchesPrefix(block.path, p));
|
|
40
|
+
});
|
|
41
|
+
return filtered.slice(0, topK).map((r) => {
|
|
42
|
+
const block = this.blocks.get(r.key);
|
|
43
|
+
return {
|
|
44
|
+
path: block?.path ?? "/",
|
|
45
|
+
blockId: block?.blockId ?? "",
|
|
46
|
+
snippet: block?.content ?? "",
|
|
47
|
+
score: r.score
|
|
48
|
+
};
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
async getIndexInfo() {
|
|
52
|
+
this.ensureOpen();
|
|
53
|
+
return { ...this.info };
|
|
54
|
+
}
|
|
55
|
+
async *search(params) {
|
|
56
|
+
this.ensureOpen();
|
|
57
|
+
const { queries, paths } = params;
|
|
58
|
+
const topK = params.topK ?? 100;
|
|
59
|
+
if (!queries || queries.length === 0) return;
|
|
60
|
+
const merged = /* @__PURE__ */ new Map();
|
|
61
|
+
for (const query of queries) {
|
|
62
|
+
const results = this.searchInternal(query, topK, paths);
|
|
63
|
+
for (const r of results) {
|
|
64
|
+
const key = compositeKey(r.path, r.blockId);
|
|
65
|
+
const existing = merged.get(key);
|
|
66
|
+
if (!existing) merged.set(key, {
|
|
67
|
+
result: r,
|
|
68
|
+
best: r.score,
|
|
69
|
+
total: r.score
|
|
70
|
+
});
|
|
71
|
+
else {
|
|
72
|
+
existing.total += r.score;
|
|
73
|
+
if (r.score > existing.best) {
|
|
74
|
+
existing.best = r.score;
|
|
75
|
+
existing.result = r;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
|
|
81
|
+
for (const { result, total } of sorted.slice(0, topK)) yield {
|
|
82
|
+
...result,
|
|
83
|
+
score: total
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
async addDocument(blocks) {
|
|
87
|
+
this.ensureOpen();
|
|
88
|
+
for (const block of blocks) {
|
|
89
|
+
const key = compositeKey(block.path, block.blockId);
|
|
90
|
+
if (this.keySet.has(key)) this.miniSearch.discard(key);
|
|
91
|
+
this.miniSearch.add({
|
|
92
|
+
key,
|
|
93
|
+
content: block.content
|
|
94
|
+
});
|
|
95
|
+
this.keySet.add(key);
|
|
96
|
+
this.blocks.set(key, {
|
|
97
|
+
path: block.path,
|
|
98
|
+
blockId: block.blockId,
|
|
99
|
+
content: block.content,
|
|
100
|
+
metadata: block.metadata
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async addDocuments(blocks) {
|
|
105
|
+
this.ensureOpen();
|
|
106
|
+
for await (const batch of blocks) await this.addDocument(batch);
|
|
107
|
+
}
|
|
108
|
+
async deleteDocuments(pathSelectors) {
|
|
109
|
+
this.ensureOpen();
|
|
110
|
+
for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
|
|
111
|
+
const key = compositeKey(sel.path, sel.blockId);
|
|
112
|
+
this.removeByKey(key);
|
|
113
|
+
} else {
|
|
114
|
+
const keysToDelete = [];
|
|
115
|
+
for (const [key, block] of this.blocks) if (matchesPrefix(block.path, sel.path)) keysToDelete.push(key);
|
|
116
|
+
for (const key of keysToDelete) this.removeByKey(key);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
removeByKey(key) {
|
|
120
|
+
if (this.keySet.has(key)) {
|
|
121
|
+
this.miniSearch.discard(key);
|
|
122
|
+
this.keySet.delete(key);
|
|
123
|
+
}
|
|
124
|
+
this.blocks.delete(key);
|
|
125
|
+
}
|
|
126
|
+
async getSize(pathPrefix) {
|
|
127
|
+
this.ensureOpen();
|
|
128
|
+
if (pathPrefix === void 0) return this.blocks.size;
|
|
129
|
+
let count = 0;
|
|
130
|
+
for (const block of this.blocks.values()) if (matchesPrefix(block.path, pathPrefix)) count++;
|
|
131
|
+
return count;
|
|
132
|
+
}
|
|
133
|
+
async *getDocumentPaths(pathPrefix) {
|
|
134
|
+
this.ensureOpen();
|
|
135
|
+
const paths = /* @__PURE__ */ new Set();
|
|
136
|
+
for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) paths.add(block.path);
|
|
137
|
+
for (const p of paths) yield p;
|
|
138
|
+
}
|
|
139
|
+
async *getDocumentBlocksRefs(pathPrefix) {
|
|
140
|
+
this.ensureOpen();
|
|
141
|
+
for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
|
|
142
|
+
path: block.path,
|
|
143
|
+
blockId: block.blockId
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
async *getDocumentsBlocks(pathPrefix) {
|
|
147
|
+
this.ensureOpen();
|
|
148
|
+
for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
|
|
149
|
+
path: block.path,
|
|
150
|
+
blockId: block.blockId,
|
|
151
|
+
content: block.content,
|
|
152
|
+
metadata: block.metadata
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
async close(_options) {
|
|
156
|
+
this.closed = true;
|
|
157
|
+
}
|
|
158
|
+
async flush() {
|
|
159
|
+
this.ensureOpen();
|
|
160
|
+
}
|
|
161
|
+
async deleteIndex() {
|
|
162
|
+
this.ensureOpen();
|
|
163
|
+
this.blocks.clear();
|
|
164
|
+
this.keySet.clear();
|
|
165
|
+
this.closed = true;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Stream the FTS sub-index's state as a single named entry. The composite
|
|
169
|
+
* prefixes this with `${indexName}/${subName}/` before it hits persistence.
|
|
170
|
+
*/
|
|
171
|
+
async *serialise() {
|
|
172
|
+
yield {
|
|
173
|
+
name: "json",
|
|
174
|
+
content: singleChunk(toBytes(this.serialize()))
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
/** Restore the FTS sub-index from a previously-produced entry set. */
|
|
178
|
+
async loadFrom(entries) {
|
|
179
|
+
this.ensureOpen();
|
|
180
|
+
for (const e of entries) {
|
|
181
|
+
if (e.name !== "json") continue;
|
|
182
|
+
const bytes = await readEntryBytes(e);
|
|
183
|
+
const json = new TextDecoder().decode(bytes);
|
|
184
|
+
const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
|
|
185
|
+
this.miniSearch = rebuilt.miniSearch;
|
|
186
|
+
this.blocks = rebuilt.blocks;
|
|
187
|
+
this.keySet = rebuilt.keySet;
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/** Serialize to JSON (v3 format with path data) */
|
|
192
|
+
serialize() {
|
|
193
|
+
const blocksData = [];
|
|
194
|
+
for (const block of this.blocks.values()) blocksData.push({
|
|
195
|
+
path: block.path,
|
|
196
|
+
blockId: block.blockId,
|
|
197
|
+
content: block.content,
|
|
198
|
+
metadata: block.metadata
|
|
199
|
+
});
|
|
200
|
+
return JSON.stringify({
|
|
201
|
+
version: 3,
|
|
202
|
+
miniSearch: this.miniSearch.toJSON(),
|
|
203
|
+
blocks: blocksData,
|
|
204
|
+
keys: [...this.keySet]
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
/** Deserialize from JSON — handles v3 format */
|
|
208
|
+
static deserialize(info, json) {
|
|
209
|
+
const parsed = JSON.parse(json);
|
|
210
|
+
if (parsed.version !== 3) throw new Error(`MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`);
|
|
211
|
+
const index = new MiniSearchFullTextIndex(info);
|
|
212
|
+
index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
|
|
213
|
+
fields: ["content"],
|
|
214
|
+
idField: "key"
|
|
215
|
+
});
|
|
216
|
+
index.keySet = new Set(parsed.keys);
|
|
217
|
+
for (const block of parsed.blocks) {
|
|
218
|
+
const key = compositeKey(block.path, block.blockId);
|
|
219
|
+
index.blocks.set(key, {
|
|
220
|
+
path: block.path,
|
|
221
|
+
blockId: block.blockId,
|
|
222
|
+
content: block.content,
|
|
223
|
+
metadata: block.metadata
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
return index;
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
//#endregion
|
|
230
|
+
export { MiniSearchFullTextIndex };
|
|
231
|
+
|
|
232
|
+
//# sourceMappingURL=minisearch-full-text-index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-full-text-index.js","names":[],"sources":["../src/minisearch-full-text-index.ts"],"sourcesContent":["import type {\n BlockReference,\n DocumentPath,\n Metadata,\n PathSelector,\n PersistableSearchIndex,\n PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n compositeKey,\n matchesPrefix,\n readEntryBytes,\n singleChunk,\n toAsyncIterable,\n toBytes,\n} from \"@statewalker/indexer-core\";\nimport type {\n FullTextBlock,\n FullTextIndex,\n FullTextIndexInfo,\n FulltextQuery as FullTextSearchParams,\n FulltextResult as FullTextSearchResult,\n} from \"@statewalker/indexer-fulltext\";\nimport MiniSearch from \"minisearch\";\n\ninterface StoredBlock {\n path: DocumentPath;\n blockId: string;\n content: string;\n metadata?: Metadata;\n}\n\nexport class MiniSearchFullTextIndex\n implements\n FullTextIndex,\n PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>\n{\n private readonly info: FullTextIndexInfo;\n private miniSearch: MiniSearch;\n private blocks = new Map<string, StoredBlock>();\n private keySet = new Set<string>();\n private closed = false;\n\n constructor(info: FullTextIndexInfo) {\n this.info = info;\n this.miniSearch = new MiniSearch({\n fields: [\"content\"],\n idField: \"key\",\n });\n }\n\n private ensureOpen(): void {\n if (this.closed) {\n throw new Error(\"FullTextIndex is closed\");\n }\n }\n\n private searchInternal(\n query: string,\n topK: number,\n pathPrefixes?: DocumentPath[],\n ): FullTextSearchResult[] {\n let results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n });\n\n if (results.length === 0) {\n results = this.miniSearch.search(query, { prefix: true });\n }\n\n if (results.length === 0) {\n const words = query.split(/\\s+/).filter((w) => w.length > 1);\n if (words.length > 1) {\n results = this.miniSearch.search(query, {\n prefix: true,\n fuzzy: 0.2,\n combineWith: \"OR\",\n });\n }\n }\n\n // Filter by path prefixes\n let filtered = results.map((r) => ({\n key: r.id as string,\n score: r.score,\n }));\n\n if (pathPrefixes && pathPrefixes.length > 0) {\n filtered = filtered.filter((r) => {\n const block = this.blocks.get(r.key);\n return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));\n });\n }\n\n return filtered.slice(0, topK).map((r) => {\n const block = this.blocks.get(r.key);\n return {\n path: block?.path ?? (\"/\" as DocumentPath),\n blockId: block?.blockId ?? \"\",\n snippet: block?.content ?? \"\",\n score: r.score,\n };\n });\n }\n\n async getIndexInfo(): Promise<FullTextIndexInfo> {\n this.ensureOpen();\n return { ...this.info };\n }\n\n async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {\n this.ensureOpen();\n const { queries, paths } = params;\n const topK = params.topK ?? 100;\n\n if (!queries || queries.length === 0) return;\n\n // Multi-query: search for each and ACCUMULATE per-block scores, so a block\n // matching more queries ranks higher (the FulltextQuery contract). Taking the\n // max instead would let a block that is the top hit for one query outrank a\n // block matching several queries. The snippet is kept from the strongest\n // single-query match.\n const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();\n\n for (const query of queries) {\n const results = this.searchInternal(query, topK, paths);\n for (const r of results) {\n const key = compositeKey(r.path, r.blockId);\n const existing = merged.get(key);\n if (!existing) {\n merged.set(key, { result: r, best: r.score, total: r.score });\n } else {\n existing.total += r.score;\n if (r.score > existing.best) {\n existing.best = r.score;\n existing.result = r;\n }\n }\n }\n }\n\n const sorted = [...merged.values()].sort((a, b) => b.total - a.total);\n for (const { result, total } of sorted.slice(0, topK)) {\n yield { ...result, score: total };\n }\n }\n\n async addDocument(blocks: FullTextBlock[]): Promise<void> {\n this.ensureOpen();\n for (const block of blocks) {\n const key = compositeKey(block.path, block.blockId);\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n }\n this.miniSearch.add({ key, content: block.content });\n this.keySet.add(key);\n this.blocks.set(key, {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n }\n\n async addDocuments(\n blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const batch of blocks) {\n await this.addDocument(batch);\n }\n }\n\n async deleteDocuments(\n pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n ): Promise<void> {\n this.ensureOpen();\n for await (const sel of toAsyncIterable(pathSelectors)) {\n if (sel.blockId !== undefined) {\n const key = compositeKey(sel.path, sel.blockId);\n this.removeByKey(key);\n } else {\n const keysToDelete: string[] = [];\n for (const [key, block] of this.blocks) {\n if (matchesPrefix(block.path, sel.path)) {\n keysToDelete.push(key);\n }\n }\n for (const key of keysToDelete) {\n this.removeByKey(key);\n }\n }\n }\n }\n\n private removeByKey(key: string): void {\n if (this.keySet.has(key)) {\n this.miniSearch.discard(key);\n this.keySet.delete(key);\n }\n this.blocks.delete(key);\n }\n\n async getSize(pathPrefix?: DocumentPath): Promise<number> {\n this.ensureOpen();\n if (pathPrefix === undefined) return this.blocks.size;\n let count = 0;\n for (const block of this.blocks.values()) {\n if (matchesPrefix(block.path, pathPrefix)) count++;\n }\n return count;\n }\n\n async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n this.ensureOpen();\n const paths = new Set<DocumentPath>();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n paths.add(block.path);\n }\n }\n for (const p of paths) yield p;\n }\n\n async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield { path: block.path, blockId: block.blockId };\n }\n }\n }\n\n async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n this.ensureOpen();\n for (const block of this.blocks.values()) {\n if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n yield {\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n };\n }\n }\n }\n\n async close(_options?: { force?: boolean }): Promise<void> {\n this.closed = true;\n }\n\n async flush(): Promise<void> {\n this.ensureOpen();\n }\n\n async deleteIndex(): Promise<void> {\n this.ensureOpen();\n this.blocks.clear();\n this.keySet.clear();\n this.closed = true;\n }\n\n // --- PersistableSearchIndex --------------------------------------------------\n\n /**\n * Stream the FTS sub-index's state as a single named entry. The composite\n * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n */\n async *serialise(): AsyncIterable<PersistenceEntry> {\n yield { name: \"json\", content: singleChunk(toBytes(this.serialize())) };\n }\n\n /** Restore the FTS sub-index from a previously-produced entry set. */\n async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n this.ensureOpen();\n for (const e of entries) {\n if (e.name !== \"json\") continue;\n const bytes = await readEntryBytes(e);\n const json = new TextDecoder().decode(bytes);\n const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);\n this.miniSearch = rebuilt.miniSearch;\n this.blocks = rebuilt.blocks;\n this.keySet = rebuilt.keySet;\n return;\n }\n }\n\n /** Serialize to JSON (v3 format with path data) */\n serialize(): string {\n const blocksData: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }> = [];\n for (const block of this.blocks.values()) {\n blocksData.push({\n path: block.path,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return JSON.stringify({\n version: 3,\n miniSearch: this.miniSearch.toJSON(),\n blocks: blocksData,\n keys: [...this.keySet],\n });\n }\n\n /** Deserialize from JSON — handles v3 format */\n static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {\n const parsed = JSON.parse(json) as {\n version?: number;\n miniSearch: unknown;\n blocks: Array<{\n path: string;\n blockId: string;\n content: string;\n metadata?: Metadata;\n }>;\n keys: string[];\n };\n\n if (parsed.version !== 3) {\n throw new Error(\n `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,\n );\n }\n\n const index = new MiniSearchFullTextIndex(info);\n index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {\n fields: [\"content\"],\n idField: \"key\",\n });\n index.keySet = new Set(parsed.keys);\n for (const block of parsed.blocks) {\n const key = compositeKey(block.path as DocumentPath, block.blockId);\n index.blocks.set(key, {\n path: block.path as DocumentPath,\n blockId: block.blockId,\n content: block.content,\n metadata: block.metadata,\n });\n }\n\n return index;\n }\n}\n"],"mappings":";;;AAgCA,IAAa,0BAAb,MAAa,wBAIb;CACE;CACA;CACA,yBAAiB,IAAI,IAAyB;CAC9C,yBAAiB,IAAI,IAAY;CACjC,SAAiB;CAEjB,YAAY,MAAyB;EACnC,KAAK,OAAO;EACZ,KAAK,aAAa,IAAI,WAAW;GAC/B,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;CACH;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,yBAAyB;CAE7C;CAEA,eACE,OACA,MACA,cACwB;EACxB,IAAI,UAAU,KAAK,WAAW,OAAO,OAAO;GAC1C,QAAQ;GACR,OAAO;EACT,CAAC;EAED,IAAI,QAAQ,WAAW,GACrB,UAAU,KAAK,WAAW,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;EAG1D,IAAI,QAAQ,WAAW,GACP;OAAA,MAAM,MAAM,KAAK,CAAC,CAAC,QAAQ,MAAM,EAAE,SAAS,CAClD,CAAC,CAAC,SAAS,GACjB,UAAU,KAAK,WAAW,OAAO,OAAO;IACtC,QAAQ;IACR,OAAO;IACP,aAAa;GACf,CAAC;EAAA;EAKL,IAAI,WAAW,QAAQ,KAAK,OAAO;GACjC,KAAK,EAAE;GACP,OAAO,EAAE;EACX,EAAE;EAEF,IAAI,gBAAgB,aAAa,SAAS,GACxC,WAAW,SAAS,QAAQ,MAAM;GAChC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO,UAAU,KAAA,KAAa,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC;EACrF,CAAC;EAGH,OAAO,SAAS,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,MAAM;GACxC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO;IACL,MAAM,OAAO,QAAS;IACtB,SAAS,OAAO,WAAW;IAC3B,SAAS,OAAO,WAAW;IAC3B,OAAO,EAAE;GACX;EACF,CAAC;CACH;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAoE;EAChF,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EAOtC,MAAM,yBAAS,IAAI,IAA2E;EAE9F,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,UAAU,KAAK,eAAe,OAAO,MAAM,KAAK;GACtD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,OAAO,IAAI,GAAG;IAC/B,IAAI,CAAC,UACH,OAAO,IAAI,KAAK;KAAE,QAAQ;KAAG,MAAM,EAAE;KAAO,OAAO,EAAE;IAAM,CAAC;SACvD;KACL,SAAS,SAAS,EAAE;KACpB,IAAI,EAAE,QAAQ,SAAS,MAAM;MAC3B,SAAS,OAAO,EAAE;MAClB,SAAS,SAAS;KACpB;IACF;GACF;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACpE,KAAK,MAAM,EAAE,QAAQ,WAAW,OAAO,MAAM,GAAG,IAAI,GAClD,MAAM;GAAE,GAAG;GAAQ,OAAO;EAAM;CAEpC;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,IAAI,KAAK,OAAO,IAAI,GAAG,GACrB,KAAK,WAAW,QAAQ,GAAG;GAE7B,KAAK,WAAW,IAAI;IAAE;IAAK,SAAS,MAAM;GAAQ,CAAC;GACnD,KAAK,OAAO,IAAI,GAAG;GACnB,KAAK,OAAO,IAAI,KAAK;IACnB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;CACF;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GAAW;GAC7B,MAAM,MAAM,aAAa,IAAI,MAAM,IAAI,OAAO;GAC9C,KAAK,YAAY,GAAG;EACtB,OAAO;GACL,MAAM,eAAyB,CAAC;GAChC,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GACpC,aAAa,KAAK,GAAG;GAGzB,KAAK,MAAM,OAAO,cAChB,KAAK,YAAY,GAAG;EAExB;CAEJ;CAEA,YAAoB,KAAmB;EACrC,IAAI,KAAK,OAAO,IAAI,GAAG,GAAG;GACxB,KAAK,WAAW,QAAQ,GAAG;GAC3B,KAAK,OAAO,OAAO,GAAG;EACxB;EACA,KAAK,OAAO,OAAO,GAAG;CACxB;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAClB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,OAAO,MAAM;EAClB,KAAK,OAAO,MAAM;EAClB,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,KAAK,UAAU,CAAC,CAAC;EAAE;CACxE;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;GAC3C,MAAM,UAAU,wBAAwB,YAAY,KAAK,MAAM,IAAI;GACnE,KAAK,aAAa,QAAQ;GAC1B,KAAK,SAAS,QAAQ;GACtB,KAAK,SAAS,QAAQ;GACtB;EACF;CACF;;CAGA,YAAoB;EAClB,MAAM,aAKD,CAAC;EACN,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,WAAW,KAAK;GACd,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB,CAAC;EAGH,OAAO,KAAK,UAAU;GACpB,SAAS;GACT,YAAY,KAAK,WAAW,OAAO;GACnC,QAAQ;GACR,MAAM,CAAC,GAAG,KAAK,MAAM;EACvB,CAAC;CACH;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,SAAS,KAAK,MAAM,IAAI;EAY9B,IAAI,OAAO,YAAY,GACrB,MAAM,IAAI,MACR,2DAA2D,OAAO,OAAO,OAAO,EAAE,cACpF;EAGF,MAAM,QAAQ,IAAI,wBAAwB,IAAI;EAC9C,MAAM,aAAa,WAAW,SAAS,KAAK,UAAU,OAAO,UAAU,GAAG;GACxE,QAAQ,CAAC,SAAS;GAClB,SAAS;EACX,CAAC;EACD,MAAM,SAAS,IAAI,IAAI,OAAO,IAAI;EAClC,KAAK,MAAM,SAAS,OAAO,QAAQ;GACjC,MAAM,MAAM,aAAa,MAAM,MAAsB,MAAM,OAAO;GAClE,MAAM,OAAO,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EAEA,OAAO;CACT;AACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
|
|
2
|
+
//#region src/minisearch-indexer.d.ts
|
|
3
|
+
export interface MiniSearchIndexerOptions {
|
|
4
|
+
persistence?: IndexerPersistence;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* In-memory Indexer combining MiniSearch full-text and brute-force vector
|
|
8
|
+
* sub-indexes via the Provider model. Multiple sub-indexes of either modality
|
|
9
|
+
* can be created on one Index (e.g. English and French FTS sub-indexes).
|
|
10
|
+
*/
|
|
11
|
+
export declare function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer;
|
|
12
|
+
//#endregion
|
|
13
|
+
//# sourceMappingURL=minisearch-indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-indexer.d.ts","names":[],"sources":["../src/minisearch-indexer.ts"],"mappings":";;iBAKiB;EACf,cAAc;;;;;;;wBAQA,wBAAwB,UAAU,2BAA2B"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { miniSearchFullTextProvider } from "./minisearch-provider.js";
|
|
2
|
+
import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
|
|
3
|
+
import { memVectorProvider } from "@statewalker/indexer-mem";
|
|
4
|
+
//#region src/minisearch-indexer.ts
|
|
5
|
+
/**
|
|
6
|
+
* In-memory Indexer combining MiniSearch full-text and brute-force vector
|
|
7
|
+
* sub-indexes via the Provider model. Multiple sub-indexes of either modality
|
|
8
|
+
* can be created on one Index (e.g. English and French FTS sub-indexes).
|
|
9
|
+
*/
|
|
10
|
+
function createMiniSearchIndexer(options) {
|
|
11
|
+
return createPersistenceBackedIndexer({
|
|
12
|
+
persistence: options?.persistence,
|
|
13
|
+
providers: [miniSearchFullTextProvider, memVectorProvider]
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
17
|
+
export { createMiniSearchIndexer };
|
|
18
|
+
|
|
19
|
+
//# sourceMappingURL=minisearch-indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-indexer.js","names":[],"sources":["../src/minisearch-indexer.ts"],"sourcesContent":["import type { Indexer, IndexerPersistence } from \"@statewalker/indexer-api\";\nimport { createPersistenceBackedIndexer } from \"@statewalker/indexer-core\";\nimport { memVectorProvider } from \"@statewalker/indexer-mem\";\nimport { miniSearchFullTextProvider } from \"./minisearch-provider.js\";\n\nexport interface MiniSearchIndexerOptions {\n persistence?: IndexerPersistence;\n}\n\n/**\n * In-memory Indexer combining MiniSearch full-text and brute-force vector\n * sub-indexes via the Provider model. Multiple sub-indexes of either modality\n * can be created on one Index (e.g. English and French FTS sub-indexes).\n */\nexport function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer {\n return createPersistenceBackedIndexer({\n persistence: options?.persistence,\n providers: [miniSearchFullTextProvider, memVectorProvider],\n });\n}\n"],"mappings":";;;;;;;;;AAcA,SAAgB,wBAAwB,SAA6C;CACnF,OAAO,+BAA+B;EACpC,aAAa,SAAS;EACtB,WAAW,CAAC,4BAA4B,iBAAiB;CAC3D,CAAC;AACH"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { FullTextProvider } from "@statewalker/indexer-fulltext";
|
|
2
|
+
//#region src/minisearch-provider.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
|
|
5
|
+
* `Indexer` alongside a `VectorProvider` to compose a full-text + vector
|
|
6
|
+
* Indexer.
|
|
7
|
+
*/
|
|
8
|
+
export declare const miniSearchFullTextProvider: FullTextProvider;
|
|
9
|
+
//#endregion
|
|
10
|
+
//# sourceMappingURL=minisearch-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-provider.d.ts","names":[],"sources":["../src/minisearch-provider.ts"],"mappings":";;;;;;;qBAYa,4BAA4B"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
|
|
2
|
+
import { FULL_TEXT_TYPE } from "@statewalker/indexer-fulltext";
|
|
3
|
+
//#region src/minisearch-provider.ts
|
|
4
|
+
/**
|
|
5
|
+
* `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
|
|
6
|
+
* `Indexer` alongside a `VectorProvider` to compose a full-text + vector
|
|
7
|
+
* Indexer.
|
|
8
|
+
*/
|
|
9
|
+
const miniSearchFullTextProvider = {
|
|
10
|
+
type: FULL_TEXT_TYPE,
|
|
11
|
+
create(config) {
|
|
12
|
+
return new MiniSearchFullTextIndex({
|
|
13
|
+
language: config.language,
|
|
14
|
+
metadata: config.metadata
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
//#endregion
|
|
19
|
+
export { miniSearchFullTextProvider };
|
|
20
|
+
|
|
21
|
+
//# sourceMappingURL=minisearch-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"minisearch-provider.js","names":[],"sources":["../src/minisearch-provider.ts"],"sourcesContent":["import {\n FULL_TEXT_TYPE,\n type FullTextConfig,\n type FullTextProvider,\n} from \"@statewalker/indexer-fulltext\";\nimport { MiniSearchFullTextIndex } from \"./minisearch-full-text-index.js\";\n\n/**\n * `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an\n * `Indexer` alongside a `VectorProvider` to compose a full-text + vector\n * Indexer.\n */\nexport const miniSearchFullTextProvider: FullTextProvider = {\n type: FULL_TEXT_TYPE,\n create(config: FullTextConfig): MiniSearchFullTextIndex {\n return new MiniSearchFullTextIndex({\n language: config.language,\n metadata: config.metadata,\n });\n },\n};\n"],"mappings":";;;;;;;;AAYA,MAAa,6BAA+C;CAC1D,MAAM;CACN,OAAO,QAAiD;EACtD,OAAO,IAAI,wBAAwB;GACjC,UAAU,OAAO;GACjB,UAAU,OAAO;EACnB,CAAC;CACH;AACF"}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@statewalker/indexer-mem-minisearch",
|
|
3
|
+
"version": "0.1.2",
|
|
4
|
+
"private": false,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"description": "In-memory indexer implementation backed by MiniSearch for full-text search.",
|
|
7
|
+
"homepage": "https://github.com/statewalker/statewalker-indexer",
|
|
8
|
+
"author": {
|
|
9
|
+
"name": "Mikhail Kotelnikov",
|
|
10
|
+
"email": "mikhail.kotelnikov@gmail.com"
|
|
11
|
+
},
|
|
12
|
+
"license": "MIT",
|
|
13
|
+
"repository": {
|
|
14
|
+
"type": "git",
|
|
15
|
+
"url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git"
|
|
16
|
+
},
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"source": "./src/index.ts",
|
|
20
|
+
"types": "./dist/index.d.ts",
|
|
21
|
+
"import": "./dist/index.js",
|
|
22
|
+
"default": "./dist/index.js"
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"files": [
|
|
26
|
+
"dist",
|
|
27
|
+
"src"
|
|
28
|
+
],
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"minisearch": "^7.2.0",
|
|
31
|
+
"@statewalker/indexer-core": "^0.1.1",
|
|
32
|
+
"@statewalker/indexer-api": "^0.1.1",
|
|
33
|
+
"@statewalker/indexer-fulltext": "^0.1.1",
|
|
34
|
+
"@statewalker/indexer-mem": "^0.1.2"
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"rimraf": "^6.1.3",
|
|
38
|
+
"tsdown": "^0.23.0",
|
|
39
|
+
"typescript": "^7.0.2",
|
|
40
|
+
"vitest": "^5.0.3",
|
|
41
|
+
"@statewalker/indexer-tests": "^0.0.0",
|
|
42
|
+
"@statewalker/indexer-vector": "^0.1.1"
|
|
43
|
+
},
|
|
44
|
+
"sideEffects": false,
|
|
45
|
+
"publishConfig": {
|
|
46
|
+
"access": "public"
|
|
47
|
+
},
|
|
48
|
+
"scripts": {
|
|
49
|
+
"build": "tsdown",
|
|
50
|
+
"dev": "tsdown --watch",
|
|
51
|
+
"test": "vitest run --passWithNoTests",
|
|
52
|
+
"test:watch": "vitest",
|
|
53
|
+
"typecheck": "tsc --noEmit",
|
|
54
|
+
"clean": "rimraf dist",
|
|
55
|
+
"lint": "biome check --write .",
|
|
56
|
+
"format": "biome format --write ."
|
|
57
|
+
}
|
|
58
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
BlockReference,
|
|
3
|
+
DocumentPath,
|
|
4
|
+
Metadata,
|
|
5
|
+
PathSelector,
|
|
6
|
+
PersistableSearchIndex,
|
|
7
|
+
PersistenceEntry,
|
|
8
|
+
} from "@statewalker/indexer-api";
|
|
9
|
+
import {
|
|
10
|
+
compositeKey,
|
|
11
|
+
matchesPrefix,
|
|
12
|
+
readEntryBytes,
|
|
13
|
+
singleChunk,
|
|
14
|
+
toAsyncIterable,
|
|
15
|
+
toBytes,
|
|
16
|
+
} from "@statewalker/indexer-core";
|
|
17
|
+
import type {
|
|
18
|
+
FullTextBlock,
|
|
19
|
+
FullTextIndex,
|
|
20
|
+
FullTextIndexInfo,
|
|
21
|
+
FulltextQuery as FullTextSearchParams,
|
|
22
|
+
FulltextResult as FullTextSearchResult,
|
|
23
|
+
} from "@statewalker/indexer-fulltext";
|
|
24
|
+
import MiniSearch from "minisearch";
|
|
25
|
+
|
|
26
|
+
interface StoredBlock {
|
|
27
|
+
path: DocumentPath;
|
|
28
|
+
blockId: string;
|
|
29
|
+
content: string;
|
|
30
|
+
metadata?: Metadata;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class MiniSearchFullTextIndex
|
|
34
|
+
implements
|
|
35
|
+
FullTextIndex,
|
|
36
|
+
PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>
|
|
37
|
+
{
|
|
38
|
+
private readonly info: FullTextIndexInfo;
|
|
39
|
+
private miniSearch: MiniSearch;
|
|
40
|
+
private blocks = new Map<string, StoredBlock>();
|
|
41
|
+
private keySet = new Set<string>();
|
|
42
|
+
private closed = false;
|
|
43
|
+
|
|
44
|
+
constructor(info: FullTextIndexInfo) {
|
|
45
|
+
this.info = info;
|
|
46
|
+
this.miniSearch = new MiniSearch({
|
|
47
|
+
fields: ["content"],
|
|
48
|
+
idField: "key",
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private ensureOpen(): void {
|
|
53
|
+
if (this.closed) {
|
|
54
|
+
throw new Error("FullTextIndex is closed");
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
private searchInternal(
|
|
59
|
+
query: string,
|
|
60
|
+
topK: number,
|
|
61
|
+
pathPrefixes?: DocumentPath[],
|
|
62
|
+
): FullTextSearchResult[] {
|
|
63
|
+
let results = this.miniSearch.search(query, {
|
|
64
|
+
prefix: true,
|
|
65
|
+
fuzzy: 0.2,
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
if (results.length === 0) {
|
|
69
|
+
results = this.miniSearch.search(query, { prefix: true });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (results.length === 0) {
|
|
73
|
+
const words = query.split(/\s+/).filter((w) => w.length > 1);
|
|
74
|
+
if (words.length > 1) {
|
|
75
|
+
results = this.miniSearch.search(query, {
|
|
76
|
+
prefix: true,
|
|
77
|
+
fuzzy: 0.2,
|
|
78
|
+
combineWith: "OR",
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Filter by path prefixes
|
|
84
|
+
let filtered = results.map((r) => ({
|
|
85
|
+
key: r.id as string,
|
|
86
|
+
score: r.score,
|
|
87
|
+
}));
|
|
88
|
+
|
|
89
|
+
if (pathPrefixes && pathPrefixes.length > 0) {
|
|
90
|
+
filtered = filtered.filter((r) => {
|
|
91
|
+
const block = this.blocks.get(r.key);
|
|
92
|
+
return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return filtered.slice(0, topK).map((r) => {
|
|
97
|
+
const block = this.blocks.get(r.key);
|
|
98
|
+
return {
|
|
99
|
+
path: block?.path ?? ("/" as DocumentPath),
|
|
100
|
+
blockId: block?.blockId ?? "",
|
|
101
|
+
snippet: block?.content ?? "",
|
|
102
|
+
score: r.score,
|
|
103
|
+
};
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async getIndexInfo(): Promise<FullTextIndexInfo> {
|
|
108
|
+
this.ensureOpen();
|
|
109
|
+
return { ...this.info };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {
|
|
113
|
+
this.ensureOpen();
|
|
114
|
+
const { queries, paths } = params;
|
|
115
|
+
const topK = params.topK ?? 100;
|
|
116
|
+
|
|
117
|
+
if (!queries || queries.length === 0) return;
|
|
118
|
+
|
|
119
|
+
// Multi-query: search for each and ACCUMULATE per-block scores, so a block
|
|
120
|
+
// matching more queries ranks higher (the FulltextQuery contract). Taking the
|
|
121
|
+
// max instead would let a block that is the top hit for one query outrank a
|
|
122
|
+
// block matching several queries. The snippet is kept from the strongest
|
|
123
|
+
// single-query match.
|
|
124
|
+
const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();
|
|
125
|
+
|
|
126
|
+
for (const query of queries) {
|
|
127
|
+
const results = this.searchInternal(query, topK, paths);
|
|
128
|
+
for (const r of results) {
|
|
129
|
+
const key = compositeKey(r.path, r.blockId);
|
|
130
|
+
const existing = merged.get(key);
|
|
131
|
+
if (!existing) {
|
|
132
|
+
merged.set(key, { result: r, best: r.score, total: r.score });
|
|
133
|
+
} else {
|
|
134
|
+
existing.total += r.score;
|
|
135
|
+
if (r.score > existing.best) {
|
|
136
|
+
existing.best = r.score;
|
|
137
|
+
existing.result = r;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
|
|
144
|
+
for (const { result, total } of sorted.slice(0, topK)) {
|
|
145
|
+
yield { ...result, score: total };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async addDocument(blocks: FullTextBlock[]): Promise<void> {
|
|
150
|
+
this.ensureOpen();
|
|
151
|
+
for (const block of blocks) {
|
|
152
|
+
const key = compositeKey(block.path, block.blockId);
|
|
153
|
+
if (this.keySet.has(key)) {
|
|
154
|
+
this.miniSearch.discard(key);
|
|
155
|
+
}
|
|
156
|
+
this.miniSearch.add({ key, content: block.content });
|
|
157
|
+
this.keySet.add(key);
|
|
158
|
+
this.blocks.set(key, {
|
|
159
|
+
path: block.path,
|
|
160
|
+
blockId: block.blockId,
|
|
161
|
+
content: block.content,
|
|
162
|
+
metadata: block.metadata,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async addDocuments(
|
|
168
|
+
blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
|
|
169
|
+
): Promise<void> {
|
|
170
|
+
this.ensureOpen();
|
|
171
|
+
for await (const batch of blocks) {
|
|
172
|
+
await this.addDocument(batch);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async deleteDocuments(
|
|
177
|
+
pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
|
|
178
|
+
): Promise<void> {
|
|
179
|
+
this.ensureOpen();
|
|
180
|
+
for await (const sel of toAsyncIterable(pathSelectors)) {
|
|
181
|
+
if (sel.blockId !== undefined) {
|
|
182
|
+
const key = compositeKey(sel.path, sel.blockId);
|
|
183
|
+
this.removeByKey(key);
|
|
184
|
+
} else {
|
|
185
|
+
const keysToDelete: string[] = [];
|
|
186
|
+
for (const [key, block] of this.blocks) {
|
|
187
|
+
if (matchesPrefix(block.path, sel.path)) {
|
|
188
|
+
keysToDelete.push(key);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
for (const key of keysToDelete) {
|
|
192
|
+
this.removeByKey(key);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
private removeByKey(key: string): void {
|
|
199
|
+
if (this.keySet.has(key)) {
|
|
200
|
+
this.miniSearch.discard(key);
|
|
201
|
+
this.keySet.delete(key);
|
|
202
|
+
}
|
|
203
|
+
this.blocks.delete(key);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async getSize(pathPrefix?: DocumentPath): Promise<number> {
|
|
207
|
+
this.ensureOpen();
|
|
208
|
+
if (pathPrefix === undefined) return this.blocks.size;
|
|
209
|
+
let count = 0;
|
|
210
|
+
for (const block of this.blocks.values()) {
|
|
211
|
+
if (matchesPrefix(block.path, pathPrefix)) count++;
|
|
212
|
+
}
|
|
213
|
+
return count;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
|
|
217
|
+
this.ensureOpen();
|
|
218
|
+
const paths = new Set<DocumentPath>();
|
|
219
|
+
for (const block of this.blocks.values()) {
|
|
220
|
+
if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
|
|
221
|
+
paths.add(block.path);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
for (const p of paths) yield p;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
|
|
228
|
+
this.ensureOpen();
|
|
229
|
+
for (const block of this.blocks.values()) {
|
|
230
|
+
if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
|
|
231
|
+
yield { path: block.path, blockId: block.blockId };
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
|
|
237
|
+
this.ensureOpen();
|
|
238
|
+
for (const block of this.blocks.values()) {
|
|
239
|
+
if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
|
|
240
|
+
yield {
|
|
241
|
+
path: block.path,
|
|
242
|
+
blockId: block.blockId,
|
|
243
|
+
content: block.content,
|
|
244
|
+
metadata: block.metadata,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async close(_options?: { force?: boolean }): Promise<void> {
|
|
251
|
+
this.closed = true;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async flush(): Promise<void> {
|
|
255
|
+
this.ensureOpen();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async deleteIndex(): Promise<void> {
|
|
259
|
+
this.ensureOpen();
|
|
260
|
+
this.blocks.clear();
|
|
261
|
+
this.keySet.clear();
|
|
262
|
+
this.closed = true;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// --- PersistableSearchIndex --------------------------------------------------
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Stream the FTS sub-index's state as a single named entry. The composite
|
|
269
|
+
* prefixes this with `${indexName}/${subName}/` before it hits persistence.
|
|
270
|
+
*/
|
|
271
|
+
async *serialise(): AsyncIterable<PersistenceEntry> {
|
|
272
|
+
yield { name: "json", content: singleChunk(toBytes(this.serialize())) };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** Restore the FTS sub-index from a previously-produced entry set. */
|
|
276
|
+
async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
|
|
277
|
+
this.ensureOpen();
|
|
278
|
+
for (const e of entries) {
|
|
279
|
+
if (e.name !== "json") continue;
|
|
280
|
+
const bytes = await readEntryBytes(e);
|
|
281
|
+
const json = new TextDecoder().decode(bytes);
|
|
282
|
+
const rebuilt = MiniSearchFullTextIndex.deserialize(this.info, json);
|
|
283
|
+
this.miniSearch = rebuilt.miniSearch;
|
|
284
|
+
this.blocks = rebuilt.blocks;
|
|
285
|
+
this.keySet = rebuilt.keySet;
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** Serialize to JSON (v3 format with path data) */
|
|
291
|
+
serialize(): string {
|
|
292
|
+
const blocksData: Array<{
|
|
293
|
+
path: string;
|
|
294
|
+
blockId: string;
|
|
295
|
+
content: string;
|
|
296
|
+
metadata?: Metadata;
|
|
297
|
+
}> = [];
|
|
298
|
+
for (const block of this.blocks.values()) {
|
|
299
|
+
blocksData.push({
|
|
300
|
+
path: block.path,
|
|
301
|
+
blockId: block.blockId,
|
|
302
|
+
content: block.content,
|
|
303
|
+
metadata: block.metadata,
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
return JSON.stringify({
|
|
308
|
+
version: 3,
|
|
309
|
+
miniSearch: this.miniSearch.toJSON(),
|
|
310
|
+
blocks: blocksData,
|
|
311
|
+
keys: [...this.keySet],
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Deserialize from JSON — handles v3 format */
|
|
316
|
+
static deserialize(info: FullTextIndexInfo, json: string): MiniSearchFullTextIndex {
|
|
317
|
+
const parsed = JSON.parse(json) as {
|
|
318
|
+
version?: number;
|
|
319
|
+
miniSearch: unknown;
|
|
320
|
+
blocks: Array<{
|
|
321
|
+
path: string;
|
|
322
|
+
blockId: string;
|
|
323
|
+
content: string;
|
|
324
|
+
metadata?: Metadata;
|
|
325
|
+
}>;
|
|
326
|
+
keys: string[];
|
|
327
|
+
};
|
|
328
|
+
|
|
329
|
+
if (parsed.version !== 3) {
|
|
330
|
+
throw new Error(
|
|
331
|
+
`MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const index = new MiniSearchFullTextIndex(info);
|
|
336
|
+
index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), {
|
|
337
|
+
fields: ["content"],
|
|
338
|
+
idField: "key",
|
|
339
|
+
});
|
|
340
|
+
index.keySet = new Set(parsed.keys);
|
|
341
|
+
for (const block of parsed.blocks) {
|
|
342
|
+
const key = compositeKey(block.path as DocumentPath, block.blockId);
|
|
343
|
+
index.blocks.set(key, {
|
|
344
|
+
path: block.path as DocumentPath,
|
|
345
|
+
blockId: block.blockId,
|
|
346
|
+
content: block.content,
|
|
347
|
+
metadata: block.metadata,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
return index;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
|
|
2
|
+
import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
|
|
3
|
+
import { memVectorProvider } from "@statewalker/indexer-mem";
|
|
4
|
+
import { miniSearchFullTextProvider } from "./minisearch-provider.js";
|
|
5
|
+
|
|
6
|
+
export interface MiniSearchIndexerOptions {
|
|
7
|
+
persistence?: IndexerPersistence;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* In-memory Indexer combining MiniSearch full-text and brute-force vector
|
|
12
|
+
* sub-indexes via the Provider model. Multiple sub-indexes of either modality
|
|
13
|
+
* can be created on one Index (e.g. English and French FTS sub-indexes).
|
|
14
|
+
*/
|
|
15
|
+
export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer {
|
|
16
|
+
return createPersistenceBackedIndexer({
|
|
17
|
+
persistence: options?.persistence,
|
|
18
|
+
providers: [miniSearchFullTextProvider, memVectorProvider],
|
|
19
|
+
});
|
|
20
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import {
|
|
2
|
+
FULL_TEXT_TYPE,
|
|
3
|
+
type FullTextConfig,
|
|
4
|
+
type FullTextProvider,
|
|
5
|
+
} from "@statewalker/indexer-fulltext";
|
|
6
|
+
import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* `FullTextProvider` backed by {@link MiniSearchFullTextIndex}. Wired into an
|
|
10
|
+
* `Indexer` alongside a `VectorProvider` to compose a full-text + vector
|
|
11
|
+
* Indexer.
|
|
12
|
+
*/
|
|
13
|
+
export const miniSearchFullTextProvider: FullTextProvider = {
|
|
14
|
+
type: FULL_TEXT_TYPE,
|
|
15
|
+
create(config: FullTextConfig): MiniSearchFullTextIndex {
|
|
16
|
+
return new MiniSearchFullTextIndex({
|
|
17
|
+
language: config.language,
|
|
18
|
+
metadata: config.metadata,
|
|
19
|
+
});
|
|
20
|
+
},
|
|
21
|
+
};
|