@delali/narsil 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -4
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +46 -2
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-XODT2F3h.d.ts → adapters-Bo2JQg8b.d.ts} +8 -1
- package/dist/chunk-ALB35PRQ.mjs +98 -0
- package/dist/{chunk-TFWBALU4.mjs → chunk-FS7XLIUS.mjs} +3219 -2063
- package/dist/chunk-GQEHCZG7.mjs +6825 -0
- package/dist/chunk-KMMJMDUA.mjs +44 -0
- package/dist/chunk-RQLXC2YS.mjs +269 -0
- package/dist/chunk-S7RJJMHI.mjs +1086 -0
- package/dist/{chunk-WYC635GI.mjs → chunk-TW6BXTOL.mjs} +10 -7
- package/dist/chunk-WISGW24C.mjs +455 -0
- package/dist/config-cQLlYR3U.d.ts +470 -0
- package/dist/direct-executor-FX56DYD3.mjs +4 -0
- package/dist/distribution/coordinator/etcd.d.ts +16 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -0
- package/dist/distribution/coordinator/in-memory.d.ts +7 -0
- package/dist/distribution/coordinator/in-memory.mjs +1 -0
- package/dist/distribution.d.ts +122 -0
- package/dist/distribution.mjs +7410 -0
- package/dist/embeddings/openai.d.ts +13 -0
- package/dist/embeddings/openai.mjs +248 -0
- package/dist/index.browser.mjs +12111 -5793
- package/dist/index.d.ts +10 -420
- package/dist/index.mjs +7 -2052
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
- package/dist/languages/arabic.d.ts +1 -1
- package/dist/languages/armenian.d.ts +1 -1
- package/dist/languages/bulgarian.d.ts +1 -1
- package/dist/languages/chinese.d.ts +1 -1
- package/dist/languages/dagbani.d.ts +1 -1
- package/dist/languages/danish.d.ts +1 -1
- package/dist/languages/dutch.d.ts +1 -1
- package/dist/languages/english.d.ts +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/ewe.d.ts +1 -1
- package/dist/languages/finnish.d.ts +1 -1
- package/dist/languages/french.d.ts +1 -1
- package/dist/languages/ga.d.ts +1 -1
- package/dist/languages/german.d.ts +1 -1
- package/dist/languages/greek.d.ts +1 -1
- package/dist/languages/hausa.d.ts +1 -1
- package/dist/languages/hindi.d.ts +1 -1
- package/dist/languages/hungarian.d.ts +1 -1
- package/dist/languages/igbo.d.ts +1 -1
- package/dist/languages/indonesian.d.ts +1 -1
- package/dist/languages/irish.d.ts +1 -1
- package/dist/languages/italian.d.ts +1 -1
- package/dist/languages/japanese.d.ts +1 -1
- package/dist/languages/nepali.d.ts +1 -1
- package/dist/languages/norwegian.d.ts +1 -1
- package/dist/languages/portuguese.d.ts +1 -1
- package/dist/languages/romanian.d.ts +1 -1
- package/dist/languages/russian.d.ts +1 -1
- package/dist/languages/sanskrit.d.ts +1 -1
- package/dist/languages/serbian.d.ts +1 -1
- package/dist/languages/slovenian.d.ts +1 -1
- package/dist/languages/spanish.d.ts +1 -1
- package/dist/languages/swahili.d.ts +1 -1
- package/dist/languages/swedish.d.ts +1 -1
- package/dist/languages/tamil.d.ts +1 -1
- package/dist/languages/turkish.d.ts +1 -1
- package/dist/languages/twi.d.ts +1 -1
- package/dist/languages/ukrainian.d.ts +1 -1
- package/dist/languages/yoruba.d.ts +1 -1
- package/dist/languages/zulu.d.ts +1 -1
- package/dist/narsil-ChTdkwPu.d.ts +68 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
- package/dist/schema-BVaw7nqz.d.ts +58 -0
- package/dist/serialization/crc32-worker.d.ts +20 -0
- package/dist/serialization/crc32-worker.mjs +61 -0
- package/dist/server.d.ts +218 -0
- package/dist/server.mjs +1185 -0
- package/dist/types-Bt7c4hDF.d.ts +73 -0
- package/dist/vector/hnsw-build-worker.d.ts +9 -2
- package/dist/vector/hnsw-build-worker.mjs +54 -10
- package/dist/workers/entry.mjs +1 -1
- package/package.json +46 -4
- package/dist/chunk-7GKPWGCN.mjs +0 -43
- package/dist/chunk-GVN2W4VA.mjs +0 -570
- package/dist/direct-executor-6TSG7GWH.mjs +0 -4
package/dist/index.mjs
CHANGED
|
@@ -1,2052 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
export { getLanguage, registerLanguage } from './chunk-
|
|
3
|
-
export { isSimdAvailable } from './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import
|
|
6
|
-
export { ErrorCodes, NarsilError } from './chunk-
|
|
7
|
-
import
|
|
8
|
-
import { decode } from '@msgpack/msgpack';
|
|
9
|
-
|
|
10
|
-
// src/workers/worker-executor.ts
|
|
11
|
-
var DEFAULT_BACKPRESSURE_LIMIT = 100;
|
|
12
|
-
var DEFAULT_REQUEST_TIMEOUT = 3e4;
|
|
13
|
-
var SHUTDOWN_TIMEOUT = 5e3;
|
|
14
|
-
function createWorkerExecutor(worker, config) {
|
|
15
|
-
const backpressureLimit = DEFAULT_BACKPRESSURE_LIMIT;
|
|
16
|
-
const requestTimeout = DEFAULT_REQUEST_TIMEOUT;
|
|
17
|
-
const pending = /* @__PURE__ */ new Map();
|
|
18
|
-
function processResponse(msg) {
|
|
19
|
-
if (!isValidWorkerResponse(msg)) {
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
|
-
const response = msg;
|
|
23
|
-
const entry = pending.get(response.requestId);
|
|
24
|
-
if (!entry) {
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
clearTimeout(entry.timeoutId);
|
|
28
|
-
pending.delete(response.requestId);
|
|
29
|
-
if (response.type === "error") {
|
|
30
|
-
entry.reject(new NarsilError(response.code, response.message));
|
|
31
|
-
} else {
|
|
32
|
-
entry.resolve(response.data);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
if (typeof worker.on === "function") {
|
|
36
|
-
worker.on("message", (msg) => processResponse(msg));
|
|
37
|
-
} else if (typeof worker.addEventListener === "function") {
|
|
38
|
-
worker.addEventListener("message", (event) => {
|
|
39
|
-
const msg = event.data;
|
|
40
|
-
processResponse(msg);
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
function execute(action) {
|
|
44
|
-
if (pending.size >= backpressureLimit) {
|
|
45
|
-
return Promise.reject(
|
|
46
|
-
new NarsilError(ErrorCodes.WORKER_BUSY, `Backpressure limit of ${backpressureLimit} pending requests reached`)
|
|
47
|
-
);
|
|
48
|
-
}
|
|
49
|
-
const requestId = createRequestId();
|
|
50
|
-
const taggedAction = { ...action, requestId };
|
|
51
|
-
return new Promise((resolve, reject) => {
|
|
52
|
-
const timeoutId = setTimeout(() => {
|
|
53
|
-
pending.delete(requestId);
|
|
54
|
-
reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, `Request ${requestId} timed out after ${requestTimeout}ms`));
|
|
55
|
-
}, requestTimeout);
|
|
56
|
-
pending.set(requestId, {
|
|
57
|
-
resolve,
|
|
58
|
-
reject,
|
|
59
|
-
timeoutId
|
|
60
|
-
});
|
|
61
|
-
worker.postMessage(taggedAction);
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
async function shutdown() {
|
|
65
|
-
const requestId = createRequestId();
|
|
66
|
-
const shutdownAction = { type: "shutdown", requestId };
|
|
67
|
-
const shutdownPromise = new Promise((resolve, reject) => {
|
|
68
|
-
const timeoutId = setTimeout(() => {
|
|
69
|
-
pending.delete(requestId);
|
|
70
|
-
reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, "Shutdown timed out"));
|
|
71
|
-
}, SHUTDOWN_TIMEOUT);
|
|
72
|
-
pending.set(requestId, {
|
|
73
|
-
resolve: () => resolve(),
|
|
74
|
-
reject,
|
|
75
|
-
timeoutId
|
|
76
|
-
});
|
|
77
|
-
worker.postMessage(shutdownAction);
|
|
78
|
-
});
|
|
79
|
-
try {
|
|
80
|
-
await shutdownPromise;
|
|
81
|
-
} finally {
|
|
82
|
-
for (const [id, entry] of pending) {
|
|
83
|
-
clearTimeout(entry.timeoutId);
|
|
84
|
-
entry.reject(new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker is shutting down"));
|
|
85
|
-
pending.delete(id);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
return { execute, shutdown };
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// src/workers/factory.ts
|
|
93
|
-
function resolveEntryPoint() {
|
|
94
|
-
const base = import.meta.url;
|
|
95
|
-
const distIndex = base.lastIndexOf("/dist/");
|
|
96
|
-
if (distIndex !== -1) {
|
|
97
|
-
return new URL("workers/entry.mjs", base.slice(0, distIndex + 6)).href;
|
|
98
|
-
}
|
|
99
|
-
return base.replace(/\/src\/workers\/[^/]+$/, "/dist/workers/entry.mjs");
|
|
100
|
-
}
|
|
101
|
-
async function createWorkerFactory(entryPoint) {
|
|
102
|
-
const runtime = detectRuntime();
|
|
103
|
-
const resolvedEntry = resolveEntryPoint();
|
|
104
|
-
if (runtime.supportsWorkerThreads) {
|
|
105
|
-
const workerThreadsModule = await import('worker_threads');
|
|
106
|
-
return function nodeFactory(_workerId) {
|
|
107
|
-
const instance = new workerThreadsModule.Worker(new URL(resolvedEntry));
|
|
108
|
-
return createWorkerExecutor(instance);
|
|
109
|
-
};
|
|
110
|
-
}
|
|
111
|
-
if (runtime.supportsWebWorkers) {
|
|
112
|
-
return function webFactory(_workerId) {
|
|
113
|
-
const instance = new Worker(resolvedEntry, { type: "module" });
|
|
114
|
-
return createWorkerExecutor(instance);
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
// src/workers/pool.ts
|
|
121
|
-
function resolveWorkerCount(requested) {
|
|
122
|
-
if (requested !== void 0 && requested > 0) {
|
|
123
|
-
return requested;
|
|
124
|
-
}
|
|
125
|
-
let cpuCount = 4;
|
|
126
|
-
try {
|
|
127
|
-
if (navigator?.hardwareConcurrency) {
|
|
128
|
-
cpuCount = navigator.hardwareConcurrency;
|
|
129
|
-
} else if (typeof process !== "undefined") {
|
|
130
|
-
const ap = process.availableParallelism;
|
|
131
|
-
if (typeof ap === "function") {
|
|
132
|
-
cpuCount = ap();
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
} catch {
|
|
136
|
-
cpuCount = 4;
|
|
137
|
-
}
|
|
138
|
-
return Math.max(2, Math.min(8, cpuCount - 1));
|
|
139
|
-
}
|
|
140
|
-
function createWorkerPool(config) {
|
|
141
|
-
const workerCount = resolveWorkerCount(config.count);
|
|
142
|
-
const workers = [];
|
|
143
|
-
const indexAssignment = /* @__PURE__ */ new Map();
|
|
144
|
-
let isShutdown = false;
|
|
145
|
-
function ensureWorker(slotIndex) {
|
|
146
|
-
if (!workers[slotIndex]) {
|
|
147
|
-
workers[slotIndex] = {
|
|
148
|
-
executor: config.workerFactory(slotIndex),
|
|
149
|
-
indexes: /* @__PURE__ */ new Set(),
|
|
150
|
-
memoryStats: { heapUsed: 0, heapTotal: 0, external: 0 }
|
|
151
|
-
};
|
|
152
|
-
}
|
|
153
|
-
return workers[slotIndex];
|
|
154
|
-
}
|
|
155
|
-
function assignSlot(indexName) {
|
|
156
|
-
return fnv1a(indexName) % workerCount;
|
|
157
|
-
}
|
|
158
|
-
function addIndex(indexName) {
|
|
159
|
-
if (isShutdown) {
|
|
160
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
161
|
-
}
|
|
162
|
-
if (indexAssignment.has(indexName)) {
|
|
163
|
-
return;
|
|
164
|
-
}
|
|
165
|
-
const slotIndex = assignSlot(indexName);
|
|
166
|
-
const slot = ensureWorker(slotIndex);
|
|
167
|
-
slot.indexes.add(indexName);
|
|
168
|
-
indexAssignment.set(indexName, slotIndex);
|
|
169
|
-
}
|
|
170
|
-
function removeIndex(indexName) {
|
|
171
|
-
const slotIndex = indexAssignment.get(indexName);
|
|
172
|
-
if (slotIndex === void 0) {
|
|
173
|
-
return;
|
|
174
|
-
}
|
|
175
|
-
const slot = workers[slotIndex];
|
|
176
|
-
if (slot) {
|
|
177
|
-
slot.indexes.delete(indexName);
|
|
178
|
-
}
|
|
179
|
-
indexAssignment.delete(indexName);
|
|
180
|
-
}
|
|
181
|
-
function getExecutor(indexName) {
|
|
182
|
-
if (isShutdown) {
|
|
183
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
184
|
-
}
|
|
185
|
-
const slotIndex = indexAssignment.get(indexName);
|
|
186
|
-
if (slotIndex === void 0) {
|
|
187
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
188
|
-
}
|
|
189
|
-
return workers[slotIndex].executor;
|
|
190
|
-
}
|
|
191
|
-
function getMemoryStats() {
|
|
192
|
-
const stats = [];
|
|
193
|
-
for (let i = 0; i < workers.length; i++) {
|
|
194
|
-
const slot = workers[i];
|
|
195
|
-
if (slot) {
|
|
196
|
-
stats.push({
|
|
197
|
-
workerId: i,
|
|
198
|
-
heapUsed: slot.memoryStats.heapUsed,
|
|
199
|
-
heapTotal: slot.memoryStats.heapTotal,
|
|
200
|
-
external: slot.memoryStats.external
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
return stats;
|
|
205
|
-
}
|
|
206
|
-
async function shutdown() {
|
|
207
|
-
if (isShutdown) {
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
isShutdown = true;
|
|
211
|
-
const shutdownPromises = workers.filter(Boolean).map((slot) => {
|
|
212
|
-
const timeoutPromise = new Promise((resolve) => {
|
|
213
|
-
setTimeout(resolve, 5e3);
|
|
214
|
-
});
|
|
215
|
-
return Promise.race([slot.executor.shutdown(), timeoutPromise]);
|
|
216
|
-
});
|
|
217
|
-
await Promise.allSettled(shutdownPromises);
|
|
218
|
-
indexAssignment.clear();
|
|
219
|
-
}
|
|
220
|
-
function getAllExecutors() {
|
|
221
|
-
return workers.filter(Boolean).map((slot) => slot.executor);
|
|
222
|
-
}
|
|
223
|
-
function addIndexToAll(indexName) {
|
|
224
|
-
if (isShutdown) {
|
|
225
|
-
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
226
|
-
}
|
|
227
|
-
for (let i = 0; i < workerCount; i++) {
|
|
228
|
-
const slot = ensureWorker(i);
|
|
229
|
-
slot.indexes.add(indexName);
|
|
230
|
-
}
|
|
231
|
-
indexAssignment.set(indexName, 0);
|
|
232
|
-
}
|
|
233
|
-
return {
|
|
234
|
-
getExecutor,
|
|
235
|
-
getAllExecutors,
|
|
236
|
-
get workerCount() {
|
|
237
|
-
return workerCount;
|
|
238
|
-
},
|
|
239
|
-
addIndex,
|
|
240
|
-
addIndexToAll,
|
|
241
|
-
removeIndex,
|
|
242
|
-
getMemoryStats,
|
|
243
|
-
shutdown
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
// src/engine/orchestration.ts
|
|
248
|
-
function createWorkerOrchestrator(config, executor, promoter, indexRegistry, callbacks) {
|
|
249
|
-
let workerPool = null;
|
|
250
|
-
let promotionInProgress = false;
|
|
251
|
-
const promotionBuffer = [];
|
|
252
|
-
const workersEnabled = config?.workers?.enabled === true;
|
|
253
|
-
async function checkPromotion() {
|
|
254
|
-
if (!workersEnabled || promotionInProgress || workerPool) return;
|
|
255
|
-
const indexMap = /* @__PURE__ */ new Map();
|
|
256
|
-
for (const [name] of indexRegistry) {
|
|
257
|
-
const mgr = executor.getManager(name);
|
|
258
|
-
indexMap.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
|
|
259
|
-
}
|
|
260
|
-
const result = promoter.check(indexMap);
|
|
261
|
-
if (result.shouldPromote) {
|
|
262
|
-
promotionInProgress = true;
|
|
263
|
-
setTimeout(() => {
|
|
264
|
-
runPromotion(result.reason).catch((err) => {
|
|
265
|
-
console.warn("Worker promotion failed:", err);
|
|
266
|
-
promotionInProgress = false;
|
|
267
|
-
});
|
|
268
|
-
}, 0);
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
async function runPromotion(reason) {
|
|
272
|
-
try {
|
|
273
|
-
const factory = await createWorkerFactory();
|
|
274
|
-
const pool = createWorkerPool({
|
|
275
|
-
count: config?.workers?.count,
|
|
276
|
-
workerFactory: factory
|
|
277
|
-
});
|
|
278
|
-
for (const [name] of indexRegistry) {
|
|
279
|
-
pool.addIndexToAll(name);
|
|
280
|
-
}
|
|
281
|
-
const allExecutors = pool.getAllExecutors();
|
|
282
|
-
for (const [name, entry] of indexRegistry) {
|
|
283
|
-
await Promise.all(
|
|
284
|
-
allExecutors.map(
|
|
285
|
-
(workerExecutor) => workerExecutor.execute({
|
|
286
|
-
type: "createIndex",
|
|
287
|
-
indexName: name,
|
|
288
|
-
config: entry.config,
|
|
289
|
-
requestId: `promote-create-${name}`
|
|
290
|
-
})
|
|
291
|
-
)
|
|
292
|
-
);
|
|
293
|
-
const manager = executor.getManager(name);
|
|
294
|
-
if (manager) {
|
|
295
|
-
for (let i = 0; i < manager.partitionCount; i++) {
|
|
296
|
-
const serialized = manager.serializePartition(i);
|
|
297
|
-
await Promise.all(
|
|
298
|
-
allExecutors.map(
|
|
299
|
-
(workerExecutor) => workerExecutor.execute({
|
|
300
|
-
type: "deserialize",
|
|
301
|
-
indexName: name,
|
|
302
|
-
partitionId: i,
|
|
303
|
-
data: serialized,
|
|
304
|
-
requestId: `promote-sync-${name}-${i}`
|
|
305
|
-
})
|
|
306
|
-
)
|
|
307
|
-
);
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
workerPool = pool;
|
|
312
|
-
promoter.markPromoted();
|
|
313
|
-
if (promotionBuffer.length > 0) {
|
|
314
|
-
const buffered = [...promotionBuffer];
|
|
315
|
-
promotionBuffer.length = 0;
|
|
316
|
-
for (const action of buffered) {
|
|
317
|
-
await replicateToWorkers(action);
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
callbacks?.onPromotion?.(pool.workerCount, reason);
|
|
321
|
-
} catch (err) {
|
|
322
|
-
promotionBuffer.length = 0;
|
|
323
|
-
throw err;
|
|
324
|
-
} finally {
|
|
325
|
-
promotionInProgress = false;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
async function replicateToWorkers(action) {
|
|
329
|
-
if (promotionInProgress) {
|
|
330
|
-
promotionBuffer.push(action);
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
333
|
-
if (!workerPool) return;
|
|
334
|
-
const allExecutors = workerPool.getAllExecutors();
|
|
335
|
-
const results = await Promise.allSettled(allExecutors.map((workerExecutor) => workerExecutor.execute(action)));
|
|
336
|
-
for (const result of results) {
|
|
337
|
-
if (result.status === "rejected") {
|
|
338
|
-
console.warn("Worker replication failed:", result.reason);
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
async function searchViaWorker(indexName, params) {
|
|
343
|
-
if (!workerPool) return null;
|
|
344
|
-
const manager = executor.getManager(indexName);
|
|
345
|
-
if (!manager) return null;
|
|
346
|
-
const allExecutors = workerPool.getAllExecutors();
|
|
347
|
-
const totalPartitions = manager.partitionCount;
|
|
348
|
-
const numWorkers = allExecutors.length;
|
|
349
|
-
if (numWorkers === 0) return null;
|
|
350
|
-
if (numWorkers === 1) {
|
|
351
|
-
try {
|
|
352
|
-
return await allExecutors[0].execute({
|
|
353
|
-
type: "query",
|
|
354
|
-
indexName,
|
|
355
|
-
params,
|
|
356
|
-
requestId: createRequestId()
|
|
357
|
-
});
|
|
358
|
-
} catch (err) {
|
|
359
|
-
console.warn("Worker search failed, falling back to local:", err);
|
|
360
|
-
return null;
|
|
361
|
-
}
|
|
362
|
-
}
|
|
363
|
-
try {
|
|
364
|
-
const workerAssignments = Array.from({ length: numWorkers }, () => []);
|
|
365
|
-
for (let p = 0; p < totalPartitions; p++) {
|
|
366
|
-
workerAssignments[p % numWorkers].push(p);
|
|
367
|
-
}
|
|
368
|
-
const results = await Promise.all(
|
|
369
|
-
allExecutors.map(
|
|
370
|
-
(workerExecutor, idx) => workerExecutor.execute({
|
|
371
|
-
type: "query",
|
|
372
|
-
indexName,
|
|
373
|
-
params,
|
|
374
|
-
requestId: createRequestId(),
|
|
375
|
-
partitionIds: workerAssignments[idx]
|
|
376
|
-
})
|
|
377
|
-
)
|
|
378
|
-
);
|
|
379
|
-
const allScored = results.map((r) => r.scored);
|
|
380
|
-
const merged = kWayMerge(allScored);
|
|
381
|
-
let totalMatched = 0;
|
|
382
|
-
for (const r of results) {
|
|
383
|
-
totalMatched += r.totalMatched;
|
|
384
|
-
}
|
|
385
|
-
return { scored: merged, totalMatched };
|
|
386
|
-
} catch (err) {
|
|
387
|
-
console.warn("Parallel worker search failed, falling back to local:", err);
|
|
388
|
-
return null;
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
function isPromoted() {
|
|
392
|
-
return workerPool !== null;
|
|
393
|
-
}
|
|
394
|
-
function getMemoryStats() {
|
|
395
|
-
const workers = workerPool ? workerPool.getMemoryStats().map((s) => ({
|
|
396
|
-
workerId: s.workerId,
|
|
397
|
-
heapUsed: s.heapUsed,
|
|
398
|
-
heapTotal: s.heapTotal,
|
|
399
|
-
external: s.external
|
|
400
|
-
})) : [];
|
|
401
|
-
const totalBytes = workers.reduce((sum, w) => sum + w.heapUsed, 0);
|
|
402
|
-
return { totalBytes, workers };
|
|
403
|
-
}
|
|
404
|
-
async function shutdown() {
|
|
405
|
-
if (workerPool) {
|
|
406
|
-
await workerPool.shutdown();
|
|
407
|
-
workerPool = null;
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
return { checkPromotion, replicateToWorkers, searchViaWorker, isPromoted, getMemoryStats, shutdown };
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
// src/highlighting/highlighter.ts
|
|
414
|
-
function buildTokenCharOffsets(text, originalTokens) {
|
|
415
|
-
const offsets = [];
|
|
416
|
-
let searchFrom = 0;
|
|
417
|
-
for (const token of originalTokens) {
|
|
418
|
-
const idx = text.indexOf(token, searchFrom);
|
|
419
|
-
if (idx === -1) continue;
|
|
420
|
-
offsets.push({ start: idx, end: idx + token.length });
|
|
421
|
-
searchFrom = idx + token.length;
|
|
422
|
-
}
|
|
423
|
-
return offsets;
|
|
424
|
-
}
|
|
425
|
-
function mergeOverlapping(ranges) {
|
|
426
|
-
if (ranges.length === 0) return [];
|
|
427
|
-
const sorted = [...ranges].sort((a, b) => a.start - b.start);
|
|
428
|
-
const merged = [sorted[0]];
|
|
429
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
430
|
-
const prev = merged[merged.length - 1];
|
|
431
|
-
const curr = sorted[i];
|
|
432
|
-
if (curr.start <= prev.end) {
|
|
433
|
-
prev.end = Math.max(prev.end, curr.end);
|
|
434
|
-
} else {
|
|
435
|
-
merged.push(curr);
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
return merged;
|
|
439
|
-
}
|
|
440
|
-
function findDensestWindow(positions, text, windowSize) {
|
|
441
|
-
if (positions.length === 0) {
|
|
442
|
-
return { start: 0, end: Math.min(windowSize, text.length) };
|
|
443
|
-
}
|
|
444
|
-
let bestStart = 0;
|
|
445
|
-
let bestCount = 0;
|
|
446
|
-
for (const pos of positions) {
|
|
447
|
-
const windowStart = Math.max(0, pos.start - Math.floor(windowSize / 2));
|
|
448
|
-
const windowEnd = Math.min(text.length, windowStart + windowSize);
|
|
449
|
-
const adjustedStart = Math.max(0, windowEnd - windowSize);
|
|
450
|
-
let count = 0;
|
|
451
|
-
for (const p of positions) {
|
|
452
|
-
if (p.start >= adjustedStart && p.end <= windowEnd) {
|
|
453
|
-
count++;
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
if (count > bestCount) {
|
|
457
|
-
bestCount = count;
|
|
458
|
-
bestStart = adjustedStart;
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
return { start: bestStart, end: Math.min(text.length, bestStart + windowSize) };
|
|
462
|
-
}
|
|
463
|
-
function applySnippetTruncation(taggedText, originalText, positions, maxLen, preTag, postTag) {
|
|
464
|
-
if (maxLen <= 0 || originalText.length <= maxLen) {
|
|
465
|
-
return { snippet: taggedText, adjustedPositions: positions };
|
|
466
|
-
}
|
|
467
|
-
const window = findDensestWindow(positions, originalText, maxLen);
|
|
468
|
-
const snippetRaw = originalText.substring(window.start, window.end);
|
|
469
|
-
const adjustedPositions = [];
|
|
470
|
-
for (const pos of positions) {
|
|
471
|
-
if (pos.start >= window.start && pos.end <= window.end) {
|
|
472
|
-
adjustedPositions.push({
|
|
473
|
-
start: pos.start - window.start,
|
|
474
|
-
end: pos.end - window.start
|
|
475
|
-
});
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
let snippet = snippetRaw;
|
|
479
|
-
const sortedDesc = [...adjustedPositions].sort((a, b) => b.start - a.start);
|
|
480
|
-
for (const pos of sortedDesc) {
|
|
481
|
-
snippet = snippet.substring(0, pos.start) + preTag + snippet.substring(pos.start, pos.end) + postTag + snippet.substring(pos.end);
|
|
482
|
-
}
|
|
483
|
-
const needsLeadingEllipsis = window.start > 0;
|
|
484
|
-
const needsTrailingEllipsis = window.end < originalText.length;
|
|
485
|
-
if (needsLeadingEllipsis) snippet = `...${snippet}`;
|
|
486
|
-
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
487
|
-
return { snippet, adjustedPositions };
|
|
488
|
-
}
|
|
489
|
-
function highlightField(text, queryTokens, language, options) {
|
|
490
|
-
const preTag = options?.preTag ?? "<mark>";
|
|
491
|
-
const postTag = options?.postTag ?? "</mark>";
|
|
492
|
-
const maxSnippetLength = options?.maxSnippetLength ?? 200;
|
|
493
|
-
if (text.length === 0) {
|
|
494
|
-
return { snippet: text, positions: [] };
|
|
495
|
-
}
|
|
496
|
-
const fieldResult = tokenize(text, language, { stem: false, removeStopWords: false });
|
|
497
|
-
if (fieldResult.tokens.length === 0) {
|
|
498
|
-
return { snippet: text, positions: [] };
|
|
499
|
-
}
|
|
500
|
-
const lowercaseText = text.normalize("NFC").toLowerCase();
|
|
501
|
-
const charOffsets = buildTokenCharOffsets(lowercaseText, fieldResult.originalTokens);
|
|
502
|
-
const stemmedQueryTokens = queryTokens.map((qt) => {
|
|
503
|
-
if (language.stemmer) {
|
|
504
|
-
return language.stemmer(qt.token.toLowerCase());
|
|
505
|
-
}
|
|
506
|
-
return qt.token.toLowerCase();
|
|
507
|
-
});
|
|
508
|
-
const matchedRanges = [];
|
|
509
|
-
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
510
|
-
if (i >= charOffsets.length) break;
|
|
511
|
-
const fieldToken = fieldResult.tokens[i].token;
|
|
512
|
-
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
513
|
-
for (const stemmedQuery of stemmedQueryTokens) {
|
|
514
|
-
if (stemmedField === stemmedQuery) {
|
|
515
|
-
matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
|
|
516
|
-
break;
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
|
-
const mergedPositions = mergeOverlapping(matchedRanges);
|
|
521
|
-
if (mergedPositions.length === 0) {
|
|
522
|
-
if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
|
|
523
|
-
const truncated = text.substring(0, maxSnippetLength);
|
|
524
|
-
return { snippet: `${truncated}...`, positions: [] };
|
|
525
|
-
}
|
|
526
|
-
return { snippet: text, positions: [] };
|
|
527
|
-
}
|
|
528
|
-
if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
|
|
529
|
-
const { snippet, adjustedPositions } = applySnippetTruncation(
|
|
530
|
-
text,
|
|
531
|
-
text,
|
|
532
|
-
mergedPositions,
|
|
533
|
-
maxSnippetLength,
|
|
534
|
-
preTag,
|
|
535
|
-
postTag
|
|
536
|
-
);
|
|
537
|
-
return { snippet, positions: adjustedPositions };
|
|
538
|
-
}
|
|
539
|
-
let result = text;
|
|
540
|
-
const sortedDesc = [...mergedPositions].sort((a, b) => b.start - a.start);
|
|
541
|
-
for (const pos of sortedDesc) {
|
|
542
|
-
result = result.substring(0, pos.start) + preTag + result.substring(pos.start, pos.end) + postTag + result.substring(pos.end);
|
|
543
|
-
}
|
|
544
|
-
return { snippet: result, positions: mergedPositions };
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
// src/search/grouping.ts
|
|
548
|
-
function getNestedValue(obj, path) {
|
|
549
|
-
const segments = path.split(".");
|
|
550
|
-
let current = obj;
|
|
551
|
-
for (const segment of segments) {
|
|
552
|
-
if (current === null || current === void 0 || typeof current !== "object") {
|
|
553
|
-
return void 0;
|
|
554
|
-
}
|
|
555
|
-
if (!Object.hasOwn(current, segment)) {
|
|
556
|
-
return void 0;
|
|
557
|
-
}
|
|
558
|
-
current = current[segment];
|
|
559
|
-
}
|
|
560
|
-
return current;
|
|
561
|
-
}
|
|
562
|
-
function applyGrouping(hits, group, getDocument) {
|
|
563
|
-
if (!group.fields || group.fields.length === 0) {
|
|
564
|
-
return [{ values: {}, hits }];
|
|
565
|
-
}
|
|
566
|
-
const groupMap = /* @__PURE__ */ new Map();
|
|
567
|
-
for (const hit of hits) {
|
|
568
|
-
const doc = getDocument(hit.id);
|
|
569
|
-
const fieldValues = [];
|
|
570
|
-
const valuesRecord = {};
|
|
571
|
-
for (const field of group.fields) {
|
|
572
|
-
const value = doc ? getNestedValue(doc, field) : void 0;
|
|
573
|
-
fieldValues.push(value);
|
|
574
|
-
valuesRecord[field] = value;
|
|
575
|
-
}
|
|
576
|
-
const compositeKey = JSON.stringify(fieldValues);
|
|
577
|
-
const existing = groupMap.get(compositeKey);
|
|
578
|
-
if (existing) {
|
|
579
|
-
existing.hits.push(hit);
|
|
580
|
-
} else {
|
|
581
|
-
groupMap.set(compositeKey, { values: valuesRecord, hits: [hit] });
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
const groups = [];
|
|
585
|
-
for (const entry of groupMap.values()) {
|
|
586
|
-
let groupHits = entry.hits;
|
|
587
|
-
if (group.maxPerGroup !== void 0 && group.maxPerGroup > 0) {
|
|
588
|
-
groupHits = groupHits.slice(0, group.maxPerGroup);
|
|
589
|
-
}
|
|
590
|
-
const result = { values: entry.values, hits: groupHits };
|
|
591
|
-
if (group.reduce) {
|
|
592
|
-
try {
|
|
593
|
-
let accumulator = group.reduce.initialValue();
|
|
594
|
-
for (const hit of groupHits) {
|
|
595
|
-
const doc = getDocument(hit.id);
|
|
596
|
-
if (doc) {
|
|
597
|
-
accumulator = group.reduce.reducer(accumulator, doc, hit.score);
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
;
|
|
601
|
-
result.reduced = accumulator;
|
|
602
|
-
} catch (err) {
|
|
603
|
-
result.reducerError = err instanceof Error ? err.message : String(err);
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
groups.push(result);
|
|
607
|
-
}
|
|
608
|
-
groups.sort((a, b) => {
|
|
609
|
-
const aScore = a.hits.length > 0 ? a.hits[0].score : 0;
|
|
610
|
-
const bScore = b.hits.length > 0 ? b.hits[0].score : 0;
|
|
611
|
-
return bScore - aScore;
|
|
612
|
-
});
|
|
613
|
-
return groups;
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
// src/search/pagination.ts
|
|
617
|
-
function encodeCursor(state) {
|
|
618
|
-
const json = JSON.stringify(state);
|
|
619
|
-
if (typeof Buffer !== "undefined") {
|
|
620
|
-
return Buffer.from(json).toString("base64");
|
|
621
|
-
}
|
|
622
|
-
return btoa(json);
|
|
623
|
-
}
|
|
624
|
-
function decodeCursor(encoded) {
|
|
625
|
-
let json;
|
|
626
|
-
try {
|
|
627
|
-
if (typeof Buffer !== "undefined") {
|
|
628
|
-
json = Buffer.from(encoded, "base64").toString("utf-8");
|
|
629
|
-
} else {
|
|
630
|
-
json = atob(encoded);
|
|
631
|
-
}
|
|
632
|
-
} catch {
|
|
633
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode cursor: invalid base64 encoding", {
|
|
634
|
-
cursor: encoded
|
|
635
|
-
});
|
|
636
|
-
}
|
|
637
|
-
let parsed;
|
|
638
|
-
try {
|
|
639
|
-
parsed = JSON.parse(json);
|
|
640
|
-
} catch {
|
|
641
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode cursor: invalid JSON", {
|
|
642
|
-
cursor: encoded
|
|
643
|
-
});
|
|
644
|
-
}
|
|
645
|
-
if (!Array.isArray(parsed)) {
|
|
646
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Invalid cursor structure: expected an array", {
|
|
647
|
-
cursor: encoded
|
|
648
|
-
});
|
|
649
|
-
}
|
|
650
|
-
for (let i = 0; i < parsed.length; i++) {
|
|
651
|
-
const entry = parsed[i];
|
|
652
|
-
if (typeof entry !== "object" || entry === null) {
|
|
653
|
-
throw new NarsilError(
|
|
654
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
655
|
-
`Invalid cursor entry at index ${i}: expected an object`,
|
|
656
|
-
{
|
|
657
|
-
cursor: encoded,
|
|
658
|
-
index: i
|
|
659
|
-
}
|
|
660
|
-
);
|
|
661
|
-
}
|
|
662
|
-
const { s, d, p } = entry;
|
|
663
|
-
if (typeof s !== "number" || !Number.isFinite(s)) {
|
|
664
|
-
throw new NarsilError(
|
|
665
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
666
|
-
`Invalid cursor entry at index ${i}: "s" must be a finite number`,
|
|
667
|
-
{ cursor: encoded, index: i }
|
|
668
|
-
);
|
|
669
|
-
}
|
|
670
|
-
if (typeof d !== "string") {
|
|
671
|
-
throw new NarsilError(
|
|
672
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
673
|
-
`Invalid cursor entry at index ${i}: "d" must be a string`,
|
|
674
|
-
{ cursor: encoded, index: i }
|
|
675
|
-
);
|
|
676
|
-
}
|
|
677
|
-
if (typeof p !== "number" || !Number.isInteger(p) || p < 0) {
|
|
678
|
-
throw new NarsilError(
|
|
679
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
680
|
-
`Invalid cursor entry at index ${i}: "p" must be a non-negative integer`,
|
|
681
|
-
{ cursor: encoded, index: i }
|
|
682
|
-
);
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
return parsed;
|
|
686
|
-
}
|
|
687
|
-
function applyPagination(results, limit, offset, cursor) {
|
|
688
|
-
if (limit <= 0) {
|
|
689
|
-
return { paginated: [] };
|
|
690
|
-
}
|
|
691
|
-
let startIndex = 0;
|
|
692
|
-
if (cursor) {
|
|
693
|
-
const cursorState = decodeCursor(cursor);
|
|
694
|
-
const anchor = cursorState[0];
|
|
695
|
-
for (let i = 0; i < results.length; i++) {
|
|
696
|
-
const result = results[i];
|
|
697
|
-
if (result.score < anchor.s || result.score === anchor.s && result.id > anchor.d) {
|
|
698
|
-
startIndex = i;
|
|
699
|
-
break;
|
|
700
|
-
}
|
|
701
|
-
if (i === results.length - 1) {
|
|
702
|
-
startIndex = results.length;
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
const afterOffset = startIndex + offset;
|
|
707
|
-
const sliced = results.slice(afterOffset, afterOffset + limit);
|
|
708
|
-
let nextCursor;
|
|
709
|
-
const hasMore = afterOffset + limit < results.length;
|
|
710
|
-
if (hasMore && sliced.length > 0) {
|
|
711
|
-
const lastResult = sliced[sliced.length - 1];
|
|
712
|
-
const cursorState = [{ s: lastResult.score, d: lastResult.id, p: 0 }];
|
|
713
|
-
nextCursor = encodeCursor(cursorState);
|
|
714
|
-
}
|
|
715
|
-
return { paginated: sliced, nextCursor };
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
// src/search/pinning.ts
|
|
719
|
-
function applyPinning(hits, pinned, resolveDoc) {
|
|
720
|
-
const result = hits.slice();
|
|
721
|
-
const pinnedDocIds = new Set(pinned.map((p) => p.docId));
|
|
722
|
-
for (let i = result.length - 1; i >= 0; i--) {
|
|
723
|
-
if (pinnedDocIds.has(result[i].id)) {
|
|
724
|
-
result.splice(i, 1);
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
const sorted = pinned.slice().sort((a, b) => a.position - b.position);
|
|
728
|
-
for (const entry of sorted) {
|
|
729
|
-
const doc = resolveDoc(entry.docId);
|
|
730
|
-
if (!doc) continue;
|
|
731
|
-
const pos = Math.max(0, Math.min(entry.position, result.length));
|
|
732
|
-
result.splice(pos, 0, doc);
|
|
733
|
-
}
|
|
734
|
-
return result;
|
|
735
|
-
}
|
|
736
|
-
|
|
737
|
-
// src/search/sorting.ts
|
|
738
|
-
function getNestedValue2(obj, path) {
|
|
739
|
-
const segments = path.split(".");
|
|
740
|
-
let current = obj;
|
|
741
|
-
for (const segment of segments) {
|
|
742
|
-
if (current === null || current === void 0 || typeof current !== "object") {
|
|
743
|
-
return void 0;
|
|
744
|
-
}
|
|
745
|
-
if (!Object.hasOwn(current, segment)) {
|
|
746
|
-
return void 0;
|
|
747
|
-
}
|
|
748
|
-
current = current[segment];
|
|
749
|
-
}
|
|
750
|
-
return current;
|
|
751
|
-
}
|
|
752
|
-
function applySorting(hits, sort, getDocument) {
|
|
753
|
-
const sortFields = Object.entries(sort);
|
|
754
|
-
if (sortFields.length === 0) return hits;
|
|
755
|
-
const collator = new Intl.Collator(void 0, { sensitivity: "base" });
|
|
756
|
-
const valueCache = /* @__PURE__ */ new Map();
|
|
757
|
-
for (const hit of hits) {
|
|
758
|
-
const doc = getDocument(hit.id);
|
|
759
|
-
if (!doc) {
|
|
760
|
-
valueCache.set(
|
|
761
|
-
hit.id,
|
|
762
|
-
sortFields.map(() => void 0)
|
|
763
|
-
);
|
|
764
|
-
continue;
|
|
765
|
-
}
|
|
766
|
-
const values = sortFields.map(([field]) => getNestedValue2(doc, field));
|
|
767
|
-
valueCache.set(hit.id, values);
|
|
768
|
-
}
|
|
769
|
-
const sorted = hits.slice();
|
|
770
|
-
sorted.sort((a, b) => {
|
|
771
|
-
const aValues = valueCache.get(a.id) ?? [];
|
|
772
|
-
const bValues = valueCache.get(b.id) ?? [];
|
|
773
|
-
for (let i = 0; i < sortFields.length; i++) {
|
|
774
|
-
const direction = sortFields[i][1];
|
|
775
|
-
const aVal = aValues[i];
|
|
776
|
-
const bVal = bValues[i];
|
|
777
|
-
const aIsNullish = aVal === void 0 || aVal === null;
|
|
778
|
-
const bIsNullish = bVal === void 0 || bVal === null;
|
|
779
|
-
if (aIsNullish && bIsNullish) continue;
|
|
780
|
-
if (aIsNullish) return 1;
|
|
781
|
-
if (bIsNullish) return -1;
|
|
782
|
-
let comparison = 0;
|
|
783
|
-
if (typeof aVal === "string" && typeof bVal === "string") {
|
|
784
|
-
comparison = collator.compare(aVal, bVal);
|
|
785
|
-
} else if (typeof aVal === "number" && typeof bVal === "number") {
|
|
786
|
-
const aIsNaN = Number.isNaN(aVal);
|
|
787
|
-
const bIsNaN = Number.isNaN(bVal);
|
|
788
|
-
if (aIsNaN && bIsNaN) continue;
|
|
789
|
-
if (aIsNaN) return 1;
|
|
790
|
-
if (bIsNaN) return -1;
|
|
791
|
-
comparison = aVal - bVal;
|
|
792
|
-
} else if (typeof aVal === "boolean" && typeof bVal === "boolean") {
|
|
793
|
-
comparison = aVal === bVal ? 0 : aVal ? 1 : -1;
|
|
794
|
-
}
|
|
795
|
-
if (comparison !== 0) {
|
|
796
|
-
return direction === "desc" ? -comparison : comparison;
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
return 0;
|
|
800
|
-
});
|
|
801
|
-
return sorted;
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
// src/engine/validation.ts
|
|
805
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
806
|
-
var MAX_INDEX_NAME_LENGTH = 256;
|
|
807
|
-
var MAX_DOC_ID_LENGTH = 512;
|
|
808
|
-
var BATCH_CHUNK_SIZE = 1e3;
|
|
809
|
-
var MAX_LIMIT = 1e4;
|
|
810
|
-
var MAX_OFFSET = 1e5;
|
|
811
|
-
var DEFAULT_LIMIT = 10;
|
|
812
|
-
var DEFAULT_OFFSET = 0;
|
|
813
|
-
function now() {
|
|
814
|
-
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
815
|
-
return performance.now();
|
|
816
|
-
}
|
|
817
|
-
return Date.now();
|
|
818
|
-
}
|
|
819
|
-
function validateIndexName(name) {
|
|
820
|
-
if (!name || name.length === 0) {
|
|
821
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
|
|
822
|
-
}
|
|
823
|
-
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
824
|
-
throw new NarsilError(
|
|
825
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
826
|
-
`Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
827
|
-
{
|
|
828
|
-
indexName: name,
|
|
829
|
-
length: name.length
|
|
830
|
-
}
|
|
831
|
-
);
|
|
832
|
-
}
|
|
833
|
-
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
834
|
-
throw new NarsilError(
|
|
835
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
836
|
-
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
837
|
-
{ indexName: name }
|
|
838
|
-
);
|
|
839
|
-
}
|
|
840
|
-
if (name.includes("..")) {
|
|
841
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
function validateDocId(docId) {
|
|
845
|
-
if (!docId || docId.length === 0) {
|
|
846
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
847
|
-
}
|
|
848
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
849
|
-
throw new NarsilError(
|
|
850
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
851
|
-
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
852
|
-
{
|
|
853
|
-
docId,
|
|
854
|
-
length: docId.length
|
|
855
|
-
}
|
|
856
|
-
);
|
|
857
|
-
}
|
|
858
|
-
if (docId.includes("\0")) {
|
|
859
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
function clampLimit(limit) {
|
|
863
|
-
if (limit === void 0) return DEFAULT_LIMIT;
|
|
864
|
-
return Math.max(0, Math.min(limit, MAX_LIMIT));
|
|
865
|
-
}
|
|
866
|
-
function clampOffset(offset) {
|
|
867
|
-
if (offset === void 0) return DEFAULT_OFFSET;
|
|
868
|
-
return Math.max(0, Math.min(offset, MAX_OFFSET));
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
// src/engine/query.ts
|
|
872
|
-
async function executeQuery(params, context) {
|
|
873
|
-
const { manager, language, config, workerSearch, indexName } = context;
|
|
874
|
-
const startTime = now();
|
|
875
|
-
const limit = clampLimit(params.limit);
|
|
876
|
-
const offset = clampOffset(params.offset);
|
|
877
|
-
let fanOutResult;
|
|
878
|
-
const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
|
|
879
|
-
if (workerResult) {
|
|
880
|
-
fanOutResult = workerResult;
|
|
881
|
-
} else {
|
|
882
|
-
const searchOptions = {
|
|
883
|
-
bm25Params: config.bm25,
|
|
884
|
-
stopWords: config.stopWords,
|
|
885
|
-
customTokenizer: config.tokenizer
|
|
886
|
-
};
|
|
887
|
-
fanOutResult = await fanOutQuery(
|
|
888
|
-
manager,
|
|
889
|
-
params,
|
|
890
|
-
language,
|
|
891
|
-
config.schema,
|
|
892
|
-
{ scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
|
|
893
|
-
searchOptions
|
|
894
|
-
);
|
|
895
|
-
}
|
|
896
|
-
const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
897
|
-
let hits;
|
|
898
|
-
if (needsFullHits) {
|
|
899
|
-
hits = fanOutResult.scored.map((scored) => ({
|
|
900
|
-
id: scored.docId,
|
|
901
|
-
score: scored.score,
|
|
902
|
-
document: void 0,
|
|
903
|
-
scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
|
|
904
|
-
}));
|
|
905
|
-
} else {
|
|
906
|
-
const end = Math.min(offset + limit + 1, fanOutResult.scored.length);
|
|
907
|
-
hits = new Array(end);
|
|
908
|
-
for (let i = 0; i < end; i++) {
|
|
909
|
-
const scored = fanOutResult.scored[i];
|
|
910
|
-
hits[i] = {
|
|
911
|
-
id: scored.docId,
|
|
912
|
-
score: scored.score,
|
|
913
|
-
document: void 0,
|
|
914
|
-
scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
|
|
915
|
-
};
|
|
916
|
-
}
|
|
917
|
-
}
|
|
918
|
-
if (params.sort) {
|
|
919
|
-
hits = applySorting(hits, params.sort, (docId) => manager.getRef(docId));
|
|
920
|
-
}
|
|
921
|
-
let groups;
|
|
922
|
-
if (params.group) {
|
|
923
|
-
groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
|
|
924
|
-
}
|
|
925
|
-
if (params.pinned) {
|
|
926
|
-
hits = applyPinning(hits, params.pinned, (docId) => {
|
|
927
|
-
const doc = manager.getRef(docId);
|
|
928
|
-
if (!doc) return void 0;
|
|
929
|
-
return { id: docId, score: 0, document: doc };
|
|
930
|
-
});
|
|
931
|
-
}
|
|
932
|
-
const { paginated, nextCursor } = applyPagination(hits, limit, offset, params.searchAfter);
|
|
933
|
-
for (const hit of paginated) {
|
|
934
|
-
hit.document = manager.get(hit.id) ?? {};
|
|
935
|
-
}
|
|
936
|
-
if (groups) {
|
|
937
|
-
for (const group of groups) {
|
|
938
|
-
for (const hit of group.hits) {
|
|
939
|
-
hit.document = manager.get(hit.id) ?? {};
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
}
|
|
943
|
-
if (params.highlight) {
|
|
944
|
-
applyHighlights(paginated, params, language, config);
|
|
945
|
-
}
|
|
946
|
-
const elapsed = now() - startTime;
|
|
947
|
-
return {
|
|
948
|
-
hits: paginated,
|
|
949
|
-
count: fanOutResult.totalMatched,
|
|
950
|
-
elapsed,
|
|
951
|
-
cursor: nextCursor,
|
|
952
|
-
facets: fanOutResult.facets,
|
|
953
|
-
groups
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
async function executePreflight(params, context) {
|
|
957
|
-
const { manager, language, config, workerSearch, indexName } = context;
|
|
958
|
-
const startTime = now();
|
|
959
|
-
let totalMatched;
|
|
960
|
-
const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
|
|
961
|
-
if (workerResult) {
|
|
962
|
-
totalMatched = workerResult.totalMatched;
|
|
963
|
-
} else {
|
|
964
|
-
const searchOptions = {
|
|
965
|
-
bm25Params: config.bm25,
|
|
966
|
-
stopWords: config.stopWords,
|
|
967
|
-
customTokenizer: config.tokenizer
|
|
968
|
-
};
|
|
969
|
-
const fanOutResult = await fanOutQuery(
|
|
970
|
-
manager,
|
|
971
|
-
params,
|
|
972
|
-
language,
|
|
973
|
-
config.schema,
|
|
974
|
-
{ scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
|
|
975
|
-
searchOptions
|
|
976
|
-
);
|
|
977
|
-
totalMatched = fanOutResult.totalMatched;
|
|
978
|
-
}
|
|
979
|
-
const elapsed = now() - startTime;
|
|
980
|
-
return { count: totalMatched, elapsed };
|
|
981
|
-
}
|
|
982
|
-
function applyHighlights(hits, params, language, config) {
|
|
983
|
-
if (!params.highlight) return;
|
|
984
|
-
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
985
|
-
stem: true,
|
|
986
|
-
removeStopWords: true,
|
|
987
|
-
customTokenizer: config.tokenizer,
|
|
988
|
-
stopWordOverride: config.stopWords
|
|
989
|
-
});
|
|
990
|
-
for (const hit of hits) {
|
|
991
|
-
const highlights = {};
|
|
992
|
-
for (const field of params.highlight.fields) {
|
|
993
|
-
const doc = hit.document;
|
|
994
|
-
const fieldValue = doc[field];
|
|
995
|
-
if (typeof fieldValue === "string") {
|
|
996
|
-
highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
|
|
997
|
-
preTag: params.highlight.preTag,
|
|
998
|
-
postTag: params.highlight.postTag,
|
|
999
|
-
maxSnippetLength: params.highlight.maxSnippetLength
|
|
1000
|
-
});
|
|
1001
|
-
}
|
|
1002
|
-
}
|
|
1003
|
-
hit.highlights = highlights;
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
// src/partitioning/rebalancer.ts
|
|
1008
|
-
var CHUNK_SIZE = 1e3;
|
|
1009
|
-
function createRebalancer() {
|
|
1010
|
-
let rebalancing = false;
|
|
1011
|
-
async function rebalance(manager, newPartitionCount, router, onProgress) {
|
|
1012
|
-
if (rebalancing) {
|
|
1013
|
-
throw new NarsilError(
|
|
1014
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1015
|
-
"A rebalance operation is already in progress"
|
|
1016
|
-
);
|
|
1017
|
-
}
|
|
1018
|
-
if (newPartitionCount <= 0) {
|
|
1019
|
-
throw new NarsilError(
|
|
1020
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1021
|
-
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
1022
|
-
{ newPartitionCount }
|
|
1023
|
-
);
|
|
1024
|
-
}
|
|
1025
|
-
if (newPartitionCount === manager.partitionCount) {
|
|
1026
|
-
throw new NarsilError(
|
|
1027
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1028
|
-
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
1029
|
-
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
1030
|
-
);
|
|
1031
|
-
}
|
|
1032
|
-
rebalancing = true;
|
|
1033
|
-
try {
|
|
1034
|
-
const collectedDocs = [];
|
|
1035
|
-
const currentPartitions = manager.getAllPartitions();
|
|
1036
|
-
for (const partition of currentPartitions) {
|
|
1037
|
-
for (const docId of partition.docIds()) {
|
|
1038
|
-
const document = partition.get(docId);
|
|
1039
|
-
if (document) {
|
|
1040
|
-
collectedDocs.push({ docId, document });
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
const documentsTotal = collectedDocs.length;
|
|
1045
|
-
onProgress?.({
|
|
1046
|
-
phase: "scanning",
|
|
1047
|
-
documentsProcessed: 0,
|
|
1048
|
-
documentsTotal
|
|
1049
|
-
});
|
|
1050
|
-
const newPartitions = [];
|
|
1051
|
-
for (let i = 0; i < newPartitionCount; i++) {
|
|
1052
|
-
newPartitions.push(createPartitionIndex(i));
|
|
1053
|
-
}
|
|
1054
|
-
let documentsProcessed = 0;
|
|
1055
|
-
for (let chunkStart = 0; chunkStart < collectedDocs.length; chunkStart += CHUNK_SIZE) {
|
|
1056
|
-
const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, collectedDocs.length);
|
|
1057
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
1058
|
-
const { docId, document } = collectedDocs[i];
|
|
1059
|
-
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
1060
|
-
const targetPartition = newPartitions[targetPartitionId];
|
|
1061
|
-
if (targetPartition.has(docId)) {
|
|
1062
|
-
continue;
|
|
1063
|
-
}
|
|
1064
|
-
targetPartition.insert(docId, document, manager.schema, manager.language, { validate: false });
|
|
1065
|
-
documentsProcessed++;
|
|
1066
|
-
}
|
|
1067
|
-
onProgress?.({
|
|
1068
|
-
phase: "moving",
|
|
1069
|
-
documentsProcessed,
|
|
1070
|
-
documentsTotal
|
|
1071
|
-
});
|
|
1072
|
-
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
1073
|
-
}
|
|
1074
|
-
onProgress?.({
|
|
1075
|
-
phase: "swapping",
|
|
1076
|
-
documentsProcessed,
|
|
1077
|
-
documentsTotal
|
|
1078
|
-
});
|
|
1079
|
-
manager.setPartitions(newPartitions);
|
|
1080
|
-
onProgress?.({
|
|
1081
|
-
phase: "complete",
|
|
1082
|
-
documentsProcessed,
|
|
1083
|
-
documentsTotal
|
|
1084
|
-
});
|
|
1085
|
-
} finally {
|
|
1086
|
-
rebalancing = false;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
return {
|
|
1090
|
-
rebalance,
|
|
1091
|
-
isRebalancing() {
|
|
1092
|
-
return rebalancing;
|
|
1093
|
-
}
|
|
1094
|
-
};
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
// src/partitioning/write-ahead-queue.ts
|
|
1098
|
-
function createWriteAheadQueue(maxSize = 1e4) {
|
|
1099
|
-
const entries = [];
|
|
1100
|
-
let nextSequence = 1;
|
|
1101
|
-
return {
|
|
1102
|
-
push(entry) {
|
|
1103
|
-
if (entries.length >= maxSize) {
|
|
1104
|
-
throw new NarsilError(
|
|
1105
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1106
|
-
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
1107
|
-
{ maxSize, currentSize: entries.length }
|
|
1108
|
-
);
|
|
1109
|
-
}
|
|
1110
|
-
const seq = nextSequence++;
|
|
1111
|
-
entries.push({ ...entry, sequenceNumber: seq });
|
|
1112
|
-
return seq;
|
|
1113
|
-
},
|
|
1114
|
-
drain() {
|
|
1115
|
-
const result = entries.splice(0);
|
|
1116
|
-
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
1117
|
-
return result;
|
|
1118
|
-
},
|
|
1119
|
-
clear() {
|
|
1120
|
-
entries.length = 0;
|
|
1121
|
-
},
|
|
1122
|
-
get size() {
|
|
1123
|
-
return entries.length;
|
|
1124
|
-
},
|
|
1125
|
-
get isFull() {
|
|
1126
|
-
return entries.length >= maxSize;
|
|
1127
|
-
}
|
|
1128
|
-
};
|
|
1129
|
-
}
|
|
1130
|
-
|
|
1131
|
-
// src/persistence/flush-manager.ts
|
|
1132
|
-
var DEFAULT_INTERVAL = 5e3;
|
|
1133
|
-
var DEFAULT_MUTATION_THRESHOLD = 1e3;
|
|
1134
|
-
var DEFAULT_MAX_RETRIES = 3;
|
|
1135
|
-
var DEFAULT_BASE_RETRY_DELAY = 100;
|
|
1136
|
-
function createFlushManager(config, getPartitionData, getInstanceId) {
|
|
1137
|
-
const {
|
|
1138
|
-
persistence,
|
|
1139
|
-
invalidation,
|
|
1140
|
-
interval = DEFAULT_INTERVAL,
|
|
1141
|
-
mutationThreshold = DEFAULT_MUTATION_THRESHOLD,
|
|
1142
|
-
maxRetries = DEFAULT_MAX_RETRIES,
|
|
1143
|
-
baseRetryDelay = DEFAULT_BASE_RETRY_DELAY,
|
|
1144
|
-
onError
|
|
1145
|
-
} = config;
|
|
1146
|
-
const dirtyPartitions = /* @__PURE__ */ new Map();
|
|
1147
|
-
let mutationCount = 0;
|
|
1148
|
-
let flushTimerId = null;
|
|
1149
|
-
let isShuttingDown = false;
|
|
1150
|
-
let flushPromise = null;
|
|
1151
|
-
function startTimer() {
|
|
1152
|
-
if (flushTimerId !== null || isShuttingDown) {
|
|
1153
|
-
return;
|
|
1154
|
-
}
|
|
1155
|
-
flushTimerId = setInterval(() => {
|
|
1156
|
-
flush();
|
|
1157
|
-
}, interval);
|
|
1158
|
-
}
|
|
1159
|
-
function addToDirtySet(indexName, partitionId) {
|
|
1160
|
-
let partitions = dirtyPartitions.get(indexName);
|
|
1161
|
-
if (partitions === void 0) {
|
|
1162
|
-
partitions = /* @__PURE__ */ new Set();
|
|
1163
|
-
dirtyPartitions.set(indexName, partitions);
|
|
1164
|
-
}
|
|
1165
|
-
partitions.add(partitionId);
|
|
1166
|
-
}
|
|
1167
|
-
function snapshotAndClearDirty() {
|
|
1168
|
-
const snapshot = /* @__PURE__ */ new Map();
|
|
1169
|
-
for (const [indexName, partitions] of dirtyPartitions) {
|
|
1170
|
-
snapshot.set(indexName, new Set(partitions));
|
|
1171
|
-
}
|
|
1172
|
-
dirtyPartitions.clear();
|
|
1173
|
-
return snapshot;
|
|
1174
|
-
}
|
|
1175
|
-
async function saveWithRetry(indexName, partitionId) {
|
|
1176
|
-
const key = `${indexName}/${partitionId}`;
|
|
1177
|
-
const data = getPartitionData(indexName, partitionId);
|
|
1178
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1179
|
-
try {
|
|
1180
|
-
await persistence.save(key, data);
|
|
1181
|
-
return true;
|
|
1182
|
-
} catch (err) {
|
|
1183
|
-
const isLastAttempt = attempt === maxRetries - 1;
|
|
1184
|
-
if (isLastAttempt) {
|
|
1185
|
-
onError?.(indexName, partitionId, err, true);
|
|
1186
|
-
return false;
|
|
1187
|
-
}
|
|
1188
|
-
const cappedAttempt = Math.min(attempt, 20);
|
|
1189
|
-
const delay = baseRetryDelay * 2 ** cappedAttempt * (0.5 + Math.random() * 0.5);
|
|
1190
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1191
|
-
}
|
|
1192
|
-
}
|
|
1193
|
-
return false;
|
|
1194
|
-
}
|
|
1195
|
-
async function performFlush() {
|
|
1196
|
-
const snapshot = snapshotAndClearDirty();
|
|
1197
|
-
for (const [indexName, partitions] of snapshot) {
|
|
1198
|
-
const succeededPartitions = [];
|
|
1199
|
-
for (const partitionId of partitions) {
|
|
1200
|
-
const saved = await saveWithRetry(indexName, partitionId);
|
|
1201
|
-
if (saved) {
|
|
1202
|
-
succeededPartitions.push(partitionId);
|
|
1203
|
-
} else {
|
|
1204
|
-
addToDirtySet(indexName, partitionId);
|
|
1205
|
-
}
|
|
1206
|
-
}
|
|
1207
|
-
if (succeededPartitions.length > 0) {
|
|
1208
|
-
await invalidation.publish({
|
|
1209
|
-
type: "partition",
|
|
1210
|
-
indexName,
|
|
1211
|
-
partitions: succeededPartitions,
|
|
1212
|
-
timestamp: Date.now(),
|
|
1213
|
-
sourceInstanceId: getInstanceId()
|
|
1214
|
-
});
|
|
1215
|
-
}
|
|
1216
|
-
}
|
|
1217
|
-
mutationCount = 0;
|
|
1218
|
-
}
|
|
1219
|
-
function flush() {
|
|
1220
|
-
if (flushPromise !== null) {
|
|
1221
|
-
return flushPromise;
|
|
1222
|
-
}
|
|
1223
|
-
flushPromise = performFlush().finally(() => {
|
|
1224
|
-
flushPromise = null;
|
|
1225
|
-
});
|
|
1226
|
-
return flushPromise;
|
|
1227
|
-
}
|
|
1228
|
-
function markDirty(indexName, partitionId) {
|
|
1229
|
-
if (isShuttingDown) return;
|
|
1230
|
-
addToDirtySet(indexName, partitionId);
|
|
1231
|
-
mutationCount++;
|
|
1232
|
-
if (mutationCount >= mutationThreshold) {
|
|
1233
|
-
flush();
|
|
1234
|
-
}
|
|
1235
|
-
startTimer();
|
|
1236
|
-
}
|
|
1237
|
-
async function shutdown() {
|
|
1238
|
-
isShuttingDown = true;
|
|
1239
|
-
if (flushTimerId !== null) {
|
|
1240
|
-
clearInterval(flushTimerId);
|
|
1241
|
-
flushTimerId = null;
|
|
1242
|
-
}
|
|
1243
|
-
await flush();
|
|
1244
|
-
}
|
|
1245
|
-
return {
|
|
1246
|
-
markDirty,
|
|
1247
|
-
flush,
|
|
1248
|
-
shutdown
|
|
1249
|
-
};
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
// src/plugins/registry.ts
|
|
1253
|
-
function isThenable(value) {
|
|
1254
|
-
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
1255
|
-
}
|
|
1256
|
-
async function continueAsync(pending, hooks, startIndex, context) {
|
|
1257
|
-
await pending;
|
|
1258
|
-
for (let i = startIndex; i < hooks.length; i++) {
|
|
1259
|
-
await hooks[i](context);
|
|
1260
|
-
}
|
|
1261
|
-
}
|
|
1262
|
-
function createPluginRegistry() {
|
|
1263
|
-
const plugins = [];
|
|
1264
|
-
return {
|
|
1265
|
-
register(plugin) {
|
|
1266
|
-
plugins.push(plugin);
|
|
1267
|
-
},
|
|
1268
|
-
hasHooks(hookName) {
|
|
1269
|
-
return plugins.some((p) => typeof p[hookName] === "function");
|
|
1270
|
-
},
|
|
1271
|
-
runHook(hookName, context) {
|
|
1272
|
-
const hooks = [];
|
|
1273
|
-
for (const plugin of plugins) {
|
|
1274
|
-
const hook = plugin[hookName];
|
|
1275
|
-
if (typeof hook === "function") {
|
|
1276
|
-
hooks.push(hook.bind(plugin));
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
if (hooks.length === 0) return;
|
|
1280
|
-
for (let i = 0; i < hooks.length; i++) {
|
|
1281
|
-
const result = hooks[i](context);
|
|
1282
|
-
if (isThenable(result)) {
|
|
1283
|
-
return continueAsync(result, hooks, i + 1, context);
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
|
-
}
|
|
1287
|
-
};
|
|
1288
|
-
}
|
|
1289
|
-
var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
|
|
1290
|
-
function validateHnswMetric(value) {
|
|
1291
|
-
if (value === void 0 || value === null) return void 0;
|
|
1292
|
-
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
1293
|
-
return value;
|
|
1294
|
-
}
|
|
1295
|
-
return void 0;
|
|
1296
|
-
}
|
|
1297
|
-
function wireToPartition(raw) {
|
|
1298
|
-
const documents = {};
|
|
1299
|
-
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
1300
|
-
documents[docId] = {
|
|
1301
|
-
fields: doc.fields,
|
|
1302
|
-
fieldLengths: doc.field_lengths ?? {}
|
|
1303
|
-
};
|
|
1304
|
-
}
|
|
1305
|
-
const invertedIndex = {};
|
|
1306
|
-
for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
|
|
1307
|
-
invertedIndex[token] = {
|
|
1308
|
-
docFrequency: list.doc_freq,
|
|
1309
|
-
postings: (list.postings ?? []).map((p) => ({
|
|
1310
|
-
docId: p.doc_id,
|
|
1311
|
-
termFrequency: p.term_freq,
|
|
1312
|
-
field: p.field,
|
|
1313
|
-
positions: p.positions ?? []
|
|
1314
|
-
}))
|
|
1315
|
-
};
|
|
1316
|
-
}
|
|
1317
|
-
const numeric = {};
|
|
1318
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
1319
|
-
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
1320
|
-
}
|
|
1321
|
-
const boolean = {};
|
|
1322
|
-
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
1323
|
-
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
1324
|
-
}
|
|
1325
|
-
const geopoint = {};
|
|
1326
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
1327
|
-
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
1328
|
-
}
|
|
1329
|
-
const vectorData = {};
|
|
1330
|
-
for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
|
|
1331
|
-
vectorData[field] = {
|
|
1332
|
-
dimension: data.dimension,
|
|
1333
|
-
vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
1334
|
-
hnswGraph: data.hnsw_graph ? {
|
|
1335
|
-
entryPoint: data.hnsw_graph.entry_point,
|
|
1336
|
-
maxLayer: data.hnsw_graph.max_layer,
|
|
1337
|
-
m: data.hnsw_graph.m,
|
|
1338
|
-
efConstruction: data.hnsw_graph.ef_construction,
|
|
1339
|
-
metric: validateHnswMetric(data.hnsw_graph.metric),
|
|
1340
|
-
nodes: data.hnsw_graph.nodes
|
|
1341
|
-
} : null
|
|
1342
|
-
};
|
|
1343
|
-
}
|
|
1344
|
-
return {
|
|
1345
|
-
indexName: raw.index_name,
|
|
1346
|
-
partitionId: raw.partition_id,
|
|
1347
|
-
totalPartitions: raw.total_partitions,
|
|
1348
|
-
language: raw.language ?? "english",
|
|
1349
|
-
schema: raw.schema ?? {},
|
|
1350
|
-
docCount: raw.doc_count ?? 0,
|
|
1351
|
-
avgDocLength: raw.avg_doc_length ?? 0,
|
|
1352
|
-
documents,
|
|
1353
|
-
invertedIndex,
|
|
1354
|
-
fieldIndexes: {
|
|
1355
|
-
numeric,
|
|
1356
|
-
boolean,
|
|
1357
|
-
enum: raw.field_indexes?.enum ?? {},
|
|
1358
|
-
geopoint
|
|
1359
|
-
},
|
|
1360
|
-
vectorData,
|
|
1361
|
-
statistics: {
|
|
1362
|
-
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
1363
|
-
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
1364
|
-
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
1365
|
-
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
1366
|
-
}
|
|
1367
|
-
};
|
|
1368
|
-
}
|
|
1369
|
-
function deserializePayloadV1(data) {
|
|
1370
|
-
const raw = decode(data);
|
|
1371
|
-
return wireToPartition(raw);
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
// src/workers/promoter.ts
|
|
1375
|
-
var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
|
|
1376
|
-
var DEFAULT_TOTAL_THRESHOLD = 5e4;
|
|
1377
|
-
function createExecutionPromoter(config) {
|
|
1378
|
-
const perIndexThreshold = config?.perIndexThreshold ?? DEFAULT_PER_INDEX_THRESHOLD;
|
|
1379
|
-
const totalThreshold = config?.totalThreshold ?? DEFAULT_TOTAL_THRESHOLD;
|
|
1380
|
-
let promoted = false;
|
|
1381
|
-
function check(indexes) {
|
|
1382
|
-
if (promoted) {
|
|
1383
|
-
return { shouldPromote: false, reason: "" };
|
|
1384
|
-
}
|
|
1385
|
-
let totalDocuments = 0;
|
|
1386
|
-
for (const [indexName, stats] of indexes) {
|
|
1387
|
-
if (stats.documentCount >= perIndexThreshold) {
|
|
1388
|
-
return {
|
|
1389
|
-
shouldPromote: true,
|
|
1390
|
-
reason: `Index "${indexName}" has ${stats.documentCount} documents, exceeding the per-index threshold of ${perIndexThreshold}`
|
|
1391
|
-
};
|
|
1392
|
-
}
|
|
1393
|
-
totalDocuments += stats.documentCount;
|
|
1394
|
-
}
|
|
1395
|
-
if (totalDocuments >= totalThreshold) {
|
|
1396
|
-
return {
|
|
1397
|
-
shouldPromote: true,
|
|
1398
|
-
reason: `Total document count of ${totalDocuments} exceeds the total threshold of ${totalThreshold}`
|
|
1399
|
-
};
|
|
1400
|
-
}
|
|
1401
|
-
return { shouldPromote: false, reason: "" };
|
|
1402
|
-
}
|
|
1403
|
-
function markPromoted() {
|
|
1404
|
-
promoted = true;
|
|
1405
|
-
}
|
|
1406
|
-
function isPromoted() {
|
|
1407
|
-
return promoted;
|
|
1408
|
-
}
|
|
1409
|
-
return { check, markPromoted, isPromoted };
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
// src/narsil.ts
|
|
1413
|
-
async function createNarsil(config) {
|
|
1414
|
-
const executor = createDirectExecutor();
|
|
1415
|
-
const promoter = createExecutionPromoter({
|
|
1416
|
-
perIndexThreshold: config?.workers?.promotionThreshold,
|
|
1417
|
-
totalThreshold: config?.workers?.totalPromotionThreshold
|
|
1418
|
-
});
|
|
1419
|
-
const pluginRegistry = createPluginRegistry();
|
|
1420
|
-
if (config?.plugins) {
|
|
1421
|
-
for (const plugin of config.plugins) {
|
|
1422
|
-
pluginRegistry.register(plugin);
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
let flushManager = null;
|
|
1426
|
-
if (config?.persistence) {
|
|
1427
|
-
const noopInvalidation = config.invalidation ?? {
|
|
1428
|
-
publish: async () => {
|
|
1429
|
-
},
|
|
1430
|
-
subscribe: async () => {
|
|
1431
|
-
},
|
|
1432
|
-
shutdown: async () => {
|
|
1433
|
-
}
|
|
1434
|
-
};
|
|
1435
|
-
flushManager = createFlushManager(
|
|
1436
|
-
{
|
|
1437
|
-
persistence: config.persistence,
|
|
1438
|
-
invalidation: noopInvalidation,
|
|
1439
|
-
interval: config.flush?.interval,
|
|
1440
|
-
mutationThreshold: config.flush?.mutationThreshold
|
|
1441
|
-
},
|
|
1442
|
-
() => new Uint8Array(0),
|
|
1443
|
-
() => "instance-0"
|
|
1444
|
-
);
|
|
1445
|
-
}
|
|
1446
|
-
const idGenerator = config?.idGenerator ?? generateId;
|
|
1447
|
-
const indexRegistry = /* @__PURE__ */ new Map();
|
|
1448
|
-
const eventHandlers = /* @__PURE__ */ new Map();
|
|
1449
|
-
let isShutdown = false;
|
|
1450
|
-
const orchestrator = createWorkerOrchestrator(config, executor, promoter, indexRegistry, {
|
|
1451
|
-
onPromotion(workerCount, reason) {
|
|
1452
|
-
const handlers = eventHandlers.get("workerPromote");
|
|
1453
|
-
if (handlers) {
|
|
1454
|
-
for (const handler of handlers) {
|
|
1455
|
-
handler({ workerCount, reason });
|
|
1456
|
-
}
|
|
1457
|
-
}
|
|
1458
|
-
}
|
|
1459
|
-
});
|
|
1460
|
-
const rebalancer = createRebalancer();
|
|
1461
|
-
const rebalanceRouter = createPartitionRouter();
|
|
1462
|
-
const rebalancingIndexes = /* @__PURE__ */ new Set();
|
|
1463
|
-
const waqMap = /* @__PURE__ */ new Map();
|
|
1464
|
-
const lastAppliedSeqMap = /* @__PURE__ */ new Map();
|
|
1465
|
-
function guardShutdown() {
|
|
1466
|
-
if (isShutdown) {
|
|
1467
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "This Narsil instance has been shut down");
|
|
1468
|
-
}
|
|
1469
|
-
}
|
|
1470
|
-
function requireIndex(indexName) {
|
|
1471
|
-
const entry = indexRegistry.get(indexName);
|
|
1472
|
-
if (!entry) {
|
|
1473
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
|
|
1474
|
-
}
|
|
1475
|
-
return entry;
|
|
1476
|
-
}
|
|
1477
|
-
function bufferIfRebalancing(indexName, entry) {
|
|
1478
|
-
if (!rebalancingIndexes.has(indexName)) return false;
|
|
1479
|
-
const waq = waqMap.get(indexName);
|
|
1480
|
-
if (!waq) return false;
|
|
1481
|
-
waq.push(entry);
|
|
1482
|
-
return true;
|
|
1483
|
-
}
|
|
1484
|
-
function requireManager(indexName) {
|
|
1485
|
-
const manager = executor.getManager(indexName);
|
|
1486
|
-
if (!manager) {
|
|
1487
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" manager not found`, { indexName });
|
|
1488
|
-
}
|
|
1489
|
-
return manager;
|
|
1490
|
-
}
|
|
1491
|
-
const narsil = {
|
|
1492
|
-
async createIndex(name, indexConfig) {
|
|
1493
|
-
guardShutdown();
|
|
1494
|
-
validateIndexName(name);
|
|
1495
|
-
if (indexRegistry.has(name)) {
|
|
1496
|
-
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
1497
|
-
}
|
|
1498
|
-
validateSchema(indexConfig.schema);
|
|
1499
|
-
const language = getLanguage(indexConfig.language ?? "english");
|
|
1500
|
-
executor.createIndex(name, indexConfig, language);
|
|
1501
|
-
indexRegistry.set(name, { config: indexConfig, language });
|
|
1502
|
-
await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
1503
|
-
},
|
|
1504
|
-
async dropIndex(name) {
|
|
1505
|
-
guardShutdown();
|
|
1506
|
-
const entry = requireIndex(name);
|
|
1507
|
-
executor.dropIndex(name);
|
|
1508
|
-
indexRegistry.delete(name);
|
|
1509
|
-
await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
1510
|
-
},
|
|
1511
|
-
listIndexes() {
|
|
1512
|
-
const infos = [];
|
|
1513
|
-
for (const [name, entry] of indexRegistry) {
|
|
1514
|
-
const manager = executor.getManager(name);
|
|
1515
|
-
infos.push({
|
|
1516
|
-
name,
|
|
1517
|
-
documentCount: manager?.countDocuments() ?? 0,
|
|
1518
|
-
partitionCount: manager?.partitionCount ?? 0,
|
|
1519
|
-
language: entry.language.name
|
|
1520
|
-
});
|
|
1521
|
-
}
|
|
1522
|
-
return infos;
|
|
1523
|
-
},
|
|
1524
|
-
getStats(indexName) {
|
|
1525
|
-
guardShutdown();
|
|
1526
|
-
const entry = requireIndex(indexName);
|
|
1527
|
-
const manager = executor.getManager(indexName);
|
|
1528
|
-
const estimatedMemory = manager?.estimateMemoryBytes() ?? 0;
|
|
1529
|
-
return {
|
|
1530
|
-
documentCount: manager?.countDocuments() ?? 0,
|
|
1531
|
-
partitionCount: manager?.partitionCount ?? 0,
|
|
1532
|
-
memoryBytes: estimatedMemory,
|
|
1533
|
-
indexSizeBytes: estimatedMemory,
|
|
1534
|
-
language: entry.language.name,
|
|
1535
|
-
schema: entry.config.schema
|
|
1536
|
-
};
|
|
1537
|
-
},
|
|
1538
|
-
getPartitionStats(indexName) {
|
|
1539
|
-
guardShutdown();
|
|
1540
|
-
requireIndex(indexName);
|
|
1541
|
-
const manager = executor.getManager(indexName);
|
|
1542
|
-
return manager?.getPartitionStats() ?? [];
|
|
1543
|
-
},
|
|
1544
|
-
async insert(indexName, document, docId, options) {
|
|
1545
|
-
guardShutdown();
|
|
1546
|
-
requireIndex(indexName);
|
|
1547
|
-
const resolvedDocId = docId ?? idGenerator();
|
|
1548
|
-
validateDocId(resolvedDocId);
|
|
1549
|
-
if (bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
|
|
1550
|
-
return resolvedDocId;
|
|
1551
|
-
}
|
|
1552
|
-
await pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document });
|
|
1553
|
-
await executor.execute({
|
|
1554
|
-
type: "insert",
|
|
1555
|
-
indexName,
|
|
1556
|
-
docId: resolvedDocId,
|
|
1557
|
-
document,
|
|
1558
|
-
requestId: resolvedDocId,
|
|
1559
|
-
skipClone: options?.skipClone
|
|
1560
|
-
});
|
|
1561
|
-
try {
|
|
1562
|
-
await pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
|
|
1563
|
-
} catch (err) {
|
|
1564
|
-
console.warn("afterInsert plugin hook error:", err);
|
|
1565
|
-
}
|
|
1566
|
-
flushManager?.markDirty(indexName, 0);
|
|
1567
|
-
await orchestrator.replicateToWorkers({
|
|
1568
|
-
type: "insert",
|
|
1569
|
-
indexName,
|
|
1570
|
-
docId: resolvedDocId,
|
|
1571
|
-
document,
|
|
1572
|
-
requestId: `replicate-insert-${resolvedDocId}`,
|
|
1573
|
-
skipClone: options?.skipClone
|
|
1574
|
-
});
|
|
1575
|
-
await orchestrator.checkPromotion();
|
|
1576
|
-
return resolvedDocId;
|
|
1577
|
-
},
|
|
1578
|
-
async insertBatch(indexName, documents, options) {
|
|
1579
|
-
guardShutdown();
|
|
1580
|
-
requireIndex(indexName);
|
|
1581
|
-
const succeeded = [];
|
|
1582
|
-
const failed = [];
|
|
1583
|
-
const hasBeforeHook = pluginRegistry.hasHooks("beforeInsert");
|
|
1584
|
-
const hasAfterHook = pluginRegistry.hasHooks("afterInsert");
|
|
1585
|
-
for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
1586
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
|
|
1587
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
1588
|
-
const docId = idGenerator();
|
|
1589
|
-
try {
|
|
1590
|
-
validateDocId(docId);
|
|
1591
|
-
if (hasBeforeHook) {
|
|
1592
|
-
await pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
1593
|
-
}
|
|
1594
|
-
const result = executor.execute({
|
|
1595
|
-
type: "insert",
|
|
1596
|
-
indexName,
|
|
1597
|
-
docId,
|
|
1598
|
-
document: documents[i],
|
|
1599
|
-
requestId: docId,
|
|
1600
|
-
skipClone: options?.skipClone
|
|
1601
|
-
});
|
|
1602
|
-
if (result && typeof result.then === "function") {
|
|
1603
|
-
await result;
|
|
1604
|
-
}
|
|
1605
|
-
if (hasAfterHook) {
|
|
1606
|
-
try {
|
|
1607
|
-
await pluginRegistry.runHook("afterInsert", { indexName, docId, document: documents[i] });
|
|
1608
|
-
} catch (err) {
|
|
1609
|
-
console.warn("afterInsert plugin hook error:", err);
|
|
1610
|
-
}
|
|
1611
|
-
}
|
|
1612
|
-
succeeded.push(docId);
|
|
1613
|
-
} catch (err) {
|
|
1614
|
-
failed.push({
|
|
1615
|
-
docId,
|
|
1616
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
|
|
1617
|
-
});
|
|
1618
|
-
}
|
|
1619
|
-
}
|
|
1620
|
-
if (chunkEnd < documents.length) {
|
|
1621
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
flushManager?.markDirty(indexName, 0);
|
|
1625
|
-
await orchestrator.checkPromotion();
|
|
1626
|
-
return { succeeded, failed };
|
|
1627
|
-
},
|
|
1628
|
-
async remove(indexName, docId) {
|
|
1629
|
-
guardShutdown();
|
|
1630
|
-
requireIndex(indexName);
|
|
1631
|
-
validateDocId(docId);
|
|
1632
|
-
if (bufferIfRebalancing(indexName, { action: "remove", docId, indexName })) {
|
|
1633
|
-
return;
|
|
1634
|
-
}
|
|
1635
|
-
await pluginRegistry.runHook("beforeRemove", { indexName, docId });
|
|
1636
|
-
await executor.execute({ type: "remove", indexName, docId, requestId: docId });
|
|
1637
|
-
try {
|
|
1638
|
-
await pluginRegistry.runHook("afterRemove", { indexName, docId });
|
|
1639
|
-
} catch (err) {
|
|
1640
|
-
console.warn("afterRemove plugin hook error:", err);
|
|
1641
|
-
}
|
|
1642
|
-
flushManager?.markDirty(indexName, 0);
|
|
1643
|
-
await orchestrator.replicateToWorkers({
|
|
1644
|
-
type: "remove",
|
|
1645
|
-
indexName,
|
|
1646
|
-
docId,
|
|
1647
|
-
requestId: `replicate-remove-${docId}`
|
|
1648
|
-
});
|
|
1649
|
-
},
|
|
1650
|
-
async removeBatch(indexName, docIds) {
|
|
1651
|
-
guardShutdown();
|
|
1652
|
-
requireIndex(indexName);
|
|
1653
|
-
const succeeded = [];
|
|
1654
|
-
const failed = [];
|
|
1655
|
-
for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
1656
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
|
|
1657
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
1658
|
-
try {
|
|
1659
|
-
await narsil.remove(indexName, docIds[i]);
|
|
1660
|
-
succeeded.push(docIds[i]);
|
|
1661
|
-
} catch (err) {
|
|
1662
|
-
failed.push({
|
|
1663
|
-
docId: docIds[i],
|
|
1664
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err))
|
|
1665
|
-
});
|
|
1666
|
-
}
|
|
1667
|
-
}
|
|
1668
|
-
if (chunkEnd < docIds.length) {
|
|
1669
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
return { succeeded, failed };
|
|
1673
|
-
},
|
|
1674
|
-
async update(indexName, docId, document) {
|
|
1675
|
-
guardShutdown();
|
|
1676
|
-
requireIndex(indexName);
|
|
1677
|
-
validateDocId(docId);
|
|
1678
|
-
if (bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
|
|
1679
|
-
return;
|
|
1680
|
-
}
|
|
1681
|
-
const manager = executor.getManager(indexName);
|
|
1682
|
-
const oldDocument = manager?.get(docId);
|
|
1683
|
-
await pluginRegistry.runHook("beforeUpdate", {
|
|
1684
|
-
indexName,
|
|
1685
|
-
docId,
|
|
1686
|
-
oldDocument: oldDocument ?? {},
|
|
1687
|
-
newDocument: document
|
|
1688
|
-
});
|
|
1689
|
-
await executor.execute({ type: "update", indexName, docId, document, requestId: docId });
|
|
1690
|
-
try {
|
|
1691
|
-
await pluginRegistry.runHook("afterUpdate", {
|
|
1692
|
-
indexName,
|
|
1693
|
-
docId,
|
|
1694
|
-
oldDocument: oldDocument ?? {},
|
|
1695
|
-
newDocument: document
|
|
1696
|
-
});
|
|
1697
|
-
} catch (err) {
|
|
1698
|
-
console.warn("afterUpdate plugin hook error:", err);
|
|
1699
|
-
}
|
|
1700
|
-
flushManager?.markDirty(indexName, 0);
|
|
1701
|
-
await orchestrator.replicateToWorkers({
|
|
1702
|
-
type: "update",
|
|
1703
|
-
indexName,
|
|
1704
|
-
docId,
|
|
1705
|
-
document,
|
|
1706
|
-
requestId: `replicate-update-${docId}`
|
|
1707
|
-
});
|
|
1708
|
-
},
|
|
1709
|
-
async updateBatch(indexName, updates) {
|
|
1710
|
-
guardShutdown();
|
|
1711
|
-
requireIndex(indexName);
|
|
1712
|
-
const succeeded = [];
|
|
1713
|
-
const failed = [];
|
|
1714
|
-
for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
|
|
1715
|
-
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
1716
|
-
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
1717
|
-
try {
|
|
1718
|
-
await narsil.update(indexName, updates[i].docId, updates[i].document);
|
|
1719
|
-
succeeded.push(updates[i].docId);
|
|
1720
|
-
} catch (err) {
|
|
1721
|
-
failed.push({
|
|
1722
|
-
docId: updates[i].docId,
|
|
1723
|
-
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err))
|
|
1724
|
-
});
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
if (chunkEnd < updates.length) {
|
|
1728
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
1729
|
-
}
|
|
1730
|
-
}
|
|
1731
|
-
return { succeeded, failed };
|
|
1732
|
-
},
|
|
1733
|
-
async get(indexName, docId) {
|
|
1734
|
-
guardShutdown();
|
|
1735
|
-
requireIndex(indexName);
|
|
1736
|
-
return executor.execute({ type: "get", indexName, docId, requestId: docId });
|
|
1737
|
-
},
|
|
1738
|
-
async getMultiple(indexName, docIds) {
|
|
1739
|
-
guardShutdown();
|
|
1740
|
-
requireIndex(indexName);
|
|
1741
|
-
const result = /* @__PURE__ */ new Map();
|
|
1742
|
-
for (const docId of docIds) {
|
|
1743
|
-
const doc = await narsil.get(indexName, docId);
|
|
1744
|
-
if (doc !== void 0) {
|
|
1745
|
-
result.set(docId, doc);
|
|
1746
|
-
}
|
|
1747
|
-
}
|
|
1748
|
-
return result;
|
|
1749
|
-
},
|
|
1750
|
-
async has(indexName, docId) {
|
|
1751
|
-
guardShutdown();
|
|
1752
|
-
requireIndex(indexName);
|
|
1753
|
-
return executor.execute({ type: "has", indexName, docId, requestId: docId });
|
|
1754
|
-
},
|
|
1755
|
-
async countDocuments(indexName) {
|
|
1756
|
-
guardShutdown();
|
|
1757
|
-
requireIndex(indexName);
|
|
1758
|
-
return executor.execute({ type: "count", indexName, requestId: indexName });
|
|
1759
|
-
},
|
|
1760
|
-
async query(indexName, params) {
|
|
1761
|
-
guardShutdown();
|
|
1762
|
-
const entry = requireIndex(indexName);
|
|
1763
|
-
const manager = requireManager(indexName);
|
|
1764
|
-
await pluginRegistry.runHook("beforeSearch", { indexName, params });
|
|
1765
|
-
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
1766
|
-
const result = await executeQuery(params, {
|
|
1767
|
-
manager,
|
|
1768
|
-
language: entry.language,
|
|
1769
|
-
config: entry.config,
|
|
1770
|
-
workerSearch,
|
|
1771
|
-
indexName
|
|
1772
|
-
});
|
|
1773
|
-
try {
|
|
1774
|
-
await pluginRegistry.runHook("afterSearch", {
|
|
1775
|
-
indexName,
|
|
1776
|
-
params,
|
|
1777
|
-
results: result
|
|
1778
|
-
});
|
|
1779
|
-
} catch (err) {
|
|
1780
|
-
console.warn("afterSearch plugin hook error:", err);
|
|
1781
|
-
}
|
|
1782
|
-
return result;
|
|
1783
|
-
},
|
|
1784
|
-
async preflight(indexName, params) {
|
|
1785
|
-
guardShutdown();
|
|
1786
|
-
const entry = requireIndex(indexName);
|
|
1787
|
-
const manager = requireManager(indexName);
|
|
1788
|
-
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
1789
|
-
return executePreflight(params, {
|
|
1790
|
-
manager,
|
|
1791
|
-
language: entry.language,
|
|
1792
|
-
config: entry.config,
|
|
1793
|
-
workerSearch,
|
|
1794
|
-
indexName
|
|
1795
|
-
});
|
|
1796
|
-
},
|
|
1797
|
-
async suggest(indexName, params) {
|
|
1798
|
-
guardShutdown();
|
|
1799
|
-
const entry = requireIndex(indexName);
|
|
1800
|
-
const manager = requireManager(indexName);
|
|
1801
|
-
const t0 = performance.now();
|
|
1802
|
-
const limit = Math.max(1, Math.min(params.limit ?? 10, 100));
|
|
1803
|
-
const rawPrefix = params.prefix.trim();
|
|
1804
|
-
if (rawPrefix.length === 0) {
|
|
1805
|
-
return { terms: [], elapsed: performance.now() - t0 };
|
|
1806
|
-
}
|
|
1807
|
-
const unstemmed = tokenize(rawPrefix, entry.language, { stem: false, removeStopWords: false });
|
|
1808
|
-
const lastToken = unstemmed.tokens.length > 0 ? unstemmed.tokens[unstemmed.tokens.length - 1].token : rawPrefix.toLowerCase();
|
|
1809
|
-
if (lastToken.length === 0) {
|
|
1810
|
-
return { terms: [], elapsed: performance.now() - t0 };
|
|
1811
|
-
}
|
|
1812
|
-
const stemmed = entry.language.stemmer ? entry.language.stemmer(lastToken) : lastToken;
|
|
1813
|
-
const prefixes = stemmed !== lastToken ? [lastToken, stemmed] : [lastToken];
|
|
1814
|
-
const partitions = manager.getAllPartitions();
|
|
1815
|
-
const merged = /* @__PURE__ */ new Map();
|
|
1816
|
-
const perPartitionLimit = limit * 2;
|
|
1817
|
-
for (const partition of partitions) {
|
|
1818
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1819
|
-
for (const prefix of prefixes) {
|
|
1820
|
-
const suggestions = partition.suggestTerms(prefix, perPartitionLimit);
|
|
1821
|
-
for (const s of suggestions) {
|
|
1822
|
-
if (seen.has(s.term)) continue;
|
|
1823
|
-
seen.add(s.term);
|
|
1824
|
-
merged.set(s.term, (merged.get(s.term) ?? 0) + s.documentFrequency);
|
|
1825
|
-
}
|
|
1826
|
-
}
|
|
1827
|
-
}
|
|
1828
|
-
const terms = Array.from(merged.entries()).map(([term, documentFrequency]) => ({ term, documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency);
|
|
1829
|
-
if (terms.length > limit) terms.length = limit;
|
|
1830
|
-
return { terms, elapsed: performance.now() - t0 };
|
|
1831
|
-
},
|
|
1832
|
-
async snapshot(indexName) {
|
|
1833
|
-
guardShutdown();
|
|
1834
|
-
const entry = requireIndex(indexName);
|
|
1835
|
-
const manager = requireManager(indexName);
|
|
1836
|
-
const partitionBuffers = [];
|
|
1837
|
-
for (let i = 0; i < manager.partitionCount; i++) {
|
|
1838
|
-
partitionBuffers.push(manager.serializePartitionToBytes(i));
|
|
1839
|
-
}
|
|
1840
|
-
const { encode: encode2 } = await import('@msgpack/msgpack');
|
|
1841
|
-
return encode2({
|
|
1842
|
-
version: 2,
|
|
1843
|
-
schema: entry.config.schema,
|
|
1844
|
-
language: entry.language.name,
|
|
1845
|
-
partitions: partitionBuffers
|
|
1846
|
-
});
|
|
1847
|
-
},
|
|
1848
|
-
async restore(indexName, data) {
|
|
1849
|
-
guardShutdown();
|
|
1850
|
-
if (!(data instanceof Uint8Array)) {
|
|
1851
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
|
|
1852
|
-
}
|
|
1853
|
-
const { decode: decode2 } = await import('@msgpack/msgpack');
|
|
1854
|
-
const envelope = decode2(data);
|
|
1855
|
-
if (envelope.version !== 1 && envelope.version !== 2) {
|
|
1856
|
-
throw new NarsilError(
|
|
1857
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
1858
|
-
`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`,
|
|
1859
|
-
{ version: envelope.version }
|
|
1860
|
-
);
|
|
1861
|
-
}
|
|
1862
|
-
if (!envelope.schema || typeof envelope.schema !== "object") {
|
|
1863
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid schema");
|
|
1864
|
-
}
|
|
1865
|
-
if (!envelope.language || typeof envelope.language !== "string") {
|
|
1866
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid language");
|
|
1867
|
-
}
|
|
1868
|
-
if (!Array.isArray(envelope.partitions)) {
|
|
1869
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing partitions");
|
|
1870
|
-
}
|
|
1871
|
-
const language = getLanguage(envelope.language);
|
|
1872
|
-
const schema = envelope.schema;
|
|
1873
|
-
validateSchema(schema);
|
|
1874
|
-
if (indexRegistry.has(indexName)) {
|
|
1875
|
-
await narsil.dropIndex(indexName);
|
|
1876
|
-
}
|
|
1877
|
-
const indexConfig = { schema, language: envelope.language };
|
|
1878
|
-
executor.createIndex(indexName, indexConfig, language);
|
|
1879
|
-
indexRegistry.set(indexName, { config: indexConfig, language });
|
|
1880
|
-
try {
|
|
1881
|
-
const manager = requireManager(indexName);
|
|
1882
|
-
while (manager.partitionCount < envelope.partitions.length) {
|
|
1883
|
-
manager.addPartition();
|
|
1884
|
-
}
|
|
1885
|
-
const deserializePartitionPayload = envelope.version === 2 ? deserializePayloadV2 : deserializePayloadV1;
|
|
1886
|
-
for (let i = 0; i < envelope.partitions.length; i++) {
|
|
1887
|
-
const partition = deserializePartitionPayload(envelope.partitions[i]);
|
|
1888
|
-
manager.deserializePartition(i, partition);
|
|
1889
|
-
}
|
|
1890
|
-
} catch (err) {
|
|
1891
|
-
try {
|
|
1892
|
-
executor.dropIndex(indexName);
|
|
1893
|
-
indexRegistry.delete(indexName);
|
|
1894
|
-
} catch (_) {
|
|
1895
|
-
}
|
|
1896
|
-
throw err;
|
|
1897
|
-
}
|
|
1898
|
-
},
|
|
1899
|
-
async clear(indexName) {
|
|
1900
|
-
guardShutdown();
|
|
1901
|
-
requireIndex(indexName);
|
|
1902
|
-
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
1903
|
-
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
1904
|
-
},
|
|
1905
|
-
async rebalance(indexName, targetPartitionCount) {
|
|
1906
|
-
guardShutdown();
|
|
1907
|
-
requireIndex(indexName);
|
|
1908
|
-
const manager = requireManager(indexName);
|
|
1909
|
-
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
1910
|
-
throw new NarsilError(
|
|
1911
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
1912
|
-
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
1913
|
-
{ targetPartitionCount }
|
|
1914
|
-
);
|
|
1915
|
-
}
|
|
1916
|
-
if (rebalancingIndexes.has(indexName)) {
|
|
1917
|
-
throw new NarsilError(
|
|
1918
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1919
|
-
`Index "${indexName}" is already being rebalanced`
|
|
1920
|
-
);
|
|
1921
|
-
}
|
|
1922
|
-
if (targetPartitionCount === manager.partitionCount) {
|
|
1923
|
-
return;
|
|
1924
|
-
}
|
|
1925
|
-
const waq = createWriteAheadQueue();
|
|
1926
|
-
waqMap.set(indexName, waq);
|
|
1927
|
-
rebalancingIndexes.add(indexName);
|
|
1928
|
-
try {
|
|
1929
|
-
const oldCount = manager.partitionCount;
|
|
1930
|
-
await rebalancer.rebalance(manager, targetPartitionCount, rebalanceRouter, (progress) => {
|
|
1931
|
-
if (progress.phase === "complete") {
|
|
1932
|
-
const handlers = eventHandlers.get("partitionRebalance");
|
|
1933
|
-
if (handlers) {
|
|
1934
|
-
for (const handler of handlers) {
|
|
1935
|
-
handler({ indexName, oldCount, newCount: targetPartitionCount });
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
});
|
|
1940
|
-
const entries = waq.drain();
|
|
1941
|
-
const appliedSeqs = lastAppliedSeqMap.get(indexName) ?? /* @__PURE__ */ new Map();
|
|
1942
|
-
for (const entry of entries) {
|
|
1943
|
-
const partitionLastSeq = appliedSeqs.get(0) ?? 0;
|
|
1944
|
-
if (entry.sequenceNumber <= partitionLastSeq) continue;
|
|
1945
|
-
try {
|
|
1946
|
-
if (entry.action === "insert" && entry.document) {
|
|
1947
|
-
manager.insert(entry.docId, entry.document);
|
|
1948
|
-
} else if (entry.action === "remove") {
|
|
1949
|
-
manager.remove(entry.docId);
|
|
1950
|
-
} else if (entry.action === "update" && entry.document) {
|
|
1951
|
-
manager.update(entry.docId, entry.document);
|
|
1952
|
-
}
|
|
1953
|
-
} catch (replayErr) {
|
|
1954
|
-
const isDuplicate = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_ALREADY_EXISTS;
|
|
1955
|
-
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
1956
|
-
if (!isDuplicate && !isMissing) {
|
|
1957
|
-
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
1958
|
-
}
|
|
1959
|
-
}
|
|
1960
|
-
appliedSeqs.set(0, entry.sequenceNumber);
|
|
1961
|
-
}
|
|
1962
|
-
lastAppliedSeqMap.set(indexName, appliedSeqs);
|
|
1963
|
-
} finally {
|
|
1964
|
-
rebalancingIndexes.delete(indexName);
|
|
1965
|
-
waqMap.delete(indexName);
|
|
1966
|
-
}
|
|
1967
|
-
},
|
|
1968
|
-
async updatePartitionConfig(indexName, partitionConfig) {
|
|
1969
|
-
guardShutdown();
|
|
1970
|
-
const entry = requireIndex(indexName);
|
|
1971
|
-
const manager = requireManager(indexName);
|
|
1972
|
-
if (rebalancingIndexes.has(indexName)) {
|
|
1973
|
-
throw new NarsilError(
|
|
1974
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
1975
|
-
`Index "${indexName}" is currently being rebalanced`
|
|
1976
|
-
);
|
|
1977
|
-
}
|
|
1978
|
-
const currentDocCount = manager.countDocuments();
|
|
1979
|
-
const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
|
|
1980
|
-
const newMaxPartitions = partitionConfig.maxPartitions ?? entry.config.partitions?.maxPartitions ?? manager.partitionCount;
|
|
1981
|
-
if (newMaxDocs !== void 0) {
|
|
1982
|
-
const newTotalCapacity = newMaxDocs * newMaxPartitions;
|
|
1983
|
-
if (newTotalCapacity < currentDocCount) {
|
|
1984
|
-
throw new NarsilError(
|
|
1985
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
1986
|
-
`New capacity (${newTotalCapacity}) is less than current document count (${currentDocCount})`,
|
|
1987
|
-
{ newTotalCapacity, currentDocCount }
|
|
1988
|
-
);
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
if (!entry.config.partitions) {
|
|
1992
|
-
entry.config.partitions = {};
|
|
1993
|
-
}
|
|
1994
|
-
if (partitionConfig.maxDocsPerPartition !== void 0) {
|
|
1995
|
-
entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
|
|
1996
|
-
}
|
|
1997
|
-
if (partitionConfig.maxPartitions !== void 0) {
|
|
1998
|
-
entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
|
|
1999
|
-
}
|
|
2000
|
-
},
|
|
2001
|
-
getMemoryStats() {
|
|
2002
|
-
const workerStats = orchestrator.getMemoryStats();
|
|
2003
|
-
let mainThreadBytes = 0;
|
|
2004
|
-
for (const [name] of indexRegistry) {
|
|
2005
|
-
const manager = executor.getManager(name);
|
|
2006
|
-
if (manager) {
|
|
2007
|
-
mainThreadBytes += manager.estimateMemoryBytes();
|
|
2008
|
-
}
|
|
2009
|
-
}
|
|
2010
|
-
return {
|
|
2011
|
-
totalBytes: mainThreadBytes + workerStats.totalBytes,
|
|
2012
|
-
workers: workerStats.workers
|
|
2013
|
-
};
|
|
2014
|
-
},
|
|
2015
|
-
on(event, handler) {
|
|
2016
|
-
const key = event;
|
|
2017
|
-
let handlers = eventHandlers.get(key);
|
|
2018
|
-
if (!handlers) {
|
|
2019
|
-
handlers = /* @__PURE__ */ new Set();
|
|
2020
|
-
eventHandlers.set(key, handlers);
|
|
2021
|
-
}
|
|
2022
|
-
handlers.add(handler);
|
|
2023
|
-
},
|
|
2024
|
-
off(event, handler) {
|
|
2025
|
-
const key = event;
|
|
2026
|
-
const handlers = eventHandlers.get(key);
|
|
2027
|
-
if (handlers) {
|
|
2028
|
-
handlers.delete(handler);
|
|
2029
|
-
if (handlers.size === 0) {
|
|
2030
|
-
eventHandlers.delete(key);
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
},
|
|
2034
|
-
async shutdown() {
|
|
2035
|
-
if (isShutdown) return;
|
|
2036
|
-
isShutdown = true;
|
|
2037
|
-
if (flushManager) {
|
|
2038
|
-
await flushManager.shutdown();
|
|
2039
|
-
}
|
|
2040
|
-
await orchestrator.shutdown();
|
|
2041
|
-
await executor.shutdown();
|
|
2042
|
-
eventHandlers.clear();
|
|
2043
|
-
indexRegistry.clear();
|
|
2044
|
-
}
|
|
2045
|
-
};
|
|
2046
|
-
return narsil;
|
|
2047
|
-
}
|
|
2048
|
-
|
|
2049
|
-
// src/index.ts
|
|
2050
|
-
var VERSION = "0.1.0";
|
|
2051
|
-
|
|
2052
|
-
export { VERSION, createNarsil };
|
|
1
|
+
export { VERSION, createNarsil } from './chunk-GQEHCZG7.mjs';
|
|
2
|
+
export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, registerLanguage, resetNormalizationCache } from './chunk-FS7XLIUS.mjs';
|
|
3
|
+
export { isSimdAvailable } from './chunk-S7RJJMHI.mjs';
|
|
4
|
+
import './chunk-KMMJMDUA.mjs';
|
|
5
|
+
import './chunk-TW6BXTOL.mjs';
|
|
6
|
+
export { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
|
|
7
|
+
import './chunk-HRABJLPB.mjs';
|