@delali/narsil 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +44 -2
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +46 -2
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-BbRkoiu9.d.ts → adapters-Bo2JQg8b.d.ts} +2 -1
  8. package/dist/chunk-ALB35PRQ.mjs +98 -0
  9. package/dist/{chunk-KHL4RPC3.mjs → chunk-FS7XLIUS.mjs} +2536 -2284
  10. package/dist/chunk-GQEHCZG7.mjs +6825 -0
  11. package/dist/chunk-KMMJMDUA.mjs +44 -0
  12. package/dist/chunk-RQLXC2YS.mjs +269 -0
  13. package/dist/chunk-S7RJJMHI.mjs +1086 -0
  14. package/dist/{chunk-I24SGLFS.mjs → chunk-TW6BXTOL.mjs} +5 -1
  15. package/dist/chunk-WISGW24C.mjs +455 -0
  16. package/dist/config-cQLlYR3U.d.ts +470 -0
  17. package/dist/direct-executor-FX56DYD3.mjs +4 -0
  18. package/dist/distribution/coordinator/etcd.d.ts +16 -0
  19. package/dist/distribution/coordinator/etcd.mjs +2 -0
  20. package/dist/distribution/coordinator/in-memory.d.ts +7 -0
  21. package/dist/distribution/coordinator/in-memory.mjs +1 -0
  22. package/dist/distribution.d.ts +122 -0
  23. package/dist/distribution.mjs +7410 -0
  24. package/dist/embeddings/openai.d.ts +1 -1
  25. package/dist/embeddings/openai.mjs +1 -1
  26. package/dist/index.browser.mjs +11469 -7481
  27. package/dist/index.d.ts +6 -450
  28. package/dist/index.mjs +7 -3236
  29. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  30. package/dist/invalidation/filesystem.d.ts +1 -1
  31. package/dist/invalidation/noop.d.ts +1 -1
  32. package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
  33. package/dist/languages/arabic.d.ts +1 -1
  34. package/dist/languages/armenian.d.ts +1 -1
  35. package/dist/languages/bulgarian.d.ts +1 -1
  36. package/dist/languages/chinese.d.ts +1 -1
  37. package/dist/languages/dagbani.d.ts +1 -1
  38. package/dist/languages/danish.d.ts +1 -1
  39. package/dist/languages/dutch.d.ts +1 -1
  40. package/dist/languages/english.d.ts +1 -1
  41. package/dist/languages/english.mjs +1 -1
  42. package/dist/languages/ewe.d.ts +1 -1
  43. package/dist/languages/finnish.d.ts +1 -1
  44. package/dist/languages/french.d.ts +1 -1
  45. package/dist/languages/ga.d.ts +1 -1
  46. package/dist/languages/german.d.ts +1 -1
  47. package/dist/languages/greek.d.ts +1 -1
  48. package/dist/languages/hausa.d.ts +1 -1
  49. package/dist/languages/hindi.d.ts +1 -1
  50. package/dist/languages/hungarian.d.ts +1 -1
  51. package/dist/languages/igbo.d.ts +1 -1
  52. package/dist/languages/indonesian.d.ts +1 -1
  53. package/dist/languages/irish.d.ts +1 -1
  54. package/dist/languages/italian.d.ts +1 -1
  55. package/dist/languages/japanese.d.ts +1 -1
  56. package/dist/languages/nepali.d.ts +1 -1
  57. package/dist/languages/norwegian.d.ts +1 -1
  58. package/dist/languages/portuguese.d.ts +1 -1
  59. package/dist/languages/romanian.d.ts +1 -1
  60. package/dist/languages/russian.d.ts +1 -1
  61. package/dist/languages/sanskrit.d.ts +1 -1
  62. package/dist/languages/serbian.d.ts +1 -1
  63. package/dist/languages/slovenian.d.ts +1 -1
  64. package/dist/languages/spanish.d.ts +1 -1
  65. package/dist/languages/swahili.d.ts +1 -1
  66. package/dist/languages/swedish.d.ts +1 -1
  67. package/dist/languages/tamil.d.ts +1 -1
  68. package/dist/languages/turkish.d.ts +1 -1
  69. package/dist/languages/twi.d.ts +1 -1
  70. package/dist/languages/ukrainian.d.ts +1 -1
  71. package/dist/languages/yoruba.d.ts +1 -1
  72. package/dist/languages/zulu.d.ts +1 -1
  73. package/dist/narsil-ChTdkwPu.d.ts +68 -0
  74. package/dist/persistence/durability/checkpoint-worker.d.ts +41 -0
  75. package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
  76. package/dist/schema-BVaw7nqz.d.ts +58 -0
  77. package/dist/serialization/crc32-worker.d.ts +20 -0
  78. package/dist/serialization/crc32-worker.mjs +61 -0
  79. package/dist/server.d.ts +218 -0
  80. package/dist/server.mjs +1185 -0
  81. package/dist/types-Bt7c4hDF.d.ts +73 -0
  82. package/dist/vector/hnsw-build-worker.mjs +1 -1
  83. package/dist/workers/entry.mjs +1 -1
  84. package/package.json +37 -2
  85. package/dist/chunk-R2C2EQGO.mjs +0 -53
  86. package/dist/chunk-VVORJOXR.mjs +0 -759
  87. package/dist/direct-executor-LBJLGMJ2.mjs +0 -4
package/dist/index.mjs CHANGED
@@ -1,3236 +1,7 @@
1
- import { createDirectExecutor, createPartitionRouter, validateSchema, getLanguage, deserializePayloadV2, extractVectorFieldsFromSchema, tokenize, fanOutQuery, validateRequiredFields, flattenSchema, getNestedValue, detectRuntime, fnv1a, kWayMerge, createPartitionIndex } from './chunk-KHL4RPC3.mjs';
2
- export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, registerLanguage, resetNormalizationCache } from './chunk-KHL4RPC3.mjs';
3
- export { isSimdAvailable } from './chunk-VVORJOXR.mjs';
4
- import './chunk-I24SGLFS.mjs';
5
- import { ErrorCodes, NarsilError } from './chunk-R2C2EQGO.mjs';
6
- export { ErrorCodes, NarsilError } from './chunk-R2C2EQGO.mjs';
7
- import { generateId, isValidWorkerResponse, createRequestId } from './chunk-HRABJLPB.mjs';
8
- import { decode } from '@msgpack/msgpack';
9
-
10
- // src/engine/embed.ts
11
- function validateVector(vector, targetField, expectedDimensions) {
12
- if (!(vector instanceof Float32Array)) {
13
- throw new NarsilError(
14
- ErrorCodes.EMBEDDING_FAILED,
15
- `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
16
- { field: targetField, returnedType: typeof vector }
17
- );
18
- }
19
- if (vector.length !== expectedDimensions) {
20
- throw new NarsilError(
21
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
22
- `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
23
- { field: targetField, expected: expectedDimensions, actual: vector.length }
24
- );
25
- }
26
- for (let i = 0; i < vector.length; i++) {
27
- if (Number.isNaN(vector[i])) {
28
- throw new NarsilError(
29
- ErrorCodes.EMBEDDING_FAILED,
30
- `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
31
- { field: targetField, index: i }
32
- );
33
- }
34
- }
35
- return vector;
36
- }
37
- function resolveFieldValue(document, path) {
38
- if (!path.includes(".")) return document[path];
39
- const segments = path.split(".");
40
- let current = document;
41
- for (const segment of segments) {
42
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
43
- current = current[segment];
44
- }
45
- return current;
46
- }
47
- function setFieldValue(document, path, value) {
48
- if (!path.includes(".")) {
49
- document[path] = value;
50
- return;
51
- }
52
- const segments = path.split(".");
53
- let current = document;
54
- for (let i = 0; i < segments.length - 1; i++) {
55
- let next = current[segments[i]];
56
- if (next === null || next === void 0 || typeof next !== "object") {
57
- next = {};
58
- current[segments[i]] = next;
59
- }
60
- current = next;
61
- }
62
- current[segments[segments.length - 1]] = value;
63
- }
64
- function deleteFieldValue(document, path) {
65
- if (!path.includes(".")) {
66
- delete document[path];
67
- return;
68
- }
69
- const segments = path.split(".");
70
- let current = document;
71
- for (let i = 0; i < segments.length - 1; i++) {
72
- const next = current[segments[i]];
73
- if (next === null || next === void 0 || typeof next !== "object") return;
74
- current = next;
75
- }
76
- delete current[segments[segments.length - 1]];
77
- }
78
- function collectSourceText(document, sourceFields, targetField, documentIndex) {
79
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
80
- const parts = [];
81
- for (const source of sources) {
82
- const value = resolveFieldValue(document, source);
83
- if (value === void 0 || value === null || value === "") continue;
84
- if (typeof value !== "string") {
85
- throw new NarsilError(
86
- ErrorCodes.DOC_VALIDATION_FAILED,
87
- `Embedding source field "${source}" must be a string, got ${typeof value}`,
88
- { field: source, targetField }
89
- );
90
- }
91
- parts.push(value);
92
- }
93
- if (parts.length === 0) {
94
- const details = { field: targetField };
95
- if (documentIndex !== void 0) {
96
- details.documentIndex = documentIndex;
97
- }
98
- throw new NarsilError(
99
- ErrorCodes.EMBEDDING_NO_SOURCE,
100
- `All source fields for embedding target "${targetField}" are missing or empty`,
101
- details
102
- );
103
- }
104
- return parts.join("\n");
105
- }
106
- function wrapAdapterError(err, targetField) {
107
- if (err instanceof NarsilError) return err;
108
- const message = err instanceof Error ? err.message : String(err);
109
- return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
110
- field: targetField,
111
- cause: message
112
- });
113
- }
114
- async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
115
- const embeddedFields = /* @__PURE__ */ new Set();
116
- try {
117
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
118
- const existing = resolveFieldValue(document, targetField);
119
- if (existing !== void 0 && existing !== null) continue;
120
- const text = collectSourceText(document, sourceFields, targetField);
121
- let vector;
122
- try {
123
- const raw = await adapter.embed(text, "document", signal);
124
- vector = validateVector(raw, targetField, adapter.dimensions);
125
- } catch (err) {
126
- throw wrapAdapterError(err, targetField);
127
- }
128
- setFieldValue(document, targetField, vector);
129
- embeddedFields.add(targetField);
130
- }
131
- } catch (err) {
132
- for (const field of embeddedFields) {
133
- deleteFieldValue(document, field);
134
- }
135
- throw err;
136
- }
137
- return embeddedFields;
138
- }
139
- var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
140
- function isDocumentSourceError(err) {
141
- return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
142
- }
143
- async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
144
- const embedded = /* @__PURE__ */ new Map();
145
- const failed = /* @__PURE__ */ new Map();
146
- try {
147
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
148
- const needsEmbedding = [];
149
- for (let i = 0; i < documents.length; i++) {
150
- if (failed.has(i)) continue;
151
- const doc = documents[i];
152
- const existing = resolveFieldValue(doc, targetField);
153
- if (existing !== void 0 && existing !== null) continue;
154
- try {
155
- const text = collectSourceText(doc, sourceFields, targetField, i);
156
- needsEmbedding.push({ docIndex: i, text });
157
- } catch (err) {
158
- if (isDocumentSourceError(err)) {
159
- failed.set(i, err);
160
- } else {
161
- throw err;
162
- }
163
- }
164
- }
165
- if (needsEmbedding.length === 0) continue;
166
- const texts = needsEmbedding.map((entry) => entry.text);
167
- let vectors;
168
- try {
169
- if (adapter.embedBatch) {
170
- vectors = await adapter.embedBatch(texts, "document", signal);
171
- } else {
172
- vectors = [];
173
- const concurrency = 8;
174
- for (let start = 0; start < texts.length; start += concurrency) {
175
- const chunk = texts.slice(start, start + concurrency);
176
- const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
177
- vectors.push(...chunkResults);
178
- }
179
- }
180
- } catch (err) {
181
- throw wrapAdapterError(err, targetField);
182
- }
183
- if (vectors.length !== needsEmbedding.length) {
184
- throw new NarsilError(
185
- ErrorCodes.EMBEDDING_FAILED,
186
- `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
187
- { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
188
- );
189
- }
190
- for (let j = 0; j < needsEmbedding.length; j++) {
191
- const { docIndex } = needsEmbedding[j];
192
- const validated = validateVector(vectors[j], targetField, adapter.dimensions);
193
- setFieldValue(documents[docIndex], targetField, validated);
194
- let fieldSet = embedded.get(docIndex);
195
- if (!fieldSet) {
196
- fieldSet = /* @__PURE__ */ new Set();
197
- embedded.set(docIndex, fieldSet);
198
- }
199
- fieldSet.add(targetField);
200
- }
201
- }
202
- } catch (err) {
203
- for (const [docIndex, fields] of embedded) {
204
- for (const field of fields) {
205
- deleteFieldValue(documents[docIndex], field);
206
- }
207
- }
208
- throw err;
209
- }
210
- return { embedded, failed };
211
- }
212
-
213
- // src/engine/validation.ts
214
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
215
- var MAX_INDEX_NAME_LENGTH = 256;
216
- var MAX_DOC_ID_LENGTH = 512;
217
- var BATCH_CHUNK_SIZE = 1e3;
218
- var MAX_LIMIT = 1e4;
219
- var MAX_OFFSET = 1e5;
220
- var DEFAULT_LIMIT = 10;
221
- var DEFAULT_OFFSET = 0;
222
- function now() {
223
- if (typeof performance !== "undefined" && typeof performance.now === "function") {
224
- return performance.now();
225
- }
226
- return Date.now();
227
- }
228
- function validateIndexName(name) {
229
- if (!name || name.length === 0) {
230
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
231
- }
232
- if (name.length > MAX_INDEX_NAME_LENGTH) {
233
- throw new NarsilError(
234
- ErrorCodes.INDEX_NOT_FOUND,
235
- `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
236
- {
237
- indexName: name,
238
- length: name.length
239
- }
240
- );
241
- }
242
- if (!INDEX_NAME_PATTERN.test(name)) {
243
- throw new NarsilError(
244
- ErrorCodes.INDEX_NOT_FOUND,
245
- `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
246
- { indexName: name }
247
- );
248
- }
249
- if (name.includes("..")) {
250
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
251
- }
252
- }
253
- function validateDocId(docId) {
254
- if (!docId || docId.length === 0) {
255
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
256
- }
257
- if (docId.length > MAX_DOC_ID_LENGTH) {
258
- throw new NarsilError(
259
- ErrorCodes.DOC_VALIDATION_FAILED,
260
- `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
261
- {
262
- docId,
263
- length: docId.length
264
- }
265
- );
266
- }
267
- if (docId.includes("\0")) {
268
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
269
- }
270
- }
271
- function clampLimit(limit) {
272
- if (limit === void 0) return DEFAULT_LIMIT;
273
- return Math.max(0, Math.min(limit, MAX_LIMIT));
274
- }
275
- function clampOffset(offset) {
276
- if (offset === void 0) return DEFAULT_OFFSET;
277
- return Math.max(0, Math.min(offset, MAX_OFFSET));
278
- }
279
-
280
- // src/engine/vector-coordinator.ts
281
- function extractVectorFromDoc(document, fieldPath) {
282
- const value = getNestedValue(document, fieldPath);
283
- if (value === void 0 || value === null) return null;
284
- if (value instanceof Float32Array) return value;
285
- if (!Array.isArray(value)) return null;
286
- const arr = new Float32Array(value.length);
287
- for (let i = 0; i < value.length; i++) {
288
- const n = value[i];
289
- if (typeof n !== "number" || !Number.isFinite(n)) return null;
290
- arr[i] = n;
291
- }
292
- return arr;
293
- }
294
- function deleteNestedValue(obj, path) {
295
- if (!path.includes(".")) {
296
- delete obj[path];
297
- return;
298
- }
299
- const segments = path.split(".");
300
- let current = obj;
301
- for (let i = 0; i < segments.length - 1; i++) {
302
- const next = current[segments[i]];
303
- if (next === null || next === void 0 || typeof next !== "object") return;
304
- current = next;
305
- }
306
- delete current[segments[segments.length - 1]];
307
- }
308
- function vectorsEqual(a, b) {
309
- if (!a) return false;
310
- if (a.length !== b.length) return false;
311
- for (let i = 0; i < a.length; i++) {
312
- if (a[i] !== b[i]) return false;
313
- }
314
- return true;
315
- }
316
- function validateVectorDimensions(vectors, vecIndexes) {
317
- for (const [fieldPath, vector] of vectors) {
318
- const vecIndex = vecIndexes.get(fieldPath);
319
- if (!vecIndex) continue;
320
- if (vector.length !== vecIndex.dimension) {
321
- throw new NarsilError(
322
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
323
- `Vector dimension mismatch for field "${fieldPath}": expected ${vecIndex.dimension}, got ${vector.length}`,
324
- { field: fieldPath, expected: vecIndex.dimension, received: vector.length }
325
- );
326
- }
327
- }
328
- }
329
- function prepareDocumentVectors(document, vectorFieldPaths, vecIndexes) {
330
- if (vecIndexes.size === 0) {
331
- return { partitionDoc: document, extractedVectors: /* @__PURE__ */ new Map() };
332
- }
333
- const extractedVectors = /* @__PURE__ */ new Map();
334
- for (const fieldPath of vectorFieldPaths) {
335
- const vec = extractVectorFromDoc(document, fieldPath);
336
- if (vec) {
337
- extractedVectors.set(fieldPath, vec);
338
- }
339
- }
340
- if (extractedVectors.size === 0) {
341
- return { partitionDoc: document, extractedVectors };
342
- }
343
- const partitionDoc = shallowCopyExcluding(document, extractedVectors.keys());
344
- return { partitionDoc, extractedVectors };
345
- }
346
- function shallowCopyExcluding(document, fieldPaths) {
347
- const topLevelExcludes = /* @__PURE__ */ new Set();
348
- const nestedPaths = [];
349
- for (const path of fieldPaths) {
350
- if (!path.includes(".")) {
351
- topLevelExcludes.add(path);
352
- } else {
353
- nestedPaths.push(path);
354
- }
355
- }
356
- const result = {};
357
- for (const key of Object.keys(document)) {
358
- if (topLevelExcludes.has(key)) continue;
359
- result[key] = document[key];
360
- }
361
- for (const path of nestedPaths) {
362
- const segments = path.split(".");
363
- const topKey = segments[0];
364
- if (!(topKey in result) || result[topKey] === null || typeof result[topKey] !== "object") continue;
365
- const source = document[topKey];
366
- let target = { ...source };
367
- result[topKey] = target;
368
- for (let i = 1; i < segments.length - 1; i++) {
369
- const seg = segments[i];
370
- const next = target[seg];
371
- if (next === null || next === void 0 || typeof next !== "object") break;
372
- const copy = { ...next };
373
- target[seg] = copy;
374
- target = copy;
375
- }
376
- delete target[segments[segments.length - 1]];
377
- }
378
- return result;
379
- }
380
- function insertDocumentVectors(docId, vectors, vecIndexes) {
381
- const insertedFields = [];
382
- try {
383
- for (const [fieldPath, vector] of vectors) {
384
- const vecIndex = vecIndexes.get(fieldPath);
385
- if (!vecIndex) continue;
386
- vecIndex.insert(docId, vector);
387
- insertedFields.push(fieldPath);
388
- }
389
- } catch (err) {
390
- for (const fieldPath of insertedFields) {
391
- const vecIndex = vecIndexes.get(fieldPath);
392
- if (vecIndex) {
393
- vecIndex.remove(docId);
394
- }
395
- }
396
- throw err;
397
- }
398
- return insertedFields;
399
- }
400
- function removeDocumentVectors(docId, vecIndexes) {
401
- for (const [, vecIndex] of vecIndexes) {
402
- vecIndex.remove(docId);
403
- }
404
- }
405
- function updateDocumentVectors(docId, vectors, vecIndexes) {
406
- const updatedFields = [];
407
- try {
408
- for (const [fieldPath, newVec] of vectors) {
409
- const vecIndex = vecIndexes.get(fieldPath);
410
- if (!vecIndex) continue;
411
- if (newVec === null) {
412
- if (vecIndex.has(docId)) {
413
- const oldVec2 = vecIndex.getVector(docId);
414
- vecIndex.remove(docId);
415
- updatedFields.push({ fieldPath, oldVec: oldVec2 });
416
- }
417
- continue;
418
- }
419
- const oldVec = vecIndex.getVector(docId);
420
- if (vectorsEqual(oldVec, newVec)) continue;
421
- vecIndex.remove(docId);
422
- try {
423
- vecIndex.insert(docId, newVec);
424
- updatedFields.push({ fieldPath, oldVec });
425
- } catch (err) {
426
- if (oldVec) {
427
- vecIndex.insert(docId, oldVec);
428
- }
429
- throw err;
430
- }
431
- }
432
- } catch (err) {
433
- for (const { fieldPath, oldVec } of updatedFields) {
434
- const vecIndex = vecIndexes.get(fieldPath);
435
- if (!vecIndex) continue;
436
- vecIndex.remove(docId);
437
- if (oldVec) {
438
- vecIndex.insert(docId, oldVec);
439
- }
440
- }
441
- throw err;
442
- }
443
- }
444
-
445
- // src/engine/mutations.ts
446
- async function insertDocument(ctx, indexName, document, docId, options) {
447
- ctx.guardShutdown();
448
- const entry = ctx.requireIndex(indexName);
449
- const resolvedDocId = docId ?? ctx.idGenerator();
450
- validateDocId(resolvedDocId);
451
- if (ctx.bufferIfRebalancing(indexName, { action: "insert", docId: resolvedDocId, document, indexName })) {
452
- return resolvedDocId;
453
- }
454
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document });
455
- if (entry.config.required && entry.config.required.length > 0) {
456
- validateRequiredFields(document, entry.config.required);
457
- }
458
- if (entry.embeddingAdapter && entry.config.embedding) {
459
- await embedDocumentFields(
460
- document,
461
- entry.config.embedding,
462
- entry.embeddingAdapter,
463
- ctx.abortController.signal
464
- );
465
- }
466
- const insertManager = ctx.requireManager(indexName);
467
- const insertVecIndexes = insertManager.getVectorIndexes();
468
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
469
- document,
470
- entry.vectorFieldPaths,
471
- insertVecIndexes
472
- );
473
- if (extractedVectors.size > 0) {
474
- validateVectorDimensions(extractedVectors, insertVecIndexes);
475
- }
476
- await ctx.executor.execute({
477
- type: "insert",
478
- indexName,
479
- docId: resolvedDocId,
480
- document: partitionDoc,
481
- requestId: resolvedDocId,
482
- skipClone: extractedVectors.size > 0 ? true : options?.skipClone
483
- });
484
- try {
485
- insertDocumentVectors(resolvedDocId, extractedVectors, insertVecIndexes);
486
- } catch (err) {
487
- try {
488
- await ctx.executor.execute({ type: "remove", indexName, docId: resolvedDocId, requestId: resolvedDocId });
489
- } catch (rollbackErr) {
490
- console.warn(
491
- `Rollback failed for doc "${resolvedDocId}" during insert atomicity:`,
492
- rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
493
- );
494
- }
495
- throw err;
496
- }
497
- try {
498
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: resolvedDocId, document });
499
- } catch (err) {
500
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
501
- }
502
- ctx.flushManager?.markDirty(indexName, 0);
503
- await ctx.orchestrator.replicateToWorkers({
504
- type: "insert",
505
- indexName,
506
- docId: resolvedDocId,
507
- document,
508
- requestId: `replicate-insert-${resolvedDocId}`,
509
- skipClone: options?.skipClone
510
- });
511
- for (const fieldPath of extractedVectors.keys()) {
512
- const vecIndex = insertVecIndexes.get(fieldPath);
513
- if (vecIndex) {
514
- vecIndex.scheduleBuild();
515
- }
516
- }
517
- await ctx.orchestrator.checkPromotion();
518
- return resolvedDocId;
519
- }
520
- async function insertDocumentBatch(ctx, indexName, documents, options) {
521
- ctx.guardShutdown();
522
- const entry = ctx.requireIndex(indexName);
523
- const succeeded = [];
524
- const succeededDocs = [];
525
- const failed = [];
526
- const hasBeforeHook = ctx.pluginRegistry.hasHooks("beforeInsert");
527
- const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
528
- const hasRequired = entry.config.required && entry.config.required.length > 0;
529
- const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
530
- const batchManager = ctx.requireManager(indexName);
531
- const batchVecIndexes = batchManager.getVectorIndexes();
532
- const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
533
- const touchedVectorFields = /* @__PURE__ */ new Set();
534
- for (let chunkStart = 0; chunkStart < documents.length; chunkStart += BATCH_CHUNK_SIZE) {
535
- if (ctx.abortController.signal.aborted) break;
536
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, documents.length);
537
- const chunkFailedIndexes = /* @__PURE__ */ new Set();
538
- if (hasRequired) {
539
- for (let i = chunkStart; i < chunkEnd; i++) {
540
- try {
541
- validateRequiredFields(documents[i], entry.config.required);
542
- } catch (err) {
543
- chunkFailedIndexes.add(i);
544
- failed.push({
545
- docId: "",
546
- error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
547
- });
548
- }
549
- }
550
- }
551
- if (hasEmbedding) {
552
- const embeddableSlice = [];
553
- const embeddableOriginalIndexes = [];
554
- for (let i = chunkStart; i < chunkEnd; i++) {
555
- if (chunkFailedIndexes.has(i)) continue;
556
- embeddableSlice.push(documents[i]);
557
- embeddableOriginalIndexes.push(i);
558
- }
559
- if (embeddableSlice.length > 0) {
560
- try {
561
- const embedResult = await embedBatchDocumentFields(
562
- embeddableSlice,
563
- entry.config.embedding,
564
- entry.embeddingAdapter,
565
- ctx.abortController.signal
566
- );
567
- for (const [sliceIndex, error] of embedResult.failed) {
568
- const originalIdx = embeddableOriginalIndexes[sliceIndex];
569
- chunkFailedIndexes.add(originalIdx);
570
- failed.push({ docId: "", error });
571
- }
572
- } catch (err) {
573
- const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
574
- for (const originalIdx of embeddableOriginalIndexes) {
575
- chunkFailedIndexes.add(originalIdx);
576
- failed.push({ docId: "", error: embeddingError });
577
- }
578
- }
579
- }
580
- }
581
- for (let i = chunkStart; i < chunkEnd; i++) {
582
- if (ctx.abortController.signal.aborted) break;
583
- if (chunkFailedIndexes.has(i)) continue;
584
- const batchDocId = ctx.idGenerator();
585
- try {
586
- validateDocId(batchDocId);
587
- if (hasBeforeHook) {
588
- await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
589
- }
590
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
591
- documents[i],
592
- batchVectorFieldPaths,
593
- batchVecIndexes
594
- );
595
- if (extractedVectors.size > 0) {
596
- validateVectorDimensions(extractedVectors, batchVecIndexes);
597
- }
598
- const result = ctx.executor.execute({
599
- type: "insert",
600
- indexName,
601
- docId: batchDocId,
602
- document: partitionDoc,
603
- requestId: batchDocId,
604
- skipClone: extractedVectors.size > 0 ? true : options?.skipClone
605
- });
606
- if (result && typeof result.then === "function") {
607
- await result;
608
- }
609
- try {
610
- insertDocumentVectors(batchDocId, extractedVectors, batchVecIndexes);
611
- } catch (vecErr) {
612
- try {
613
- await ctx.executor.execute({ type: "remove", indexName, docId: batchDocId, requestId: batchDocId });
614
- } catch (rollbackErr) {
615
- console.warn(
616
- `Rollback failed for doc "${batchDocId}" during batch insert atomicity:`,
617
- rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
618
- );
619
- }
620
- throw vecErr;
621
- }
622
- for (const fieldPath of extractedVectors.keys()) {
623
- touchedVectorFields.add(fieldPath);
624
- }
625
- if (hasAfterHook) {
626
- try {
627
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: batchDocId, document: documents[i] });
628
- } catch (err) {
629
- console.warn("afterInsert plugin hook error:", err instanceof Error ? err.message : String(err));
630
- }
631
- }
632
- succeeded.push(batchDocId);
633
- succeededDocs.push(documents[i]);
634
- } catch (err) {
635
- failed.push({
636
- docId: batchDocId,
637
- error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
638
- });
639
- }
640
- }
641
- if (chunkEnd < documents.length) {
642
- await new Promise((r) => setTimeout(r, 0));
643
- }
644
- }
645
- ctx.flushManager?.markDirty(indexName, 0);
646
- for (let i = 0; i < succeeded.length; i++) {
647
- await ctx.orchestrator.replicateToWorkers({
648
- type: "insert",
649
- indexName,
650
- docId: succeeded[i],
651
- document: succeededDocs[i],
652
- requestId: `replicate-insert-${succeeded[i]}`,
653
- skipClone: options?.skipClone
654
- });
655
- }
656
- for (const fieldPath of touchedVectorFields) {
657
- const vecIndex = batchVecIndexes.get(fieldPath);
658
- if (vecIndex) {
659
- vecIndex.scheduleBuild();
660
- }
661
- }
662
- await ctx.orchestrator.checkPromotion();
663
- return { succeeded, failed };
664
- }
665
- async function removeDocument(ctx, indexName, docId) {
666
- ctx.guardShutdown();
667
- ctx.requireIndex(indexName);
668
- validateDocId(docId);
669
- if (ctx.bufferIfRebalancing(indexName, { action: "remove", docId, indexName })) {
670
- return;
671
- }
672
- await ctx.pluginRegistry.runHook("beforeRemove", { indexName, docId });
673
- await ctx.executor.execute({ type: "remove", indexName, docId, requestId: docId });
674
- const removeManager = ctx.requireManager(indexName);
675
- const removeVecIndexes = removeManager.getVectorIndexes();
676
- removeDocumentVectors(docId, removeVecIndexes);
677
- try {
678
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId });
679
- } catch (err) {
680
- console.warn("afterRemove plugin hook error:", err instanceof Error ? err.message : String(err));
681
- }
682
- ctx.flushManager?.markDirty(indexName, 0);
683
- await ctx.orchestrator.replicateToWorkers({
684
- type: "remove",
685
- indexName,
686
- docId,
687
- requestId: `replicate-remove-${docId}`
688
- });
689
- }
690
- async function removeDocumentBatch(ctx, indexName, docIds) {
691
- ctx.guardShutdown();
692
- ctx.requireIndex(indexName);
693
- const succeeded = [];
694
- const failed = [];
695
- const manager = ctx.requireManager(indexName);
696
- manager.beginBatchRemove();
697
- try {
698
- for (let chunkStart = 0; chunkStart < docIds.length; chunkStart += BATCH_CHUNK_SIZE) {
699
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, docIds.length);
700
- for (let i = chunkStart; i < chunkEnd; i++) {
701
- try {
702
- await removeDocument(ctx, indexName, docIds[i]);
703
- succeeded.push(docIds[i]);
704
- } catch (err) {
705
- failed.push({
706
- docId: docIds[i],
707
- error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err))
708
- });
709
- }
710
- }
711
- if (chunkEnd < docIds.length) {
712
- await new Promise((r) => setTimeout(r, 0));
713
- }
714
- }
715
- } finally {
716
- manager.endBatchRemove();
717
- }
718
- return { succeeded, failed };
719
- }
720
- async function updateDocument(ctx, indexName, docId, document) {
721
- ctx.guardShutdown();
722
- const entry = ctx.requireIndex(indexName);
723
- validateDocId(docId);
724
- if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
725
- return;
726
- }
727
- if (entry.embeddingAdapter && entry.config.embedding) {
728
- await embedDocumentFields(
729
- document,
730
- entry.config.embedding,
731
- entry.embeddingAdapter,
732
- ctx.abortController.signal
733
- );
734
- }
735
- const updateManager = ctx.requireManager(indexName);
736
- const oldDocument = updateManager.get(docId);
737
- const oldPartitionDoc = updateManager.getRef(docId);
738
- const rollbackDoc = oldPartitionDoc ? structuredClone(oldPartitionDoc) : void 0;
739
- await ctx.pluginRegistry.runHook("beforeUpdate", {
740
- indexName,
741
- docId,
742
- oldDocument: oldDocument ?? {},
743
- newDocument: document
744
- });
745
- const updateVecIndexes = updateManager.getVectorIndexes();
746
- const updateExtractedVectors = /* @__PURE__ */ new Map();
747
- if (updateVecIndexes.size > 0) {
748
- const dimensionCheckVectors = /* @__PURE__ */ new Map();
749
- for (const fieldPath of entry.vectorFieldPaths) {
750
- const newVec = extractVectorFromDocForUpdate(document, fieldPath);
751
- updateExtractedVectors.set(fieldPath, newVec);
752
- if (newVec) {
753
- dimensionCheckVectors.set(fieldPath, newVec);
754
- }
755
- }
756
- if (dimensionCheckVectors.size > 0) {
757
- validateVectorDimensions(dimensionCheckVectors, updateVecIndexes);
758
- }
759
- }
760
- const { partitionDoc } = prepareUpdatePartitionDoc(document, updateExtractedVectors);
761
- await ctx.executor.execute({
762
- type: "update",
763
- indexName,
764
- docId,
765
- document: partitionDoc,
766
- requestId: docId
767
- });
768
- try {
769
- updateDocumentVectors(docId, updateExtractedVectors, updateVecIndexes);
770
- } catch (err) {
771
- if (rollbackDoc) {
772
- try {
773
- await ctx.executor.execute({
774
- type: "update",
775
- indexName,
776
- docId,
777
- document: rollbackDoc,
778
- requestId: docId
779
- });
780
- } catch (rollbackErr) {
781
- console.warn(
782
- `Rollback failed for doc "${docId}" during update atomicity:`,
783
- rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)
784
- );
785
- }
786
- }
787
- throw err;
788
- }
789
- try {
790
- await ctx.pluginRegistry.runHook("afterUpdate", {
791
- indexName,
792
- docId,
793
- oldDocument: oldDocument ?? {},
794
- newDocument: document
795
- });
796
- } catch (err) {
797
- console.warn("afterUpdate plugin hook error:", err instanceof Error ? err.message : String(err));
798
- }
799
- ctx.flushManager?.markDirty(indexName, 0);
800
- await ctx.orchestrator.replicateToWorkers({
801
- type: "update",
802
- indexName,
803
- docId,
804
- document,
805
- requestId: `replicate-update-${docId}`
806
- });
807
- for (const [fieldPath, vec] of updateExtractedVectors) {
808
- if (vec === null) continue;
809
- const vecIndex = updateVecIndexes.get(fieldPath);
810
- if (vecIndex) {
811
- vecIndex.scheduleBuild();
812
- }
813
- }
814
- }
815
- async function updateDocumentBatch(ctx, indexName, updates) {
816
- ctx.guardShutdown();
817
- const entry = ctx.requireIndex(indexName);
818
- const succeeded = [];
819
- const failed = [];
820
- const updateBatchManager = ctx.requireManager(indexName);
821
- const updateBatchVecIndexes = updateBatchManager.getVectorIndexes();
822
- const touchedVectorFields = /* @__PURE__ */ new Set();
823
- for (let chunkStart = 0; chunkStart < updates.length; chunkStart += BATCH_CHUNK_SIZE) {
824
- const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
825
- for (let i = chunkStart; i < chunkEnd; i++) {
826
- try {
827
- await updateDocument(ctx, indexName, updates[i].docId, updates[i].document);
828
- succeeded.push(updates[i].docId);
829
- if (updateBatchVecIndexes.size > 0) {
830
- for (const fieldPath of entry.vectorFieldPaths) {
831
- const vec = extractVectorFromDocForUpdate(updates[i].document, fieldPath);
832
- if (vec !== null) {
833
- touchedVectorFields.add(fieldPath);
834
- }
835
- }
836
- }
837
- } catch (err) {
838
- failed.push({
839
- docId: updates[i].docId,
840
- error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_NOT_FOUND, String(err))
841
- });
842
- }
843
- }
844
- if (chunkEnd < updates.length) {
845
- await new Promise((r) => setTimeout(r, 0));
846
- }
847
- }
848
- for (const fieldPath of touchedVectorFields) {
849
- const vecIndex = updateBatchVecIndexes.get(fieldPath);
850
- if (vecIndex) {
851
- vecIndex.scheduleBuild();
852
- }
853
- }
854
- return { succeeded, failed };
855
- }
856
- function extractVectorFromDocForUpdate(document, fieldPath) {
857
- return extractVectorFromDoc(document, fieldPath);
858
- }
859
- function prepareUpdatePartitionDoc(document, extractedVectors) {
860
- if (extractedVectors.size === 0) {
861
- return { partitionDoc: document };
862
- }
863
- const partitionDoc = structuredClone(document);
864
- for (const fieldPath of extractedVectors.keys()) {
865
- deleteNestedValue(partitionDoc, fieldPath);
866
- }
867
- return { partitionDoc };
868
- }
869
-
870
- // src/workers/worker-executor.ts
871
- var DEFAULT_BACKPRESSURE_LIMIT = 100;
872
- var DEFAULT_REQUEST_TIMEOUT = 3e4;
873
- var SHUTDOWN_TIMEOUT = 5e3;
874
- function createWorkerExecutor(worker, config) {
875
- const backpressureLimit = DEFAULT_BACKPRESSURE_LIMIT;
876
- const requestTimeout = DEFAULT_REQUEST_TIMEOUT;
877
- const pending = /* @__PURE__ */ new Map();
878
- function processResponse(msg) {
879
- if (!isValidWorkerResponse(msg)) {
880
- return;
881
- }
882
- const response = msg;
883
- const entry = pending.get(response.requestId);
884
- if (!entry) {
885
- return;
886
- }
887
- clearTimeout(entry.timeoutId);
888
- pending.delete(response.requestId);
889
- if (response.type === "error") {
890
- entry.reject(new NarsilError(response.code, response.message));
891
- } else {
892
- entry.resolve(response.data);
893
- }
894
- }
895
- if (typeof worker.on === "function") {
896
- worker.on("message", (msg) => processResponse(msg));
897
- } else if (typeof worker.addEventListener === "function") {
898
- worker.addEventListener("message", (event) => {
899
- const msg = event.data;
900
- processResponse(msg);
901
- });
902
- }
903
- function execute(action) {
904
- if (pending.size >= backpressureLimit) {
905
- return Promise.reject(
906
- new NarsilError(ErrorCodes.WORKER_BUSY, `Backpressure limit of ${backpressureLimit} pending requests reached`)
907
- );
908
- }
909
- const requestId = createRequestId();
910
- const taggedAction = { ...action, requestId };
911
- return new Promise((resolve, reject) => {
912
- const timeoutId = setTimeout(() => {
913
- pending.delete(requestId);
914
- reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, `Request ${requestId} timed out after ${requestTimeout}ms`));
915
- }, requestTimeout);
916
- pending.set(requestId, {
917
- resolve,
918
- reject,
919
- timeoutId
920
- });
921
- worker.postMessage(taggedAction);
922
- });
923
- }
924
- async function shutdown() {
925
- const requestId = createRequestId();
926
- const shutdownAction = { type: "shutdown", requestId };
927
- const shutdownPromise = new Promise((resolve, reject) => {
928
- const timeoutId = setTimeout(() => {
929
- pending.delete(requestId);
930
- reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, "Shutdown timed out"));
931
- }, SHUTDOWN_TIMEOUT);
932
- pending.set(requestId, {
933
- resolve: () => resolve(),
934
- reject,
935
- timeoutId
936
- });
937
- worker.postMessage(shutdownAction);
938
- });
939
- try {
940
- await shutdownPromise;
941
- } finally {
942
- for (const [id, entry] of pending) {
943
- clearTimeout(entry.timeoutId);
944
- entry.reject(new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker is shutting down"));
945
- pending.delete(id);
946
- }
947
- }
948
- }
949
- return { execute, shutdown };
950
- }
951
-
952
- // src/workers/factory.ts
953
- function resolveEntryPoint() {
954
- const base = import.meta.url;
955
- const distIndex = base.lastIndexOf("/dist/");
956
- if (distIndex !== -1) {
957
- return new URL("workers/entry.mjs", base.slice(0, distIndex + 6)).href;
958
- }
959
- return base.replace(/\/src\/workers\/[^/]+$/, "/dist/workers/entry.mjs");
960
- }
961
- async function createWorkerFactory(entryPoint) {
962
- const runtime = detectRuntime();
963
- const resolvedEntry = resolveEntryPoint();
964
- if (runtime.supportsWorkerThreads) {
965
- const workerThreadsModule = await import('worker_threads');
966
- return function nodeFactory(_workerId) {
967
- const instance = new workerThreadsModule.Worker(new URL(resolvedEntry));
968
- return createWorkerExecutor(instance);
969
- };
970
- }
971
- if (runtime.supportsWebWorkers) {
972
- return function webFactory(_workerId) {
973
- const instance = new Worker(resolvedEntry, { type: "module" });
974
- return createWorkerExecutor(instance);
975
- };
976
- }
977
- throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
978
- }
979
-
980
- // src/workers/pool.ts
981
- function resolveWorkerCount(requested) {
982
- if (requested !== void 0 && requested > 0) {
983
- return requested;
984
- }
985
- let cpuCount = 4;
986
- try {
987
- if (navigator?.hardwareConcurrency) {
988
- cpuCount = navigator.hardwareConcurrency;
989
- } else if (typeof process !== "undefined") {
990
- const ap = process.availableParallelism;
991
- if (typeof ap === "function") {
992
- cpuCount = ap();
993
- }
994
- }
995
- } catch {
996
- cpuCount = 4;
997
- }
998
- return Math.max(2, Math.min(8, cpuCount - 1));
999
- }
1000
- function createWorkerPool(config) {
1001
- const workerCount = resolveWorkerCount(config.count);
1002
- const workers = [];
1003
- const indexAssignment = /* @__PURE__ */ new Map();
1004
- let isShutdown = false;
1005
- function ensureWorker(slotIndex) {
1006
- if (!workers[slotIndex]) {
1007
- workers[slotIndex] = {
1008
- executor: config.workerFactory(slotIndex),
1009
- indexes: /* @__PURE__ */ new Set(),
1010
- memoryStats: { heapUsed: 0, heapTotal: 0, external: 0 }
1011
- };
1012
- }
1013
- return workers[slotIndex];
1014
- }
1015
- function assignSlot(indexName) {
1016
- return fnv1a(indexName) % workerCount;
1017
- }
1018
- function addIndex(indexName) {
1019
- if (isShutdown) {
1020
- throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
1021
- }
1022
- if (indexAssignment.has(indexName)) {
1023
- return;
1024
- }
1025
- const slotIndex = assignSlot(indexName);
1026
- const slot = ensureWorker(slotIndex);
1027
- slot.indexes.add(indexName);
1028
- indexAssignment.set(indexName, slotIndex);
1029
- }
1030
- function removeIndex(indexName) {
1031
- const slotIndex = indexAssignment.get(indexName);
1032
- if (slotIndex === void 0) {
1033
- return;
1034
- }
1035
- const slot = workers[slotIndex];
1036
- if (slot) {
1037
- slot.indexes.delete(indexName);
1038
- }
1039
- indexAssignment.delete(indexName);
1040
- }
1041
- function getExecutor(indexName) {
1042
- if (isShutdown) {
1043
- throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
1044
- }
1045
- const slotIndex = indexAssignment.get(indexName);
1046
- if (slotIndex === void 0) {
1047
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
1048
- }
1049
- return workers[slotIndex].executor;
1050
- }
1051
- function getMemoryStats() {
1052
- const stats = [];
1053
- for (let i = 0; i < workers.length; i++) {
1054
- const slot = workers[i];
1055
- if (slot) {
1056
- stats.push({
1057
- workerId: i,
1058
- heapUsed: slot.memoryStats.heapUsed,
1059
- heapTotal: slot.memoryStats.heapTotal,
1060
- external: slot.memoryStats.external
1061
- });
1062
- }
1063
- }
1064
- return stats;
1065
- }
1066
- async function shutdown() {
1067
- if (isShutdown) {
1068
- return;
1069
- }
1070
- isShutdown = true;
1071
- const shutdownPromises = workers.filter(Boolean).map((slot) => {
1072
- const timeoutPromise = new Promise((resolve) => {
1073
- setTimeout(resolve, 5e3);
1074
- });
1075
- return Promise.race([slot.executor.shutdown(), timeoutPromise]);
1076
- });
1077
- await Promise.allSettled(shutdownPromises);
1078
- indexAssignment.clear();
1079
- }
1080
- function getAllExecutors() {
1081
- return workers.filter(Boolean).map((slot) => slot.executor);
1082
- }
1083
- function addIndexToAll(indexName) {
1084
- if (isShutdown) {
1085
- throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
1086
- }
1087
- for (let i = 0; i < workerCount; i++) {
1088
- const slot = ensureWorker(i);
1089
- slot.indexes.add(indexName);
1090
- }
1091
- indexAssignment.set(indexName, 0);
1092
- }
1093
- return {
1094
- getExecutor,
1095
- getAllExecutors,
1096
- get workerCount() {
1097
- return workerCount;
1098
- },
1099
- addIndex,
1100
- addIndexToAll,
1101
- removeIndex,
1102
- getMemoryStats,
1103
- shutdown
1104
- };
1105
- }
1106
-
1107
- // src/engine/orchestration.ts
1108
- function createWorkerOrchestrator(config, executor, promoter, indexRegistry, callbacks) {
1109
- let workerPool = null;
1110
- let promotionInProgress = false;
1111
- const promotionBuffer = [];
1112
- const workersEnabled = config?.workers?.enabled === true;
1113
- async function checkPromotion() {
1114
- if (!workersEnabled || promotionInProgress || workerPool) return;
1115
- const indexMap = /* @__PURE__ */ new Map();
1116
- for (const [name] of indexRegistry) {
1117
- const mgr = executor.getManager(name);
1118
- indexMap.set(name, { documentCount: mgr?.countDocuments() ?? 0 });
1119
- }
1120
- const result = promoter.check(indexMap);
1121
- if (result.shouldPromote) {
1122
- promotionInProgress = true;
1123
- setTimeout(() => {
1124
- runPromotion(result.reason).catch((err) => {
1125
- console.warn("Worker promotion failed:", err);
1126
- promotionInProgress = false;
1127
- });
1128
- }, 0);
1129
- }
1130
- }
1131
- async function runPromotion(reason) {
1132
- try {
1133
- const factory = await createWorkerFactory();
1134
- const pool = createWorkerPool({
1135
- count: config?.workers?.count,
1136
- workerFactory: factory
1137
- });
1138
- for (const [name] of indexRegistry) {
1139
- pool.addIndexToAll(name);
1140
- }
1141
- const allExecutors = pool.getAllExecutors();
1142
- for (const [name, entry] of indexRegistry) {
1143
- await Promise.all(
1144
- allExecutors.map(
1145
- (workerExecutor) => workerExecutor.execute({
1146
- type: "createIndex",
1147
- indexName: name,
1148
- config: entry.config,
1149
- requestId: `promote-create-${name}`
1150
- })
1151
- )
1152
- );
1153
- const manager = executor.getManager(name);
1154
- if (manager) {
1155
- for (let i = 0; i < manager.partitionCount; i++) {
1156
- const serialized = manager.serializePartition(i);
1157
- await Promise.all(
1158
- allExecutors.map(
1159
- (workerExecutor) => workerExecutor.execute({
1160
- type: "deserialize",
1161
- indexName: name,
1162
- partitionId: i,
1163
- data: serialized,
1164
- requestId: `promote-sync-${name}-${i}`
1165
- })
1166
- )
1167
- );
1168
- }
1169
- }
1170
- }
1171
- workerPool = pool;
1172
- promoter.markPromoted();
1173
- if (promotionBuffer.length > 0) {
1174
- const buffered = [...promotionBuffer];
1175
- promotionBuffer.length = 0;
1176
- for (const action of buffered) {
1177
- await replicateToWorkers(action);
1178
- }
1179
- }
1180
- callbacks?.onPromotion?.(pool.workerCount, reason);
1181
- } catch (err) {
1182
- promotionBuffer.length = 0;
1183
- throw err;
1184
- } finally {
1185
- promotionInProgress = false;
1186
- }
1187
- }
1188
- async function replicateToWorkers(action) {
1189
- if (promotionInProgress) {
1190
- promotionBuffer.push(action);
1191
- return;
1192
- }
1193
- if (!workerPool) return;
1194
- const allExecutors = workerPool.getAllExecutors();
1195
- const results = await Promise.allSettled(allExecutors.map((workerExecutor) => workerExecutor.execute(action)));
1196
- for (const result of results) {
1197
- if (result.status === "rejected") {
1198
- console.warn("Worker replication failed:", result.reason);
1199
- }
1200
- }
1201
- }
1202
- async function searchViaWorker(indexName, params) {
1203
- if (!workerPool) return null;
1204
- const manager = executor.getManager(indexName);
1205
- if (!manager) return null;
1206
- const allExecutors = workerPool.getAllExecutors();
1207
- const totalPartitions = manager.partitionCount;
1208
- const numWorkers = allExecutors.length;
1209
- if (numWorkers === 0) return null;
1210
- if (numWorkers === 1) {
1211
- try {
1212
- return await allExecutors[0].execute({
1213
- type: "query",
1214
- indexName,
1215
- params,
1216
- requestId: createRequestId()
1217
- });
1218
- } catch (err) {
1219
- console.warn("Worker search failed, falling back to local:", err);
1220
- return null;
1221
- }
1222
- }
1223
- try {
1224
- const workerAssignments = Array.from({ length: numWorkers }, () => []);
1225
- for (let p = 0; p < totalPartitions; p++) {
1226
- workerAssignments[p % numWorkers].push(p);
1227
- }
1228
- const results = await Promise.all(
1229
- allExecutors.map(
1230
- (workerExecutor, idx) => workerExecutor.execute({
1231
- type: "query",
1232
- indexName,
1233
- params,
1234
- requestId: createRequestId(),
1235
- partitionIds: workerAssignments[idx]
1236
- })
1237
- )
1238
- );
1239
- const allScored = results.map((r) => r.scored);
1240
- const merged = kWayMerge(allScored);
1241
- let totalMatched = 0;
1242
- for (const r of results) {
1243
- totalMatched += r.totalMatched;
1244
- }
1245
- return { scored: merged, totalMatched };
1246
- } catch (err) {
1247
- console.warn("Parallel worker search failed, falling back to local:", err);
1248
- return null;
1249
- }
1250
- }
1251
- function isPromoted() {
1252
- return workerPool !== null;
1253
- }
1254
- function getMemoryStats() {
1255
- const workers = workerPool ? workerPool.getMemoryStats().map((s) => ({
1256
- workerId: s.workerId,
1257
- heapUsed: s.heapUsed,
1258
- heapTotal: s.heapTotal,
1259
- external: s.external
1260
- })) : [];
1261
- const totalBytes = workers.reduce((sum, w) => sum + w.heapUsed, 0);
1262
- return { totalBytes, workers };
1263
- }
1264
- async function shutdown() {
1265
- if (workerPool) {
1266
- await workerPool.shutdown();
1267
- workerPool = null;
1268
- }
1269
- }
1270
- return { checkPromotion, replicateToWorkers, searchViaWorker, isPromoted, getMemoryStats, shutdown };
1271
- }
1272
-
1273
- // src/highlighting/highlighter.ts
1274
- function buildTokenCharOffsets(text, originalTokens) {
1275
- const offsets = [];
1276
- let searchFrom = 0;
1277
- for (const token of originalTokens) {
1278
- const idx = text.indexOf(token, searchFrom);
1279
- if (idx === -1) continue;
1280
- offsets.push({ start: idx, end: idx + token.length });
1281
- searchFrom = idx + token.length;
1282
- }
1283
- return offsets;
1284
- }
1285
- function mergeOverlapping(ranges) {
1286
- if (ranges.length === 0) return [];
1287
- const sorted = [...ranges].sort((a, b) => a.start - b.start);
1288
- const merged = [sorted[0]];
1289
- for (let i = 1; i < sorted.length; i++) {
1290
- const prev = merged[merged.length - 1];
1291
- const curr = sorted[i];
1292
- if (curr.start <= prev.end) {
1293
- prev.end = Math.max(prev.end, curr.end);
1294
- } else {
1295
- merged.push(curr);
1296
- }
1297
- }
1298
- return merged;
1299
- }
1300
- function findDensestWindow(positions, text, windowSize) {
1301
- if (positions.length === 0) {
1302
- return { start: 0, end: Math.min(windowSize, text.length) };
1303
- }
1304
- let bestStart = 0;
1305
- let bestCount = 0;
1306
- for (const pos of positions) {
1307
- const windowStart = Math.max(0, pos.start - Math.floor(windowSize / 2));
1308
- const windowEnd = Math.min(text.length, windowStart + windowSize);
1309
- const adjustedStart = Math.max(0, windowEnd - windowSize);
1310
- let count = 0;
1311
- for (const p of positions) {
1312
- if (p.start >= adjustedStart && p.end <= windowEnd) {
1313
- count++;
1314
- }
1315
- }
1316
- if (count > bestCount) {
1317
- bestCount = count;
1318
- bestStart = adjustedStart;
1319
- }
1320
- }
1321
- return { start: bestStart, end: Math.min(text.length, bestStart + windowSize) };
1322
- }
1323
- function applySnippetTruncation(taggedText, originalText, positions, maxLen, preTag, postTag) {
1324
- if (maxLen <= 0 || originalText.length <= maxLen) {
1325
- return { snippet: taggedText, adjustedPositions: positions };
1326
- }
1327
- const window = findDensestWindow(positions, originalText, maxLen);
1328
- const snippetRaw = originalText.substring(window.start, window.end);
1329
- const adjustedPositions = [];
1330
- for (const pos of positions) {
1331
- if (pos.start >= window.start && pos.end <= window.end) {
1332
- adjustedPositions.push({
1333
- start: pos.start - window.start,
1334
- end: pos.end - window.start
1335
- });
1336
- }
1337
- }
1338
- let snippet = snippetRaw;
1339
- const sortedDesc = [...adjustedPositions].sort((a, b) => b.start - a.start);
1340
- for (const pos of sortedDesc) {
1341
- snippet = snippet.substring(0, pos.start) + preTag + snippet.substring(pos.start, pos.end) + postTag + snippet.substring(pos.end);
1342
- }
1343
- const needsLeadingEllipsis = window.start > 0;
1344
- const needsTrailingEllipsis = window.end < originalText.length;
1345
- if (needsLeadingEllipsis) snippet = `...${snippet}`;
1346
- if (needsTrailingEllipsis) snippet = `${snippet}...`;
1347
- return { snippet, adjustedPositions };
1348
- }
1349
- function highlightField(text, queryTokens, language, options) {
1350
- const preTag = options?.preTag ?? "<mark>";
1351
- const postTag = options?.postTag ?? "</mark>";
1352
- const maxSnippetLength = options?.maxSnippetLength ?? 200;
1353
- if (text.length === 0) {
1354
- return { snippet: text, positions: [] };
1355
- }
1356
- const fieldResult = tokenize(text, language, { stem: false, removeStopWords: false });
1357
- if (fieldResult.tokens.length === 0) {
1358
- return { snippet: text, positions: [] };
1359
- }
1360
- const lowercaseText = text.normalize("NFC").toLowerCase();
1361
- const charOffsets = buildTokenCharOffsets(lowercaseText, fieldResult.originalTokens);
1362
- const stemmedQueryTokens = queryTokens.map((qt) => {
1363
- if (language.stemmer) {
1364
- return language.stemmer(qt.token.toLowerCase());
1365
- }
1366
- return qt.token.toLowerCase();
1367
- });
1368
- const matchedRanges = [];
1369
- for (let i = 0; i < fieldResult.tokens.length; i++) {
1370
- if (i >= charOffsets.length) break;
1371
- const fieldToken = fieldResult.tokens[i].token;
1372
- const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
1373
- for (const stemmedQuery of stemmedQueryTokens) {
1374
- if (stemmedField === stemmedQuery) {
1375
- matchedRanges.push({ start: charOffsets[i].start, end: charOffsets[i].end });
1376
- break;
1377
- }
1378
- }
1379
- }
1380
- const mergedPositions = mergeOverlapping(matchedRanges);
1381
- if (mergedPositions.length === 0) {
1382
- if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
1383
- const truncated = text.substring(0, maxSnippetLength);
1384
- return { snippet: `${truncated}...`, positions: [] };
1385
- }
1386
- return { snippet: text, positions: [] };
1387
- }
1388
- if (maxSnippetLength > 0 && text.length > maxSnippetLength) {
1389
- const { snippet, adjustedPositions } = applySnippetTruncation(
1390
- text,
1391
- text,
1392
- mergedPositions,
1393
- maxSnippetLength,
1394
- preTag,
1395
- postTag
1396
- );
1397
- return { snippet, positions: adjustedPositions };
1398
- }
1399
- let result = text;
1400
- const sortedDesc = [...mergedPositions].sort((a, b) => b.start - a.start);
1401
- for (const pos of sortedDesc) {
1402
- result = result.substring(0, pos.start) + preTag + result.substring(pos.start, pos.end) + postTag + result.substring(pos.end);
1403
- }
1404
- return { snippet: result, positions: mergedPositions };
1405
- }
1406
-
1407
- // src/search/fusion.ts
1408
- function reciprocalRankFusion(lists, options) {
1409
- const k = options.k > 0 ? options.k : 60;
1410
- const scores = /* @__PURE__ */ new Map();
1411
- const docData = /* @__PURE__ */ new Map();
1412
- for (const list of lists) {
1413
- for (let rank = 0; rank < list.length; rank++) {
1414
- const doc = list[rank];
1415
- const rrfContribution = 1 / (k + rank + 1);
1416
- scores.set(doc.docId, (scores.get(doc.docId) ?? 0) + rrfContribution);
1417
- if (!docData.has(doc.docId)) {
1418
- docData.set(doc.docId, doc);
1419
- }
1420
- }
1421
- }
1422
- const result = [];
1423
- for (const [docId, score] of scores) {
1424
- const original = docData.get(docId);
1425
- if (!original) continue;
1426
- result.push({
1427
- docId,
1428
- score,
1429
- termFrequencies: original.termFrequencies,
1430
- fieldLengths: original.fieldLengths,
1431
- idf: original.idf
1432
- });
1433
- }
1434
- result.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
1435
- return result;
1436
- }
1437
- function linearCombination(textResults, vectorResults, options) {
1438
- const { alpha } = options;
1439
- const normalizedText = minMaxNormalize(textResults);
1440
- const normalizedVector = minMaxNormalize(vectorResults);
1441
- const textScoreMap = /* @__PURE__ */ new Map();
1442
- for (const doc of normalizedText) {
1443
- textScoreMap.set(doc.docId, doc.score);
1444
- }
1445
- const textDocMap = /* @__PURE__ */ new Map();
1446
- for (const doc of textResults) {
1447
- textDocMap.set(doc.docId, doc);
1448
- }
1449
- const vectorScoreMap = /* @__PURE__ */ new Map();
1450
- for (const doc of normalizedVector) {
1451
- vectorScoreMap.set(doc.docId, doc.score);
1452
- }
1453
- const allDocIds = /* @__PURE__ */ new Set();
1454
- for (const doc of textResults) allDocIds.add(doc.docId);
1455
- for (const doc of vectorResults) allDocIds.add(doc.docId);
1456
- const result = [];
1457
- for (const docId of allDocIds) {
1458
- const tScore = textScoreMap.get(docId) ?? 0;
1459
- const vScore = vectorScoreMap.get(docId) ?? 0;
1460
- const combined = alpha * vScore + (1 - alpha) * tScore;
1461
- const original = textDocMap.get(docId);
1462
- result.push({
1463
- docId,
1464
- score: combined,
1465
- termFrequencies: original?.termFrequencies ?? {},
1466
- fieldLengths: original?.fieldLengths ?? {},
1467
- idf: original?.idf ?? {}
1468
- });
1469
- }
1470
- result.sort((a, b) => b.score - a.score || a.docId.localeCompare(b.docId));
1471
- return result;
1472
- }
1473
- function minMaxNormalize(docs) {
1474
- if (docs.length === 0) return [];
1475
- let min = docs[0].score;
1476
- let max = docs[0].score;
1477
- for (let i = 1; i < docs.length; i++) {
1478
- if (docs[i].score < min) min = docs[i].score;
1479
- if (docs[i].score > max) max = docs[i].score;
1480
- }
1481
- const range = max - min;
1482
- if (range === 0) {
1483
- return docs.map((d) => ({ docId: d.docId, score: 1 }));
1484
- }
1485
- return docs.map((d) => ({ docId: d.docId, score: (d.score - min) / range }));
1486
- }
1487
-
1488
- // src/search/grouping.ts
1489
- function getNestedValue2(obj, path) {
1490
- const segments = path.split(".");
1491
- let current = obj;
1492
- for (const segment of segments) {
1493
- if (current === null || current === void 0 || typeof current !== "object") {
1494
- return void 0;
1495
- }
1496
- if (!Object.hasOwn(current, segment)) {
1497
- return void 0;
1498
- }
1499
- current = current[segment];
1500
- }
1501
- return current;
1502
- }
1503
- function applyGrouping(hits, group, getDocument) {
1504
- if (!group.fields || group.fields.length === 0) {
1505
- return [{ values: {}, hits }];
1506
- }
1507
- const groupMap = /* @__PURE__ */ new Map();
1508
- for (const hit of hits) {
1509
- const doc = getDocument(hit.id);
1510
- const fieldValues = [];
1511
- const valuesRecord = {};
1512
- for (const field of group.fields) {
1513
- const value = doc ? getNestedValue2(doc, field) : void 0;
1514
- fieldValues.push(value);
1515
- valuesRecord[field] = value;
1516
- }
1517
- const compositeKey = JSON.stringify(fieldValues);
1518
- const existing = groupMap.get(compositeKey);
1519
- if (existing) {
1520
- existing.hits.push(hit);
1521
- } else {
1522
- groupMap.set(compositeKey, { values: valuesRecord, hits: [hit] });
1523
- }
1524
- }
1525
- const groups = [];
1526
- for (const entry of groupMap.values()) {
1527
- let groupHits = entry.hits;
1528
- if (group.maxPerGroup !== void 0 && group.maxPerGroup > 0) {
1529
- groupHits = groupHits.slice(0, group.maxPerGroup);
1530
- }
1531
- const result = { values: entry.values, hits: groupHits };
1532
- if (group.reduce) {
1533
- try {
1534
- let accumulator = group.reduce.initialValue();
1535
- for (const hit of groupHits) {
1536
- const doc = getDocument(hit.id);
1537
- if (doc) {
1538
- accumulator = group.reduce.reducer(accumulator, doc, hit.score);
1539
- }
1540
- }
1541
- ;
1542
- result.reduced = accumulator;
1543
- } catch (err) {
1544
- result.reducerError = err instanceof Error ? err.message : String(err);
1545
- }
1546
- }
1547
- groups.push(result);
1548
- }
1549
- groups.sort((a, b) => {
1550
- const aScore = a.hits.length > 0 ? a.hits[0].score : 0;
1551
- const bScore = b.hits.length > 0 ? b.hits[0].score : 0;
1552
- return bScore - aScore;
1553
- });
1554
- return groups;
1555
- }
1556
-
1557
- // src/search/pagination.ts
1558
- function encodeCursor(state) {
1559
- const json = JSON.stringify(state);
1560
- if (typeof Buffer !== "undefined") {
1561
- return Buffer.from(json).toString("base64");
1562
- }
1563
- return btoa(json);
1564
- }
1565
- function decodeCursor(encoded) {
1566
- let json;
1567
- try {
1568
- if (typeof Buffer !== "undefined") {
1569
- json = Buffer.from(encoded, "base64").toString("utf-8");
1570
- } else {
1571
- json = atob(encoded);
1572
- }
1573
- } catch {
1574
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode cursor: invalid base64 encoding", {
1575
- cursor: encoded
1576
- });
1577
- }
1578
- let parsed;
1579
- try {
1580
- parsed = JSON.parse(json);
1581
- } catch {
1582
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode cursor: invalid JSON", {
1583
- cursor: encoded
1584
- });
1585
- }
1586
- if (!Array.isArray(parsed)) {
1587
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Invalid cursor structure: expected an array", {
1588
- cursor: encoded
1589
- });
1590
- }
1591
- for (let i = 0; i < parsed.length; i++) {
1592
- const entry = parsed[i];
1593
- if (typeof entry !== "object" || entry === null) {
1594
- throw new NarsilError(
1595
- ErrorCodes.SEARCH_INVALID_CURSOR,
1596
- `Invalid cursor entry at index ${i}: expected an object`,
1597
- {
1598
- cursor: encoded,
1599
- index: i
1600
- }
1601
- );
1602
- }
1603
- const { s, d, p } = entry;
1604
- if (typeof s !== "number" || !Number.isFinite(s)) {
1605
- throw new NarsilError(
1606
- ErrorCodes.SEARCH_INVALID_CURSOR,
1607
- `Invalid cursor entry at index ${i}: "s" must be a finite number`,
1608
- { cursor: encoded, index: i }
1609
- );
1610
- }
1611
- if (typeof d !== "string") {
1612
- throw new NarsilError(
1613
- ErrorCodes.SEARCH_INVALID_CURSOR,
1614
- `Invalid cursor entry at index ${i}: "d" must be a string`,
1615
- { cursor: encoded, index: i }
1616
- );
1617
- }
1618
- if (typeof p !== "number" || !Number.isInteger(p) || p < 0) {
1619
- throw new NarsilError(
1620
- ErrorCodes.SEARCH_INVALID_CURSOR,
1621
- `Invalid cursor entry at index ${i}: "p" must be a non-negative integer`,
1622
- { cursor: encoded, index: i }
1623
- );
1624
- }
1625
- }
1626
- return parsed;
1627
- }
1628
- function applyPagination(results, limit, offset, cursor) {
1629
- if (limit <= 0) {
1630
- return { paginated: [] };
1631
- }
1632
- let startIndex = 0;
1633
- if (cursor) {
1634
- const cursorState = decodeCursor(cursor);
1635
- const anchor = cursorState[0];
1636
- for (let i = 0; i < results.length; i++) {
1637
- const result = results[i];
1638
- if (result.score < anchor.s || result.score === anchor.s && result.id > anchor.d) {
1639
- startIndex = i;
1640
- break;
1641
- }
1642
- if (i === results.length - 1) {
1643
- startIndex = results.length;
1644
- }
1645
- }
1646
- }
1647
- const afterOffset = startIndex + offset;
1648
- const sliced = results.slice(afterOffset, afterOffset + limit);
1649
- let nextCursor;
1650
- const hasMore = afterOffset + limit < results.length;
1651
- if (hasMore && sliced.length > 0) {
1652
- const lastResult = sliced[sliced.length - 1];
1653
- const cursorState = [{ s: lastResult.score, d: lastResult.id, p: 0 }];
1654
- nextCursor = encodeCursor(cursorState);
1655
- }
1656
- return { paginated: sliced, nextCursor };
1657
- }
1658
-
1659
- // src/search/pinning.ts
1660
- function applyPinning(hits, pinned, resolveDoc) {
1661
- const result = hits.slice();
1662
- const pinnedDocIds = new Set(pinned.map((p) => p.docId));
1663
- for (let i = result.length - 1; i >= 0; i--) {
1664
- if (pinnedDocIds.has(result[i].id)) {
1665
- result.splice(i, 1);
1666
- }
1667
- }
1668
- const sorted = pinned.slice().sort((a, b) => a.position - b.position);
1669
- for (const entry of sorted) {
1670
- const doc = resolveDoc(entry.docId);
1671
- if (!doc) continue;
1672
- const pos = Math.max(0, Math.min(entry.position, result.length));
1673
- result.splice(pos, 0, doc);
1674
- }
1675
- return result;
1676
- }
1677
-
1678
- // src/search/sorting.ts
1679
- function getNestedValue3(obj, path) {
1680
- const segments = path.split(".");
1681
- let current = obj;
1682
- for (const segment of segments) {
1683
- if (current === null || current === void 0 || typeof current !== "object") {
1684
- return void 0;
1685
- }
1686
- if (!Object.hasOwn(current, segment)) {
1687
- return void 0;
1688
- }
1689
- current = current[segment];
1690
- }
1691
- return current;
1692
- }
1693
- function applySorting(hits, sort, getDocument) {
1694
- const sortFields = Object.entries(sort);
1695
- if (sortFields.length === 0) return hits;
1696
- const collator = new Intl.Collator(void 0, { sensitivity: "base" });
1697
- const valueCache = /* @__PURE__ */ new Map();
1698
- for (const hit of hits) {
1699
- const doc = getDocument(hit.id);
1700
- if (!doc) {
1701
- valueCache.set(
1702
- hit.id,
1703
- sortFields.map(() => void 0)
1704
- );
1705
- continue;
1706
- }
1707
- const values = sortFields.map(([field]) => getNestedValue3(doc, field));
1708
- valueCache.set(hit.id, values);
1709
- }
1710
- const sorted = hits.slice();
1711
- sorted.sort((a, b) => {
1712
- const aValues = valueCache.get(a.id) ?? [];
1713
- const bValues = valueCache.get(b.id) ?? [];
1714
- for (let i = 0; i < sortFields.length; i++) {
1715
- const direction = sortFields[i][1];
1716
- const aVal = aValues[i];
1717
- const bVal = bValues[i];
1718
- const aIsNullish = aVal === void 0 || aVal === null;
1719
- const bIsNullish = bVal === void 0 || bVal === null;
1720
- if (aIsNullish && bIsNullish) continue;
1721
- if (aIsNullish) return 1;
1722
- if (bIsNullish) return -1;
1723
- let comparison = 0;
1724
- if (typeof aVal === "string" && typeof bVal === "string") {
1725
- comparison = collator.compare(aVal, bVal);
1726
- } else if (typeof aVal === "number" && typeof bVal === "number") {
1727
- const aIsNaN = Number.isNaN(aVal);
1728
- const bIsNaN = Number.isNaN(bVal);
1729
- if (aIsNaN && bIsNaN) continue;
1730
- if (aIsNaN) return 1;
1731
- if (bIsNaN) return -1;
1732
- comparison = aVal - bVal;
1733
- } else if (typeof aVal === "boolean" && typeof bVal === "boolean") {
1734
- comparison = aVal === bVal ? 0 : aVal ? 1 : -1;
1735
- }
1736
- if (comparison !== 0) {
1737
- return direction === "desc" ? -comparison : comparison;
1738
- }
1739
- }
1740
- return 0;
1741
- });
1742
- return sorted;
1743
- }
1744
-
1745
- // src/engine/query.ts
1746
- function collectFilterDocIds(manager, params, schema) {
1747
- const filterDocIds = /* @__PURE__ */ new Set();
1748
- if (!params.filters) return filterDocIds;
1749
- for (const partition of manager.getAllPartitions()) {
1750
- const partitionFiltered = partition.applyFilters(params.filters, schema);
1751
- for (const docId of partitionFiltered) {
1752
- filterDocIds.add(docId);
1753
- }
1754
- }
1755
- return filterDocIds;
1756
- }
1757
- function vectorResultsToScored(results) {
1758
- return results.map((r) => ({
1759
- docId: r.docId,
1760
- score: r.score,
1761
- termFrequencies: {},
1762
- fieldLengths: {},
1763
- idf: {}
1764
- }));
1765
- }
1766
- function resolveVectorIndex(manager, fieldName) {
1767
- return manager.getVectorIndexes().get(fieldName);
1768
- }
1769
- function executeVectorSearch(params, manager, config, limit, offset) {
1770
- const vectorConfig = params.vector;
1771
- if (!vectorConfig || !vectorConfig.value) {
1772
- return { scored: [], totalMatched: 0 };
1773
- }
1774
- const vecIndex = resolveVectorIndex(manager, vectorConfig.field);
1775
- if (!vecIndex) {
1776
- return { scored: [], totalMatched: 0 };
1777
- }
1778
- let filterDocIds;
1779
- if (params.filters) {
1780
- filterDocIds = collectFilterDocIds(manager, params, config.schema);
1781
- if (filterDocIds.size === 0) {
1782
- return { scored: [], totalMatched: 0 };
1783
- }
1784
- }
1785
- const queryVec = new Float32Array(vectorConfig.value);
1786
- const k = limit + offset + 1;
1787
- const results = vecIndex.search(queryVec, k, {
1788
- metric: vectorConfig.metric ?? "cosine",
1789
- minSimilarity: vectorConfig.similarity ?? 0,
1790
- filterDocIds,
1791
- efSearch: vectorConfig.efSearch
1792
- });
1793
- const scored = vectorResultsToScored(results);
1794
- return { scored, totalMatched: scored.length };
1795
- }
1796
- async function executeHybridSearch(params, manager, language, config, workerSearch, indexName, limit, offset) {
1797
- const { vector: _vector, mode: _mode, hybrid: _hybrid, ...textOnlyParams } = params;
1798
- let filterDocIds;
1799
- if (params.filters) {
1800
- filterDocIds = collectFilterDocIds(manager, params, config.schema);
1801
- if (filterDocIds.size === 0) {
1802
- return { scored: [], totalMatched: 0 };
1803
- }
1804
- }
1805
- let textFanOutResult;
1806
- const textWorkerResult = workerSearch ? await workerSearch(indexName, textOnlyParams) : null;
1807
- if (textWorkerResult) {
1808
- textFanOutResult = textWorkerResult;
1809
- } else {
1810
- const searchOptions = {
1811
- bm25Params: config.bm25,
1812
- stopWords: config.stopWords,
1813
- customTokenizer: config.tokenizer
1814
- };
1815
- textFanOutResult = await fanOutQuery(
1816
- manager,
1817
- textOnlyParams,
1818
- language,
1819
- config.schema,
1820
- { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
1821
- searchOptions
1822
- );
1823
- }
1824
- const vectorConfig = params.vector;
1825
- if (!vectorConfig || !vectorConfig.value) {
1826
- return {
1827
- scored: textFanOutResult.scored,
1828
- totalMatched: textFanOutResult.totalMatched,
1829
- facets: textFanOutResult.facets
1830
- };
1831
- }
1832
- let vectorScored = [];
1833
- const vecIndex = resolveVectorIndex(manager, vectorConfig.field);
1834
- if (vecIndex) {
1835
- const queryVec = new Float32Array(vectorConfig.value);
1836
- const vectorK = limit + offset + 1;
1837
- const vectorResults = vecIndex.search(queryVec, vectorK, {
1838
- metric: vectorConfig.metric ?? "cosine",
1839
- minSimilarity: vectorConfig.similarity ?? 0,
1840
- filterDocIds,
1841
- efSearch: vectorConfig.efSearch
1842
- });
1843
- vectorScored = vectorResultsToScored(vectorResults);
1844
- }
1845
- const hybridConfig = params.hybrid ?? {};
1846
- const strategy = hybridConfig.strategy ?? "rrf";
1847
- let fusedScored;
1848
- if (strategy === "rrf") {
1849
- const rrfK = hybridConfig.k !== void 0 && hybridConfig.k > 0 ? hybridConfig.k : 60;
1850
- fusedScored = reciprocalRankFusion([textFanOutResult.scored, vectorScored], { k: rrfK });
1851
- } else {
1852
- const alpha = clampAlpha(hybridConfig.alpha);
1853
- fusedScored = linearCombination(textFanOutResult.scored, vectorScored, { alpha });
1854
- }
1855
- if (params.minScore !== void 0 && params.minScore > 0) {
1856
- const threshold = params.minScore;
1857
- fusedScored = fusedScored.filter((doc) => doc.score >= threshold);
1858
- }
1859
- if (filterDocIds) {
1860
- fusedScored = fusedScored.filter((doc) => filterDocIds.has(doc.docId));
1861
- }
1862
- return {
1863
- scored: fusedScored,
1864
- totalMatched: fusedScored.length,
1865
- facets: textFanOutResult.facets
1866
- };
1867
- }
1868
- function clampAlpha(alpha) {
1869
- if (alpha === void 0) return 0.5;
1870
- if (!Number.isFinite(alpha)) return 0.5;
1871
- if (alpha < 0) return 0;
1872
- if (alpha > 1) return 1;
1873
- return alpha;
1874
- }
1875
- async function executeQuery(params, context) {
1876
- const { manager, language, config, workerSearch, indexName } = context;
1877
- const startTime = now();
1878
- const limit = clampLimit(params.limit);
1879
- const offset = clampOffset(params.offset);
1880
- const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
1881
- const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
1882
- const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
1883
- const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
1884
- const requestedVectorField = params.vector?.field;
1885
- const hasGlobalVectorIndex = requestedVectorField !== void 0 && manager.getVectorIndexes().has(requestedVectorField);
1886
- let fanOutResult;
1887
- if (isVectorOnly && hasGlobalVectorIndex) {
1888
- fanOutResult = executeVectorSearch(params, manager, config, limit, offset);
1889
- } else if (isHybridMode && hasGlobalVectorIndex) {
1890
- fanOutResult = await executeHybridSearch(params, manager, language, config, workerSearch, indexName, limit, offset);
1891
- } else {
1892
- const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
1893
- if (workerResult) {
1894
- fanOutResult = workerResult;
1895
- } else {
1896
- const searchOptions = {
1897
- bm25Params: config.bm25,
1898
- stopWords: config.stopWords,
1899
- customTokenizer: config.tokenizer
1900
- };
1901
- fanOutResult = await fanOutQuery(
1902
- manager,
1903
- params,
1904
- language,
1905
- config.schema,
1906
- { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
1907
- searchOptions
1908
- );
1909
- }
1910
- }
1911
- const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
1912
- let hits;
1913
- if (needsFullHits) {
1914
- hits = fanOutResult.scored.map((scored) => ({
1915
- id: scored.docId,
1916
- score: scored.score,
1917
- document: void 0,
1918
- scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
1919
- }));
1920
- } else {
1921
- const end = Math.min(offset + limit + 1, fanOutResult.scored.length);
1922
- hits = new Array(end);
1923
- for (let i = 0; i < end; i++) {
1924
- const scored = fanOutResult.scored[i];
1925
- hits[i] = {
1926
- id: scored.docId,
1927
- score: scored.score,
1928
- document: void 0,
1929
- scoreComponents: params.includeScoreComponents ? { termFrequencies: scored.termFrequencies, fieldLengths: scored.fieldLengths, idf: scored.idf } : void 0
1930
- };
1931
- }
1932
- }
1933
- if (params.sort) {
1934
- hits = applySorting(hits, params.sort, (docId) => manager.getRef(docId));
1935
- }
1936
- let groups;
1937
- if (params.group) {
1938
- groups = applyGrouping(hits, params.group, (docId) => manager.getRef(docId));
1939
- }
1940
- if (params.pinned) {
1941
- hits = applyPinning(hits, params.pinned, (docId) => {
1942
- const doc = manager.getRef(docId);
1943
- if (!doc) return void 0;
1944
- return { id: docId, score: 0, document: doc };
1945
- });
1946
- }
1947
- const { paginated, nextCursor } = applyPagination(hits, limit, offset, params.searchAfter);
1948
- for (const hit of paginated) {
1949
- hit.document = manager.get(hit.id) ?? {};
1950
- }
1951
- if (groups) {
1952
- for (const group of groups) {
1953
- for (const hit of group.hits) {
1954
- hit.document = manager.get(hit.id) ?? {};
1955
- }
1956
- }
1957
- }
1958
- if (params.highlight) {
1959
- applyHighlights(paginated, params, language, config);
1960
- }
1961
- const elapsed = now() - startTime;
1962
- return {
1963
- hits: paginated,
1964
- count: fanOutResult.totalMatched,
1965
- elapsed,
1966
- cursor: nextCursor,
1967
- facets: fanOutResult.facets,
1968
- groups
1969
- };
1970
- }
1971
- async function executePreflight(params, context) {
1972
- const { manager, language, config, workerSearch, indexName } = context;
1973
- const startTime = now();
1974
- const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
1975
- const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
1976
- const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
1977
- const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
1978
- const requestedVectorField = params.vector?.field;
1979
- const hasGlobalVectorIndex = requestedVectorField !== void 0 && manager.getVectorIndexes().has(requestedVectorField);
1980
- let totalMatched;
1981
- const preflightLimit = 1e3;
1982
- const preflightOffset = 0;
1983
- if (isVectorOnly && hasGlobalVectorIndex) {
1984
- const result = executeVectorSearch(params, manager, config, preflightLimit, preflightOffset);
1985
- totalMatched = result.totalMatched;
1986
- } else if (isHybridMode && hasGlobalVectorIndex) {
1987
- const result = await executeHybridSearch(
1988
- params,
1989
- manager,
1990
- language,
1991
- config,
1992
- workerSearch,
1993
- indexName,
1994
- preflightLimit,
1995
- preflightOffset
1996
- );
1997
- totalMatched = result.totalMatched;
1998
- } else {
1999
- const workerResult = workerSearch ? await workerSearch(indexName, params) : null;
2000
- if (workerResult) {
2001
- totalMatched = workerResult.totalMatched;
2002
- } else {
2003
- const searchOptions = {
2004
- bm25Params: config.bm25,
2005
- stopWords: config.stopWords,
2006
- customTokenizer: config.tokenizer
2007
- };
2008
- const fanOutResult = await fanOutQuery(
2009
- manager,
2010
- params,
2011
- language,
2012
- config.schema,
2013
- { scoringMode: params.scoring ?? config.defaultScoring ?? "local" },
2014
- searchOptions
2015
- );
2016
- totalMatched = fanOutResult.totalMatched;
2017
- }
2018
- }
2019
- const elapsed = now() - startTime;
2020
- return { count: totalMatched, elapsed };
2021
- }
2022
- function applyHighlights(hits, params, language, config) {
2023
- if (!params.highlight) return;
2024
- const queryTokenResult = tokenize(params.term ?? "", language, {
2025
- stem: true,
2026
- removeStopWords: true,
2027
- customTokenizer: config.tokenizer,
2028
- stopWordOverride: config.stopWords
2029
- });
2030
- for (const hit of hits) {
2031
- const highlights = {};
2032
- for (const field of params.highlight.fields) {
2033
- const doc = hit.document;
2034
- const fieldValue = doc[field];
2035
- if (typeof fieldValue === "string") {
2036
- highlights[field] = highlightField(fieldValue, queryTokenResult.tokens, language, {
2037
- preTag: params.highlight.preTag,
2038
- postTag: params.highlight.postTag,
2039
- maxSnippetLength: params.highlight.maxSnippetLength
2040
- });
2041
- }
2042
- }
2043
- hit.highlights = highlights;
2044
- }
2045
- }
2046
-
2047
- // src/partitioning/write-ahead-queue.ts
2048
- function createWriteAheadQueue(maxSize = 1e4) {
2049
- const entries = [];
2050
- let nextSequence = 1;
2051
- return {
2052
- push(entry) {
2053
- if (entries.length >= maxSize) {
2054
- throw new NarsilError(
2055
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
2056
- `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
2057
- { maxSize, currentSize: entries.length }
2058
- );
2059
- }
2060
- const seq = nextSequence++;
2061
- entries.push({ ...entry, sequenceNumber: seq });
2062
- return seq;
2063
- },
2064
- drain() {
2065
- const result = entries.splice(0);
2066
- result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
2067
- return result;
2068
- },
2069
- clear() {
2070
- entries.length = 0;
2071
- },
2072
- get size() {
2073
- return entries.length;
2074
- },
2075
- get isFull() {
2076
- return entries.length >= maxSize;
2077
- }
2078
- };
2079
- }
2080
-
2081
- // src/engine/rebalance-executor.ts
2082
- async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
2083
- if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
2084
- throw new NarsilError(
2085
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
2086
- `Target partition count must be a positive integer, got ${targetPartitionCount}`,
2087
- { targetPartitionCount }
2088
- );
2089
- }
2090
- if (ctx.rebalancingIndexes.has(indexName)) {
2091
- throw new NarsilError(
2092
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
2093
- `Index "${indexName}" is already being rebalanced`
2094
- );
2095
- }
2096
- if (targetPartitionCount === manager.partitionCount) {
2097
- return;
2098
- }
2099
- const waq = createWriteAheadQueue();
2100
- ctx.waqMap.set(indexName, waq);
2101
- ctx.rebalancingIndexes.add(indexName);
2102
- try {
2103
- const oldCount = manager.partitionCount;
2104
- await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
2105
- if (progress.phase === "complete") {
2106
- const handlers = ctx.eventHandlers.get("partitionRebalance");
2107
- if (handlers) {
2108
- for (const handler of handlers) {
2109
- handler({ indexName, oldCount, newCount: targetPartitionCount });
2110
- }
2111
- }
2112
- }
2113
- });
2114
- const entries = waq.drain();
2115
- const appliedSeqs = ctx.lastAppliedSeqMap.get(indexName) ?? /* @__PURE__ */ new Map();
2116
- const rebalanceEntry = ctx.requireIndex(indexName);
2117
- const rebalanceVecIndexes = manager.getVectorIndexes();
2118
- const rebalanceVectorFieldPaths = rebalanceVecIndexes.size > 0 ? rebalanceEntry.vectorFieldPaths : /* @__PURE__ */ new Set();
2119
- for (const entry of entries) {
2120
- const partitionLastSeq = appliedSeqs.get(0) ?? 0;
2121
- if (entry.sequenceNumber <= partitionLastSeq) continue;
2122
- try {
2123
- if (entry.action === "insert" && entry.document) {
2124
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
2125
- entry.document,
2126
- rebalanceVectorFieldPaths,
2127
- rebalanceVecIndexes
2128
- );
2129
- manager.insert(entry.docId, partitionDoc);
2130
- if (extractedVectors.size > 0) {
2131
- insertDocumentVectors(entry.docId, extractedVectors, rebalanceVecIndexes);
2132
- }
2133
- } else if (entry.action === "remove") {
2134
- manager.remove(entry.docId);
2135
- removeDocumentVectors(entry.docId, rebalanceVecIndexes);
2136
- } else if (entry.action === "update" && entry.document) {
2137
- const replayUpdateVectors = /* @__PURE__ */ new Map();
2138
- for (const fieldPath of rebalanceVectorFieldPaths) {
2139
- const newVec = extractVectorFromDoc(entry.document, fieldPath);
2140
- replayUpdateVectors.set(fieldPath, newVec);
2141
- }
2142
- const replayUpdateDoc = replayUpdateVectors.size > 0 ? structuredClone(entry.document) : entry.document;
2143
- if (replayUpdateVectors.size > 0) {
2144
- for (const fieldPath of replayUpdateVectors.keys()) {
2145
- deleteNestedValue(replayUpdateDoc, fieldPath);
2146
- }
2147
- }
2148
- manager.update(entry.docId, replayUpdateDoc);
2149
- for (const [fieldPath, newVec] of replayUpdateVectors) {
2150
- const vecIndex = rebalanceVecIndexes.get(fieldPath);
2151
- if (!vecIndex) continue;
2152
- if (newVec === null) {
2153
- if (vecIndex.has(entry.docId)) {
2154
- vecIndex.remove(entry.docId);
2155
- }
2156
- } else {
2157
- const oldVec = vecIndex.getVector(entry.docId);
2158
- if (!vectorsEqual(oldVec, newVec)) {
2159
- vecIndex.remove(entry.docId);
2160
- vecIndex.insert(entry.docId, newVec);
2161
- }
2162
- }
2163
- }
2164
- }
2165
- } catch (replayErr) {
2166
- const isDuplicate = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_ALREADY_EXISTS;
2167
- const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
2168
- if (!isDuplicate && !isMissing) {
2169
- console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
2170
- }
2171
- }
2172
- appliedSeqs.set(0, entry.sequenceNumber);
2173
- }
2174
- ctx.lastAppliedSeqMap.set(indexName, appliedSeqs);
2175
- } finally {
2176
- ctx.rebalancingIndexes.delete(indexName);
2177
- ctx.waqMap.delete(indexName);
2178
- }
2179
- }
2180
-
2181
- // src/engine/resolve-vector-text.ts
2182
- async function resolveVectorText(params, embeddingAdapter, signal) {
2183
- if (!params.vector) return params;
2184
- if (params.vector.text === void 0 && params.vector.value === void 0) return params;
2185
- if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
2186
- if (params.vector.text !== void 0 && params.vector.value !== void 0) {
2187
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
2188
- }
2189
- if (!embeddingAdapter) {
2190
- throw new NarsilError(
2191
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2192
- "Vector query with 'text' requires an embedding adapter on the index or instance"
2193
- );
2194
- }
2195
- if (typeof params.vector.text !== "string") {
2196
- throw new NarsilError(
2197
- ErrorCodes.DOC_VALIDATION_FAILED,
2198
- `Vector query 'text' must be a string, got ${typeof params.vector.text}`
2199
- );
2200
- }
2201
- if (params.vector.text.length === 0) {
2202
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
2203
- }
2204
- const raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
2205
- if (!(raw instanceof Float32Array)) {
2206
- throw new NarsilError(
2207
- ErrorCodes.EMBEDDING_FAILED,
2208
- `Adapter returned ${typeof raw} for query embedding, expected Float32Array`
2209
- );
2210
- }
2211
- if (raw.length !== embeddingAdapter.dimensions) {
2212
- throw new NarsilError(
2213
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
2214
- `Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
2215
- { expected: embeddingAdapter.dimensions, actual: raw.length }
2216
- );
2217
- }
2218
- const resolved = { ...params.vector, value: Array.from(raw) };
2219
- delete resolved.text;
2220
- return { ...params, vector: resolved };
2221
- }
2222
- var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
2223
- function validateHnswMetric(value) {
2224
- if (value === void 0 || value === null) return void 0;
2225
- if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
2226
- return value;
2227
- }
2228
- return void 0;
2229
- }
2230
- function wireToPartition(raw) {
2231
- const documents = {};
2232
- for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
2233
- documents[docId] = {
2234
- fields: doc.fields,
2235
- fieldLengths: doc.field_lengths ?? {}
2236
- };
2237
- }
2238
- const invertedIndex = {};
2239
- for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
2240
- invertedIndex[token] = {
2241
- docFrequency: list.doc_freq,
2242
- postings: (list.postings ?? []).map((p) => ({
2243
- docId: p.doc_id,
2244
- termFrequency: p.term_freq,
2245
- field: p.field,
2246
- positions: p.positions ?? []
2247
- }))
2248
- };
2249
- }
2250
- const numeric = {};
2251
- for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
2252
- numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
2253
- }
2254
- const boolean = {};
2255
- for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
2256
- boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
2257
- }
2258
- const geopoint = {};
2259
- for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
2260
- geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
2261
- }
2262
- const vectorData = {};
2263
- for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
2264
- vectorData[field] = {
2265
- dimension: data.dimension,
2266
- vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
2267
- hnswGraph: data.hnsw_graph ? {
2268
- entryPoint: data.hnsw_graph.entry_point,
2269
- maxLayer: data.hnsw_graph.max_layer,
2270
- m: data.hnsw_graph.m,
2271
- efConstruction: data.hnsw_graph.ef_construction,
2272
- metric: validateHnswMetric(data.hnsw_graph.metric),
2273
- nodes: data.hnsw_graph.nodes
2274
- } : null,
2275
- sq8: data.sq8 ? {
2276
- alpha: data.sq8.alpha,
2277
- offset: data.sq8.offset,
2278
- quantizedVectors: data.sq8.quantized_vectors,
2279
- vectorSums: data.sq8.vector_sums,
2280
- vectorSumSqs: data.sq8.vector_sum_sqs
2281
- } : null
2282
- };
2283
- }
2284
- return {
2285
- indexName: raw.index_name,
2286
- partitionId: raw.partition_id,
2287
- totalPartitions: raw.total_partitions,
2288
- language: raw.language ?? "english",
2289
- schema: raw.schema ?? {},
2290
- docCount: raw.doc_count ?? 0,
2291
- avgDocLength: raw.avg_doc_length ?? 0,
2292
- documents,
2293
- invertedIndex,
2294
- fieldIndexes: {
2295
- numeric,
2296
- boolean,
2297
- enum: raw.field_indexes?.enum ?? {},
2298
- geopoint
2299
- },
2300
- vectorData,
2301
- statistics: {
2302
- totalDocuments: raw.statistics?.total_documents ?? 0,
2303
- totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
2304
- averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
2305
- docFrequencies: raw.statistics?.doc_frequencies ?? {}
2306
- }
2307
- };
2308
- }
2309
- function deserializePayloadV1(data) {
2310
- const raw = decode(data);
2311
- return wireToPartition(raw);
2312
- }
2313
-
2314
- // src/engine/snapshot.ts
2315
- async function createSnapshot(manager, entry) {
2316
- const partitionBuffers = [];
2317
- for (let i = 0; i < manager.partitionCount; i++) {
2318
- partitionBuffers.push(manager.serializePartitionToBytes(i));
2319
- }
2320
- const snapshotVecIndexes = manager.getVectorIndexes();
2321
- const vectorPayloads = {};
2322
- for (const [fieldPath, vecIndex] of snapshotVecIndexes) {
2323
- vectorPayloads[fieldPath] = vecIndex.serialize();
2324
- }
2325
- const { encode: encode2 } = await import('@msgpack/msgpack');
2326
- return encode2({
2327
- version: 2,
2328
- schema: entry.config.schema,
2329
- language: entry.language.name,
2330
- partitions: partitionBuffers,
2331
- vectorIndexes: vectorPayloads
2332
- });
2333
- }
2334
- async function restoreFromSnapshot(indexName, data, executor, indexRegistry, getVectorFieldPaths2, dropIndex, requireManager) {
2335
- if (!(data instanceof Uint8Array)) {
2336
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
2337
- }
2338
- const { decode: decode2 } = await import('@msgpack/msgpack');
2339
- const envelope = decode2(data);
2340
- if (envelope.version !== 1 && envelope.version !== 2) {
2341
- throw new NarsilError(
2342
- ErrorCodes.DOC_VALIDATION_FAILED,
2343
- `Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`,
2344
- { version: envelope.version }
2345
- );
2346
- }
2347
- if (!envelope.schema || typeof envelope.schema !== "object") {
2348
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid schema");
2349
- }
2350
- if (!envelope.language || typeof envelope.language !== "string") {
2351
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing or invalid language");
2352
- }
2353
- if (!Array.isArray(envelope.partitions)) {
2354
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Invalid snapshot: missing partitions");
2355
- }
2356
- const language = getLanguage(envelope.language);
2357
- const schema = envelope.schema;
2358
- validateSchema(schema);
2359
- if (indexRegistry.has(indexName)) {
2360
- await dropIndex(indexName);
2361
- }
2362
- const indexConfig = { schema, language: envelope.language };
2363
- executor.createIndex(indexName, indexConfig, language);
2364
- const vectorFieldPaths = getVectorFieldPaths2(schema);
2365
- indexRegistry.set(indexName, { config: indexConfig, language, embeddingAdapter: null, vectorFieldPaths });
2366
- try {
2367
- const manager = requireManager(indexName);
2368
- while (manager.partitionCount < envelope.partitions.length) {
2369
- manager.addPartition();
2370
- }
2371
- const deserializePartitionPayload = envelope.version === 2 ? deserializePayloadV2 : deserializePayloadV1;
2372
- for (let i = 0; i < envelope.partitions.length; i++) {
2373
- const partition = deserializePartitionPayload(envelope.partitions[i]);
2374
- manager.deserializePartition(i, partition);
2375
- }
2376
- if (envelope.vectorIndexes) {
2377
- const restoreVecIndexes = manager.getVectorIndexes();
2378
- for (const [fieldPath, payload] of Object.entries(envelope.vectorIndexes)) {
2379
- const vecIndex = restoreVecIndexes.get(fieldPath);
2380
- if (vecIndex) {
2381
- vecIndex.deserialize(payload);
2382
- }
2383
- }
2384
- }
2385
- } catch (err) {
2386
- try {
2387
- executor.dropIndex(indexName);
2388
- indexRegistry.delete(indexName);
2389
- } catch (_) {
2390
- }
2391
- throw err;
2392
- }
2393
- }
2394
-
2395
- // src/engine/suggest.ts
2396
- function executeSuggest(manager, language, params) {
2397
- const t0 = performance.now();
2398
- const limit = Math.max(1, Math.min(params.limit ?? 10, 100));
2399
- const rawPrefix = params.prefix.trim();
2400
- if (rawPrefix.length === 0) {
2401
- return { terms: [], elapsed: performance.now() - t0 };
2402
- }
2403
- const unstemmed = tokenize(rawPrefix, language, { stem: false, removeStopWords: false });
2404
- const lastToken = unstemmed.tokens.length > 0 ? unstemmed.tokens[unstemmed.tokens.length - 1].token : rawPrefix.toLowerCase();
2405
- if (lastToken.length === 0) {
2406
- return { terms: [], elapsed: performance.now() - t0 };
2407
- }
2408
- const stemmed = language.stemmer ? language.stemmer(lastToken) : lastToken;
2409
- const prefixes = stemmed !== lastToken ? [lastToken, stemmed] : [lastToken];
2410
- const partitions = manager.getAllPartitions();
2411
- const merged = /* @__PURE__ */ new Map();
2412
- const perPartitionLimit = limit * 2;
2413
- for (const partition of partitions) {
2414
- const seen = /* @__PURE__ */ new Set();
2415
- for (const prefix of prefixes) {
2416
- const suggestions = partition.suggestTerms(prefix, perPartitionLimit);
2417
- for (const s of suggestions) {
2418
- if (seen.has(s.term)) continue;
2419
- seen.add(s.term);
2420
- merged.set(s.term, (merged.get(s.term) ?? 0) + s.documentFrequency);
2421
- }
2422
- }
2423
- }
2424
- const terms = Array.from(merged.entries()).map(([term, documentFrequency]) => ({ term, documentFrequency })).sort((a, b) => b.documentFrequency - a.documentFrequency);
2425
- if (terms.length > limit) terms.length = limit;
2426
- return { terms, elapsed: performance.now() - t0 };
2427
- }
2428
-
2429
- // src/engine/vector-maintenance.ts
2430
- function resolveVectorIndexes(manager, indexName, fieldName) {
2431
- const all = manager.getVectorIndexes();
2432
- if (fieldName === void 0) return all;
2433
- const idx = all.get(fieldName);
2434
- if (!idx) {
2435
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Vector field "${fieldName}" not found in index "${indexName}"`, {
2436
- indexName,
2437
- fieldName
2438
- });
2439
- }
2440
- const single = /* @__PURE__ */ new Map();
2441
- single.set(fieldName, idx);
2442
- return single;
2443
- }
2444
- function compactVectors(manager, indexName, fieldName) {
2445
- const indexes = resolveVectorIndexes(manager, indexName, fieldName);
2446
- for (const [, idx] of indexes) {
2447
- idx.compact();
2448
- }
2449
- }
2450
- async function optimizeVectors(manager, indexName, fieldName) {
2451
- const indexes = resolveVectorIndexes(manager, indexName, fieldName);
2452
- for (const [, idx] of indexes) {
2453
- await idx.optimize();
2454
- }
2455
- }
2456
- function getVectorMaintenanceStatus(manager) {
2457
- const indexes = manager.getVectorIndexes();
2458
- const results = [];
2459
- for (const [field, idx] of indexes) {
2460
- const status = idx.maintenanceStatus();
2461
- results.push({
2462
- fieldName: field,
2463
- tombstoneRatio: status.tombstoneRatio,
2464
- graphCount: status.graphCount,
2465
- bufferSize: status.bufferSize,
2466
- building: status.building,
2467
- estimatedCompactMs: status.estimatedCompactMs,
2468
- estimatedOptimizeMs: status.estimatedOptimizeMs
2469
- });
2470
- }
2471
- return results;
2472
- }
2473
-
2474
- // src/partitioning/rebalancer.ts
2475
- var CHUNK_SIZE = 1e3;
2476
- function createRebalancer() {
2477
- let rebalancing = false;
2478
- async function rebalance(manager, newPartitionCount, router, onProgress) {
2479
- if (rebalancing) {
2480
- throw new NarsilError(
2481
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
2482
- "A rebalance operation is already in progress"
2483
- );
2484
- }
2485
- if (newPartitionCount <= 0) {
2486
- throw new NarsilError(
2487
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
2488
- `New partition count must be greater than 0, received ${newPartitionCount}`,
2489
- { newPartitionCount }
2490
- );
2491
- }
2492
- if (newPartitionCount === manager.partitionCount) {
2493
- throw new NarsilError(
2494
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
2495
- `New partition count (${newPartitionCount}) is the same as the current count`,
2496
- { newPartitionCount, currentPartitionCount: manager.partitionCount }
2497
- );
2498
- }
2499
- rebalancing = true;
2500
- try {
2501
- const collectedDocs = [];
2502
- const currentPartitions = manager.getAllPartitions();
2503
- for (const partition of currentPartitions) {
2504
- for (const docId of partition.docIds()) {
2505
- const document = partition.get(docId);
2506
- if (document) {
2507
- collectedDocs.push({ docId, document });
2508
- }
2509
- }
2510
- }
2511
- const documentsTotal = collectedDocs.length;
2512
- onProgress?.({
2513
- phase: "scanning",
2514
- documentsProcessed: 0,
2515
- documentsTotal
2516
- });
2517
- const newPartitions = [];
2518
- for (let i = 0; i < newPartitionCount; i++) {
2519
- newPartitions.push(createPartitionIndex(i));
2520
- }
2521
- let documentsProcessed = 0;
2522
- for (let chunkStart = 0; chunkStart < collectedDocs.length; chunkStart += CHUNK_SIZE) {
2523
- const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, collectedDocs.length);
2524
- for (let i = chunkStart; i < chunkEnd; i++) {
2525
- const { docId, document } = collectedDocs[i];
2526
- const targetPartitionId = router.route(docId, newPartitionCount);
2527
- const targetPartition = newPartitions[targetPartitionId];
2528
- if (targetPartition.has(docId)) {
2529
- continue;
2530
- }
2531
- targetPartition.insert(docId, document, manager.schema, manager.language, { validate: false });
2532
- documentsProcessed++;
2533
- }
2534
- onProgress?.({
2535
- phase: "moving",
2536
- documentsProcessed,
2537
- documentsTotal
2538
- });
2539
- await new Promise((resolve) => setTimeout(resolve, 0));
2540
- }
2541
- onProgress?.({
2542
- phase: "swapping",
2543
- documentsProcessed,
2544
- documentsTotal
2545
- });
2546
- manager.setPartitions(newPartitions);
2547
- onProgress?.({
2548
- phase: "complete",
2549
- documentsProcessed,
2550
- documentsTotal
2551
- });
2552
- } finally {
2553
- rebalancing = false;
2554
- }
2555
- }
2556
- return {
2557
- rebalance,
2558
- isRebalancing() {
2559
- return rebalancing;
2560
- }
2561
- };
2562
- }
2563
-
2564
- // src/persistence/flush-manager.ts
2565
- var DEFAULT_INTERVAL = 5e3;
2566
- var DEFAULT_MUTATION_THRESHOLD = 1e3;
2567
- var DEFAULT_MAX_RETRIES = 3;
2568
- var DEFAULT_BASE_RETRY_DELAY = 100;
2569
- function createFlushManager(config, getPartitionData, getInstanceId) {
2570
- const {
2571
- persistence,
2572
- invalidation,
2573
- interval = DEFAULT_INTERVAL,
2574
- mutationThreshold = DEFAULT_MUTATION_THRESHOLD,
2575
- maxRetries = DEFAULT_MAX_RETRIES,
2576
- baseRetryDelay = DEFAULT_BASE_RETRY_DELAY,
2577
- onError
2578
- } = config;
2579
- const dirtyPartitions = /* @__PURE__ */ new Map();
2580
- let mutationCount = 0;
2581
- let flushTimerId = null;
2582
- let isShuttingDown = false;
2583
- let flushPromise = null;
2584
- function startTimer() {
2585
- if (flushTimerId !== null || isShuttingDown) {
2586
- return;
2587
- }
2588
- flushTimerId = setInterval(() => {
2589
- flush();
2590
- }, interval);
2591
- }
2592
- function addToDirtySet(indexName, partitionId) {
2593
- let partitions = dirtyPartitions.get(indexName);
2594
- if (partitions === void 0) {
2595
- partitions = /* @__PURE__ */ new Set();
2596
- dirtyPartitions.set(indexName, partitions);
2597
- }
2598
- partitions.add(partitionId);
2599
- }
2600
- function snapshotAndClearDirty() {
2601
- const snapshot = /* @__PURE__ */ new Map();
2602
- for (const [indexName, partitions] of dirtyPartitions) {
2603
- snapshot.set(indexName, new Set(partitions));
2604
- }
2605
- dirtyPartitions.clear();
2606
- return snapshot;
2607
- }
2608
- async function saveWithRetry(indexName, partitionId) {
2609
- const key = `${indexName}/${partitionId}`;
2610
- const data = getPartitionData(indexName, partitionId);
2611
- for (let attempt = 0; attempt < maxRetries; attempt++) {
2612
- try {
2613
- await persistence.save(key, data);
2614
- return true;
2615
- } catch (err) {
2616
- const isLastAttempt = attempt === maxRetries - 1;
2617
- if (isLastAttempt) {
2618
- onError?.(indexName, partitionId, err, true);
2619
- return false;
2620
- }
2621
- const cappedAttempt = Math.min(attempt, 20);
2622
- const delay = baseRetryDelay * 2 ** cappedAttempt * (0.5 + Math.random() * 0.5);
2623
- await new Promise((resolve) => setTimeout(resolve, delay));
2624
- }
2625
- }
2626
- return false;
2627
- }
2628
- async function performFlush() {
2629
- const snapshot = snapshotAndClearDirty();
2630
- for (const [indexName, partitions] of snapshot) {
2631
- const succeededPartitions = [];
2632
- for (const partitionId of partitions) {
2633
- const saved = await saveWithRetry(indexName, partitionId);
2634
- if (saved) {
2635
- succeededPartitions.push(partitionId);
2636
- } else {
2637
- addToDirtySet(indexName, partitionId);
2638
- }
2639
- }
2640
- if (succeededPartitions.length > 0) {
2641
- await invalidation.publish({
2642
- type: "partition",
2643
- indexName,
2644
- partitions: succeededPartitions,
2645
- timestamp: Date.now(),
2646
- sourceInstanceId: getInstanceId()
2647
- });
2648
- }
2649
- }
2650
- mutationCount = 0;
2651
- }
2652
- function flush() {
2653
- if (flushPromise !== null) {
2654
- return flushPromise;
2655
- }
2656
- flushPromise = performFlush().finally(() => {
2657
- flushPromise = null;
2658
- });
2659
- return flushPromise;
2660
- }
2661
- function markDirty(indexName, partitionId) {
2662
- if (isShuttingDown) return;
2663
- addToDirtySet(indexName, partitionId);
2664
- mutationCount++;
2665
- if (mutationCount >= mutationThreshold) {
2666
- flush();
2667
- }
2668
- startTimer();
2669
- }
2670
- async function shutdown() {
2671
- isShuttingDown = true;
2672
- if (flushTimerId !== null) {
2673
- clearInterval(flushTimerId);
2674
- flushTimerId = null;
2675
- }
2676
- await flush();
2677
- }
2678
- return {
2679
- markDirty,
2680
- flush,
2681
- shutdown
2682
- };
2683
- }
2684
-
2685
- // src/plugins/registry.ts
2686
- function isThenable(value) {
2687
- return value !== null && typeof value === "object" && typeof value.then === "function";
2688
- }
2689
- async function continueAsync(pending, hooks, startIndex, context) {
2690
- await pending;
2691
- for (let i = startIndex; i < hooks.length; i++) {
2692
- await hooks[i](context);
2693
- }
2694
- }
2695
- function createPluginRegistry() {
2696
- const plugins = [];
2697
- return {
2698
- register(plugin) {
2699
- plugins.push(plugin);
2700
- },
2701
- hasHooks(hookName) {
2702
- return plugins.some((p) => typeof p[hookName] === "function");
2703
- },
2704
- runHook(hookName, context) {
2705
- const hooks = [];
2706
- for (const plugin of plugins) {
2707
- const hook = plugin[hookName];
2708
- if (typeof hook === "function") {
2709
- hooks.push(hook.bind(plugin));
2710
- }
2711
- }
2712
- if (hooks.length === 0) return;
2713
- for (let i = 0; i < hooks.length; i++) {
2714
- const result = hooks[i](context);
2715
- if (isThenable(result)) {
2716
- return continueAsync(result, hooks, i + 1, context);
2717
- }
2718
- }
2719
- }
2720
- };
2721
- }
2722
-
2723
- // src/schema/embedding-validator.ts
2724
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
2725
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
2726
- const fieldEntries = Object.keys(embeddingConfig.fields);
2727
- if (fieldEntries.length === 0) {
2728
- throw new NarsilError(
2729
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2730
- "Embedding config must define at least one field mapping"
2731
- );
2732
- }
2733
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
2734
- if (!resolvedAdapter) {
2735
- throw new NarsilError(
2736
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2737
- "Embedding config requires an adapter (set on the index or the instance)"
2738
- );
2739
- }
2740
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
2741
- throw new NarsilError(
2742
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2743
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
2744
- );
2745
- }
2746
- const flatFields = flattenSchema(schema);
2747
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
2748
- const targetType = flatFields[targetField];
2749
- if (targetType === void 0) {
2750
- throw new NarsilError(
2751
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2752
- `Embedding target field "${targetField}" does not exist in the schema`,
2753
- { field: targetField }
2754
- );
2755
- }
2756
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
2757
- if (!vectorMatch) {
2758
- throw new NarsilError(
2759
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2760
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
2761
- { field: targetField, type: targetType }
2762
- );
2763
- }
2764
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
2765
- if (schemaDimension !== resolvedAdapter.dimensions) {
2766
- throw new NarsilError(
2767
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
2768
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
2769
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
2770
- );
2771
- }
2772
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
2773
- if (sources.length === 0) {
2774
- throw new NarsilError(
2775
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2776
- `Embedding target field "${targetField}" must have at least one source field`,
2777
- { field: targetField }
2778
- );
2779
- }
2780
- for (const source of sources) {
2781
- const sourceType = flatFields[source];
2782
- if (sourceType === void 0) {
2783
- throw new NarsilError(
2784
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2785
- `Embedding source field "${source}" does not exist in the schema`,
2786
- { field: source, targetField }
2787
- );
2788
- }
2789
- if (sourceType !== "string") {
2790
- throw new NarsilError(
2791
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2792
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
2793
- { field: source, type: sourceType, targetField }
2794
- );
2795
- }
2796
- }
2797
- }
2798
- return resolvedAdapter;
2799
- }
2800
- function validateRequiredFieldsInSchema(required, schema) {
2801
- if (required.length === 0) return;
2802
- const flatFields = flattenSchema(schema);
2803
- for (const field of required) {
2804
- if (flatFields[field] === void 0) {
2805
- throw new NarsilError(
2806
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2807
- `Required field "${field}" does not exist in the schema`,
2808
- { field }
2809
- );
2810
- }
2811
- }
2812
- }
2813
-
2814
- // src/workers/promoter.ts
2815
- var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
2816
- var DEFAULT_TOTAL_THRESHOLD = 5e4;
2817
- function createExecutionPromoter(config) {
2818
- const perIndexThreshold = config?.perIndexThreshold ?? DEFAULT_PER_INDEX_THRESHOLD;
2819
- const totalThreshold = config?.totalThreshold ?? DEFAULT_TOTAL_THRESHOLD;
2820
- let promoted = false;
2821
- function check(indexes) {
2822
- if (promoted) {
2823
- return { shouldPromote: false, reason: "" };
2824
- }
2825
- let totalDocuments = 0;
2826
- for (const [indexName, stats] of indexes) {
2827
- if (stats.documentCount >= perIndexThreshold) {
2828
- return {
2829
- shouldPromote: true,
2830
- reason: `Index "${indexName}" has ${stats.documentCount} documents, exceeding the per-index threshold of ${perIndexThreshold}`
2831
- };
2832
- }
2833
- totalDocuments += stats.documentCount;
2834
- }
2835
- if (totalDocuments >= totalThreshold) {
2836
- return {
2837
- shouldPromote: true,
2838
- reason: `Total document count of ${totalDocuments} exceeds the total threshold of ${totalThreshold}`
2839
- };
2840
- }
2841
- return { shouldPromote: false, reason: "" };
2842
- }
2843
- function markPromoted() {
2844
- promoted = true;
2845
- }
2846
- function isPromoted() {
2847
- return promoted;
2848
- }
2849
- return { check, markPromoted, isPromoted };
2850
- }
2851
-
2852
- // src/narsil.ts
2853
- function getVectorFieldPaths(schema) {
2854
- return new Set(extractVectorFieldsFromSchema(schema).keys());
2855
- }
2856
- async function createNarsil(config) {
2857
- const executor = createDirectExecutor();
2858
- const promoter = createExecutionPromoter({
2859
- perIndexThreshold: config?.workers?.promotionThreshold,
2860
- totalThreshold: config?.workers?.totalPromotionThreshold
2861
- });
2862
- const pluginRegistry = createPluginRegistry();
2863
- if (config?.plugins) {
2864
- for (const plugin of config.plugins) pluginRegistry.register(plugin);
2865
- }
2866
- let flushManager = null;
2867
- if (config?.persistence) {
2868
- const noopInvalidation = config.invalidation ?? {
2869
- publish: async () => {
2870
- },
2871
- subscribe: async () => {
2872
- },
2873
- shutdown: async () => {
2874
- }
2875
- };
2876
- flushManager = createFlushManager(
2877
- {
2878
- persistence: config.persistence,
2879
- invalidation: noopInvalidation,
2880
- interval: config.flush?.interval,
2881
- mutationThreshold: config.flush?.mutationThreshold
2882
- },
2883
- () => new Uint8Array(0),
2884
- () => "instance-0"
2885
- );
2886
- }
2887
- const idGenerator = config?.idGenerator ?? generateId;
2888
- const indexRegistry = /* @__PURE__ */ new Map();
2889
- const eventHandlers = /* @__PURE__ */ new Map();
2890
- let isShutdown = false;
2891
- const abortController = new AbortController();
2892
- const orchestrator = createWorkerOrchestrator(config, executor, promoter, indexRegistry, {
2893
- onPromotion(workerCount, reason) {
2894
- const handlers = eventHandlers.get("workerPromote");
2895
- if (handlers) {
2896
- for (const handler of handlers) handler({ workerCount, reason });
2897
- }
2898
- }
2899
- });
2900
- const rebalancer = createRebalancer();
2901
- const rebalanceRouter = createPartitionRouter();
2902
- const rebalancingIndexes = /* @__PURE__ */ new Set();
2903
- const waqMap = /* @__PURE__ */ new Map();
2904
- const lastAppliedSeqMap = /* @__PURE__ */ new Map();
2905
- function guardShutdown() {
2906
- if (isShutdown) {
2907
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "This Narsil instance has been shut down");
2908
- }
2909
- }
2910
- function requireIndex(indexName) {
2911
- const entry = indexRegistry.get(indexName);
2912
- if (!entry) {
2913
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" does not exist`, { indexName });
2914
- }
2915
- return entry;
2916
- }
2917
- function bufferIfRebalancing(indexName, entry) {
2918
- if (!rebalancingIndexes.has(indexName)) return false;
2919
- const waq = waqMap.get(indexName);
2920
- if (!waq) return false;
2921
- const clonedEntry = entry.document ? { ...entry, document: structuredClone(entry.document) } : entry;
2922
- waq.push(clonedEntry);
2923
- return true;
2924
- }
2925
- function requireManager(indexName) {
2926
- const manager = executor.getManager(indexName);
2927
- if (!manager) {
2928
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" manager not found`, { indexName });
2929
- }
2930
- return manager;
2931
- }
2932
- const mutationCtx = {
2933
- executor,
2934
- pluginRegistry,
2935
- flushManager,
2936
- orchestrator,
2937
- idGenerator,
2938
- abortController,
2939
- guardShutdown,
2940
- requireIndex,
2941
- requireManager,
2942
- bufferIfRebalancing
2943
- };
2944
- const rebalanceCtx = {
2945
- rebalancer,
2946
- router: rebalanceRouter,
2947
- waqMap,
2948
- rebalancingIndexes,
2949
- lastAppliedSeqMap,
2950
- eventHandlers,
2951
- requireIndex
2952
- };
2953
- const narsil = {
2954
- async createIndex(name, indexConfig) {
2955
- guardShutdown();
2956
- validateIndexName(name);
2957
- if (indexRegistry.has(name)) {
2958
- throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
2959
- }
2960
- validateSchema(indexConfig.schema);
2961
- let resolvedEmbeddingAdapter = null;
2962
- if (indexConfig.embedding) {
2963
- resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
2964
- }
2965
- if (indexConfig.required && indexConfig.required.length > 0) {
2966
- validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
2967
- }
2968
- const language = getLanguage(indexConfig.language ?? "english");
2969
- executor.createIndex(name, indexConfig, language);
2970
- const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
2971
- indexRegistry.set(name, {
2972
- config: indexConfig,
2973
- language,
2974
- embeddingAdapter: resolvedEmbeddingAdapter,
2975
- vectorFieldPaths
2976
- });
2977
- await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
2978
- },
2979
- async dropIndex(name) {
2980
- guardShutdown();
2981
- const entry = requireIndex(name);
2982
- executor.dropIndex(name);
2983
- indexRegistry.delete(name);
2984
- await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
2985
- },
2986
- listIndexes() {
2987
- const infos = [];
2988
- for (const [name, entry] of indexRegistry) {
2989
- const manager = executor.getManager(name);
2990
- infos.push({
2991
- name,
2992
- documentCount: manager?.countDocuments() ?? 0,
2993
- partitionCount: manager?.partitionCount ?? 0,
2994
- language: entry.language.name
2995
- });
2996
- }
2997
- return infos;
2998
- },
2999
- getStats(indexName) {
3000
- guardShutdown();
3001
- const entry = requireIndex(indexName);
3002
- const manager = executor.getManager(indexName);
3003
- const mem = manager?.estimateMemoryBytes() ?? 0;
3004
- return {
3005
- documentCount: manager?.countDocuments() ?? 0,
3006
- partitionCount: manager?.partitionCount ?? 0,
3007
- memoryBytes: mem,
3008
- indexSizeBytes: mem,
3009
- language: entry.language.name,
3010
- schema: entry.config.schema
3011
- };
3012
- },
3013
- getPartitionStats(indexName) {
3014
- guardShutdown();
3015
- requireIndex(indexName);
3016
- return executor.getManager(indexName)?.getPartitionStats() ?? [];
3017
- },
3018
- insert(indexName, document, docId, options) {
3019
- return insertDocument(mutationCtx, indexName, document, docId, options);
3020
- },
3021
- insertBatch(indexName, documents, options) {
3022
- return insertDocumentBatch(mutationCtx, indexName, documents, options);
3023
- },
3024
- remove(indexName, docId) {
3025
- return removeDocument(mutationCtx, indexName, docId);
3026
- },
3027
- removeBatch(indexName, docIds) {
3028
- return removeDocumentBatch(mutationCtx, indexName, docIds);
3029
- },
3030
- update(indexName, docId, document) {
3031
- return updateDocument(mutationCtx, indexName, docId, document);
3032
- },
3033
- updateBatch(indexName, updates) {
3034
- return updateDocumentBatch(mutationCtx, indexName, updates);
3035
- },
3036
- async get(indexName, docId) {
3037
- guardShutdown();
3038
- requireIndex(indexName);
3039
- return executor.execute({ type: "get", indexName, docId, requestId: docId });
3040
- },
3041
- async getMultiple(indexName, docIds) {
3042
- guardShutdown();
3043
- requireIndex(indexName);
3044
- const result = /* @__PURE__ */ new Map();
3045
- for (const docId of docIds) {
3046
- const doc = await narsil.get(indexName, docId);
3047
- if (doc !== void 0) result.set(docId, doc);
3048
- }
3049
- return result;
3050
- },
3051
- async has(indexName, docId) {
3052
- guardShutdown();
3053
- requireIndex(indexName);
3054
- return executor.execute({ type: "has", indexName, docId, requestId: docId });
3055
- },
3056
- async countDocuments(indexName) {
3057
- guardShutdown();
3058
- requireIndex(indexName);
3059
- return executor.execute({ type: "count", indexName, requestId: indexName });
3060
- },
3061
- async query(indexName, params) {
3062
- guardShutdown();
3063
- const entry = requireIndex(indexName);
3064
- const manager = requireManager(indexName);
3065
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
3066
- await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
3067
- const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
3068
- const result = await executeQuery(resolvedParams, {
3069
- manager,
3070
- language: entry.language,
3071
- config: entry.config,
3072
- workerSearch,
3073
- indexName
3074
- });
3075
- try {
3076
- await pluginRegistry.runHook("afterSearch", {
3077
- indexName,
3078
- params: resolvedParams,
3079
- results: result
3080
- });
3081
- } catch (err) {
3082
- console.warn("afterSearch plugin hook error:", err);
3083
- }
3084
- return result;
3085
- },
3086
- async preflight(indexName, params) {
3087
- guardShutdown();
3088
- const entry = requireIndex(indexName);
3089
- const manager = requireManager(indexName);
3090
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
3091
- const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
3092
- return executePreflight(resolvedParams, {
3093
- manager,
3094
- language: entry.language,
3095
- config: entry.config,
3096
- workerSearch,
3097
- indexName
3098
- });
3099
- },
3100
- async suggest(indexName, params) {
3101
- guardShutdown();
3102
- return executeSuggest(requireManager(indexName), requireIndex(indexName).language, params);
3103
- },
3104
- async snapshot(indexName) {
3105
- guardShutdown();
3106
- return createSnapshot(requireManager(indexName), requireIndex(indexName));
3107
- },
3108
- async restore(indexName, data) {
3109
- guardShutdown();
3110
- return restoreFromSnapshot(
3111
- indexName,
3112
- data,
3113
- executor,
3114
- indexRegistry,
3115
- getVectorFieldPaths,
3116
- narsil.dropIndex.bind(narsil),
3117
- requireManager
3118
- );
3119
- },
3120
- async clear(indexName) {
3121
- guardShutdown();
3122
- requireIndex(indexName);
3123
- await executor.execute({ type: "clear", indexName, requestId: indexName });
3124
- await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
3125
- },
3126
- async rebalance(indexName, targetPartitionCount) {
3127
- guardShutdown();
3128
- requireIndex(indexName);
3129
- return executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
3130
- },
3131
- async updatePartitionConfig(indexName, partitionConfig) {
3132
- guardShutdown();
3133
- const entry = requireIndex(indexName);
3134
- const manager = requireManager(indexName);
3135
- if (rebalancingIndexes.has(indexName)) {
3136
- throw new NarsilError(
3137
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
3138
- `Index "${indexName}" is currently being rebalanced`
3139
- );
3140
- }
3141
- const currentDocCount = manager.countDocuments();
3142
- const newMaxDocs = partitionConfig.maxDocsPerPartition ?? entry.config.partitions?.maxDocsPerPartition;
3143
- const newMaxPartitions = partitionConfig.maxPartitions ?? entry.config.partitions?.maxPartitions ?? manager.partitionCount;
3144
- if (newMaxDocs !== void 0) {
3145
- const newTotalCapacity = newMaxDocs * newMaxPartitions;
3146
- if (newTotalCapacity < currentDocCount) {
3147
- throw new NarsilError(
3148
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
3149
- `New capacity (${newTotalCapacity}) is less than current document count (${currentDocCount})`,
3150
- { newTotalCapacity, currentDocCount }
3151
- );
3152
- }
3153
- }
3154
- if (!entry.config.partitions) entry.config.partitions = {};
3155
- if (partitionConfig.maxDocsPerPartition !== void 0)
3156
- entry.config.partitions.maxDocsPerPartition = partitionConfig.maxDocsPerPartition;
3157
- if (partitionConfig.maxPartitions !== void 0)
3158
- entry.config.partitions.maxPartitions = partitionConfig.maxPartitions;
3159
- },
3160
- getMemoryStats() {
3161
- const workerStats = orchestrator.getMemoryStats();
3162
- let mainThreadBytes = 0;
3163
- for (const [name] of indexRegistry) {
3164
- const mgr = executor.getManager(name);
3165
- if (mgr) mainThreadBytes += mgr.estimateMemoryBytes();
3166
- }
3167
- return { totalBytes: mainThreadBytes + workerStats.totalBytes, workers: workerStats.workers };
3168
- },
3169
- async compactVectors(indexName, fieldName) {
3170
- guardShutdown();
3171
- requireIndex(indexName);
3172
- compactVectors(requireManager(indexName), indexName, fieldName);
3173
- },
3174
- async optimizeVectors(indexName, fieldName) {
3175
- guardShutdown();
3176
- requireIndex(indexName);
3177
- await optimizeVectors(requireManager(indexName), indexName, fieldName);
3178
- },
3179
- vectorMaintenanceStatus(indexName) {
3180
- guardShutdown();
3181
- requireIndex(indexName);
3182
- return getVectorMaintenanceStatus(requireManager(indexName));
3183
- },
3184
- on(event, handler) {
3185
- const key = event;
3186
- let handlers = eventHandlers.get(key);
3187
- if (!handlers) {
3188
- handlers = /* @__PURE__ */ new Set();
3189
- eventHandlers.set(key, handlers);
3190
- }
3191
- handlers.add(handler);
3192
- },
3193
- off(event, handler) {
3194
- const key = event;
3195
- const handlers = eventHandlers.get(key);
3196
- if (handlers) {
3197
- handlers.delete(handler);
3198
- if (handlers.size === 0) eventHandlers.delete(key);
3199
- }
3200
- },
3201
- async shutdown() {
3202
- if (isShutdown) return;
3203
- isShutdown = true;
3204
- abortController.abort();
3205
- for (const [name] of indexRegistry) {
3206
- const mgr = executor.getManager(name);
3207
- if (mgr) {
3208
- for (const [, vecIdx] of mgr.getVectorIndexes()) {
3209
- vecIdx.dispose();
3210
- }
3211
- }
3212
- }
3213
- if (flushManager) await flushManager.shutdown();
3214
- const adaptersToShutdown = /* @__PURE__ */ new Set();
3215
- for (const [, entry] of indexRegistry) {
3216
- if (entry.embeddingAdapter?.shutdown) adaptersToShutdown.add(entry.embeddingAdapter);
3217
- }
3218
- for (const adapter of adaptersToShutdown) {
3219
- try {
3220
- await adapter.shutdown?.();
3221
- } catch (_) {
3222
- }
3223
- }
3224
- await orchestrator.shutdown();
3225
- await executor.shutdown();
3226
- eventHandlers.clear();
3227
- indexRegistry.clear();
3228
- }
3229
- };
3230
- return narsil;
3231
- }
3232
-
3233
- // src/index.ts
3234
- var VERSION = "0.1.0";
3235
-
3236
- export { VERSION, createNarsil };
1
+ export { VERSION, createNarsil } from './chunk-GQEHCZG7.mjs';
2
+ export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, registerLanguage, resetNormalizationCache } from './chunk-FS7XLIUS.mjs';
3
+ export { isSimdAvailable } from './chunk-S7RJJMHI.mjs';
4
+ import './chunk-KMMJMDUA.mjs';
5
+ import './chunk-TW6BXTOL.mjs';
6
+ export { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
7
+ import './chunk-HRABJLPB.mjs';