@delali/narsil 0.2.4-next.48 → 0.2.4-next.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +2 -2
  2. package/dist/adapters/indexeddb.mjs +15 -10
  3. package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
  4. package/dist/{chunk-UZTT2BKQ.mjs → chunk-56PJBIJ6.mjs} +891 -542
  5. package/dist/chunk-CBLS46IE.mjs +10 -0
  6. package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
  7. package/dist/{chunk-4COAUBLJ.mjs → chunk-HGJC5L65.mjs} +4 -7
  8. package/dist/{chunk-DFZDV5FV.mjs → chunk-JXHUHFJ5.mjs} +27 -14
  9. package/dist/{chunk-XWICBGYT.mjs → chunk-KGRYQVZ5.mjs} +2 -5
  10. package/dist/{chunk-EV45L35L.mjs → chunk-M6JGMKA5.mjs} +391 -183
  11. package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
  12. package/dist/chunk-ORBLD3EL.mjs +6 -0
  13. package/dist/chunk-OWCFK42Q.mjs +30 -0
  14. package/dist/{chunk-WTONLDU3.mjs → chunk-RDGXYRF4.mjs} +1 -0
  15. package/dist/chunk-UNEEFTEU.mjs +7 -0
  16. package/dist/chunk-YEO6G2NG.mjs +6 -0
  17. package/dist/{client-C5BkUqB8.d.ts → client-DX7xPcNA.d.ts} +1 -1
  18. package/dist/client.d.ts +5 -5
  19. package/dist/client.mjs +224 -222
  20. package/dist/{config-B-ZI-ft5.d.ts → config-B41W23Us.d.ts} +1 -1
  21. package/dist/{config-BZRExglL.d.ts → config-Co-zAXqO.d.ts} +27 -11
  22. package/dist/direct-executor-HRTDWMVB.mjs +9 -0
  23. package/dist/distribution/coordinator/etcd.mjs +2 -3
  24. package/dist/distribution/coordinator/in-memory.mjs +2 -2
  25. package/dist/distribution/transport/grpc.mjs +7 -9
  26. package/dist/distribution/transport/in-memory.mjs +2 -1
  27. package/dist/distribution/transport/tcp.mjs +3 -2
  28. package/dist/distribution.d.ts +5 -5
  29. package/dist/distribution.mjs +121 -97
  30. package/dist/embeddings/openai.mjs +5 -3
  31. package/dist/{engine-Cjr1uBLa.d.ts → engine-CABZg25p.d.ts} +12 -8
  32. package/dist/index.browser.mjs +1514 -894
  33. package/dist/index.d.ts +5 -5
  34. package/dist/index.mjs +9 -8
  35. package/dist/invalidation/broadcast-channel.mjs +2 -1
  36. package/dist/invalidation/filesystem.mjs +3 -4
  37. package/dist/{options-BC68VPoE.d.ts → options-Dtk99qUs.d.ts} +1 -1
  38. package/dist/persistence/durability/checkpoint-worker.mjs +9 -8
  39. package/dist/react.d.ts +3 -3
  40. package/dist/react.mjs +7 -5
  41. package/dist/{search-DDHjfXhJ.d.ts → search-FjWhZN3f.d.ts} +20 -3
  42. package/dist/server.d.ts +6 -6
  43. package/dist/server.mjs +43 -30
  44. package/dist/vector/hnsw-build-worker.mjs +2 -4
  45. package/dist/vector/search-worker.mjs +3 -4
  46. package/dist/workers/entry.mjs +2 -2
  47. package/package.json +1 -1
  48. package/dist/direct-executor-EOTFGXTO.mjs +0 -8
@@ -202,6 +202,20 @@ function resolveIndexAnalysis(config) {
202
202
  };
203
203
  }
204
204
 
205
+ // src/core/tokenizer/constants.ts
206
+ var TOKEN_CACHE_SIZE_FLOOR = 5e4;
207
+ var TOKEN_CACHE_SIZE_CEILING = 2e6;
208
+ var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
209
+ var NODE_TOKEN_CACHE_ENTRIES = 1e6;
210
+ var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
211
+ var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
212
+ var SMALL_DEVICE_MEMORY_GB = 1;
213
+ var MEDIUM_DEVICE_MEMORY_GB = 4;
214
+ var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
215
+ var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
216
+ var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
217
+ var DEFAULT_MIN_TOKEN_LENGTH = 1;
218
+
205
219
  // src/core/tokenizer/normalize.ts
206
220
  var DOTTED_CAPITAL_I = "\u0130";
207
221
  var COMBINING_DOT_ABOVE = "\u0307";
@@ -248,37 +262,33 @@ function stripDiacritics(text) {
248
262
  }
249
263
 
250
264
  // src/core/tokenizer/cache.ts
251
- var CACHE_SIZE_FLOOR = 5e4;
252
- var CACHE_SIZE_CEILING = 2e6;
253
- var BYTES_PER_ENTRY = 200;
254
- var NODE_CACHE_ENTRIES = 1e6;
255
- var UNKNOWN_MEMORY_CACHE_ENTRIES = 2e5;
256
265
  function clampCacheSize(entries) {
257
- return Math.max(CACHE_SIZE_FLOOR, Math.min(entries, CACHE_SIZE_CEILING));
266
+ return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
258
267
  }
259
268
  function computeDefaultCacheSize() {
260
269
  try {
261
270
  if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
262
271
  const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
263
272
  if (constrainedMemory > 0) {
264
- return clampCacheSize(Math.floor(constrainedMemory * 0.05 / BYTES_PER_ENTRY));
273
+ const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
274
+ return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
265
275
  }
266
- return clampCacheSize(NODE_CACHE_ENTRIES);
276
+ return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
267
277
  }
268
278
  if (typeof navigator !== "undefined") {
269
279
  const mem = navigator.deviceMemory;
270
280
  if (typeof mem === "number" && mem > 0) {
271
281
  let entries;
272
- if (mem <= 1) entries = 1e5;
273
- else if (mem <= 4) entries = 25e4;
274
- else entries = 5e5;
282
+ if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
283
+ else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
284
+ else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
275
285
  return clampCacheSize(entries);
276
286
  }
277
287
  }
278
288
  } catch {
279
- return clampCacheSize(UNKNOWN_MEMORY_CACHE_ENTRIES);
289
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
280
290
  }
281
- return clampCacheSize(UNKNOWN_MEMORY_CACHE_ENTRIES);
291
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
282
292
  }
283
293
  var normalizationCache = /* @__PURE__ */ new Map();
284
294
  var normalizationCacheSize = 0;
@@ -306,7 +316,7 @@ function configureNormalizationCache(maxSize) {
306
316
  }
307
317
  );
308
318
  }
309
- maxCacheSize = Math.max(CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), CACHE_SIZE_CEILING));
319
+ maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
310
320
  if (normalizationCacheSize > maxCacheSize) {
311
321
  evictOldestEntries(normalizationCacheSize - maxCacheSize);
312
322
  }
@@ -519,7 +529,6 @@ function expandNgrams(part, size, out) {
519
529
 
520
530
  // src/core/tokenizer/tokenize.ts
521
531
  var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
522
- var DEFAULT_MIN_TOKEN_LENGTH = 1;
523
532
  function resolveStopWords(language, override) {
524
533
  if (!override) return language.stopWords;
525
534
  if (override instanceof Set) return override;
@@ -1830,7 +1839,6 @@ var VECTOR_PATTERN = /^vector\[(\d+)]$/;
1830
1839
  var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
1831
1840
  var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
1832
1841
  var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
1833
- var MAX_NESTING_DEPTH = 4;
1834
1842
  function isPlainObject(value) {
1835
1843
  return typeof value === "object" && value !== null && !Array.isArray(value);
1836
1844
  }
@@ -2002,8 +2010,11 @@ function validateFieldValue(path, value, type) {
2002
2010
  }
2003
2011
  }
2004
2012
 
2005
- // src/schema/validator/storable.ts
2013
+ // src/schema/constants.ts
2014
+ var MAX_NESTING_DEPTH = 4;
2006
2015
  var MAX_DOCUMENT_NESTING_DEPTH = 32;
2016
+
2017
+ // src/schema/validator/storable.ts
2007
2018
  var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
2008
2019
  function reject(path, reason) {
2009
2020
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
@@ -2256,194 +2267,15 @@ function extractVectorFieldsFromSchema(schema) {
2256
2267
  return result;
2257
2268
  }
2258
2269
 
2259
- // src/vector/hnsw/adjacency.ts
2270
+ // src/vector/hnsw/constants.ts
2260
2271
  var MAX_LAYER_CAP = 32;
2261
2272
  var MAX_M = 512;
2262
- var ABSENT = -1;
2263
2273
  var INITIAL_UPPER_SLOTS = 64;
2264
- function createAdjacency(m, mMax0) {
2265
- return {
2266
- m,
2267
- mMax0,
2268
- level0Stride: mMax0 + 2,
2269
- upperStride: m + 2,
2270
- capacity: 0,
2271
- slots: 0,
2272
- nodeLevels: new Int8Array(0),
2273
- level0: new Int32Array(0),
2274
- upperBase: new Int32Array(0),
2275
- upper: new Int32Array(0),
2276
- upperUsed: 0,
2277
- freeBlocks: Array.from({ length: MAX_LAYER_CAP + 1 }, () => [])
2278
- };
2279
- }
2280
- function ensureAdjacencyCapacity(adj, needed) {
2281
- if (needed <= adj.capacity) return;
2282
- let newCapacity = adj.capacity === 0 ? 16 : adj.capacity;
2283
- while (newCapacity < needed) newCapacity *= 2;
2284
- const nextLevels = new Int8Array(newCapacity).fill(ABSENT);
2285
- nextLevels.set(adj.nodeLevels);
2286
- adj.nodeLevels = nextLevels;
2287
- const nextLevel0 = new Int32Array(newCapacity * adj.level0Stride);
2288
- nextLevel0.set(adj.level0);
2289
- adj.level0 = nextLevel0;
2290
- const nextUpperBase = new Int32Array(newCapacity).fill(ABSENT);
2291
- nextUpperBase.set(adj.upperBase);
2292
- adj.upperBase = nextUpperBase;
2293
- adj.capacity = newCapacity;
2294
- }
2295
- function allocateUpperBlock(adj, levels) {
2296
- const reusable = adj.freeBlocks[levels].pop();
2297
- const size = levels * adj.upperStride;
2298
- if (reusable !== void 0) {
2299
- adj.upper.fill(0, reusable, reusable + size);
2300
- return reusable;
2301
- }
2302
- if (adj.upperUsed + size > adj.upper.length) {
2303
- const required = adj.upperUsed + size;
2304
- let nextLength = adj.upper.length === 0 ? INITIAL_UPPER_SLOTS * adj.upperStride : adj.upper.length;
2305
- while (nextLength < required) nextLength *= 2;
2306
- const nextUpper = new Int32Array(nextLength);
2307
- nextUpper.set(adj.upper);
2308
- adj.upper = nextUpper;
2309
- }
2310
- const offset = adj.upperUsed;
2311
- adj.upperUsed += size;
2312
- adj.upper.fill(0, offset, offset + size);
2313
- return offset;
2314
- }
2315
- function createNode(adj, ord, maxLayer) {
2316
- ensureAdjacencyCapacity(adj, ord + 1);
2317
- if (ord >= adj.slots) adj.slots = ord + 1;
2318
- const existingLevel = adj.nodeLevels[ord];
2319
- const existingBase = adj.upperBase[ord];
2320
- if (existingLevel >= 1 && existingBase !== ABSENT) {
2321
- adj.freeBlocks[existingLevel].push(existingBase);
2322
- }
2323
- adj.nodeLevels[ord] = maxLayer;
2324
- adj.level0[ord * adj.level0Stride] = 0;
2325
- adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) : ABSENT;
2326
- }
2327
- function deleteNode(adj, ord) {
2328
- if (ord < 0 || ord >= adj.capacity) return;
2329
- const level = adj.nodeLevels[ord];
2330
- const base = adj.upperBase[ord];
2331
- if (level >= 1 && base !== ABSENT) {
2332
- adj.freeBlocks[level].push(base);
2333
- }
2334
- adj.nodeLevels[ord] = ABSENT;
2335
- adj.upperBase[ord] = ABSENT;
2336
- adj.level0[ord * adj.level0Stride] = 0;
2337
- }
2338
- function resetAdjacency(adj) {
2339
- adj.nodeLevels = new Int8Array(0);
2340
- adj.level0 = new Int32Array(0);
2341
- adj.upperBase = new Int32Array(0);
2342
- adj.upper = new Int32Array(0);
2343
- adj.upperUsed = 0;
2344
- adj.capacity = 0;
2345
- adj.slots = 0;
2346
- for (const list of adj.freeBlocks) list.length = 0;
2347
- }
2348
- function nodeLevel(adj, ord) {
2349
- if (ord < 0 || ord >= adj.capacity) return ABSENT;
2350
- return adj.nodeLevels[ord];
2351
- }
2352
- function hasNode(adj, ord) {
2353
- return nodeLevel(adj, ord) !== ABSENT;
2354
- }
2355
- function layerArray(adj, layer) {
2356
- return layer === 0 ? adj.level0 : adj.upper;
2357
- }
2358
- function layerBase(adj, ord, layer) {
2359
- const level = nodeLevel(adj, ord);
2360
- if (level === ABSENT || layer > level) return ABSENT;
2361
- if (layer === 0) return ord * adj.level0Stride;
2362
- const base = adj.upperBase[ord];
2363
- if (base === ABSENT) return ABSENT;
2364
- return base + (layer - 1) * adj.upperStride;
2365
- }
2366
- function neighborCount(adj, ord, layer) {
2367
- const base = layerBase(adj, ord, layer);
2368
- if (base === ABSENT) return 0;
2369
- return layerArray(adj, layer)[base];
2370
- }
2371
- function collectNeighbors(adj, ord, layer) {
2372
- const base = layerBase(adj, ord, layer);
2373
- if (base === ABSENT) return [];
2374
- const array = layerArray(adj, layer);
2375
- const count = array[base];
2376
- const collected = new Array(count);
2377
- for (let i = 0; i < count; i++) {
2378
- collected[i] = array[base + i + 1];
2379
- }
2380
- return collected;
2381
- }
2382
- function addNeighbor(adj, ord, layer, neighborOrd) {
2383
- const base = layerBase(adj, ord, layer);
2384
- if (base === ABSENT) return;
2385
- const array = layerArray(adj, layer);
2386
- const count = array[base];
2387
- for (let i = 1; i <= count; i++) {
2388
- if (array[base + i] === neighborOrd) return;
2389
- }
2390
- const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
2391
- if (count + 1 > stride - 1) return;
2392
- array[base + count + 1] = neighborOrd;
2393
- array[base] = count + 1;
2394
- }
2395
- function removeNeighbor(adj, ord, layer, neighborOrd) {
2396
- const base = layerBase(adj, ord, layer);
2397
- if (base === ABSENT) return;
2398
- const array = layerArray(adj, layer);
2399
- const count = array[base];
2400
- for (let i = 1; i <= count; i++) {
2401
- if (array[base + i] !== neighborOrd) continue;
2402
- for (let j = i; j < count; j++) {
2403
- array[base + j] = array[base + j + 1];
2404
- }
2405
- array[base] = count - 1;
2406
- return;
2407
- }
2408
- }
2409
- function replaceNeighbors(adj, ord, layer, neighbors, count) {
2410
- const base = layerBase(adj, ord, layer);
2411
- if (base === ABSENT) return;
2412
- const array = layerArray(adj, layer);
2413
- const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
2414
- const limit = Math.min(count, stride - 1);
2415
- for (let i = 0; i < limit; i++) {
2416
- array[base + i + 1] = neighbors[i];
2417
- }
2418
- array[base] = limit;
2419
- }
2420
- function exportAdjacency(adj) {
2421
- return {
2422
- m: adj.m,
2423
- mMax0: adj.mMax0,
2424
- capacity: adj.capacity,
2425
- slots: adj.slots,
2426
- upperUsed: adj.upperUsed,
2427
- nodeLevels: adj.nodeLevels.slice(),
2428
- level0: adj.level0.slice(),
2429
- upperBase: adj.upperBase.slice(),
2430
- upper: adj.upper.slice(0, adj.upperUsed)
2431
- };
2432
- }
2433
- function importAdjacency(snapshot) {
2434
- const adj = createAdjacency(snapshot.m, snapshot.mMax0);
2435
- adj.capacity = snapshot.capacity;
2436
- adj.slots = snapshot.slots;
2437
- adj.upperUsed = snapshot.upperUsed;
2438
- adj.nodeLevels = snapshot.nodeLevels;
2439
- adj.level0 = snapshot.level0;
2440
- adj.upperBase = snapshot.upperBase;
2441
- adj.upper = snapshot.upper;
2442
- return adj;
2443
- }
2444
- function estimateAdjacencyBytes(adj) {
2445
- return adj.nodeLevels.byteLength + adj.level0.byteLength + adj.upperBase.byteLength + adj.upper.byteLength;
2446
- }
2274
+ var INITIAL_LIST_CAPACITY = 64;
2275
+ var COMPACTION_TOMBSTONE_RATIO = 0.1;
2276
+ var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
2277
+ var SQ8_OVERSELECTION_FACTOR = 2;
2278
+ var SQ8_RERANK_FLOOR = 10;
2447
2279
 
2448
2280
  // src/schema/validator/vector-promotion.ts
2449
2281
  function fail(message, details) {
@@ -2476,6 +2308,11 @@ function validateVectorPromotion(config) {
2476
2308
  fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
2477
2309
  }
2478
2310
  }
2311
+
2312
+ // src/serialization/constants.ts
2313
+ var CHECKSUM_TIMEOUT_MS = 12e4;
2314
+ var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
2315
+ var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
2479
2316
  function metadataToWire(meta) {
2480
2317
  const wire = {
2481
2318
  index_name: meta.indexName,
@@ -2718,7 +2555,6 @@ function sanitizeSurfaceForms(raw) {
2718
2555
  }
2719
2556
  return count > 0 ? result : void 0;
2720
2557
  }
2721
- var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
2722
2558
  function validateHnswMetric(value) {
2723
2559
  if (value === void 0 || value === null) return void 0;
2724
2560
  if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
@@ -2816,10 +2652,9 @@ function deserializePayloadV1(data) {
2816
2652
  function encodeRawPayloadV2(wire) {
2817
2653
  return encode(wire);
2818
2654
  }
2819
- var VALID_HNSW_METRICS2 = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
2820
2655
  function validateHnswMetric2(value) {
2821
2656
  if (value === void 0 || value === null) return void 0;
2822
- if (typeof value === "string" && VALID_HNSW_METRICS2.has(value)) {
2657
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
2823
2658
  return value;
2824
2659
  }
2825
2660
  return void 0;
@@ -3023,8 +2858,10 @@ function compareSortStrings(a, b) {
3023
2858
  return compareCodePoints(a, b);
3024
2859
  }
3025
2860
 
3026
- // src/core/ordering/sort-values.ts
2861
+ // src/core/ordering/constants.ts
3027
2862
  var SORT_VALUE_MAX_CODE_POINTS = 512;
2863
+
2864
+ // src/core/ordering/sort-values.ts
3028
2865
  function truncateSortString(value) {
3029
2866
  if (value.length <= SORT_VALUE_MAX_CODE_POINTS) return value;
3030
2867
  let index = 0;
@@ -3344,6 +3181,11 @@ function createDocumentStore() {
3344
3181
  };
3345
3182
  }
3346
3183
 
3184
+ // src/core/constants.ts
3185
+ var POSTING_LIST_INITIAL_CAPACITY = 8;
3186
+ var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
3187
+ var MAX_TERM_FREQUENCY = 65535;
3188
+
3347
3189
  // src/core/fuzzy.ts
3348
3190
  function boundedLevenshtein(a, b, tolerance) {
3349
3191
  if (tolerance < 0) return { distance: -1, withinTolerance: false };
@@ -3394,15 +3236,12 @@ function boundedLevenshtein(a, b, tolerance) {
3394
3236
  }
3395
3237
 
3396
3238
  // src/core/posting-list.ts
3397
- var INITIAL_CAPACITY = 8;
3398
- var MAX_TERM_FREQUENCY = 65535;
3399
- var COMPACTION_THRESHOLD = 0.3;
3400
3239
  function createPostingList() {
3401
3240
  return {
3402
3241
  length: 0,
3403
3242
  docIds: [],
3404
- termFrequencies: new Uint16Array(INITIAL_CAPACITY),
3405
- fieldNameIndices: new Uint8Array(INITIAL_CAPACITY),
3243
+ termFrequencies: new Uint16Array(POSTING_LIST_INITIAL_CAPACITY),
3244
+ fieldNameIndices: new Uint8Array(POSTING_LIST_INITIAL_CAPACITY),
3406
3245
  positions: null,
3407
3246
  docIdSet: /* @__PURE__ */ new Set(),
3408
3247
  deletedDocs: /* @__PURE__ */ new Set(),
@@ -3570,7 +3409,7 @@ function createInvertedIndex(fieldNameTable) {
3570
3409
  batchDirtyTokens.add(token);
3571
3410
  return;
3572
3411
  }
3573
- if (list.deletedDocs.size / list.length > COMPACTION_THRESHOLD) {
3412
+ if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
3574
3413
  compactList(list);
3575
3414
  }
3576
3415
  },
@@ -3583,7 +3422,7 @@ function createInvertedIndex(fieldNameTable) {
3583
3422
  for (const token of batchDirtyTokens) {
3584
3423
  const list = index.get(token);
3585
3424
  if (!list) continue;
3586
- if (list.deletedDocs.size / list.length > COMPACTION_THRESHOLD) {
3425
+ if (list.deletedDocs.size / list.length > POSTING_LIST_COMPACTION_THRESHOLD) {
3587
3426
  compactList(list);
3588
3427
  }
3589
3428
  }
@@ -4335,9 +4174,13 @@ function computeFacets(state, matched, config, schema) {
4335
4174
  return result;
4336
4175
  }
4337
4176
 
4177
+ // src/geo/constants.ts
4178
+ var DEG_TO_RAD = Math.PI / 180;
4179
+ var VINCENTY_MAX_ITERATIONS = 200;
4180
+ var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
4181
+
4338
4182
  // src/geo/haversine.ts
4339
4183
  var EARTH_RADIUS_METERS = 63710088e-1;
4340
- var DEG_TO_RAD = Math.PI / 180;
4341
4184
  function haversineDistance(lat1, lon1, lat2, lon2) {
4342
4185
  const phi1 = lat1 * DEG_TO_RAD;
4343
4186
  const phi2 = lat2 * DEG_TO_RAD;
@@ -4375,13 +4218,10 @@ function isPointInPolygon(lat, lon, polygon) {
4375
4218
  var WGS84_A = 6378137;
4376
4219
  var WGS84_F = 1 / 298.257223563;
4377
4220
  var WGS84_B = WGS84_A * (1 - WGS84_F);
4378
- var DEG_TO_RAD2 = Math.PI / 180;
4379
- var MAX_ITERATIONS = 200;
4380
- var CONVERGENCE_THRESHOLD = 1e-12;
4381
4221
  function vincentyDistance(lat1, lon1, lat2, lon2) {
4382
- const phi1 = lat1 * DEG_TO_RAD2;
4383
- const phi2 = lat2 * DEG_TO_RAD2;
4384
- const L = (lon2 - lon1) * DEG_TO_RAD2;
4222
+ const phi1 = lat1 * DEG_TO_RAD;
4223
+ const phi2 = lat2 * DEG_TO_RAD;
4224
+ const L = (lon2 - lon1) * DEG_TO_RAD;
4385
4225
  const U1 = Math.atan((1 - WGS84_F) * Math.tan(phi1));
4386
4226
  const U2 = Math.atan((1 - WGS84_F) * Math.tan(phi2));
4387
4227
  const sinU1 = Math.sin(U1);
@@ -4396,7 +4236,7 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
4396
4236
  let sinAlpha = 0;
4397
4237
  let cos2Alpha = 0;
4398
4238
  let cos2SigmaM = 0;
4399
- let iterations = MAX_ITERATIONS;
4239
+ let iterations = VINCENTY_MAX_ITERATIONS;
4400
4240
  do {
4401
4241
  const sinLambda = Math.sin(lambda);
4402
4242
  const cosLambda = Math.cos(lambda);
@@ -4412,7 +4252,7 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
4412
4252
  const C = WGS84_F / 16 * cos2Alpha * (4 + WGS84_F * (4 - 3 * cos2Alpha));
4413
4253
  prevLambda = lambda;
4414
4254
  lambda = L + (1 - C) * WGS84_F * sinAlpha * (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1 + 2 * cos2SigmaM * cos2SigmaM)));
4415
- } while (Math.abs(lambda - prevLambda) > CONVERGENCE_THRESHOLD && --iterations > 0);
4255
+ } while (Math.abs(lambda - prevLambda) > VINCENTY_CONVERGENCE_THRESHOLD && --iterations > 0);
4416
4256
  if (iterations === 0) {
4417
4257
  return haversineDistance(lat1, lon1, lat2, lon2);
4418
4258
  }
@@ -5697,14 +5537,27 @@ function partitionFilterMatches(state, filters, schema) {
5697
5537
  };
5698
5538
  }
5699
5539
 
5540
+ // src/core/partition/posting-columns.ts
5541
+ function postingColumns(list) {
5542
+ const deletedDocs = list.deletedDocs;
5543
+ return {
5544
+ docIds: list.docIds,
5545
+ termFrequencies: list.termFrequencies,
5546
+ fieldNameIndices: list.fieldNameIndices,
5547
+ deletedDocs,
5548
+ hasDeleted: deletedDocs.size > 0,
5549
+ count: list.length
5550
+ };
5551
+ }
5552
+
5700
5553
  // src/core/partition/matches.ts
5701
5554
  function markPostingList(matched, list, fieldNames, fields, filterBitset) {
5702
- const hasDeleted = list.deletedDocs.size > 0;
5703
- for (let pi = 0; pi < list.length; pi++) {
5704
- const internalId = list.docIds[pi];
5705
- if (hasDeleted && list.deletedDocs.has(internalId)) continue;
5555
+ const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
5556
+ for (let pi = 0; pi < count; pi++) {
5557
+ const internalId = docIds[pi];
5558
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
5706
5559
  if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
5707
- if (fields && !fields.includes(fieldNames[list.fieldNameIndices[pi]])) continue;
5560
+ if (fields && !fields.includes(fieldNames[fieldNameIndices[pi]])) continue;
5708
5561
  bitsetSet(matched, internalId);
5709
5562
  }
5710
5563
  }
@@ -5775,6 +5628,12 @@ function searchFulltextMatches(state, params) {
5775
5628
  };
5776
5629
  }
5777
5630
 
5631
+ // src/core/partition/constants.ts
5632
+ var TARGET_BLOCK_ENTRIES = 128;
5633
+ var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
5634
+ var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
5635
+ var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
5636
+
5778
5637
  // src/core/partition/sort-columns/order.ts
5779
5638
  var MISSING_RANK = -1;
5780
5639
  var BYTES_PER_RANK = 4;
@@ -6000,8 +5859,6 @@ function createValueStore(initialKind) {
6000
5859
  }
6001
5860
 
6002
5861
  // src/core/partition/sort-columns/index.ts
6003
- var MINIMUM_REBUILD_THRESHOLD = 1024;
6004
- var REBUILD_FRACTION_SHIFT = 2;
6005
5862
  function liveInternalIds(docStore) {
6006
5863
  const ids = [];
6007
5864
  for (const internalId of docStore.allInternalIds()) ids.push(internalId);
@@ -6011,8 +5868,8 @@ function createSortColumnSet(docStore) {
6011
5868
  const columns = /* @__PURE__ */ new Map();
6012
5869
  function rebuildThreshold() {
6013
5870
  const count = docStore.count();
6014
- const fraction = count >> REBUILD_FRACTION_SHIFT;
6015
- return fraction > MINIMUM_REBUILD_THRESHOLD ? fraction : MINIMUM_REBUILD_THRESHOLD;
5871
+ const fraction = count >> SORT_COLUMN_REBUILD_FRACTION_SHIFT;
5872
+ return fraction > SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD ? fraction : SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD;
6016
5873
  }
6017
5874
  function rebuild2(entry) {
6018
5875
  entry.order = buildOrder(entry.store, docStore.allInternalIds(), docStore.internalIdCapacity());
@@ -6641,13 +6498,12 @@ function sortSelection(heap) {
6641
6498
  }
6642
6499
 
6643
6500
  // src/core/partition/score-buffer.ts
6644
- var INITIAL_TOUCHED_CAPACITY = 1024;
6645
6501
  var MAX_GENERATION = 2147483647;
6646
6502
  function createScoreBuffer(capacity) {
6647
6503
  return {
6648
6504
  scores: new Float64Array(capacity),
6649
6505
  stamps: new Int32Array(capacity),
6650
- touched: new Int32Array(Math.max(1, Math.min(capacity, INITIAL_TOUCHED_CAPACITY))),
6506
+ touched: new Int32Array(Math.max(1, Math.min(capacity, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY))),
6651
6507
  touchedCount: 0,
6652
6508
  generation: 0
6653
6509
  };
@@ -6732,7 +6588,6 @@ function topKFromBuffer(buffer, k, resolver, components) {
6732
6588
  }
6733
6589
 
6734
6590
  // src/core/partition/block-bounds.ts
6735
- var TARGET_BLOCK_ENTRIES = 128;
6736
6591
  var ABSENT_FIELD_LENGTH_BOUND = 0;
6737
6592
  var boundsByList = /* @__PURE__ */ new WeakMap();
6738
6593
  function fieldLengthBound(columns, fieldIndex, internalId) {
@@ -6841,12 +6696,27 @@ function bestSearchable(searchable, values) {
6841
6696
  }
6842
6697
  return best;
6843
6698
  }
6699
+ function prunableSingleTermList(params, index) {
6700
+ if (params.queryTokens.length !== 1) return null;
6701
+ if (params.prefixExpansion !== void 0) return null;
6702
+ if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
6703
+ if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
6704
+ if (params.collectComponents !== false) return null;
6705
+ if (params.collectMatchedSet !== void 0) return null;
6706
+ if (params.maxResults === void 0) return null;
6707
+ if (params.fields !== void 0) return null;
6708
+ if (params.filterBitset !== void 0) return null;
6709
+ if (!bm25PruningSound(params.bm25Params)) return null;
6710
+ const list = index.lookup(params.queryTokens[0].token);
6711
+ if (list === void 0) return null;
6712
+ if (!list.ordered) return null;
6713
+ return list;
6714
+ }
6844
6715
  function singleTermTopK(request) {
6845
6716
  const { list, docFrequency, totalDocs, bm25Params, limit, resolver } = request;
6846
6717
  const { fieldSearchable, fieldBoosts, fieldAvgLengths, fieldLengthColumns } = request;
6847
6718
  const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
6848
- const deleted = list.deletedDocs;
6849
- const hasDeleted = deleted.size > 0;
6719
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
6850
6720
  const bounds = blockBoundsFor(list, fieldLengthColumns);
6851
6721
  const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
6852
6722
  const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
@@ -6873,19 +6743,19 @@ function singleTermTopK(request) {
6873
6743
  }
6874
6744
  let entry = start;
6875
6745
  while (entry < end) {
6876
- const internalId = list.docIds[entry];
6746
+ const internalId = docIds[entry];
6877
6747
  if (hasDeleted && deleted.has(internalId)) {
6878
- while (entry < end && list.docIds[entry] === internalId) entry++;
6748
+ while (entry < end && docIds[entry] === internalId) entry++;
6879
6749
  continue;
6880
6750
  }
6881
6751
  let score = 0;
6882
6752
  let scored2 = false;
6883
- while (entry < end && list.docIds[entry] === internalId) {
6884
- const fieldIndex = list.fieldNameIndices[entry];
6753
+ while (entry < end && docIds[entry] === internalId) {
6754
+ const fieldIndex = fieldNameIndices[entry];
6885
6755
  if (fieldSearchable[fieldIndex] === 1) {
6886
6756
  const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
6887
6757
  score += computeBM25(
6888
- list.termFrequencies[entry],
6758
+ termFrequencies[entry],
6889
6759
  docFrequency,
6890
6760
  totalDocs,
6891
6761
  fieldLength,
@@ -6937,21 +6807,10 @@ function singleTermTopK(request) {
6937
6807
  function globalDocFreqFor(docFreqs, term, fallback) {
6938
6808
  return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
6939
6809
  }
6940
- function prunableSingleTermList(params, index) {
6941
- if (params.queryTokens.length !== 1) return null;
6942
- if (params.prefixExpansion !== void 0) return null;
6943
- if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
6944
- if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
6945
- if (params.collectComponents !== false) return null;
6946
- if (params.collectMatchedSet !== void 0) return null;
6947
- if (params.maxResults === void 0) return null;
6948
- if (params.fields !== void 0) return null;
6949
- if (params.filterBitset !== void 0) return null;
6950
- if (!bm25PruningSound(params.bm25Params)) return null;
6951
- const list = index.lookup(params.queryTokens[0].token);
6952
- if (list === void 0) return null;
6953
- if (!list.ordered) return null;
6954
- return list;
6810
+ function matchesFor(index, token, exact, tolerance, prefixLength) {
6811
+ if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
6812
+ const postingList = index.lookup(token);
6813
+ return postingList ? [{ token, postingList }] : [];
6955
6814
  }
6956
6815
  function searchFulltext(state, params) {
6957
6816
  const {
@@ -7032,17 +6891,18 @@ function searchFulltext(state, params) {
7032
6891
  const best = /* @__PURE__ */ new Map();
7033
6892
  for (const match of matches) {
7034
6893
  const perTerm = /* @__PURE__ */ new Map();
7035
- const list = match.postingList;
7036
- const hasDeleted = list.deletedDocs.size > 0;
7037
- for (let pi = 0; pi < list.length; pi++) {
7038
- const internalId = list.docIds[pi];
7039
- if (hasDeleted && list.deletedDocs.has(internalId)) continue;
6894
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
6895
+ match.postingList
6896
+ );
6897
+ for (let pi = 0; pi < count; pi++) {
6898
+ const internalId = docIds[pi];
6899
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
7040
6900
  if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
7041
- const fieldIndex = list.fieldNameIndices[pi];
6901
+ const fieldIndex = fieldNameIndices[pi];
7042
6902
  loadFieldMeta(fieldIndex);
7043
6903
  if (fieldSearchable[fieldIndex] === 0) continue;
7044
6904
  const fieldName = fieldNames[fieldIndex];
7045
- const termFrequency = list.termFrequencies[pi];
6905
+ const termFrequency = termFrequencies[pi];
7046
6906
  const fieldBoost = fieldBoosts[fieldIndex];
7047
6907
  const avgLen = fieldAvgLengths[fieldIndex];
7048
6908
  const actualFieldLength = resolveFieldLength(internalId, fieldIndex, avgLen);
@@ -7114,10 +6974,7 @@ function searchFulltext(state, params) {
7114
6974
  resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
7115
6975
  continue;
7116
6976
  }
7117
- const rawMatches = exact ? (() => {
7118
- const postingList = state.invertedIdx.lookup(qt.token);
7119
- return postingList ? [{ token: qt.token, postingList }] : [];
7120
- })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
6977
+ const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
7121
6978
  let totalPostings = 0;
7122
6979
  const matches = [];
7123
6980
  for (const m of rawMatches) {
@@ -7139,17 +6996,18 @@ function searchFulltext(state, params) {
7139
6996
  continue;
7140
6997
  }
7141
6998
  for (const match of resolved[tokenIndex].matches) {
7142
- const list = match.postingList;
7143
- const hasDeleted = list.deletedDocs.size > 0;
7144
- for (let pi = 0; pi < list.length; pi++) {
7145
- const internalId = list.docIds[pi];
7146
- if (hasDeleted && list.deletedDocs.has(internalId)) continue;
6999
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
7000
+ match.postingList
7001
+ );
7002
+ for (let pi = 0; pi < count; pi++) {
7003
+ const internalId = docIds[pi];
7004
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
7147
7005
  if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
7148
7006
  if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
7149
- const fieldIndex = list.fieldNameIndices[pi];
7007
+ const fieldIndex = fieldNameIndices[pi];
7150
7008
  loadFieldMeta(fieldIndex);
7151
7009
  if (fieldSearchable[fieldIndex] === 0) continue;
7152
- const termFrequency = list.termFrequencies[pi];
7010
+ const termFrequency = termFrequencies[pi];
7153
7011
  const fieldBoost = fieldBoosts[fieldIndex];
7154
7012
  const avgLen = fieldAvgLengths[fieldIndex];
7155
7013
  const actualFieldLength = resolveFieldLength(internalId, fieldIndex, avgLen);
@@ -7182,23 +7040,21 @@ function searchFulltext(state, params) {
7182
7040
  }
7183
7041
  continue;
7184
7042
  }
7185
- const matchingPostings = exact ? (() => {
7186
- const postingList = state.invertedIdx.lookup(qt.token);
7187
- return postingList ? [{ token: qt.token, postingList }] : [];
7188
- })() : state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength);
7043
+ const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
7189
7044
  for (const match of matchingPostings) {
7190
7045
  const docFreq = globalStats ? globalDocFreqFor(globalDocFreqs, match.token, match.postingList.docIdSet.size) : match.postingList.docIdSet.size;
7191
7046
  const idf = computeIDF(docFreq, totalDocs);
7192
- const list = match.postingList;
7193
- const hasDeleted = list.deletedDocs.size > 0;
7194
- for (let pi = 0; pi < list.length; pi++) {
7195
- const internalId = list.docIds[pi];
7196
- if (hasDeleted && list.deletedDocs.has(internalId)) continue;
7047
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
7048
+ match.postingList
7049
+ );
7050
+ for (let pi = 0; pi < count; pi++) {
7051
+ const internalId = docIds[pi];
7052
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
7197
7053
  if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
7198
- const fieldIndex = list.fieldNameIndices[pi];
7054
+ const fieldIndex = fieldNameIndices[pi];
7199
7055
  loadFieldMeta(fieldIndex);
7200
7056
  if (fieldSearchable[fieldIndex] === 0) continue;
7201
- const termFrequency = list.termFrequencies[pi];
7057
+ const termFrequency = termFrequencies[pi];
7202
7058
  const fieldBoost = fieldBoosts[fieldIndex];
7203
7059
  const avgLen = fieldAvgLengths[fieldIndex];
7204
7060
  const actualFieldLength = resolveFieldLength(internalId, fieldIndex, avgLen);
@@ -8111,8 +7967,11 @@ function createPartitionIndex(partitionId, trackPositions = true) {
8111
7967
  return partition;
8112
7968
  }
8113
7969
 
7970
+ // src/partitioning/constants.ts
7971
+ var REBALANCE_CHUNK_SIZE = 1e3;
7972
+ var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
7973
+
8114
7974
  // src/partitioning/rebalancer.ts
8115
- var CHUNK_SIZE = 1e3;
8116
7975
  function createRebalancer() {
8117
7976
  const activeManagers = /* @__PURE__ */ new WeakSet();
8118
7977
  let activeCount = 0;
@@ -8175,7 +8034,7 @@ function createRebalancer() {
8175
8034
  documentsProcessed++;
8176
8035
  }
8177
8036
  chunkFill++;
8178
- if (chunkFill >= CHUNK_SIZE) {
8037
+ if (chunkFill >= REBALANCE_CHUNK_SIZE) {
8179
8038
  chunkFill = 0;
8180
8039
  onProgress?.({
8181
8040
  phase: "moving",
@@ -9152,6 +9011,9 @@ function createFrozenTombstones() {
9152
9011
  removed.add(internalId);
9153
9012
  revision++;
9154
9013
  return true;
9014
+ },
9015
+ values() {
9016
+ return removed.values();
9155
9017
  }
9156
9018
  };
9157
9019
  }
@@ -9275,7 +9137,7 @@ function buildGeoReaders(entries) {
9275
9137
  }
9276
9138
  return readers;
9277
9139
  }
9278
- function assembleFrozenSegment(segmentId, source, documentSource) {
9140
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot) {
9279
9141
  const tombstones = createFrozenTombstones();
9280
9142
  const postingViews = createFrozenPostingViews(source, tombstones);
9281
9143
  const lengthColumns = /* @__PURE__ */ new Map();
@@ -9296,6 +9158,7 @@ function assembleFrozenSegment(segmentId, source, documentSource) {
9296
9158
  const segment = {
9297
9159
  segmentId,
9298
9160
  documentSource,
9161
+ sharedSnapshot,
9299
9162
  invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews),
9300
9163
  docStore,
9301
9164
  stats: buildStatsView(source),
@@ -9326,6 +9189,11 @@ function assembleFrozenSegment(segmentId, source, documentSource) {
9326
9189
  segment.scoreBuffer = null;
9327
9190
  }
9328
9191
  return added;
9192
+ },
9193
+ tombstonedDocIds() {
9194
+ const docIds = [];
9195
+ for (const ordinal of tombstones.values()) docIds.push(source.idTable.idAt(ordinal));
9196
+ return docIds;
9329
9197
  }
9330
9198
  };
9331
9199
  return segment;
@@ -9339,7 +9207,8 @@ function createFrozenSegment(payload, documents, segmentId) {
9339
9207
  idTable: buildExternalIdTable(payload.docIds),
9340
9208
  docFrequencies: () => payload.docFrequencies
9341
9209
  },
9342
- wrapDocumentArray(documents)
9210
+ wrapDocumentArray(documents),
9211
+ null
9343
9212
  );
9344
9213
  }
9345
9214
  function createSharedFrozenSegment(snapshot) {
@@ -9358,7 +9227,8 @@ function createSharedFrozenSegment(snapshot) {
9358
9227
  return frequencies;
9359
9228
  }
9360
9229
  },
9361
- wrapEncodedDocumentTable(snapshot.documentTable)
9230
+ wrapEncodedDocumentTable(snapshot.documentTable),
9231
+ snapshot
9362
9232
  );
9363
9233
  }
9364
9234
 
@@ -9400,6 +9270,15 @@ function buildCompactedSegmentPayload(segments) {
9400
9270
  }
9401
9271
  return { payload: scratch.encodeSegment(), documents };
9402
9272
  }
9273
+ function freezeLiveTailInto(live, liveState, frozen, freeze) {
9274
+ if (live.count() === 0) return null;
9275
+ const payload = live.encodeSegment();
9276
+ const segment = freeze(payload, survivorDocumentsOf(liveState, payload));
9277
+ if (segment === null) return null;
9278
+ live.clear();
9279
+ frozen.push(segment);
9280
+ return segment;
9281
+ }
9403
9282
  function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, partitionId) {
9404
9283
  const dropped = resolveFrozenSegments(frozen, dropSegmentIds, partitionId);
9405
9284
  const dropSet = new Set(dropSegmentIds);
@@ -9439,11 +9318,15 @@ function subBitsetView(composite, layout, subIndex) {
9439
9318
  const endWord = subIndex + 1 < layout.bases.length ? layout.bases[subIndex + 1] >> 5 : Math.ceil(layout.totalCapacity / WORD_BITS);
9440
9319
  return composite.subarray(startWord, endWord);
9441
9320
  }
9321
+ function placeSubBitset(composite, layout, subIndex, subBits) {
9322
+ const words = subBitsetView(composite, layout, subIndex).length;
9323
+ if (words === 0) return;
9324
+ composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
9325
+ }
9442
9326
  function compositeFiltersBitset(subs, layout, filters, schema) {
9443
9327
  const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
9444
9328
  for (let i = 0; i < subs.length; i++) {
9445
- const subBits = applyPartitionFiltersBitset(subs[i], filters, schema);
9446
- composite.set(subBits, layout.bases[i] >> 5);
9329
+ placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema));
9447
9330
  }
9448
9331
  return composite;
9449
9332
  }
@@ -9705,10 +9588,30 @@ function subParams(params, layout, subIndex, globalStats) {
9705
9588
  filterBitset: params.filterBitset === void 0 ? void 0 : subBitsetView(params.filterBitset, layout, subIndex)
9706
9589
  };
9707
9590
  }
9591
+ function onlyPopulatedSub(subs) {
9592
+ let found = -1;
9593
+ for (let i = 0; i < subs.length; i++) {
9594
+ if (subs[i].stats.totalDocuments === 0) continue;
9595
+ if (found !== -1) return -1;
9596
+ found = i;
9597
+ }
9598
+ return found;
9599
+ }
9600
+ function searchSingleSub(subs, layout, params, index) {
9601
+ const result = searchFulltext(subs[index], subParams(params, layout, index, params.globalStats));
9602
+ if (result.matchedOrdinalBitset === void 0) return result;
9603
+ const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
9604
+ placeSubBitset(matchedOrdinalBitset, layout, index, result.matchedOrdinalBitset);
9605
+ return { ...result, matchedOrdinalBitset };
9606
+ }
9708
9607
  function compositeSearchFulltext(subs, layout, params) {
9709
9608
  if (params.queryTokens.length === 0) {
9710
9609
  return { scored: [], totalMatched: 0 };
9711
9610
  }
9611
+ const single = onlyPopulatedSub(subs);
9612
+ if (single !== -1) {
9613
+ return searchSingleSub(subs, layout, params, single);
9614
+ }
9712
9615
  const globalStats = compositeQueryStats(subs, params);
9713
9616
  const results = subs.map((sub, index) => searchFulltext(sub, subParams(params, layout, index, globalStats)));
9714
9617
  const scored = kWayMerge(results.map((result) => result.scored));
@@ -9726,7 +9629,7 @@ function compositeSearchFulltext(subs, layout, params) {
9726
9629
  const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
9727
9630
  for (let i = 0; i < results.length; i++) {
9728
9631
  const subBitset = results[i].matchedOrdinalBitset;
9729
- if (subBitset !== void 0) matchedOrdinalBitset.set(subBitset, layout.bases[i] >> 5);
9632
+ if (subBitset !== void 0) placeSubBitset(matchedOrdinalBitset, layout, i, subBitset);
9730
9633
  }
9731
9634
  return { scored, totalMatched, matchedOrdinalBitset };
9732
9635
  }
@@ -9770,7 +9673,7 @@ function compositeSearchMatches(subs, layout, params) {
9770
9673
  ordinalBitset() {
9771
9674
  const combined = createBitSet(layout.totalCapacity);
9772
9675
  for (let i = 0; i < subMatches.length; i++) {
9773
- combined.set(subMatches[i].ordinalBitset(), layout.bases[i] >> 5);
9676
+ placeSubBitset(combined, layout, i, subMatches[i].ordinalBitset());
9774
9677
  }
9775
9678
  return combined;
9776
9679
  }
@@ -9853,17 +9756,6 @@ function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansi
9853
9756
  function isCompositePartition(partition) {
9854
9757
  return "attachFrozenSegment" in partition;
9855
9758
  }
9856
- function survivorDocuments(sub, payload) {
9857
- return payload.docIds.map((docId) => {
9858
- const stored = sub.docStore.get(docId);
9859
- if (stored === void 0) {
9860
- throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while composing a segment`, {
9861
- docId
9862
- });
9863
- }
9864
- return stored.fields;
9865
- });
9866
- }
9867
9759
  function createCompositePartition(partitionId, trackPositions = true, existingLive) {
9868
9760
  const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
9869
9761
  const liveState = partitionStateOf(live);
@@ -9886,7 +9778,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
9886
9778
  for (const sub of subs()) {
9887
9779
  const payload = encodeSegmentState(sub);
9888
9780
  if (payload.documentCount === 0) continue;
9889
- scratch.mergeSegmentPayload(payload, survivorDocuments(sub, payload));
9781
+ scratch.mergeSegmentPayload(payload, survivorDocumentsOf(sub, payload));
9890
9782
  }
9891
9783
  return scratch;
9892
9784
  }
@@ -9922,6 +9814,16 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
9922
9814
  swapFrozenSegmentList(frozen, dropSegmentIds, replacement, partitionId);
9923
9815
  invalidateDocFrequencies();
9924
9816
  },
9817
+ freezeLiveTail(freeze) {
9818
+ const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
9819
+ if (segment !== null) invalidateDocFrequencies();
9820
+ return segment;
9821
+ },
9822
+ replaceLiveTail(segment) {
9823
+ live.clear();
9824
+ frozen.push(segment);
9825
+ invalidateDocFrequencies();
9826
+ },
9925
9827
  appendFrozenSegment(payload, documents) {
9926
9828
  for (const docId of payload.docIds) {
9927
9829
  if (live.has(docId) || frozenOwner(docId) !== void 0) {
@@ -9966,7 +9868,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
9966
9868
  for (const segment of frozen) {
9967
9869
  const payload = encodeSegmentState(segment);
9968
9870
  if (payload.documentCount === 0) continue;
9969
- live.mergeSegmentPayload(payload, survivorDocuments(segment, payload));
9871
+ live.mergeSegmentPayload(payload, survivorDocumentsOf(segment, payload));
9970
9872
  }
9971
9873
  frozen.length = 0;
9972
9874
  }
@@ -10231,6 +10133,15 @@ function mergePartitionStats(statsArray) {
10231
10133
  };
10232
10134
  }
10233
10135
 
10136
+ // src/search/constants.ts
10137
+ var RESULT_WINDOW = 1e4;
10138
+ var DEFAULT_PAGE_SIZE = 10;
10139
+ var MAX_CURSOR_LENGTH = 40960;
10140
+ var MAX_SORT_FIELDS = 8;
10141
+ var MAX_SORT_FIELD_NAME_LENGTH = 255;
10142
+ var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
10143
+ var PREFIX_MAX_EXPANSIONS = 50;
10144
+
10234
10145
  // src/search/cursor-codec.ts
10235
10146
  function encodeCursorText(text) {
10236
10147
  if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
@@ -10288,11 +10199,6 @@ function applySorting(hits, sort, sortKeyOf) {
10288
10199
 
10289
10200
  // src/search/cursor.ts
10290
10201
  var CURSOR_VERSION = 3;
10291
- var MAX_CURSOR_LENGTH = 40960;
10292
- var MAX_SORT_FIELDS = 8;
10293
- var MAX_SORT_FIELD_NAME_LENGTH = 255;
10294
- var MAX_ANCHOR_CODE_POINTS = 512;
10295
- var MAX_SORT_VALUE_CODE_POINTS = 512;
10296
10202
  function invalidCursor(cursor, reason) {
10297
10203
  const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
10298
10204
  return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
@@ -10360,7 +10266,7 @@ function decodeSortKey(value, cursor) {
10360
10266
  continue;
10361
10267
  }
10362
10268
  if (typeof entry === "string") {
10363
- if (exceedsCodePoints(entry, MAX_SORT_VALUE_CODE_POINTS)) {
10269
+ if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
10364
10270
  throw invalidCursor(cursor, '"k" holds an oversized sort value');
10365
10271
  }
10366
10272
  key.push(entry);
@@ -10393,7 +10299,7 @@ function decodePageCursor(cursor) {
10393
10299
  throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
10394
10300
  }
10395
10301
  if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
10396
- if (exceedsCodePoints(a, MAX_ANCHOR_CODE_POINTS)) {
10302
+ if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
10397
10303
  throw invalidCursor(cursor, '"a" is too long to be a document id');
10398
10304
  }
10399
10305
  if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
@@ -10435,8 +10341,6 @@ function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, bindin
10435
10341
  }
10436
10342
 
10437
10343
  // src/search/pagination.ts
10438
- var RESULT_WINDOW = 1e4;
10439
- var DEFAULT_PAGE_SIZE = 10;
10440
10344
  function clampRowCount(value, fallback) {
10441
10345
  if (value === void 0 || !Number.isFinite(value)) return fallback;
10442
10346
  return Math.max(0, Math.floor(value));
@@ -10508,7 +10412,6 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
10508
10412
  }
10509
10413
 
10510
10414
  // src/search/fulltext.ts
10511
- var PREFIX_MAX_EXPANSIONS = 50;
10512
10415
  function resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken) {
10513
10416
  const term = params.term;
10514
10417
  if (term === void 0) return void 0;
@@ -11096,6 +10999,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
11096
10999
  if (docId !== void 0) docPartitionMap.set(docId, partitionId);
11097
11000
  }
11098
11001
  },
11002
+ freezeLiveTail(partitionId, freeze) {
11003
+ validatePartitionId(partitionId);
11004
+ return asCompositePartition(partitionId).freezeLiveTail(freeze);
11005
+ },
11006
+ replaceLiveTail(partitionId, segment) {
11007
+ validatePartitionId(partitionId);
11008
+ asCompositePartition(partitionId).replaceLiveTail(segment);
11009
+ },
11099
11010
  deserializePartition(partitionId, data) {
11100
11011
  validatePartitionId(partitionId);
11101
11012
  partitions[partitionId].deserialize(data, config.schema);
@@ -11160,8 +11071,16 @@ function fanOutMatchCount(manager, params, language, schema, options) {
11160
11071
  return count;
11161
11072
  }
11162
11073
 
11074
+ // src/vector/constants.ts
11075
+ var WASM_PAGE_BYTES = 65536;
11076
+ var MAX_WASM_PAGES = 65536;
11077
+ var VECTOR_STORE_INITIAL_CAPACITY = 16;
11078
+ var HNSW_BUILD_TIMEOUT_MS = 12e4;
11079
+ var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
11080
+ var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
11081
+ var QUANTIZATION_PADDING_FACTOR = 0.01;
11082
+
11163
11083
  // src/vector/scalar-quantization-calibration.ts
11164
- var PADDING_FACTOR = 0.01;
11165
11084
  function computeCalibrationBounds(vectors, dimensions) {
11166
11085
  let globalMin = Number.POSITIVE_INFINITY;
11167
11086
  let globalMax = Number.NEGATIVE_INFINITY;
@@ -11176,7 +11095,7 @@ function computeCalibrationBounds(vectors, dimensions) {
11176
11095
  }
11177
11096
  if (count === 0) return null;
11178
11097
  const range = globalMax - globalMin;
11179
- const pad = range * PADDING_FACTOR;
11098
+ const pad = range * QUANTIZATION_PADDING_FACTOR;
11180
11099
  globalMin -= pad;
11181
11100
  globalMax += pad;
11182
11101
  if (globalMin === globalMax) {
@@ -11372,8 +11291,6 @@ function arenaFloat32Distance(simd, byteA, byteB, dimension, metric, magnitudeA,
11372
11291
  }
11373
11292
 
11374
11293
  // src/vector/scalar-quantization.ts
11375
- var INITIAL_CAPACITY2 = 16;
11376
- var PAGE_BYTES = 65536;
11377
11294
  function createScalarQuantizer(dimensions, ordinalSource) {
11378
11295
  const docToOrd = /* @__PURE__ */ new Map();
11379
11296
  let simd = createArenaSimd();
@@ -11391,7 +11308,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
11391
11308
  let calibrated = false;
11392
11309
  function ensureCapacity3(needed) {
11393
11310
  if (needed <= capacity) return;
11394
- let newCap = capacity === 0 ? INITIAL_CAPACITY2 : capacity;
11311
+ let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
11395
11312
  while (newCap < needed) newCap *= 2;
11396
11313
  const nextSums = new Float64Array(newCap);
11397
11314
  nextSums.set(sums);
@@ -11410,7 +11327,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
11410
11327
  const have = simd.memory.buffer.byteLength;
11411
11328
  if (requiredBytes > have) {
11412
11329
  try {
11413
- simd.memory.grow(Math.ceil((requiredBytes - have) / PAGE_BYTES));
11330
+ simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
11414
11331
  } catch {
11415
11332
  const migrated = new Uint8Array(newCap * dimensions);
11416
11333
  migrated.set(quantizedArena.subarray(0, capacity * dimensions));
@@ -11721,8 +11638,6 @@ function euclideanDistance(a, b) {
11721
11638
  }
11722
11639
 
11723
11640
  // src/vector/vector-store/index.ts
11724
- var INITIAL_CAPACITY3 = 16;
11725
- var PAGE_BYTES2 = 65536;
11726
11641
  var UNKNOWN_PARTITION = -1;
11727
11642
  var NO_PARTITION = -2;
11728
11643
  function createVectorStore() {
@@ -11749,7 +11664,7 @@ function createVectorStore() {
11749
11664
  }
11750
11665
  function ensureCapacity3(needed) {
11751
11666
  if (needed <= capacity) return;
11752
- let newCap = capacity === 0 ? INITIAL_CAPACITY3 : capacity;
11667
+ let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
11753
11668
  while (newCap < needed) newCap *= 2;
11754
11669
  const nextMags = new Float64Array(newCap);
11755
11670
  nextMags.set(mags);
@@ -11759,7 +11674,7 @@ function createVectorStore() {
11759
11674
  const have = simd.memory.buffer.byteLength;
11760
11675
  if (requiredBytes > have) {
11761
11676
  try {
11762
- simd.memory.grow(Math.ceil((requiredBytes - have) / PAGE_BYTES2));
11677
+ simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
11763
11678
  } catch {
11764
11679
  const migrated = new Float32Array(newCap * dimension);
11765
11680
  migrated.set(arena.subarray(scratchFloatLength, scratchFloatLength + capacity * dimension));
@@ -11789,7 +11704,7 @@ function createVectorStore() {
11789
11704
  }
11790
11705
  function ensurePartitionSlots(ord) {
11791
11706
  if (ord < partitionOf.length) return;
11792
- let length = partitionOf.length === 0 ? INITIAL_CAPACITY3 : partitionOf.length;
11707
+ let length = partitionOf.length === 0 ? VECTOR_STORE_INITIAL_CAPACITY : partitionOf.length;
11793
11708
  while (length <= ord) length *= 2;
11794
11709
  const grown = new Int32Array(length).fill(UNKNOWN_PARTITION);
11795
11710
  grown.set(partitionOf);
@@ -12031,10 +11946,193 @@ function createVectorStore() {
12031
11946
  };
12032
11947
  }
12033
11948
 
11949
+ // src/vector/hnsw/adjacency.ts
11950
+ var ABSENT = -1;
11951
+ function createAdjacency(m, mMax0) {
11952
+ return {
11953
+ m,
11954
+ mMax0,
11955
+ level0Stride: mMax0 + 2,
11956
+ upperStride: m + 2,
11957
+ capacity: 0,
11958
+ slots: 0,
11959
+ nodeLevels: new Int8Array(0),
11960
+ level0: new Int32Array(0),
11961
+ upperBase: new Int32Array(0),
11962
+ upper: new Int32Array(0),
11963
+ upperUsed: 0,
11964
+ freeBlocks: Array.from({ length: MAX_LAYER_CAP + 1 }, () => [])
11965
+ };
11966
+ }
11967
+ function ensureAdjacencyCapacity(adj, needed) {
11968
+ if (needed <= adj.capacity) return;
11969
+ let newCapacity = adj.capacity === 0 ? 16 : adj.capacity;
11970
+ while (newCapacity < needed) newCapacity *= 2;
11971
+ const nextLevels = new Int8Array(newCapacity).fill(ABSENT);
11972
+ nextLevels.set(adj.nodeLevels);
11973
+ adj.nodeLevels = nextLevels;
11974
+ const nextLevel0 = new Int32Array(newCapacity * adj.level0Stride);
11975
+ nextLevel0.set(adj.level0);
11976
+ adj.level0 = nextLevel0;
11977
+ const nextUpperBase = new Int32Array(newCapacity).fill(ABSENT);
11978
+ nextUpperBase.set(adj.upperBase);
11979
+ adj.upperBase = nextUpperBase;
11980
+ adj.capacity = newCapacity;
11981
+ }
11982
+ function allocateUpperBlock(adj, levels) {
11983
+ const reusable = adj.freeBlocks[levels].pop();
11984
+ const size = levels * adj.upperStride;
11985
+ if (reusable !== void 0) {
11986
+ adj.upper.fill(0, reusable, reusable + size);
11987
+ return reusable;
11988
+ }
11989
+ if (adj.upperUsed + size > adj.upper.length) {
11990
+ const required = adj.upperUsed + size;
11991
+ let nextLength = adj.upper.length === 0 ? INITIAL_UPPER_SLOTS * adj.upperStride : adj.upper.length;
11992
+ while (nextLength < required) nextLength *= 2;
11993
+ const nextUpper = new Int32Array(nextLength);
11994
+ nextUpper.set(adj.upper);
11995
+ adj.upper = nextUpper;
11996
+ }
11997
+ const offset = adj.upperUsed;
11998
+ adj.upperUsed += size;
11999
+ adj.upper.fill(0, offset, offset + size);
12000
+ return offset;
12001
+ }
12002
+ function createNode(adj, ord, maxLayer) {
12003
+ ensureAdjacencyCapacity(adj, ord + 1);
12004
+ if (ord >= adj.slots) adj.slots = ord + 1;
12005
+ const existingLevel = adj.nodeLevels[ord];
12006
+ const existingBase = adj.upperBase[ord];
12007
+ if (existingLevel >= 1 && existingBase !== ABSENT) {
12008
+ adj.freeBlocks[existingLevel].push(existingBase);
12009
+ }
12010
+ adj.nodeLevels[ord] = maxLayer;
12011
+ adj.level0[ord * adj.level0Stride] = 0;
12012
+ adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) : ABSENT;
12013
+ }
12014
+ function deleteNode(adj, ord) {
12015
+ if (ord < 0 || ord >= adj.capacity) return;
12016
+ const level = adj.nodeLevels[ord];
12017
+ const base = adj.upperBase[ord];
12018
+ if (level >= 1 && base !== ABSENT) {
12019
+ adj.freeBlocks[level].push(base);
12020
+ }
12021
+ adj.nodeLevels[ord] = ABSENT;
12022
+ adj.upperBase[ord] = ABSENT;
12023
+ adj.level0[ord * adj.level0Stride] = 0;
12024
+ }
12025
+ function resetAdjacency(adj) {
12026
+ adj.nodeLevels = new Int8Array(0);
12027
+ adj.level0 = new Int32Array(0);
12028
+ adj.upperBase = new Int32Array(0);
12029
+ adj.upper = new Int32Array(0);
12030
+ adj.upperUsed = 0;
12031
+ adj.capacity = 0;
12032
+ adj.slots = 0;
12033
+ for (const list of adj.freeBlocks) list.length = 0;
12034
+ }
12035
+ function nodeLevel(adj, ord) {
12036
+ if (ord < 0 || ord >= adj.capacity) return ABSENT;
12037
+ return adj.nodeLevels[ord];
12038
+ }
12039
+ function hasNode(adj, ord) {
12040
+ return nodeLevel(adj, ord) !== ABSENT;
12041
+ }
12042
+ function layerArray(adj, layer) {
12043
+ return layer === 0 ? adj.level0 : adj.upper;
12044
+ }
12045
+ function layerBase(adj, ord, layer) {
12046
+ const level = nodeLevel(adj, ord);
12047
+ if (level === ABSENT || layer > level) return ABSENT;
12048
+ if (layer === 0) return ord * adj.level0Stride;
12049
+ const base = adj.upperBase[ord];
12050
+ if (base === ABSENT) return ABSENT;
12051
+ return base + (layer - 1) * adj.upperStride;
12052
+ }
12053
+ function neighborCount(adj, ord, layer) {
12054
+ const base = layerBase(adj, ord, layer);
12055
+ if (base === ABSENT) return 0;
12056
+ return layerArray(adj, layer)[base];
12057
+ }
12058
+ function collectNeighbors(adj, ord, layer) {
12059
+ const base = layerBase(adj, ord, layer);
12060
+ if (base === ABSENT) return [];
12061
+ const array = layerArray(adj, layer);
12062
+ const count = array[base];
12063
+ const collected = new Array(count);
12064
+ for (let i = 0; i < count; i++) {
12065
+ collected[i] = array[base + i + 1];
12066
+ }
12067
+ return collected;
12068
+ }
12069
+ function addNeighbor(adj, ord, layer, neighborOrd) {
12070
+ const base = layerBase(adj, ord, layer);
12071
+ if (base === ABSENT) return;
12072
+ const array = layerArray(adj, layer);
12073
+ const count = array[base];
12074
+ for (let i = 1; i <= count; i++) {
12075
+ if (array[base + i] === neighborOrd) return;
12076
+ }
12077
+ const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
12078
+ if (count + 1 > stride - 1) return;
12079
+ array[base + count + 1] = neighborOrd;
12080
+ array[base] = count + 1;
12081
+ }
12082
+ function removeNeighbor(adj, ord, layer, neighborOrd) {
12083
+ const base = layerBase(adj, ord, layer);
12084
+ if (base === ABSENT) return;
12085
+ const array = layerArray(adj, layer);
12086
+ const count = array[base];
12087
+ for (let i = 1; i <= count; i++) {
12088
+ if (array[base + i] !== neighborOrd) continue;
12089
+ for (let j = i; j < count; j++) {
12090
+ array[base + j] = array[base + j + 1];
12091
+ }
12092
+ array[base] = count - 1;
12093
+ return;
12094
+ }
12095
+ }
12096
+ function replaceNeighbors(adj, ord, layer, neighbors, count) {
12097
+ const base = layerBase(adj, ord, layer);
12098
+ if (base === ABSENT) return;
12099
+ const array = layerArray(adj, layer);
12100
+ const stride = layer === 0 ? adj.level0Stride : adj.upperStride;
12101
+ const limit = Math.min(count, stride - 1);
12102
+ for (let i = 0; i < limit; i++) {
12103
+ array[base + i + 1] = neighbors[i];
12104
+ }
12105
+ array[base] = limit;
12106
+ }
12107
+ function exportAdjacency(adj) {
12108
+ return {
12109
+ m: adj.m,
12110
+ mMax0: adj.mMax0,
12111
+ capacity: adj.capacity,
12112
+ slots: adj.slots,
12113
+ upperUsed: adj.upperUsed,
12114
+ nodeLevels: adj.nodeLevels.slice(),
12115
+ level0: adj.level0.slice(),
12116
+ upperBase: adj.upperBase.slice(),
12117
+ upper: adj.upper.slice(0, adj.upperUsed)
12118
+ };
12119
+ }
12120
+ function importAdjacency(snapshot) {
12121
+ const adj = createAdjacency(snapshot.m, snapshot.mMax0);
12122
+ adj.capacity = snapshot.capacity;
12123
+ adj.slots = snapshot.slots;
12124
+ adj.upperUsed = snapshot.upperUsed;
12125
+ adj.nodeLevels = snapshot.nodeLevels;
12126
+ adj.level0 = snapshot.level0;
12127
+ adj.upperBase = snapshot.upperBase;
12128
+ adj.upper = snapshot.upper;
12129
+ return adj;
12130
+ }
12131
+ function estimateAdjacencyBytes(adj) {
12132
+ return adj.nodeLevels.byteLength + adj.level0.byteLength + adj.upperBase.byteLength + adj.upper.byteLength;
12133
+ }
12134
+
12034
12135
  // src/vector/hnsw/shared.ts
12035
- var COMPACTION_TOMBSTONE_RATIO = 0.1;
12036
- var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
12037
- var SQ8_OVERSELECTION_FACTOR = 2;
12038
12136
  function ensureCapacity(state, needed) {
12039
12137
  ensureAdjacencyCapacity(state.adjacency, needed);
12040
12138
  if (needed <= state.capacity) return;
@@ -12108,7 +12206,6 @@ function addConnection(connections, ord) {
12108
12206
  }
12109
12207
 
12110
12208
  // src/vector/hnsw/workspace.ts
12111
- var INITIAL_LIST_CAPACITY = 64;
12112
12209
  function createList(capacity) {
12113
12210
  return { ords: new Int32Array(capacity), distances: new Float64Array(capacity), size: 0 };
12114
12211
  }
@@ -12699,7 +12796,7 @@ function collectHits(state, query, k, searchMetric, minSimilarity, filter, efSea
12699
12796
  }
12700
12797
  function rerankWithFullPrecision(state, candidates, query, qMag, arenaQuery, k, metric, minSimilarity, filter, hasDocument) {
12701
12798
  const reranked = [];
12702
- const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, 10);
12799
+ const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR);
12703
12800
  for (let i = 0; i < candidates.size; i++) {
12704
12801
  const ord = candidates.ords[i];
12705
12802
  if (filter && !ordinalFilterHas(filter, ord)) continue;
@@ -12866,7 +12963,7 @@ function createHNSWIndex(dimension, store, config, quantizer) {
12866
12963
  }
12867
12964
 
12868
12965
  // src/workers/node-worker.browser.ts
12869
- async function spawnNodeWorker(_entryPoint) {
12966
+ async function spawnNodeWorker(_entryPoint, _resourceLimits) {
12870
12967
  return null;
12871
12968
  }
12872
12969
  async function isNodeMainThread() {
@@ -12908,7 +13005,6 @@ function detectRuntime() {
12908
13005
  }
12909
13006
 
12910
13007
  // src/vector/hnsw-worker-dispatch.ts
12911
- var BUILD_TIMEOUT_MS = 12e4;
12912
13008
  function resolveWorkerEntryPoint() {
12913
13009
  const base = import.meta.url;
12914
13010
  const distIndex = base.lastIndexOf("/dist/");
@@ -12974,7 +13070,7 @@ async function dispatchWorkerBuild(docIds, vectorData, dimension, config, timeou
12974
13070
  message: "No worker thread support available in the current runtime"
12975
13071
  };
12976
13072
  }
12977
- const effectiveTimeout = timeoutMs ?? BUILD_TIMEOUT_MS;
13073
+ const effectiveTimeout = timeoutMs ?? HNSW_BUILD_TIMEOUT_MS;
12978
13074
  let transferBuffer;
12979
13075
  if (skipCopy) {
12980
13076
  transferBuffer = vectorData;
@@ -13044,7 +13140,7 @@ async function dispatchWorkerBuild(docIds, vectorData, dimension, config, timeou
13044
13140
  });
13045
13141
  }
13046
13142
 
13047
- // src/vector/vector-index/shared.ts
13143
+ // src/vector/vector-index/constants.ts
13048
13144
  var DEFAULT_PROMOTION_THRESHOLD = 1024;
13049
13145
  var DEFAULT_FILTER_THRESHOLD = 0.03;
13050
13146
  var ESTIMATED_MS_PER_TOMBSTONE = 0.05;
@@ -13053,6 +13149,9 @@ var WORKER_BUILD_SIZE_THRESHOLD = 5e3;
13053
13149
  var WORKER_COPY_MIN_VECTORS = 1024;
13054
13150
  var BUILD_CHUNK_SIZE = 100;
13055
13151
  var REBUILD_REMOVED_RATIO = 0.2;
13152
+
13153
+ // src/vector/vector-index/shared.ts
13154
+ var VECTOR_WORKER_COPIES_ALLOWED = { enabled: true };
13056
13155
  function liveSize(state) {
13057
13156
  return state.store.size - state.tombstones.size;
13058
13157
  }
@@ -13164,6 +13263,16 @@ function recalibrateFromStore(state) {
13164
13263
  sq8.recalibrateAll(storeVectors());
13165
13264
  }
13166
13265
 
13266
+ // src/workers/constants.ts
13267
+ var WORKER_YOUNG_GENERATION_MB = 24;
13268
+ var MIN_WORKER_COUNT = 2;
13269
+ var MAX_WORKER_COUNT = 8;
13270
+ var FALLBACK_CPU_COUNT = 4;
13271
+ var DEFAULT_BACKPRESSURE_LIMIT = 100;
13272
+ var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
13273
+ var SHUTDOWN_TIMEOUT_MS = 5e3;
13274
+ var CONSECUTIVE_TIMEOUTS_BEFORE_DEATH = 3;
13275
+
13167
13276
  // src/workers/protocol.ts
13168
13277
  function createRequestId() {
13169
13278
  return generateId();
@@ -13190,7 +13299,7 @@ function resolveWorkerCount(requested) {
13190
13299
  if (requested !== void 0 && requested > 0) {
13191
13300
  return requested;
13192
13301
  }
13193
- let cpuCount = 4;
13302
+ let cpuCount = FALLBACK_CPU_COUNT;
13194
13303
  try {
13195
13304
  if (navigator?.hardwareConcurrency) {
13196
13305
  cpuCount = navigator.hardwareConcurrency;
@@ -13201,9 +13310,13 @@ function resolveWorkerCount(requested) {
13201
13310
  }
13202
13311
  }
13203
13312
  } catch {
13204
- cpuCount = 4;
13313
+ cpuCount = FALLBACK_CPU_COUNT;
13205
13314
  }
13206
- return Math.max(2, Math.min(8, cpuCount - 1));
13315
+ return Math.max(MIN_WORKER_COUNT, Math.min(MAX_WORKER_COUNT, cpuCount - 1));
13316
+ }
13317
+ function splitWorkerBudget(total) {
13318
+ const keyword = Math.ceil(total / 2);
13319
+ return { keyword, vector: total - keyword };
13207
13320
  }
13208
13321
  function createWorkerPool(config) {
13209
13322
  const workerCount = resolveWorkerCount(config.count);
@@ -13212,28 +13325,60 @@ function createWorkerPool(config) {
13212
13325
  const indexAssignment = /* @__PURE__ */ new Map();
13213
13326
  let isShutdown = false;
13214
13327
  function handleWorkerDeath(slotIndex, error) {
13215
- if (isShutdown || deadSlots.has(slotIndex)) {
13328
+ if (isShutdown) {
13216
13329
  return;
13217
13330
  }
13218
- deadSlots.add(slotIndex);
13219
13331
  const slot = workers.get(slotIndex);
13332
+ if (slot !== void 0 && !slot.serving) {
13333
+ workers.delete(slotIndex);
13334
+ return;
13335
+ }
13336
+ if (deadSlots.has(slotIndex)) {
13337
+ return;
13338
+ }
13339
+ deadSlots.add(slotIndex);
13220
13340
  const indexNames = slot ? [...slot.indexes].sort() : [];
13221
13341
  workers.delete(slotIndex);
13222
13342
  config.onWorkerCrash?.(slotIndex, indexNames, error);
13223
13343
  }
13344
+ function spawnSlot(slotIndex, serving) {
13345
+ const slot = {
13346
+ executor: config.workerFactory(slotIndex, (error) => handleWorkerDeath(slotIndex, error)),
13347
+ indexes: /* @__PURE__ */ new Set(),
13348
+ inFlight: 0,
13349
+ serving
13350
+ };
13351
+ workers.set(slotIndex, slot);
13352
+ return slot;
13353
+ }
13224
13354
  function ensureWorker(slotIndex) {
13225
- if (deadSlots.has(slotIndex)) {
13226
- return void 0;
13227
- }
13228
- let slot = workers.get(slotIndex);
13229
- if (!slot) {
13230
- slot = {
13231
- executor: config.workerFactory(slotIndex, (error) => handleWorkerDeath(slotIndex, error)),
13232
- indexes: /* @__PURE__ */ new Set()
13233
- };
13234
- workers.set(slotIndex, slot);
13235
- }
13236
- return slot;
13355
+ const slot = workers.get(slotIndex);
13356
+ if (slot !== void 0) return slot;
13357
+ if (deadSlots.has(slotIndex)) return void 0;
13358
+ return spawnSlot(slotIndex, true);
13359
+ }
13360
+ function deadWorkerIds() {
13361
+ return [...deadSlots].filter((slotIndex) => !workers.has(slotIndex)).sort((a, b) => a - b);
13362
+ }
13363
+ function spawnReplacement(slotIndex) {
13364
+ if (isShutdown || !deadSlots.has(slotIndex) || workers.has(slotIndex)) return null;
13365
+ const slot = spawnSlot(slotIndex, false);
13366
+ return {
13367
+ workerId: slotIndex,
13368
+ executor: slot.executor,
13369
+ hold(indexName) {
13370
+ slot.indexes.add(indexName);
13371
+ },
13372
+ admit() {
13373
+ if (workers.get(slotIndex) !== slot) return;
13374
+ deadSlots.delete(slotIndex);
13375
+ slot.serving = true;
13376
+ },
13377
+ abandon() {
13378
+ if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
13379
+ void slot.executor.shutdown().catch(() => void 0);
13380
+ }
13381
+ };
13237
13382
  }
13238
13383
  function assignSlot(indexName) {
13239
13384
  const preferred = fnv1a(indexName) % workerCount;
@@ -13273,11 +13418,11 @@ function createWorkerPool(config) {
13273
13418
  throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
13274
13419
  }
13275
13420
  const slot = workers.get(slotIndex);
13276
- if (slot) {
13421
+ if (slot?.serving) {
13277
13422
  return slot.executor;
13278
13423
  }
13279
13424
  for (const survivor of workers.values()) {
13280
- if (survivor.indexes.has(indexName)) {
13425
+ if (survivor.serving && survivor.indexes.has(indexName)) {
13281
13426
  return survivor.executor;
13282
13427
  }
13283
13428
  }
@@ -13322,6 +13467,54 @@ function createWorkerPool(config) {
13322
13467
  function getAllExecutors() {
13323
13468
  return [...workers.values()].map((slot) => slot.executor);
13324
13469
  }
13470
+ function executorsHolding(indexName) {
13471
+ const holders = [];
13472
+ for (const slot of workers.values()) {
13473
+ if (slot.indexes.has(indexName)) holders.push(slot.executor);
13474
+ }
13475
+ return holders;
13476
+ }
13477
+ function lease(workerId, slot) {
13478
+ slot.inFlight += 1;
13479
+ let released = false;
13480
+ return {
13481
+ workerId,
13482
+ executor: slot.executor,
13483
+ release() {
13484
+ if (released) return;
13485
+ released = true;
13486
+ slot.inFlight -= 1;
13487
+ }
13488
+ };
13489
+ }
13490
+ function leaseLeastBusy() {
13491
+ let chosenId = -1;
13492
+ let chosen = null;
13493
+ for (const [workerId, slot] of workers) {
13494
+ if (!slot.serving) continue;
13495
+ if (chosen === null || slot.inFlight < chosen.inFlight) {
13496
+ chosenId = workerId;
13497
+ chosen = slot;
13498
+ }
13499
+ if (chosen.inFlight === 0) break;
13500
+ }
13501
+ return chosen === null ? null : lease(chosenId, chosen);
13502
+ }
13503
+ function queriesInFlight() {
13504
+ let total = 0;
13505
+ for (const slot of workers.values()) {
13506
+ if (slot.serving) total += slot.inFlight;
13507
+ }
13508
+ return total;
13509
+ }
13510
+ function leaseIdle(limit) {
13511
+ const leases = [];
13512
+ for (const [workerId, slot] of workers) {
13513
+ if (leases.length >= limit) break;
13514
+ if (slot.serving && slot.inFlight === 0) leases.push(lease(workerId, slot));
13515
+ }
13516
+ return leases;
13517
+ }
13325
13518
  function addIndexToAll(indexName) {
13326
13519
  if (isShutdown) {
13327
13520
  throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
@@ -13331,9 +13524,22 @@ function createWorkerPool(config) {
13331
13524
  }
13332
13525
  indexAssignment.set(indexName, assignSlot(indexName));
13333
13526
  }
13527
+ function spawnAll() {
13528
+ if (isShutdown) {
13529
+ throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
13530
+ }
13531
+ for (let i = 0; i < workerCount; i++) ensureWorker(i);
13532
+ }
13334
13533
  return {
13335
13534
  getExecutor,
13336
13535
  getAllExecutors,
13536
+ executorsHolding,
13537
+ leaseLeastBusy,
13538
+ leaseIdle,
13539
+ queriesInFlight,
13540
+ spawnAll,
13541
+ deadWorkerIds,
13542
+ spawnReplacement,
13337
13543
  get workerCount() {
13338
13544
  return workerCount;
13339
13545
  },
@@ -13345,6 +13551,9 @@ function createWorkerPool(config) {
13345
13551
  };
13346
13552
  }
13347
13553
 
13554
+ // src/workers/resource-limits.ts
13555
+ var WORKER_RESOURCE_LIMITS = { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
13556
+
13348
13557
  // src/vector/search-pool/spawn.ts
13349
13558
  function resolveWorkerEntryPoint2() {
13350
13559
  const base = import.meta.url;
@@ -13358,7 +13567,7 @@ async function spawnWorker2(entryPoint) {
13358
13567
  const runtime = detectRuntime();
13359
13568
  if (runtime.supportsWorkerThreads) {
13360
13569
  try {
13361
- return await spawnNodeWorker(new URL(entryPoint));
13570
+ return await spawnNodeWorker(new URL(entryPoint), WORKER_RESOURCE_LIMITS);
13362
13571
  } catch {
13363
13572
  return null;
13364
13573
  }
@@ -13393,8 +13602,6 @@ function listenForFailure(worker, handler) {
13393
13602
  }
13394
13603
 
13395
13604
  // src/vector/search-pool/index.ts
13396
- var SEARCH_TIMEOUT_MS = 3e4;
13397
- var LOAD_TIMEOUT_MS = 3e5;
13398
13605
  async function createVectorSearchPool(requestedCount) {
13399
13606
  const entryPoint = resolveWorkerEntryPoint2();
13400
13607
  const count = resolveWorkerCount(requestedCount);
@@ -13457,7 +13664,7 @@ async function createVectorSearchPool(requestedCount) {
13457
13664
  async function sendSearch(slot, request) {
13458
13665
  slot.outstanding += 1;
13459
13666
  try {
13460
- return await send(slot, request.requestId, request, SEARCH_TIMEOUT_MS);
13667
+ return await send(slot, request.requestId, request, VECTOR_SEARCH_TIMEOUT_MS);
13461
13668
  } finally {
13462
13669
  slot.outstanding -= 1;
13463
13670
  }
@@ -13474,7 +13681,7 @@ async function createVectorSearchPool(requestedCount) {
13474
13681
  slots.map((slot) => {
13475
13682
  requestCounter += 1;
13476
13683
  const requestId = `${requestCounter}`;
13477
- return send(slot, requestId, { type: "load", requestId, handle, snapshot }, LOAD_TIMEOUT_MS);
13684
+ return send(slot, requestId, { type: "load", requestId, handle, snapshot }, VECTOR_SEARCH_LOAD_TIMEOUT_MS);
13478
13685
  })
13479
13686
  );
13480
13687
  return outcomes.every(
@@ -13490,7 +13697,7 @@ async function createVectorSearchPool(requestedCount) {
13490
13697
  slot,
13491
13698
  requestId,
13492
13699
  { type: "loadShared", requestId, handle, scratchSlot, snapshot },
13493
- LOAD_TIMEOUT_MS
13700
+ VECTOR_SEARCH_LOAD_TIMEOUT_MS
13494
13701
  );
13495
13702
  })
13496
13703
  );
@@ -13503,7 +13710,7 @@ async function createVectorSearchPool(requestedCount) {
13503
13710
  slots.map((slot) => {
13504
13711
  requestCounter += 1;
13505
13712
  const requestId = `${requestCounter}`;
13506
- return send(slot, requestId, { type: "drop", requestId, handle }, SEARCH_TIMEOUT_MS);
13713
+ return send(slot, requestId, { type: "drop", requestId, handle }, VECTOR_SEARCH_TIMEOUT_MS);
13507
13714
  })
13508
13715
  );
13509
13716
  },
@@ -13571,11 +13778,11 @@ async function createVectorSearchPool(requestedCount) {
13571
13778
  var sharedPool = null;
13572
13779
  var poolHolders = 0;
13573
13780
  var workersUnavailable = false;
13574
- function acquireVectorSearchPool() {
13781
+ function acquireVectorSearchPool(requestedCount) {
13575
13782
  poolHolders += 1;
13576
13783
  if (workersUnavailable) return Promise.resolve(null);
13577
13784
  if (sharedPool === null) {
13578
- sharedPool = createVectorSearchPool().then(
13785
+ sharedPool = createVectorSearchPool(requestedCount).then(
13579
13786
  (pool) => {
13580
13787
  if (pool === null) workersUnavailable = true;
13581
13788
  return pool;
@@ -13599,8 +13806,6 @@ async function releaseVectorSearchPool() {
13599
13806
  }
13600
13807
 
13601
13808
  // src/vector/shared-generation/layout.ts
13602
- var WASM_PAGE_BYTES = 65536;
13603
- var MAX_WASM_PAGES = 65536;
13604
13809
  var SCRATCH_ALIGNMENT = 16;
13605
13810
  function alignUp(bytes) {
13606
13811
  return Math.ceil(bytes / SCRATCH_ALIGNMENT) * SCRATCH_ALIGNMENT;
@@ -13742,6 +13947,7 @@ function captureCloneSnapshot(state) {
13742
13947
  };
13743
13948
  }
13744
13949
  function scheduleWorkerCopyLoad(state) {
13950
+ if (!state.workerCopies.enabled) return;
13745
13951
  if (state.disposed || state.workerCopyLoading || state.building) return;
13746
13952
  if (state.workerCopyHandle !== null) return;
13747
13953
  if (!state.hnsw || state.buffer.size > 0) return;
@@ -13753,7 +13959,7 @@ function scheduleWorkerCopyLoad(state) {
13753
13959
  }
13754
13960
  async function loadWorkerCopies(state) {
13755
13961
  const revision = state.revision;
13756
- const pool = await acquireVectorSearchPool();
13962
+ const pool = await acquireVectorSearchPool(state.workerCopies.count);
13757
13963
  if (pool === null) {
13758
13964
  await releaseVectorSearchPool();
13759
13965
  return;
@@ -14354,7 +14560,7 @@ function searchWithFilter(state, query, k, options, filter) {
14354
14560
  }
14355
14561
 
14356
14562
  // src/vector/vector-index/index.ts
14357
- function createVectorIndex(fieldName, dimension, config) {
14563
+ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_WORKER_COPIES_ALLOWED) {
14358
14564
  if (!Number.isInteger(dimension) || dimension <= 0) {
14359
14565
  throw new NarsilError(
14360
14566
  ErrorCodes.VECTOR_DIMENSION_MISMATCH,
@@ -14377,6 +14583,7 @@ function createVectorIndex(fieldName, dimension, config) {
14377
14583
  filterThreshold,
14378
14584
  quantizationMode,
14379
14585
  hnswConfig,
14586
+ workerCopies,
14380
14587
  store,
14381
14588
  tombstones: /* @__PURE__ */ new Set(),
14382
14589
  buffer: /* @__PURE__ */ new Set(),
@@ -14519,8 +14726,15 @@ function createVectorIndex(fieldName, dimension, config) {
14519
14726
  }
14520
14727
 
14521
14728
  // src/workers/direct-executor.ts
14522
- function createDirectExecutor() {
14729
+ var NO_VECTOR_WORKER_COPIES = { enabled: false };
14730
+ function growPartitionsTo(manager, partitionId) {
14731
+ while (manager.partitionCount <= partitionId) {
14732
+ manager.addPartition();
14733
+ }
14734
+ }
14735
+ function createDirectExecutor(options) {
14523
14736
  const indexes = /* @__PURE__ */ new Map();
14737
+ const vectorWorkerCopies = options?.vectorWorkerCopies ?? NO_VECTOR_WORKER_COPIES;
14524
14738
  function requireIndex(indexName) {
14525
14739
  const entry = indexes.get(indexName);
14526
14740
  if (!entry) {
@@ -14541,7 +14755,7 @@ function createDirectExecutor() {
14541
14755
  const vectorFields = extractVectorFieldsFromSchema(config.schema);
14542
14756
  const vectorIndexes = /* @__PURE__ */ new Map();
14543
14757
  for (const [fieldPath, dim] of vectorFields) {
14544
- vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, config.vectorPromotion));
14758
+ vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, config.vectorPromotion, vectorWorkerCopies));
14545
14759
  }
14546
14760
  const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
14547
14761
  indexes.set(indexName, {
@@ -14598,10 +14812,10 @@ function createDirectExecutor() {
14598
14812
  case "buildSegment": {
14599
14813
  const entry = requireIndex(action.indexName);
14600
14814
  const segment = createPartitionIndex(0, entry.config.trackPositions ?? true);
14601
- const options = resolvePartitionInsertOptions(entry.config, entry.manager.analysis, action.options);
14815
+ const options2 = resolvePartitionInsertOptions(entry.config, entry.manager.analysis, action.options);
14602
14816
  segment.beginBatch();
14603
14817
  for (const doc of action.documents) {
14604
- segment.insert(doc.docId, doc.document, entry.config.schema, entry.language, options);
14818
+ segment.insert(doc.docId, doc.document, entry.config.schema, entry.language, options2);
14605
14819
  }
14606
14820
  segment.endBatch();
14607
14821
  return segment.encodeSegment();
@@ -14622,10 +14836,10 @@ function createDirectExecutor() {
14622
14836
  case "attachSegments": {
14623
14837
  const entry = requireIndex(action.indexName);
14624
14838
  for (const segment of action.segments) {
14625
- while (entry.manager.partitionCount <= segment.partitionId) {
14626
- entry.manager.addPartition();
14627
- }
14628
- entry.manager.attachFrozenSegment(segment.partitionId, createSharedFrozenSegment(segment.snapshot));
14839
+ growPartitionsTo(entry.manager, segment.partitionId);
14840
+ const frozen = createSharedFrozenSegment(segment.snapshot);
14841
+ for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId);
14842
+ entry.manager.attachFrozenSegment(segment.partitionId, frozen);
14629
14843
  }
14630
14844
  return void 0;
14631
14845
  }
@@ -14656,6 +14870,19 @@ function createDirectExecutor() {
14656
14870
  partition.swapFrozenSegments(action.dropSegmentIds, createSharedFrozenSegment(action.snapshot));
14657
14871
  return void 0;
14658
14872
  }
14873
+ case "freezeLiveTail": {
14874
+ const entry = requireIndex(action.indexName);
14875
+ growPartitionsTo(entry.manager, action.partitionId);
14876
+ const tail = entry.manager.getPartition(action.partitionId);
14877
+ const held = isCompositePartition(tail) ? tail.live.count() : tail.count();
14878
+ if (held !== action.snapshot.documentCount) {
14879
+ console.warn(
14880
+ `Partition ${action.partitionId} of "${action.indexName}" held ${held} live documents where the frozen tail holds ${action.snapshot.documentCount}`
14881
+ );
14882
+ }
14883
+ entry.manager.replaceLiveTail(action.partitionId, createSharedFrozenSegment(action.snapshot));
14884
+ return void 0;
14885
+ }
14659
14886
  case "remove": {
14660
14887
  const entry = requireIndex(action.indexName);
14661
14888
  entry.manager.remove(action.docId);
@@ -14731,7 +14958,10 @@ function createDirectExecutor() {
14731
14958
  const vectorFields = extractVectorFieldsFromSchema(entry.config.schema);
14732
14959
  const newVectorIndexes = /* @__PURE__ */ new Map();
14733
14960
  for (const [fieldPath, dim] of vectorFields) {
14734
- newVectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dim, entry.config.vectorPromotion));
14961
+ newVectorIndexes.set(
14962
+ fieldPath,
14963
+ createVectorIndex(fieldPath, dim, entry.config.vectorPromotion, vectorWorkerCopies)
14964
+ );
14735
14965
  }
14736
14966
  entry.manager.resetVectorIndexes(newVectorIndexes);
14737
14967
  entry.vectorIndexes = entry.manager.getVectorIndexes();
@@ -14743,9 +14973,7 @@ function createDirectExecutor() {
14743
14973
  }
14744
14974
  case "deserialize": {
14745
14975
  const entry = requireIndex(action.indexName);
14746
- while (entry.manager.partitionCount <= action.partitionId) {
14747
- entry.manager.addPartition();
14748
- }
14976
+ growPartitionsTo(entry.manager, action.partitionId);
14749
14977
  entry.manager.deserializePartition(action.partitionId, action.data);
14750
14978
  return void 0;
14751
14979
  }
@@ -14777,44 +15005,6 @@ function createDirectExecutor() {
14777
15005
  };
14778
15006
  }
14779
15007
 
14780
- // src/workers/promoter.ts
14781
- var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
14782
- var DEFAULT_TOTAL_THRESHOLD = 5e4;
14783
- function createExecutionPromoter(config) {
14784
- const perIndexThreshold = config?.perIndexThreshold ?? DEFAULT_PER_INDEX_THRESHOLD;
14785
- const totalThreshold = config?.totalThreshold ?? DEFAULT_TOTAL_THRESHOLD;
14786
- let promoted = false;
14787
- function check(indexes) {
14788
- if (promoted) {
14789
- return { shouldPromote: false, reason: "" };
14790
- }
14791
- let totalDocuments = 0;
14792
- for (const [indexName, stats] of indexes) {
14793
- if (stats.documentCount >= perIndexThreshold) {
14794
- return {
14795
- shouldPromote: true,
14796
- reason: `Index "${indexName}" has ${stats.documentCount} documents, exceeding the per-index threshold of ${perIndexThreshold}`
14797
- };
14798
- }
14799
- totalDocuments += stats.documentCount;
14800
- }
14801
- if (totalDocuments >= totalThreshold) {
14802
- return {
14803
- shouldPromote: true,
14804
- reason: `Total document count of ${totalDocuments} exceeds the total threshold of ${totalThreshold}`
14805
- };
14806
- }
14807
- return { shouldPromote: false, reason: "" };
14808
- }
14809
- function markPromoted() {
14810
- promoted = true;
14811
- }
14812
- function isPromoted() {
14813
- return promoted;
14814
- }
14815
- return { check, markPromoted, isPromoted };
14816
- }
14817
-
14818
15008
  // src/engine/analysis-rebuild.ts
14819
15009
  function yieldToEventLoop2() {
14820
15010
  return new Promise((resolve) => {
@@ -14891,7 +15081,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
14891
15081
  throw error;
14892
15082
  }
14893
15083
  }
14894
- function enqueue2(indexName, entry) {
15084
+ function enqueue(indexName, entry) {
14895
15085
  if (entry.run !== null) {
14896
15086
  return entry.run;
14897
15087
  }
@@ -14905,7 +15095,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
14905
15095
  }
14906
15096
  async function announce(indexName, entry) {
14907
15097
  const documentCount = deps.getManager(indexName)?.countDocuments() ?? 0;
14908
- await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue2(indexName, entry));
15098
+ await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
14909
15099
  }
14910
15100
  async function review(indexName, entry) {
14911
15101
  if (entry.run !== null || deps.getManager(indexName) === void 0) return;
@@ -14921,7 +15111,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
14921
15111
  });
14922
15112
  }
14923
15113
  if (rebuildMode === "auto") {
14924
- void enqueue2(indexName, entry).catch(() => void 0);
15114
+ void enqueue(indexName, entry).catch(() => void 0);
14925
15115
  }
14926
15116
  }
14927
15117
  return {
@@ -14955,7 +15145,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
14955
15145
  if (entry === void 0) {
14956
15146
  return Promise.resolve();
14957
15147
  }
14958
- return enqueue2(indexName, entry);
15148
+ return enqueue(indexName, entry);
14959
15149
  }
14960
15150
  };
14961
15151
  }
@@ -15040,6 +15230,15 @@ function resolveDurabilityTier(config) {
15040
15230
  return { kind: "snapshot", config: {}, adapter: config.persistence };
15041
15231
  }
15042
15232
 
15233
+ // src/persistence/durability/constants.ts
15234
+ var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
15235
+ var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
15236
+ var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
15237
+ var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
15238
+ var DEFAULT_COMPACTION_THRESHOLD = 12;
15239
+ var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
15240
+ var CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS = 50;
15241
+
15043
15242
  // src/persistence/durability/node-fs.browser.ts
15044
15243
  function filesystemUnavailable() {
15045
15244
  throw new NarsilError(
@@ -15360,8 +15559,6 @@ function buildEntry(fields) {
15360
15559
  }
15361
15560
 
15362
15561
  // src/serialization/checksum-dispatch.ts
15363
- var CHECKSUM_TIMEOUT_MS = 12e4;
15364
- var YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
15365
15562
  var workerUsable = true;
15366
15563
  var mainThreadResolved = false;
15367
15564
  var onMainThread = true;
@@ -15395,8 +15592,8 @@ function yieldToEventLoop3() {
15395
15592
  }
15396
15593
  async function chunkedChecksum(payload) {
15397
15594
  let state = crc32Init();
15398
- for (let offset = 0; offset < payload.length; offset += YIELD_CHUNK_BYTES) {
15399
- const end = Math.min(offset + YIELD_CHUNK_BYTES, payload.length);
15595
+ for (let offset = 0; offset < payload.length; offset += CHECKSUM_YIELD_CHUNK_BYTES) {
15596
+ const end = Math.min(offset + CHECKSUM_YIELD_CHUNK_BYTES, payload.length);
15400
15597
  state = crc32Update(state, payload.subarray(offset, end));
15401
15598
  if (end < payload.length) {
15402
15599
  await yieldToEventLoop3();
@@ -15988,8 +16185,6 @@ function formatSegmentStart(startSeqNo) {
15988
16185
  }
15989
16186
 
15990
16187
  // src/persistence/durability/checkpoint-worker-dispatch.ts
15991
- var CHECKPOINT_WORKER_TIMEOUT_MS = 6e5;
15992
- var TIMEOUT_RECOVERY_BACKOFF_MS = 50;
15993
16188
  var workerUsable2 = true;
15994
16189
  var pooledWorker = null;
15995
16190
  var workerBusy = false;
@@ -16102,7 +16297,7 @@ async function runCheckpointOnWorker(request) {
16102
16297
  return true;
16103
16298
  }
16104
16299
  if (outcome.timedOut) {
16105
- await delay(TIMEOUT_RECOVERY_BACKOFF_MS);
16300
+ await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
16106
16301
  }
16107
16302
  return false;
16108
16303
  } finally {
@@ -16122,7 +16317,6 @@ function terminateCheckpointWorker() {
16122
16317
  }
16123
16318
 
16124
16319
  // src/persistence/durability/segment/layout.ts
16125
- var DEFAULT_COMPACTION_THRESHOLD = 12;
16126
16320
  var SEGMENT_ID_WIDTH = 16;
16127
16321
  function manifestKey(indexName) {
16128
16322
  return `${indexName}/manifest`;
@@ -17578,7 +17772,10 @@ async function writePartitionVectors(input) {
17578
17772
  }
17579
17773
  const vectorIndexes = /* @__PURE__ */ new Map();
17580
17774
  for (const [fieldPath, dimension] of input.vectorFields) {
17581
- vectorIndexes.set(fieldPath, createVectorIndex(fieldPath, dimension, input.config.vectorPromotion));
17775
+ vectorIndexes.set(
17776
+ fieldPath,
17777
+ createVectorIndex(fieldPath, dimension, input.config.vectorPromotion, { enabled: false })
17778
+ );
17582
17779
  }
17583
17780
  for (const ref of input.priorVectors) {
17584
17781
  const vecIndex = vectorIndexes.get(ref.fieldPath);
@@ -17996,11 +18193,6 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
17996
18193
  }
17997
18194
  }
17998
18195
 
17999
- // src/persistence/durability/types.ts
18000
- var DEFAULT_CHECKPOINT_INTERVAL_MS = 3e5;
18001
- var DEFAULT_CHECKPOINT_MUTATION_THRESHOLD = 1e5;
18002
- var DEFAULT_ASYNC_FLUSH_INTERVAL_MS = 1e3;
18003
-
18004
18196
  // src/persistence/durability/group-commit.ts
18005
18197
  function createGroupCommitCoordinator(sync) {
18006
18198
  let inFlight = null;
@@ -18074,7 +18266,6 @@ async function createMarkerWriter(directory, indexName, partitionId) {
18074
18266
  }
18075
18267
 
18076
18268
  // src/persistence/durability/wal-writer.ts
18077
- var DEFAULT_SEGMENT_MAX_BYTES = 67108864;
18078
18269
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
18079
18270
  function segmentKey2(indexName, partitionId, startSeqNo) {
18080
18271
  const padded = startSeqNo.toString().padStart(16, "0");
@@ -18992,11 +19183,26 @@ function createDurabilityFromTier(tier, wiring) {
18992
19183
  });
18993
19184
  }
18994
19185
 
18995
- // src/engine/index-state/entry.ts
18996
- var DEFAULT_MAX_REOPEN_WAITERS = 64;
18997
- var MAX_REOPEN_FAILURES = 5;
19186
+ // src/engine/constants.ts
19187
+ var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
19188
+ var BATCH_CHUNK_SIZE = 1e3;
19189
+ var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
19190
+ var DEFAULT_OFFSET = 0;
19191
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
19192
+ var MAX_INDEX_NAME_LENGTH = 255;
19193
+ var MAX_DOC_ID_LENGTH = 512;
18998
19194
  var MIN_SWEEP_INTERVAL_MS = 100;
18999
19195
  var MAX_SWEEP_INTERVAL_MS = 6e4;
19196
+ var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
19197
+ var FOREIGN_STATS_TTL_MS = 6e4;
19198
+ var DEFAULT_SUGGEST_LIMIT = 10;
19199
+ var MAX_SUGGEST_LIMIT = 100;
19200
+
19201
+ // src/engine/index-state/constants.ts
19202
+ var DEFAULT_MAX_REOPEN_WAITERS = 64;
19203
+ var MAX_REOPEN_FAILURES = 5;
19204
+
19205
+ // src/engine/index-state/entry.ts
19000
19206
  function createLifecycleEntry(state) {
19001
19207
  return {
19002
19208
  state,
@@ -19270,7 +19476,10 @@ function createIndexStateCoordinator(config, callbacks) {
19270
19476
  await reopen(indexName, false);
19271
19477
  entry = entryOf(indexName);
19272
19478
  }
19273
- if (markActive) entry.lastAccessAt = Date.now();
19479
+ if (markActive) {
19480
+ entry.lastAccessAt = Date.now();
19481
+ callbacks.onAccess?.(indexName);
19482
+ }
19274
19483
  entry.activeOperations += 1;
19275
19484
  try {
19276
19485
  if (opened) await enforceLimits(indexName);
@@ -19373,6 +19582,9 @@ function wireIndexState(wiring) {
19373
19582
  }
19374
19583
  }
19375
19584
  },
19585
+ onAccess(indexName) {
19586
+ wiring.onAccess?.(indexName);
19587
+ },
19376
19588
  canCloseAutomatically(indexName) {
19377
19589
  return !rebalancingIndexes.has(indexName) && !analysisRebuild.isRunning(indexName) && !orchestrator.isIndexBusy(indexName);
19378
19590
  },
@@ -19383,8 +19595,6 @@ function wireIndexState(wiring) {
19383
19595
  }
19384
19596
 
19385
19597
  // src/engine/invalidation.ts
19386
- var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
19387
- var FOREIGN_STATS_TTL_MS = 6e4;
19388
19598
  function createInvalidationFromConfig(config, tierKind, deps) {
19389
19599
  if (config?.invalidation === void 0) {
19390
19600
  return null;
@@ -19538,17 +19748,99 @@ function createInvalidationIntegration(deps) {
19538
19748
  };
19539
19749
  }
19540
19750
 
19751
+ // src/core/partition/frozen/share.ts
19752
+ function shareFrozenSegment(segment) {
19753
+ if (segment.sharedSnapshot !== null) {
19754
+ return { snapshot: segment.sharedSnapshot, tombstonedDocIds: segment.tombstonedDocIds() };
19755
+ }
19756
+ const payload = encodeSegmentState(segment);
19757
+ const documents = payload.docIds.map((docId) => {
19758
+ const stored = segment.docStore.get(docId);
19759
+ if (stored === void 0) {
19760
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while sharing a segment`, {
19761
+ docId
19762
+ });
19763
+ }
19764
+ return stored.fields;
19765
+ });
19766
+ const snapshot = freezeSegmentShared(payload, documents, segment.segmentId);
19767
+ return snapshot === null ? null : { snapshot, tombstonedDocIds: [] };
19768
+ }
19769
+
19541
19770
  // src/engine/worker-resync.ts
19771
+ function captureComposite(manager, partitionId) {
19772
+ const partition = manager.getPartition(partitionId);
19773
+ if (!isCompositePartition(partition)) return null;
19774
+ const segmentIds = partition.frozenSegmentSizes().map((size) => size.segmentId);
19775
+ const frozen = [];
19776
+ for (const segment of partition.frozenSegmentsById(segmentIds)) {
19777
+ const shared2 = shareFrozenSegment(segment);
19778
+ if (shared2 === null) return null;
19779
+ frozen.push(shared2);
19780
+ }
19781
+ const live = partition.live.serialize(
19782
+ manager.indexName,
19783
+ manager.partitionCount,
19784
+ manager.language.name,
19785
+ manager.schema
19786
+ );
19787
+ return { partitionId, live, frozen };
19788
+ }
19789
+ function capturePartitions(manager) {
19790
+ const captured = [];
19791
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
19792
+ captured.push(
19793
+ captureComposite(manager, partitionId) ?? {
19794
+ partitionId,
19795
+ live: manager.serializePartition(partitionId),
19796
+ frozen: []
19797
+ }
19798
+ );
19799
+ }
19800
+ return captured;
19801
+ }
19802
+ async function sendPartition(indexName, executors, captured) {
19803
+ await Promise.all(
19804
+ executors.map(
19805
+ (workerExecutor) => workerExecutor.execute({
19806
+ type: "deserialize",
19807
+ indexName,
19808
+ partitionId: captured.partitionId,
19809
+ data: captured.live,
19810
+ requestId: `resync-${indexName}-${captured.partitionId}`
19811
+ })
19812
+ )
19813
+ );
19814
+ if (captured.frozen.length === 0) return;
19815
+ const segments = captured.frozen.map((shared2) => ({
19816
+ partitionId: captured.partitionId,
19817
+ snapshot: shared2.snapshot,
19818
+ tombstonedDocIds: shared2.tombstonedDocIds
19819
+ }));
19820
+ await Promise.all(
19821
+ executors.map(
19822
+ (workerExecutor) => workerExecutor.execute({
19823
+ type: "attachSegments",
19824
+ indexName,
19825
+ segments,
19826
+ requestId: `resync-attach-${indexName}-${captured.partitionId}`
19827
+ })
19828
+ )
19829
+ );
19830
+ }
19542
19831
  async function transferIndexToPool(indexName, pool, config, manager) {
19543
- const allExecutors = pool.getAllExecutors();
19832
+ await loadIndexOntoWorkers(indexName, pool.getAllExecutors(), config, manager, () => pool.addIndexToAll(indexName));
19833
+ }
19834
+ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDropped) {
19835
+ const captured = capturePartitions(manager);
19544
19836
  await Promise.allSettled(
19545
- allExecutors.map(
19837
+ executors.map(
19546
19838
  (workerExecutor) => workerExecutor.execute({ type: "dropIndex", indexName, requestId: `resync-drop-${indexName}` })
19547
19839
  )
19548
19840
  );
19549
- pool.addIndexToAll(indexName);
19841
+ onDropped?.();
19550
19842
  await Promise.all(
19551
- allExecutors.map(
19843
+ executors.map(
19552
19844
  (workerExecutor) => workerExecutor.execute({
19553
19845
  type: "createIndex",
19554
19846
  indexName,
@@ -19557,22 +19849,25 @@ async function transferIndexToPool(indexName, pool, config, manager) {
19557
19849
  })
19558
19850
  )
19559
19851
  );
19560
- for (let i = 0; i < manager.partitionCount; i++) {
19561
- const serialized = manager.serializePartition(i);
19562
- await Promise.all(
19563
- allExecutors.map(
19564
- (workerExecutor) => workerExecutor.execute({
19565
- type: "deserialize",
19566
- indexName,
19567
- partitionId: i,
19568
- data: serialized,
19569
- requestId: `resync-${indexName}-${i}`
19570
- })
19571
- )
19572
- );
19852
+ for (const partition of captured) {
19853
+ await sendPartition(indexName, executors, partition);
19573
19854
  }
19574
19855
  }
19575
19856
 
19857
+ // src/engine/orchestration/constants.ts
19858
+ var DEFAULT_COPY_THRESHOLD = 1e3;
19859
+ var DEFAULT_COPY_IDLE_TIMEOUT_MS = 3e5;
19860
+ var COMPACTION_SEGMENT_TRIGGER = 8;
19861
+ var IDLE_MERGE_DELAY_MS = 1e3;
19862
+ var LIVE_TAIL_FREEZE_FLOOR = MIN_DOCUMENTS_FOR_SEGMENTS;
19863
+ var LIVE_TAIL_FLUSH_DOCUMENTS = 5e3;
19864
+ var MAX_PENDING_REPLICATION_DOCUMENTS = 2e4;
19865
+ var REPLICATION_WINDOW = 32;
19866
+ var LOAD_BUFFER_YIELD_INTERVAL = 1e3;
19867
+ var MAIN_COPY_LONE_QUERY_DOCUMENTS = 5e4;
19868
+ var POOL_RESTART_DELAY_MS = 1e3;
19869
+ var POOL_RESTART_DELAY_MAX_MS = 6e4;
19870
+
19576
19871
  // src/engine/orchestration/eligibility.ts
19577
19872
  function toError3(value) {
19578
19873
  return value instanceof Error ? value : new Error(String(value));
@@ -19621,7 +19916,7 @@ function workerIneligibility(indexName, config, bootstrapModule) {
19621
19916
  function reportIneligible(state, indexName, error) {
19622
19917
  if (state.reportedIneligible.has(indexName)) return;
19623
19918
  state.reportedIneligible.add(indexName);
19624
- state.callbacks?.onPromotionFailure?.("index-excluded", error, false);
19919
+ state.callbacks?.onCopyLoadFailure?.("index-excluded", error, false);
19625
19920
  }
19626
19921
  function eligibleIndexNames(state) {
19627
19922
  const names = [];
@@ -19635,33 +19930,26 @@ function eligibleIndexNames(state) {
19635
19930
  }
19636
19931
  return names;
19637
19932
  }
19638
- function collectEligibleIndexes(state) {
19639
- const eligible = /* @__PURE__ */ new Map();
19640
- for (const name of eligibleIndexNames(state)) {
19641
- const manager = state.executor.getManager(name);
19642
- eligible.set(name, { documentCount: manager?.countDocuments() ?? 0 });
19933
+
19934
+ // src/engine/orchestration/turn.ts
19935
+ function afterCurrentTurn(callback) {
19936
+ if (typeof setImmediate === "function") {
19937
+ setImmediate(callback);
19938
+ return;
19643
19939
  }
19644
- return eligible;
19940
+ setTimeout(callback, 0);
19941
+ }
19942
+ function yieldToEventLoop4() {
19943
+ return new Promise((resolve) => afterCurrentTurn(resolve));
19645
19944
  }
19646
19945
 
19647
19946
  // src/engine/orchestration/replication.ts
19648
- var MAX_PENDING_REPLICATION_DOCUMENTS = 2e4;
19649
19947
  async function dispatchToWorkers(state, action) {
19650
19948
  const pool = state.workerPool;
19651
19949
  if (!pool) return;
19652
- if (action.type === "createIndex") {
19653
- const ineligibility = workerIneligibility(action.indexName, action.config, state.bootstrapModule);
19654
- if (ineligibility) {
19655
- reportIneligible(state, action.indexName, ineligibility);
19656
- return;
19657
- }
19658
- pool.addIndexToAll(action.indexName);
19659
- state.promotedIndexes.add(action.indexName);
19660
- } else if ("indexName" in action && !state.promotedIndexes.has(action.indexName)) {
19661
- return;
19662
- }
19663
- const allExecutors = pool.getAllExecutors();
19664
- const results = await Promise.allSettled(allExecutors.map((workerExecutor) => workerExecutor.execute(action)));
19950
+ const executors = "indexName" in action ? pool.executorsHolding(action.indexName) : pool.getAllExecutors();
19951
+ if ("indexName" in action && !state.scaledOutIndexes.has(action.indexName)) return;
19952
+ const results = await Promise.allSettled(executors.map((workerExecutor) => workerExecutor.execute(action)));
19665
19953
  for (const result of results) {
19666
19954
  if (result.status === "rejected") {
19667
19955
  if (action.type === "insert" && alreadyPresentOnWorker(result.reason)) {
@@ -19729,6 +20017,13 @@ function recordSegmentBroadcast(state, action) {
19729
20017
  }
19730
20018
  return;
19731
20019
  }
20020
+ if (action.type === "freezeLiveTail") {
20021
+ partitionLedger(state, action.indexName, action.partitionId).push({
20022
+ segmentId: action.snapshot.segmentId,
20023
+ documentCount: action.snapshot.documentCount
20024
+ });
20025
+ return;
20026
+ }
19732
20027
  if (action.type === "swapSegments") {
19733
20028
  const entries = partitionLedger(state, action.indexName, action.partitionId);
19734
20029
  const dropped = new Set(action.dropSegmentIds);
@@ -19741,48 +20036,71 @@ function recordSegmentBroadcast(state, action) {
19741
20036
  function queueFor(state, indexName) {
19742
20037
  let queue = state.replicationQueues.get(indexName);
19743
20038
  if (queue === void 0) {
19744
- queue = { tail: Promise.resolve(), pendingActions: 0, pendingDocuments: 0 };
20039
+ queue = { tail: Promise.resolve(), inFlight: [], pendingActions: 0, pendingDocuments: 0, drainWaiters: [] };
19745
20040
  state.replicationQueues.set(indexName, queue);
19746
20041
  }
19747
20042
  return queue;
19748
20043
  }
19749
- function enqueue(state, indexName, action, weight) {
20044
+ function dispatchGate(queue) {
20045
+ if (queue.inFlight.length < REPLICATION_WINDOW) return Promise.resolve();
20046
+ return queue.inFlight[queue.inFlight.length - REPLICATION_WINDOW];
20047
+ }
20048
+ function enqueueReplication(state, indexName, action) {
20049
+ const weight = documentWeight(action);
20050
+ recordSegmentBroadcast(state, action);
19750
20051
  const queue = queueFor(state, indexName);
19751
20052
  queue.pendingActions += 1;
19752
20053
  queue.pendingDocuments += weight;
19753
- queue.tail = queue.tail.then(async () => {
20054
+ const run = dispatchGate(queue).then(async () => {
19754
20055
  try {
19755
20056
  await dispatchToWorkers(state, action);
19756
20057
  } catch (err) {
19757
20058
  console.warn("Worker replication failed:", err);
19758
20059
  } finally {
20060
+ const at = queue.inFlight.indexOf(run);
20061
+ if (at !== -1) queue.inFlight.splice(at, 1);
19759
20062
  queue.pendingActions -= 1;
19760
20063
  queue.pendingDocuments -= weight;
20064
+ if (queue.pendingDocuments < MAX_PENDING_REPLICATION_DOCUMENTS && queue.drainWaiters.length > 0) {
20065
+ for (const resume of queue.drainWaiters.splice(0)) resume();
20066
+ }
19761
20067
  if (queue.pendingActions === 0 && state.replicationQueues.get(indexName) === queue) {
19762
20068
  state.replicationQueues.delete(indexName);
19763
20069
  }
19764
20070
  }
19765
20071
  });
20072
+ queue.inFlight.push(run);
20073
+ queue.tail = run;
19766
20074
  }
19767
20075
  async function replicateToWorkers(state, action) {
19768
- if (state.promotionInProgress) {
19769
- state.promotionBuffer.push(detachCallerDocuments(action));
20076
+ if (!("indexName" in action)) {
20077
+ if (state.workerPool === null) return;
20078
+ await awaitReplicationIdle(state);
20079
+ await dispatchToWorkers(state, action);
19770
20080
  return;
19771
20081
  }
19772
- if (state.workerPool === null) return;
19773
20082
  const detached = detachCallerDocuments(action);
19774
- if (!("indexName" in detached)) {
19775
- await awaitReplicationIdle(state);
19776
- await dispatchToWorkers(state, detached);
20083
+ const loading = state.copyLoadBuffers.get(action.indexName);
20084
+ if (loading !== void 0) {
20085
+ queueForCopies(state, detached);
20086
+ if (loading.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop4();
19777
20087
  return;
19778
20088
  }
19779
- const weight = documentWeight(detached);
19780
- const queue = state.replicationQueues.get(detached.indexName);
19781
- if (queue !== void 0 && queue.pendingDocuments + weight > MAX_PENDING_REPLICATION_DOCUMENTS) {
19782
- await queue.tail;
20089
+ const queue = state.replicationQueues.get(action.indexName);
20090
+ if (queue !== void 0 && queue.pendingDocuments + documentWeight(detached) > MAX_PENDING_REPLICATION_DOCUMENTS) {
20091
+ await new Promise((resume) => queue.drainWaiters.push(resume));
20092
+ }
20093
+ queueForCopies(state, detached);
20094
+ }
20095
+ function queueForCopies(state, action) {
20096
+ if (!("indexName" in action)) return;
20097
+ const loading = state.copyLoadBuffers.get(action.indexName);
20098
+ if (loading !== void 0) {
20099
+ loading.push(action);
20100
+ return;
19783
20101
  }
19784
- recordSegmentBroadcast(state, detached);
19785
- enqueue(state, detached.indexName, detached, weight);
20102
+ if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
20103
+ enqueueReplication(state, action.indexName, action);
19786
20104
  }
19787
20105
  async function awaitReplicationIdle(state, indexName) {
19788
20106
  if (indexName !== void 0) {
@@ -19795,63 +20113,101 @@ async function awaitReplicationIdle(state, indexName) {
19795
20113
  await Promise.all(tails);
19796
20114
  }
19797
20115
  }
19798
- async function drainPromotionBuffer(state) {
19799
- while (state.promotionBuffer.length > 0) {
19800
- const action = state.promotionBuffer.shift();
19801
- if (action === void 0) return;
19802
- await dispatchToWorkers(state, action);
20116
+
20117
+ // src/engine/orchestration/live-tail.ts
20118
+ function liveTailCount(manager, partitionId) {
20119
+ if (partitionId >= manager.partitionCount) return 0;
20120
+ const partition = manager.getPartition(partitionId);
20121
+ return isCompositePartition(partition) ? partition.live.count() : partition.count();
20122
+ }
20123
+ function freezeTail(state, indexName, manager, partitionId) {
20124
+ const segment = manager.freezeLiveTail(partitionId, (payload, documents) => {
20125
+ const snapshot = freezeSegmentShared(payload, documents);
20126
+ return snapshot === null ? null : createSharedFrozenSegment(snapshot);
20127
+ });
20128
+ if (segment === null || segment.sharedSnapshot === null) return;
20129
+ queueForCopies(state, {
20130
+ type: "freezeLiveTail",
20131
+ indexName,
20132
+ partitionId,
20133
+ snapshot: segment.sharedSnapshot,
20134
+ requestId: createRequestId()
20135
+ });
20136
+ }
20137
+ function freezeLiveTail(state, indexName, manager, partitionId, floor) {
20138
+ if (!state.scaledOutIndexes.has(indexName)) return;
20139
+ if (liveTailCount(manager, partitionId) < floor) return;
20140
+ try {
20141
+ freezeTail(state, indexName, manager, partitionId);
20142
+ } catch (err) {
20143
+ console.warn(`Freezing the live tail of partition ${partitionId} of "${indexName}" failed:`, err);
20144
+ }
20145
+ }
20146
+ function flushGrownTails(state, indexName) {
20147
+ const manager = state.executor.getManager(indexName);
20148
+ if (manager === void 0) return;
20149
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
20150
+ freezeLiveTail(state, indexName, manager, partitionId, LIVE_TAIL_FLUSH_DOCUMENTS);
19803
20151
  }
19804
20152
  }
19805
20153
 
19806
20154
  // src/engine/orchestration/compaction.ts
19807
- var COMPACTION_SEGMENT_TRIGGER = 8;
19808
20155
  function smallestSegmentIds(entries) {
19809
20156
  return [...entries].sort((a, b) => a.documentCount - b.documentCount).slice(0, COMPACTION_SEGMENT_TRIGGER).map((entry) => entry.segmentId);
19810
20157
  }
20158
+ function pickSegmentIds(entries, policy) {
20159
+ if (policy === "idle") {
20160
+ return entries.length >= 2 ? entries.map((entry) => entry.segmentId) : null;
20161
+ }
20162
+ return entries.length >= COMPACTION_SEGMENT_TRIGGER ? smallestSegmentIds(entries) : null;
20163
+ }
19811
20164
  function mainCompositeOf(manager, partitionId) {
19812
20165
  if (partitionId >= manager.partitionCount) return null;
19813
20166
  const partition = manager.getPartition(partitionId);
19814
20167
  return isCompositePartition(partition) ? partition : null;
19815
20168
  }
19816
- function candidateSegmentIds(state, indexName, manager, partitionId) {
20169
+ function candidateSegmentIds(state, indexName, manager, partitionId, policy) {
19817
20170
  const composite = mainCompositeOf(manager, partitionId);
19818
20171
  if (composite !== null) {
19819
20172
  const sizes = composite.frozenSegmentSizes();
19820
- if (sizes.length >= COMPACTION_SEGMENT_TRIGGER) {
19821
- return smallestSegmentIds(
19822
- sizes.map((size) => ({ segmentId: size.segmentId, documentCount: size.liveDocumentCount }))
19823
- );
19824
- }
20173
+ const picks = pickSegmentIds(
20174
+ sizes.map((size) => ({ segmentId: size.segmentId, documentCount: size.liveDocumentCount })),
20175
+ policy
20176
+ );
20177
+ if (picks !== null) return picks;
19825
20178
  }
19826
20179
  const ledger = state.segmentLedger.get(indexName)?.get(partitionId);
19827
- if (ledger !== void 0 && ledger.length >= COMPACTION_SEGMENT_TRIGGER) {
19828
- return smallestSegmentIds(ledger);
19829
- }
19830
- return null;
20180
+ return ledger === void 0 ? null : pickSegmentIds(ledger, policy);
19831
20181
  }
19832
20182
  function mainHoldsAll(composite, segmentIds) {
19833
20183
  if (composite === null) return false;
19834
20184
  const held = new Set(composite.frozenSegmentSizes().map((size) => size.segmentId));
19835
20185
  return segmentIds.every((segmentId) => held.has(segmentId));
19836
20186
  }
19837
- async function compactPartitionSegments2(state, indexName, manager, partitionId, segmentIds) {
19838
- await awaitReplicationIdle(state, indexName);
19839
- let snapshot = null;
20187
+ async function compactOnWorker(state, indexName, partitionId, segmentIds) {
19840
20188
  const pool = state.workerPool;
19841
- const executors = pool !== null && state.promotedIndexes.has(indexName) ? pool.getAllExecutors() : [];
19842
- if (executors.length > 0) {
19843
- try {
19844
- snapshot = await executors[0].execute({
19845
- type: "compactSegments",
19846
- indexName,
19847
- partitionId,
19848
- segmentIds: [...segmentIds],
19849
- requestId: createRequestId()
19850
- });
19851
- } catch (err) {
19852
- console.warn("Worker segment compaction failed, compacting on the main thread:", err);
19853
- }
20189
+ if (pool === null || !state.scaledOutIndexes.has(indexName)) return null;
20190
+ const lease = pool.leaseLeastBusy();
20191
+ if (lease === null) return null;
20192
+ try {
20193
+ return await lease.executor.execute({
20194
+ type: "compactSegments",
20195
+ indexName,
20196
+ partitionId,
20197
+ segmentIds: [...segmentIds],
20198
+ requestId: createRequestId()
20199
+ });
20200
+ } catch (err) {
20201
+ console.warn("Worker segment compaction failed, compacting on the main thread:", err);
20202
+ return null;
20203
+ } finally {
20204
+ lease.release();
19854
20205
  }
20206
+ }
20207
+ async function compactPartitionSegments2(state, indexName, manager, partitionId, segmentIds, policy) {
20208
+ await awaitReplicationIdle(state, indexName);
20209
+ if (policy === "idle" && !state.scaledOutIndexes.has(indexName)) return false;
20210
+ let snapshot = await compactOnWorker(state, indexName, partitionId, segmentIds);
19855
20211
  const composite = mainCompositeOf(manager, partitionId);
19856
20212
  const holdsAll = mainHoldsAll(composite, segmentIds);
19857
20213
  let replacement = null;
@@ -19878,30 +20234,52 @@ async function compactPartitionSegments2(state, indexName, manager, partitionId,
19878
20234
  }
19879
20235
  return true;
19880
20236
  }
19881
- async function compactIndexSegments(state, indexName) {
20237
+ async function compactIndexSegments(state, indexName, policy) {
19882
20238
  const manager = state.executor.getManager(indexName);
19883
20239
  if (!manager) return;
19884
20240
  const partitionIds = /* @__PURE__ */ new Set();
19885
20241
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
19886
20242
  for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
19887
20243
  for (const partitionId of partitionIds) {
19888
- let picks = candidateSegmentIds(state, indexName, manager, partitionId);
20244
+ if (policy === "idle") {
20245
+ await awaitReplicationIdle(state, indexName);
20246
+ freezeLiveTail(state, indexName, manager, partitionId, LIVE_TAIL_FREEZE_FLOOR);
20247
+ }
20248
+ let picks = candidateSegmentIds(state, indexName, manager, partitionId, policy);
19889
20249
  while (picks !== null) {
19890
- const compacted = await compactPartitionSegments2(state, indexName, manager, partitionId, picks);
20250
+ const compacted = await compactPartitionSegments2(state, indexName, manager, partitionId, picks, policy);
19891
20251
  if (!compacted) break;
19892
- picks = candidateSegmentIds(state, indexName, manager, partitionId);
20252
+ picks = candidateSegmentIds(state, indexName, manager, partitionId, policy);
19893
20253
  }
19894
20254
  }
19895
20255
  }
19896
- function maybeCompactSegments(state, indexName) {
19897
- if (state.compactionsInFlight.has(indexName)) return;
19898
- const run = compactIndexSegments(state, indexName).catch((err) => {
20256
+ function maybeCompactSegments(state, indexName, policy = "ingest") {
20257
+ if (state.compactionsInFlight.has(indexName)) {
20258
+ if (policy === "idle") scheduleIdleMerge(state, indexName);
20259
+ return;
20260
+ }
20261
+ const run = compactIndexSegments(state, indexName, policy).catch((err) => {
19899
20262
  console.warn("Segment compaction failed:", err);
19900
20263
  }).finally(() => {
19901
20264
  state.compactionsInFlight.delete(indexName);
19902
20265
  });
19903
20266
  state.compactionsInFlight.set(indexName, run);
19904
20267
  }
20268
+ function scheduleIdleMerge(state, indexName) {
20269
+ cancelIdleMerge(state, indexName);
20270
+ const timer = setTimeout(() => {
20271
+ state.idleMergeTimers.delete(indexName);
20272
+ maybeCompactSegments(state, indexName, "idle");
20273
+ }, IDLE_MERGE_DELAY_MS);
20274
+ if (typeof timer.unref === "function") timer.unref();
20275
+ state.idleMergeTimers.set(indexName, timer);
20276
+ }
20277
+ function cancelIdleMerge(state, indexName) {
20278
+ const timer = state.idleMergeTimers.get(indexName);
20279
+ if (timer === void 0) return;
20280
+ clearTimeout(timer);
20281
+ state.idleMergeTimers.delete(indexName);
20282
+ }
19905
20283
  async function awaitCompactions(state) {
19906
20284
  while (state.compactionsInFlight.size > 0) {
19907
20285
  await Promise.all([...state.compactionsInFlight.values()]);
@@ -19909,10 +20287,6 @@ async function awaitCompactions(state) {
19909
20287
  }
19910
20288
 
19911
20289
  // src/workers/worker-executor.ts
19912
- var DEFAULT_BACKPRESSURE_LIMIT = 100;
19913
- var DEFAULT_REQUEST_TIMEOUT = 3e4;
19914
- var SHUTDOWN_TIMEOUT = 5e3;
19915
- var CONSECUTIVE_TIMEOUTS_BEFORE_DEATH = 3;
19916
20290
  function errorFromEventLike(event) {
19917
20291
  if (event instanceof Error) {
19918
20292
  return event;
@@ -19930,7 +20304,7 @@ function errorFromEventLike(event) {
19930
20304
  }
19931
20305
  function createWorkerExecutor(worker, config) {
19932
20306
  const backpressureLimit = config?.backpressureLimit ?? DEFAULT_BACKPRESSURE_LIMIT;
19933
- const requestTimeout = config?.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT;
20307
+ const requestTimeout = config?.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT_MS;
19934
20308
  const pending = /* @__PURE__ */ new Map();
19935
20309
  let deathError = null;
19936
20310
  let shutdownRequested = false;
@@ -20018,7 +20392,7 @@ function createWorkerExecutor(worker, config) {
20018
20392
  const timeoutId = setTimeout(() => {
20019
20393
  pending.delete(requestId);
20020
20394
  reject2(new NarsilError(ErrorCodes.WORKER_TIMEOUT, "Shutdown timed out"));
20021
- }, SHUTDOWN_TIMEOUT);
20395
+ }, SHUTDOWN_TIMEOUT_MS);
20022
20396
  pending.set(requestId, {
20023
20397
  resolve: () => resolve(),
20024
20398
  reject: reject2,
@@ -20060,141 +20434,328 @@ async function createWorkerFactory(entryPoint) {
20060
20434
  throw new NarsilError(ErrorCodes.WORKER_CRASHED, `No worker support available on runtime "${runtime.runtime}"`);
20061
20435
  }
20062
20436
 
20063
- // src/engine/orchestration/promotion.ts
20064
- async function bootstrapWorkers(state, pool, indexNames) {
20065
- const allExecutors = pool.getAllExecutors();
20066
- if (state.bootstrapModule !== void 0) {
20067
- const moduleUrl = state.bootstrapModule;
20068
- await Promise.all(
20069
- allExecutors.map(
20070
- (workerExecutor) => workerExecutor.execute({ type: "bootstrap", moduleUrl, requestId: "promote-bootstrap" })
20071
- )
20072
- );
20437
+ // src/engine/orchestration/repair.ts
20438
+ var COPY_RESTART_REASON = "A request arrived after every worker crashed and the restart delay passed";
20439
+ function nextRestartDelay(state) {
20440
+ const delay2 = state.poolRetryDelayMs;
20441
+ state.poolRetryDelayMs = Math.min(delay2 * 2, POOL_RESTART_DELAY_MAX_MS);
20442
+ return delay2;
20443
+ }
20444
+ function deferPoolRestart(state) {
20445
+ state.poolRetryAt = Date.now() + nextRestartDelay(state);
20446
+ }
20447
+ function retirePool(state, pool) {
20448
+ if (state.workerPool !== pool) return;
20449
+ state.workerPool = null;
20450
+ cancelRepair(state);
20451
+ deferPoolRestart(state);
20452
+ for (const indexName of state.scaledOutIndexes) state.droppedCopies.set(indexName, COPY_RESTART_REASON);
20453
+ state.scaledOutIndexes.clear();
20454
+ state.segmentLedger.clear();
20455
+ void pool.shutdown().catch(() => void 0);
20456
+ }
20457
+ function handleWorkerCrash(state, pool, workerId, indexNames, error) {
20458
+ state.callbacks?.onWorkerCrash?.(workerId, indexNames, error);
20459
+ if (pool.getAllExecutors().length === 0) {
20460
+ retirePool(state, pool);
20461
+ return;
20073
20462
  }
20074
- for (const name of indexNames) {
20075
- const entry = state.indexRegistry.get(name);
20076
- if (!entry) continue;
20077
- await Promise.all(
20078
- allExecutors.map(
20079
- (workerExecutor) => workerExecutor.execute({
20080
- type: "createIndex",
20081
- indexName: name,
20082
- config: entry.config,
20083
- requestId: `promote-create-${name}`
20084
- })
20085
- )
20086
- );
20087
- const manager = state.executor.getManager(name);
20088
- if (!manager) continue;
20089
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
20090
- const serialized = manager.serializePartition(partitionId);
20091
- await Promise.all(
20092
- allExecutors.map(
20093
- (workerExecutor) => workerExecutor.execute({
20094
- type: "deserialize",
20095
- indexName: name,
20096
- partitionId,
20097
- data: serialized,
20098
- requestId: `promote-sync-${name}-${partitionId}`
20099
- })
20100
- )
20101
- );
20102
- }
20463
+ scheduleRepair(state, pool);
20464
+ }
20465
+ function scheduleRepair(state, pool) {
20466
+ if (state.workerPool !== pool || state.repairTimer !== null || state.poolRepair !== null) return;
20467
+ const timer = setTimeout(() => {
20468
+ state.repairTimer = null;
20469
+ void repairPool(state, pool);
20470
+ }, state.poolRetryDelayMs);
20471
+ if (typeof timer.unref === "function") timer.unref();
20472
+ state.repairTimer = timer;
20473
+ }
20474
+ function cancelRepair(state) {
20475
+ if (state.repairTimer === null) return;
20476
+ clearTimeout(state.repairTimer);
20477
+ state.repairTimer = null;
20478
+ }
20479
+ async function loadOntoReplacement(state, replacement, indexName) {
20480
+ const entry = state.indexRegistry.get(indexName);
20481
+ const manager = state.executor.getManager(indexName);
20482
+ if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
20483
+ const buffered = [];
20484
+ state.copyLoadBuffers.set(indexName, buffered);
20485
+ try {
20486
+ await loadIndexOntoWorkers(indexName, [replacement.executor], entry.config, manager);
20487
+ replacement.hold(indexName);
20488
+ } finally {
20489
+ state.copyLoadBuffers.delete(indexName);
20490
+ for (const action of buffered) enqueueReplication(state, indexName, action);
20103
20491
  }
20104
20492
  }
20105
- async function runPromotion(state, reason) {
20493
+ async function replaceWorker(state, pool, workerId) {
20494
+ const replacement = pool.spawnReplacement(workerId);
20495
+ if (replacement === null) return;
20106
20496
  try {
20107
- const promotable = eligibleIndexNames(state);
20108
- if (promotable.length === 0) {
20109
- state.promotionBuffer.length = 0;
20110
- return;
20497
+ if (state.bootstrapModule !== void 0) {
20498
+ await replacement.executor.execute({
20499
+ type: "bootstrap",
20500
+ moduleUrl: state.bootstrapModule,
20501
+ requestId: createRequestId()
20502
+ });
20111
20503
  }
20112
- const factory = await createWorkerFactory();
20113
- const pool = createWorkerPool({
20114
- count: state.config?.workers?.count,
20115
- workerFactory: factory,
20116
- onWorkerCrash(workerId, indexNames, error) {
20117
- state.callbacks?.onWorkerCrash?.(workerId, indexNames, error);
20118
- }
20119
- });
20120
- for (const name of promotable) {
20121
- pool.addIndexToAll(name);
20122
- }
20123
- await bootstrapWorkers(state, pool, promotable);
20124
- state.workerPool = pool;
20125
- for (const name of promotable) {
20126
- state.promotedIndexes.add(name);
20127
- state.awaitingBufferedWrites.add(name);
20128
- }
20129
- state.promoter.markPromoted();
20130
- await drainPromotionBuffer(state);
20131
- state.awaitingBufferedWrites.clear();
20132
- state.callbacks?.onPromotion?.(pool.workerCount, reason);
20504
+ for (const indexName of [...state.scaledOutIndexes]) {
20505
+ await loadOntoReplacement(state, replacement, indexName);
20506
+ }
20507
+ replacement.admit();
20133
20508
  } catch (err) {
20134
- state.promotionBuffer.length = 0;
20509
+ replacement.abandon();
20135
20510
  throw err;
20136
- } finally {
20137
- state.awaitingBufferedWrites.clear();
20138
- state.promotionInProgress = false;
20139
20511
  }
20140
20512
  }
20141
- async function promoteBeforeBatch(state, indexName, incomingCount) {
20142
- if (!state.workersEnabled || state.promotionBlocked || state.workerPool || incomingCount <= 0) return;
20143
- if (state.promotionInProgress) {
20144
- if (state.promotionRun) await state.promotionRun;
20145
- return;
20513
+ async function replaceDeadWorkers(state, pool) {
20514
+ for (const workerId of pool.deadWorkerIds()) {
20515
+ if (state.workerPool !== pool) return;
20516
+ await replaceWorker(state, pool, workerId);
20146
20517
  }
20147
- if (state.callbacks?.shouldDeferPromotion?.()) return;
20148
- const indexMap = collectEligibleIndexes(state);
20149
- const anticipated = indexMap.get(indexName);
20150
- if (anticipated === void 0) return;
20151
- anticipated.documentCount += incomingCount;
20152
- const result = state.promoter.check(indexMap);
20153
- if (!result.shouldPromote) return;
20154
- state.promotionInProgress = true;
20155
- const run = runPromotion(state, result.reason).catch((err) => {
20156
- const error = toError3(err);
20157
- state.promotionBlocked = isDeterministicFailure(error);
20158
- state.promotionInProgress = false;
20159
- state.callbacks?.onPromotionFailure?.(result.reason, error, !state.promotionBlocked);
20160
- });
20161
- const tracked = run.then(() => {
20162
- state.promotionRun = null;
20518
+ state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
20519
+ }
20520
+ function repairPool(state, pool) {
20521
+ if (state.workerPool !== pool) return Promise.resolve();
20522
+ if (state.poolRepair !== null) return state.poolRepair;
20523
+ const run = replaceDeadWorkers(state, pool).catch((err) => {
20524
+ nextRestartDelay(state);
20525
+ console.warn("Replacing a crashed worker failed:", err);
20526
+ }).finally(() => {
20527
+ state.poolRepair = null;
20528
+ if (pool.deadWorkerIds().length > 0) scheduleRepair(state, pool);
20163
20529
  });
20164
- state.promotionRun = tracked;
20165
- await tracked;
20166
- }
20167
- async function checkPromotion(state) {
20168
- if (!state.workersEnabled || state.promotionInProgress || state.promotionBlocked || state.workerPool) return;
20169
- if (state.callbacks?.shouldDeferPromotion?.()) return;
20170
- const indexMap = collectEligibleIndexes(state);
20171
- if (indexMap.size === 0) return;
20172
- const result = state.promoter.check(indexMap);
20173
- if (!result.shouldPromote) return;
20174
- state.promotionInProgress = true;
20175
- setTimeout(() => {
20176
- const run = runPromotion(state, result.reason).catch((err) => {
20177
- const error = toError3(err);
20178
- state.promotionBlocked = isDeterministicFailure(error);
20179
- state.promotionInProgress = false;
20180
- state.callbacks?.onPromotionFailure?.(result.reason, error, !state.promotionBlocked);
20181
- });
20182
- state.promotionRun = run.then(() => {
20183
- state.promotionRun = null;
20184
- });
20185
- }, 0);
20530
+ state.poolRepair = run;
20531
+ return run;
20186
20532
  }
20187
20533
 
20188
- // src/engine/orchestration/search.ts
20189
- function partitionsPerWorker(totalPartitions, workerCount) {
20190
- const assignments = Array.from({ length: workerCount }, () => []);
20191
- for (let partitionId = 0; partitionId < totalPartitions; partitionId++) {
20192
- assignments[partitionId % workerCount].push(partitionId);
20534
+ // src/engine/orchestration/scale-out.ts
20535
+ var COPY_RELOAD_REASON = "A request arrived after an idle spell dropped the worker copies";
20536
+ function copiesAllowed(state) {
20537
+ if (!state.workersEnabled || state.scaleOutBlocked) return false;
20538
+ return state.workerPool !== null || Date.now() >= state.poolRetryAt;
20539
+ }
20540
+ async function startPool(state) {
20541
+ eligibleIndexNames(state);
20542
+ const factory = await createWorkerFactory();
20543
+ let started = null;
20544
+ const pool = createWorkerPool({
20545
+ count: state.keywordWorkerCount,
20546
+ workerFactory: factory,
20547
+ onWorkerCrash(workerId, indexNames, error) {
20548
+ if (started !== null) handleWorkerCrash(state, started, workerId, indexNames, error);
20549
+ }
20550
+ });
20551
+ started = pool;
20552
+ try {
20553
+ pool.spawnAll();
20554
+ if (state.bootstrapModule !== void 0) {
20555
+ const moduleUrl = state.bootstrapModule;
20556
+ await Promise.all(
20557
+ pool.getAllExecutors().map(
20558
+ (workerExecutor) => workerExecutor.execute({ type: "bootstrap", moduleUrl, requestId: "copies-bootstrap" })
20559
+ )
20560
+ );
20561
+ }
20562
+ } catch (err) {
20563
+ await pool.shutdown().catch(() => void 0);
20564
+ throw err;
20193
20565
  }
20194
- return assignments;
20566
+ return pool;
20195
20567
  }
20196
- function mergeWorkerResults(results) {
20197
- const merged = kWayMerge(results.map((result) => result.scored));
20568
+ async function ensurePool(state) {
20569
+ if (state.workerPool !== null) return state.workerPool;
20570
+ if (state.poolStart === null) {
20571
+ state.poolStart = startPool(state).then(
20572
+ (pool) => {
20573
+ state.workerPool = pool;
20574
+ state.poolStart = null;
20575
+ return pool;
20576
+ },
20577
+ (err) => {
20578
+ state.poolStart = null;
20579
+ deferPoolRestart(state);
20580
+ throw err;
20581
+ }
20582
+ );
20583
+ }
20584
+ return state.poolStart;
20585
+ }
20586
+ function documentCountOf(state, indexName) {
20587
+ return state.executor.getManager(indexName)?.countDocuments() ?? 0;
20588
+ }
20589
+ function copyThresholdReason(state, indexName) {
20590
+ return `Index "${indexName}" holds ${documentCountOf(state, indexName)} documents, reaching the copy threshold of ${state.copyThreshold}`;
20591
+ }
20592
+ function indexReadyForCopies(state, indexName, incomingCount = 0) {
20593
+ if (!copiesAllowed(state)) return false;
20594
+ if (state.scaledOutIndexes.has(indexName) || state.desyncedIndexes.has(indexName)) return false;
20595
+ if (state.copyTransitions.has(indexName)) return false;
20596
+ if (state.callbacks?.shouldDeferCopies?.()) return false;
20597
+ const entry = state.indexRegistry.get(indexName);
20598
+ if (entry === void 0 || state.executor.getManager(indexName) === void 0) return false;
20599
+ if (documentCountOf(state, indexName) + incomingCount < state.copyThreshold) return false;
20600
+ const ineligibility = workerIneligibility(indexName, entry.config, state.bootstrapModule);
20601
+ if (ineligibility !== null) {
20602
+ reportIneligible(state, indexName, ineligibility);
20603
+ return false;
20604
+ }
20605
+ return true;
20606
+ }
20607
+ async function loadCopies(state, indexName, reason) {
20608
+ const entry = state.indexRegistry.get(indexName);
20609
+ const manager = state.executor.getManager(indexName);
20610
+ if (entry === void 0 || manager === void 0) return;
20611
+ const ineligibility = workerIneligibility(indexName, entry.config, state.bootstrapModule);
20612
+ if (ineligibility !== null) {
20613
+ reportIneligible(state, indexName, ineligibility);
20614
+ return;
20615
+ }
20616
+ const pool = await ensurePool(state);
20617
+ if (state.executor.getManager(indexName) === void 0) return;
20618
+ const dropReason = state.droppedCopies.get(indexName);
20619
+ const reload = state.droppedCopies.delete(indexName);
20620
+ const buffered = [];
20621
+ state.copyLoadBuffers.set(indexName, buffered);
20622
+ try {
20623
+ await transferIndexToPool(indexName, pool, entry.config, manager);
20624
+ state.scaledOutIndexes.add(indexName);
20625
+ state.lastAccessAt.set(indexName, Date.now());
20626
+ state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
20627
+ for (const action of buffered) enqueueReplication(state, indexName, action);
20628
+ } catch (err) {
20629
+ if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
20630
+ throw err;
20631
+ } finally {
20632
+ state.copyLoadBuffers.delete(indexName);
20633
+ }
20634
+ if (reload) state.copyReloadCounts.set(indexName, (state.copyReloadCounts.get(indexName) ?? 0) + 1);
20635
+ scheduleIdleMerge(state, indexName);
20636
+ state.callbacks?.onCopiesLoaded?.(pool.workerCount, reason);
20637
+ }
20638
+ async function loadAfter(previous, state, indexName, reason) {
20639
+ if (previous !== void 0) await previous.done;
20640
+ if (state.poolRepair !== null) await state.poolRepair;
20641
+ if (!copiesAllowed(state)) return;
20642
+ if (state.scaledOutIndexes.has(indexName) || state.desyncedIndexes.has(indexName)) return;
20643
+ await loadCopies(state, indexName, reason);
20644
+ }
20645
+ function scaleOutIndex(state, indexName, reason) {
20646
+ const previous = state.copyTransitions.get(indexName);
20647
+ if (previous !== void 0 && previous.kind !== "drop") return previous.done;
20648
+ const kind = state.droppedCopies.has(indexName) ? "reload" : "load";
20649
+ const done = loadAfter(previous, state, indexName, reason).catch((err) => {
20650
+ const error = toError3(err);
20651
+ state.scaleOutBlocked = isDeterministicFailure(error);
20652
+ state.callbacks?.onCopyLoadFailure?.(reason, error, !state.scaleOutBlocked);
20653
+ }).finally(() => {
20654
+ if (state.copyTransitions.get(indexName)?.done === done) state.copyTransitions.delete(indexName);
20655
+ });
20656
+ state.copyTransitions.set(indexName, { kind, done });
20657
+ return done;
20658
+ }
20659
+ async function scaleOutReadyIndexes(state) {
20660
+ if (!copiesAllowed(state)) return;
20661
+ for (const indexName of state.indexRegistry.keys()) {
20662
+ if (!indexReadyForCopies(state, indexName)) continue;
20663
+ const reason = copyThresholdReason(state, indexName);
20664
+ setTimeout(() => {
20665
+ void scaleOutIndex(state, indexName, reason);
20666
+ }, 0);
20667
+ }
20668
+ }
20669
+ async function scaleOutBeforeBatch(state, indexName, incomingCount) {
20670
+ if (incomingCount <= 0) return;
20671
+ const pending = state.copyTransitions.get(indexName);
20672
+ if (pending !== void 0) {
20673
+ if (pending.kind === "load") await pending.done;
20674
+ return;
20675
+ }
20676
+ if (!indexReadyForCopies(state, indexName, incomingCount)) return;
20677
+ await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
20678
+ }
20679
+
20680
+ // src/engine/orchestration/idle.ts
20681
+ function isIndexBusy(state, indexName) {
20682
+ return state.poolRepair !== null || state.copyTransitions.has(indexName) || state.replicationQueues.has(indexName) || state.compactionsInFlight.has(indexName);
20683
+ }
20684
+ function noteAccess(state, indexName) {
20685
+ state.lastAccessAt.set(indexName, Date.now());
20686
+ if (!copiesAllowed(state)) return;
20687
+ const reloadReason = state.droppedCopies.get(indexName);
20688
+ if (reloadReason !== void 0) {
20689
+ const transition = state.copyTransitions.get(indexName);
20690
+ if (transition !== void 0 && transition.kind !== "drop") return;
20691
+ void scaleOutIndex(state, indexName, reloadReason);
20692
+ return;
20693
+ }
20694
+ if (indexReadyForCopies(state, indexName)) {
20695
+ void scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
20696
+ }
20697
+ }
20698
+ async function releaseCopies(state, indexName) {
20699
+ const pool = state.workerPool;
20700
+ if (pool === null) return;
20701
+ await awaitReplicationIdle(state, indexName);
20702
+ const cutoff = Date.now() - state.copyIdleTimeoutMs;
20703
+ if ((state.lastAccessAt.get(indexName) ?? Date.now()) > cutoff) return;
20704
+ if (!state.scaledOutIndexes.has(indexName) || state.compactionsInFlight.has(indexName)) return;
20705
+ state.scaledOutIndexes.delete(indexName);
20706
+ state.droppedCopies.set(indexName, COPY_RELOAD_REASON);
20707
+ cancelIdleMerge(state, indexName);
20708
+ const outcomes = await Promise.allSettled(
20709
+ pool.executorsHolding(indexName).map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `idle-drop-${indexName}` }))
20710
+ );
20711
+ for (const outcome of outcomes) {
20712
+ if (outcome.status === "rejected") console.warn("Dropping an idle worker copy failed:", outcome.reason);
20713
+ }
20714
+ pool.removeIndex(indexName);
20715
+ state.segmentLedger.delete(indexName);
20716
+ }
20717
+ async function dropIdleCopies(state, indexName) {
20718
+ const previous = state.copyTransitions.get(indexName);
20719
+ if (previous !== void 0) return;
20720
+ const done = releaseCopies(state, indexName).catch((err) => {
20721
+ console.warn("Dropping idle worker copies failed:", err);
20722
+ }).finally(() => {
20723
+ if (state.copyTransitions.get(indexName)?.done === done) state.copyTransitions.delete(indexName);
20724
+ });
20725
+ state.copyTransitions.set(indexName, { kind: "drop", done });
20726
+ await done;
20727
+ }
20728
+ async function sweepIdleCopies(state) {
20729
+ const cutoff = Date.now() - state.copyIdleTimeoutMs;
20730
+ for (const indexName of [...state.scaledOutIndexes]) {
20731
+ if ((state.lastAccessAt.get(indexName) ?? Date.now()) > cutoff) continue;
20732
+ if (isIndexBusy(state, indexName)) continue;
20733
+ await dropIdleCopies(state, indexName);
20734
+ }
20735
+ }
20736
+ function startIdleSweep(state) {
20737
+ if (!state.workersEnabled || state.idleSweep !== null) return;
20738
+ const interval = Math.min(MAX_SWEEP_INTERVAL_MS, Math.max(MIN_SWEEP_INTERVAL_MS, state.copyIdleTimeoutMs / 2));
20739
+ const timer = setInterval(() => void sweepIdleCopies(state).catch(() => void 0), interval);
20740
+ if (typeof timer.unref === "function") timer.unref();
20741
+ state.idleSweep = timer;
20742
+ }
20743
+ function stopIdleSweep(state) {
20744
+ if (state.idleSweep === null) return;
20745
+ clearInterval(state.idleSweep);
20746
+ state.idleSweep = null;
20747
+ }
20748
+
20749
+ // src/engine/orchestration/search.ts
20750
+ function partitionsPerLease(scope, leaseCount) {
20751
+ const assignments = Array.from({ length: leaseCount }, () => []);
20752
+ for (let at = 0; at < scope.length; at++) {
20753
+ assignments[at % leaseCount].push(scope[at]);
20754
+ }
20755
+ return assignments;
20756
+ }
20757
+ function mergeWorkerResults(results) {
20758
+ const merged = kWayMerge(results.map((result) => result.scored));
20198
20759
  let totalMatched = 0;
20199
20760
  const workerFacets = [];
20200
20761
  for (const result of results) {
@@ -20207,67 +20768,76 @@ function mergeWorkerResults(results) {
20207
20768
  facets: workerFacets.length > 0 ? mergeFacets(workerFacets) : void 0
20208
20769
  };
20209
20770
  }
20771
+ function scoresPerPartition(state, indexName, params, globalStats) {
20772
+ const mode = params.scoring ?? state.indexRegistry.get(indexName)?.config.defaultScoring ?? "local";
20773
+ if (mode === "local") return true;
20774
+ return mode === "broadcast" && globalStats !== void 0;
20775
+ }
20776
+ function queryAction(indexName, params, globalStats, partitionIds) {
20777
+ return {
20778
+ type: "query",
20779
+ indexName,
20780
+ params,
20781
+ requestId: createRequestId(),
20782
+ ...partitionIds !== void 0 ? { partitionIds } : {},
20783
+ ...globalStats !== void 0 ? { globalStats } : {}
20784
+ };
20785
+ }
20786
+ async function runSplit(leases, scope, indexName, params, globalStats) {
20787
+ const assignments = partitionsPerLease(scope, leases.length);
20788
+ const results = await Promise.all(
20789
+ assignments.map(
20790
+ (partitionIds, at) => leases[at].executor.execute(queryAction(indexName, params, globalStats, partitionIds)).finally(() => leases[at].release())
20791
+ )
20792
+ );
20793
+ return mergeWorkerResults(results);
20794
+ }
20795
+ function takeMainCopyTurn(state) {
20796
+ if (state.mainCopyTurnTaken) return false;
20797
+ state.mainCopyTurnTaken = true;
20798
+ afterCurrentTurn(() => {
20799
+ state.mainCopyTurnTaken = false;
20800
+ });
20801
+ return true;
20802
+ }
20803
+ function answersFasterOnMainCopy(state, pool, manager) {
20804
+ if (pool.queriesInFlight() > 0) return false;
20805
+ if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
20806
+ return takeMainCopyTurn(state);
20807
+ }
20210
20808
  async function searchViaWorker(state, indexName, params, globalStats, partitionIds) {
20211
20809
  const pool = state.workerPool;
20212
20810
  if (!pool) return null;
20213
- if (!state.promotedIndexes.has(indexName)) return null;
20214
- if (state.awaitingBufferedWrites.has(indexName)) return null;
20811
+ if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
20215
20812
  const pendingReplication = state.replicationQueues.get(indexName);
20216
20813
  if (pendingReplication !== void 0 && pendingReplication.pendingActions > 0) return null;
20217
20814
  const manager = state.executor.getManager(indexName);
20218
20815
  if (!manager) return null;
20219
- const allExecutors = pool.getAllExecutors();
20220
- if (allExecutors.length === 0) return null;
20221
- const stats = globalStats !== void 0 ? { globalStats } : {};
20222
- if (allExecutors.length === 1) {
20223
- try {
20224
- return await allExecutors[0].execute({
20225
- type: "query",
20226
- indexName,
20227
- params,
20228
- requestId: createRequestId(),
20229
- ...partitionIds !== void 0 ? { partitionIds } : {},
20230
- ...stats
20231
- });
20232
- } catch (err) {
20233
- console.warn("Worker search failed, falling back to local:", err);
20234
- return null;
20235
- }
20236
- }
20237
- const assignments = partitionsPerWorker(manager.partitionCount, allExecutors.length);
20238
- const scopedAssignments = partitionIds === void 0 ? assignments : assignments.map((assigned) => assigned.filter((partitionId) => partitionIds.includes(partitionId)));
20239
- const activeAssignments = [];
20240
- for (let index = 0; index < allExecutors.length; index++) {
20241
- if (scopedAssignments[index].length > 0) {
20242
- activeAssignments.push({ executor: allExecutors[index], partitionIds: scopedAssignments[index] });
20243
- }
20244
- }
20245
- if (activeAssignments.length === 0) {
20246
- return { scored: [], totalMatched: 0 };
20247
- }
20816
+ if (answersFasterOnMainCopy(state, pool, manager)) return null;
20817
+ const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
20818
+ const idle = pool.leaseIdle(scope.length);
20819
+ const leases = [];
20248
20820
  try {
20249
- const results = await Promise.all(
20250
- activeAssignments.map(
20251
- (assignment) => assignment.executor.execute({
20252
- type: "query",
20253
- indexName,
20254
- params,
20255
- requestId: createRequestId(),
20256
- partitionIds: assignment.partitionIds,
20257
- ...stats
20258
- })
20259
- )
20260
- );
20261
- return mergeWorkerResults(results);
20821
+ if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
20822
+ leases.push(...idle);
20823
+ return await runSplit(leases, scope, indexName, params, globalStats);
20824
+ }
20825
+ for (const lease2 of idle.slice(1)) lease2.release();
20826
+ const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
20827
+ if (lease === null) return null;
20828
+ leases.push(lease);
20829
+ return await lease.executor.execute(queryAction(indexName, params, globalStats, partitionIds));
20262
20830
  } catch (err) {
20263
- console.warn("Parallel worker search failed, falling back to local:", err);
20831
+ console.warn("Worker search failed, falling back to local:", err);
20264
20832
  return null;
20833
+ } finally {
20834
+ for (const lease of leases) lease.release();
20265
20835
  }
20266
20836
  }
20267
20837
 
20268
20838
  // src/engine/orchestration/segments.ts
20269
20839
  function segmentBuildConcurrency(state, indexName) {
20270
- if (!state.promotedIndexes.has(indexName)) return 0;
20840
+ if (!state.scaledOutIndexes.has(indexName)) return 0;
20271
20841
  return state.workerPool?.workerCount ?? 0;
20272
20842
  }
20273
20843
  async function buildSegments(state, requests) {
@@ -20284,44 +20854,55 @@ async function buildSegments(state, requests) {
20284
20854
  }
20285
20855
 
20286
20856
  // src/engine/orchestration/index.ts
20287
- function createWorkerOrchestrator(config, executor, promoter, indexRegistry, callbacks) {
20857
+ function workersEnabledByDefault() {
20858
+ return detectRuntime().runtime !== "browser";
20859
+ }
20860
+ function copyIdleTimeoutBeforeClose(closeAfterMs) {
20861
+ return closeAfterMs === void 0 ? DEFAULT_COPY_IDLE_TIMEOUT_MS : Math.min(DEFAULT_COPY_IDLE_TIMEOUT_MS, closeAfterMs);
20862
+ }
20863
+ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks) {
20288
20864
  const state = {
20289
20865
  config,
20290
20866
  executor,
20291
- promoter,
20292
20867
  indexRegistry,
20293
20868
  callbacks,
20294
- workersEnabled: config?.workers?.enabled === true,
20869
+ workersEnabled: config?.workers?.enabled ?? workersEnabledByDefault(),
20870
+ keywordWorkerCount: splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).keyword,
20871
+ copyThreshold: config?.workers?.promotionThreshold ?? DEFAULT_COPY_THRESHOLD,
20872
+ copyIdleTimeoutMs: config?.workers?.idleTimeoutMs ?? copyIdleTimeoutBeforeClose(config?.lifecycle?.idleTimeoutMs),
20295
20873
  bootstrapModule: config?.workers?.bootstrapModule,
20296
- promotionBuffer: [],
20297
- awaitingBufferedWrites: /* @__PURE__ */ new Set(),
20298
20874
  reportedIneligible: /* @__PURE__ */ new Set(),
20299
- promotedIndexes: /* @__PURE__ */ new Set(),
20875
+ scaledOutIndexes: /* @__PURE__ */ new Set(),
20876
+ desyncedIndexes: /* @__PURE__ */ new Set(),
20877
+ copyLoadBuffers: /* @__PURE__ */ new Map(),
20878
+ copyTransitions: /* @__PURE__ */ new Map(),
20879
+ droppedCopies: /* @__PURE__ */ new Map(),
20880
+ lastAccessAt: /* @__PURE__ */ new Map(),
20881
+ copyReloadCounts: /* @__PURE__ */ new Map(),
20300
20882
  replicationQueues: /* @__PURE__ */ new Map(),
20301
20883
  segmentLedger: /* @__PURE__ */ new Map(),
20302
20884
  compactionsInFlight: /* @__PURE__ */ new Map(),
20885
+ idleMergeTimers: /* @__PURE__ */ new Map(),
20303
20886
  workerPool: null,
20304
- promotionInProgress: false,
20305
- promotionBlocked: false,
20306
- promotionRun: null
20887
+ poolStart: null,
20888
+ poolRetryAt: 0,
20889
+ poolRetryDelayMs: POOL_RESTART_DELAY_MS,
20890
+ poolRepair: null,
20891
+ mainCopyTurnTaken: false,
20892
+ repairTimer: null,
20893
+ scaleOutBlocked: false,
20894
+ idleSweep: null
20307
20895
  };
20308
- async function resyncIndex(indexName, wasPromoted) {
20309
- if (state.promotionRun) {
20310
- await state.promotionRun.catch(() => void 0);
20311
- }
20312
- const pool = state.workerPool;
20313
- if (!pool) return;
20314
- if (!wasPromoted && !state.promotedIndexes.has(indexName)) return;
20315
- const entry = indexRegistry.get(indexName);
20316
- if (!entry) return;
20317
- if (workerIneligibility(indexName, entry.config, state.bootstrapModule)) return;
20318
- const manager = executor.getManager(indexName);
20319
- if (!manager) return;
20320
- state.promotedIndexes.delete(indexName);
20321
- await awaitReplicationIdle(state, indexName);
20322
- state.segmentLedger.delete(indexName);
20323
- await transferIndexToPool(indexName, pool, entry.config, manager);
20324
- state.promotedIndexes.add(indexName);
20896
+ startIdleSweep(state);
20897
+ async function resyncIndex(indexName, wasScaledOut) {
20898
+ state.desyncedIndexes.delete(indexName);
20899
+ if (!wasScaledOut) return;
20900
+ await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
20901
+ }
20902
+ async function openIndex(indexName) {
20903
+ state.droppedCopies.delete(indexName);
20904
+ if (!indexReadyForCopies(state, indexName)) return;
20905
+ await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
20325
20906
  }
20326
20907
  async function getWorkerMemoryStats() {
20327
20908
  if (!state.workerPool) return [];
@@ -20333,93 +20914,222 @@ function createWorkerOrchestrator(config, executor, promoter, indexRegistry, cal
20333
20914
  external: report.external
20334
20915
  }));
20335
20916
  }
20917
+ function workerCopies() {
20918
+ const reports = [];
20919
+ for (const indexName of indexRegistry.keys()) {
20920
+ reports.push({
20921
+ indexName,
20922
+ scaledOut: state.scaledOutIndexes.has(indexName),
20923
+ reloadCount: state.copyReloadCounts.get(indexName) ?? 0
20924
+ });
20925
+ }
20926
+ return reports;
20927
+ }
20336
20928
  async function shutdown() {
20929
+ stopIdleSweep(state);
20930
+ cancelRepair(state);
20931
+ for (const indexName of [...state.idleMergeTimers.keys()]) cancelIdleMerge(state, indexName);
20932
+ await Promise.allSettled([...state.copyTransitions.values()].map((transition) => transition.done));
20933
+ if (state.poolRepair !== null) await state.poolRepair;
20934
+ if (state.poolStart !== null) await state.poolStart.catch(() => void 0);
20337
20935
  if (state.workerPool) {
20338
20936
  await awaitCompactions(state);
20339
20937
  await awaitReplicationIdle(state);
20340
20938
  await state.workerPool.shutdown();
20341
20939
  state.workerPool = null;
20342
- state.promotedIndexes.clear();
20940
+ state.scaledOutIndexes.clear();
20343
20941
  state.replicationQueues.clear();
20344
20942
  state.segmentLedger.clear();
20345
20943
  }
20346
20944
  }
20347
- async function awaitPromotionIdle() {
20348
- while (state.promotionInProgress) {
20349
- const run = state.promotionRun;
20350
- if (run !== null) {
20351
- await run;
20352
- } else {
20353
- await new Promise((resolve) => setTimeout(resolve, 0));
20945
+ async function restoreAfterFailedClose(indexName, failure) {
20946
+ const pool = state.workerPool;
20947
+ const entry = indexRegistry.get(indexName);
20948
+ const manager = executor.getManager(indexName);
20949
+ if (pool !== null && entry !== void 0 && manager !== void 0) {
20950
+ try {
20951
+ await transferIndexToPool(indexName, pool, entry.config, manager);
20952
+ state.scaledOutIndexes.add(indexName);
20953
+ } catch (repairError) {
20954
+ throw new NarsilError(
20955
+ ErrorCodes.WORKER_CRASHED,
20956
+ `Worker copies for index "${indexName}" could not be restored after close failed`,
20957
+ {
20958
+ indexName,
20959
+ closeError: failure.reason instanceof Error ? failure.reason.message : String(failure.reason),
20960
+ repairError: repairError instanceof Error ? repairError.message : String(repairError)
20961
+ }
20962
+ );
20354
20963
  }
20355
20964
  }
20965
+ if (failure.reason instanceof NarsilError) throw failure.reason;
20966
+ throw new NarsilError(ErrorCodes.WORKER_CRASHED, `A worker could not close index "${indexName}"`, {
20967
+ indexName,
20968
+ cause: failure.reason instanceof Error ? failure.reason.message : String(failure.reason)
20969
+ });
20356
20970
  }
20357
20971
  async function closeIndex(indexName) {
20358
- await awaitPromotionIdle();
20972
+ if (state.poolRepair !== null) await state.poolRepair;
20973
+ const transition = state.copyTransitions.get(indexName);
20974
+ if (transition !== void 0) await transition.done;
20359
20975
  await awaitReplicationIdle(state, indexName);
20360
20976
  await awaitCompactions(state);
20977
+ cancelIdleMerge(state, indexName);
20361
20978
  const pool = state.workerPool;
20362
- if (pool !== null && state.promotedIndexes.has(indexName)) {
20979
+ if (pool !== null && state.scaledOutIndexes.has(indexName)) {
20980
+ state.scaledOutIndexes.delete(indexName);
20363
20981
  const outcomes = await Promise.allSettled(
20364
- pool.getAllExecutors().map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `close-${indexName}` }))
20982
+ pool.executorsHolding(indexName).map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `close-${indexName}` }))
20365
20983
  );
20366
20984
  const failure = outcomes.find((outcome) => outcome.status === "rejected");
20367
- if (failure?.status === "rejected") {
20368
- const entry = indexRegistry.get(indexName);
20369
- const manager = executor.getManager(indexName);
20370
- if (entry !== void 0 && manager !== void 0) {
20371
- try {
20372
- await transferIndexToPool(indexName, pool, entry.config, manager);
20373
- } catch (repairError) {
20374
- throw new NarsilError(
20375
- ErrorCodes.WORKER_CRASHED,
20376
- `Worker copies for index "${indexName}" could not be restored after close failed`,
20377
- {
20378
- indexName,
20379
- closeError: failure.reason instanceof Error ? failure.reason.message : String(failure.reason),
20380
- repairError: repairError instanceof Error ? repairError.message : String(repairError)
20381
- }
20382
- );
20383
- }
20384
- }
20385
- if (failure.reason instanceof NarsilError) throw failure.reason;
20386
- throw new NarsilError(ErrorCodes.WORKER_CRASHED, `A worker could not close index "${indexName}"`, {
20387
- indexName,
20388
- cause: failure.reason instanceof Error ? failure.reason.message : String(failure.reason)
20389
- });
20390
- }
20985
+ if (failure?.status === "rejected") await restoreAfterFailedClose(indexName, failure);
20391
20986
  pool.removeIndex(indexName);
20392
20987
  }
20393
- state.promotedIndexes.delete(indexName);
20988
+ state.droppedCopies.delete(indexName);
20989
+ state.desyncedIndexes.delete(indexName);
20990
+ state.lastAccessAt.delete(indexName);
20394
20991
  state.replicationQueues.delete(indexName);
20395
20992
  state.segmentLedger.delete(indexName);
20396
20993
  }
20397
20994
  async function replicate(action) {
20398
20995
  await replicateToWorkers(state, action);
20399
- if (action.type === "attachSegments") {
20400
- maybeCompactSegments(state, action.indexName);
20996
+ if (!("indexName" in action)) return;
20997
+ if (action.type === "attachSegments") maybeCompactSegments(state, action.indexName);
20998
+ if (action.type === "insert" || action.type === "update") flushGrownTails(state, action.indexName);
20999
+ if (state.scaledOutIndexes.has(action.indexName) || state.copyLoadBuffers.has(action.indexName)) {
21000
+ scheduleIdleMerge(state, action.indexName);
20401
21001
  }
20402
21002
  }
20403
21003
  return {
20404
- checkPromotion: () => checkPromotion(state),
20405
- promoteBeforeBatch: (indexName, incomingCount) => promoteBeforeBatch(state, indexName, incomingCount),
21004
+ scaleOutReadyIndexes: () => scaleOutReadyIndexes(state),
21005
+ scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
20406
21006
  replicateToWorkers: replicate,
20407
21007
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
20408
21008
  awaitCompactions: () => awaitCompactions(state),
20409
- openIndex: (indexName) => resyncIndex(indexName, true),
21009
+ openIndex,
20410
21010
  closeIndex,
20411
- isIndexBusy: (indexName) => state.promotionInProgress || state.replicationQueues.has(indexName) || state.compactionsInFlight.has(indexName),
21011
+ isIndexBusy: (indexName) => isIndexBusy(state, indexName),
20412
21012
  buildSegments: (requests) => buildSegments(state, requests),
20413
21013
  segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
20414
21014
  searchViaWorker: (indexName, params, globalStats, partitionIds) => searchViaWorker(state, indexName, params, globalStats, partitionIds),
20415
- isPromoted: () => state.workerPool !== null,
20416
- desyncIndex: (indexName) => state.promotedIndexes.delete(indexName),
21015
+ hasWorkerPool: () => state.workerPool !== null,
21016
+ desyncIndex: (indexName) => {
21017
+ state.desyncedIndexes.add(indexName);
21018
+ return state.scaledOutIndexes.delete(indexName);
21019
+ },
20417
21020
  resyncIndex,
21021
+ noteAccess: (indexName) => noteAccess(state, indexName),
21022
+ workerCopies,
20418
21023
  getWorkerMemoryStats,
20419
21024
  shutdown
20420
21025
  };
20421
21026
  }
20422
21027
 
21028
+ // src/engine/validation.ts
21029
+ function now() {
21030
+ if (typeof performance !== "undefined" && typeof performance.now === "function") {
21031
+ return performance.now();
21032
+ }
21033
+ return Date.now();
21034
+ }
21035
+ function validateIndexName(name) {
21036
+ if (!name || name.length === 0) {
21037
+ throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
21038
+ }
21039
+ if (name.length > MAX_INDEX_NAME_LENGTH) {
21040
+ throw new NarsilError(
21041
+ ErrorCodes.INDEX_NOT_FOUND,
21042
+ `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
21043
+ {
21044
+ indexName: name,
21045
+ length: name.length
21046
+ }
21047
+ );
21048
+ }
21049
+ if (!INDEX_NAME_PATTERN.test(name)) {
21050
+ throw new NarsilError(
21051
+ ErrorCodes.INDEX_NOT_FOUND,
21052
+ `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
21053
+ { indexName: name }
21054
+ );
21055
+ }
21056
+ if (name.includes("..")) {
21057
+ throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
21058
+ }
21059
+ }
21060
+ function validateDocId(docId) {
21061
+ if (!docId || docId.length === 0) {
21062
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
21063
+ }
21064
+ if (docId.length > MAX_DOC_ID_LENGTH) {
21065
+ throw new NarsilError(
21066
+ ErrorCodes.DOC_VALIDATION_FAILED,
21067
+ `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
21068
+ {
21069
+ docId,
21070
+ length: docId.length
21071
+ }
21072
+ );
21073
+ }
21074
+ if (docId.includes("\0")) {
21075
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
21076
+ }
21077
+ }
21078
+ function validatePartitionConfig(partitions) {
21079
+ const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
21080
+ if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
21081
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
21082
+ maxDocsPerPartition
21083
+ });
21084
+ }
21085
+ if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
21086
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
21087
+ maxPartitions
21088
+ });
21089
+ }
21090
+ if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
21091
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
21092
+ watermark
21093
+ });
21094
+ }
21095
+ }
21096
+ function requirePositiveSafeInteger(field, value) {
21097
+ if (value === void 0) return;
21098
+ if (!Number.isSafeInteger(value) || value < 1) {
21099
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${field} must be a positive safe integer`, { field, value });
21100
+ }
21101
+ }
21102
+ function validateWorkerConfig(workers, lifecycle) {
21103
+ if (workers === void 0) return;
21104
+ requirePositiveSafeInteger("workers.count", workers.count);
21105
+ requirePositiveSafeInteger("workers.promotionThreshold", workers.promotionThreshold);
21106
+ requirePositiveSafeInteger("workers.idleTimeoutMs", workers.idleTimeoutMs);
21107
+ const closeAfter = lifecycle?.idleTimeoutMs;
21108
+ if (workers.idleTimeoutMs !== void 0 && closeAfter !== void 0 && workers.idleTimeoutMs > closeAfter) {
21109
+ throw new NarsilError(
21110
+ ErrorCodes.CONFIG_INVALID,
21111
+ "workers.idleTimeoutMs must be at most lifecycle.idleTimeoutMs, so that an index drops its copies before it closes",
21112
+ { workersIdleTimeoutMs: workers.idleTimeoutMs, lifecycleIdleTimeoutMs: closeAfter }
21113
+ );
21114
+ }
21115
+ }
21116
+ function validateBM25Params(bm25) {
21117
+ if (bm25 === void 0) return;
21118
+ const { k1, b } = bm25;
21119
+ if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
21120
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
21121
+ }
21122
+ if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
21123
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
21124
+ }
21125
+ }
21126
+ function clampLimit(limit) {
21127
+ return clampRowCount(limit, DEFAULT_LIMIT);
21128
+ }
21129
+ function clampOffset(offset) {
21130
+ return clampRowCount(offset, DEFAULT_OFFSET);
21131
+ }
21132
+
20423
21133
  // src/engine/vector-fields.ts
20424
21134
  function getVectorFieldPaths(schema) {
20425
21135
  return new Set(extractVectorFieldsFromSchema(schema).keys());
@@ -20461,10 +21171,13 @@ function createWatermarkNotifier(deps) {
20461
21171
 
20462
21172
  // src/engine/core.ts
20463
21173
  function createEngineCore(config, hooks) {
20464
- const executor = createDirectExecutor();
20465
- const promoter = createExecutionPromoter({
20466
- perIndexThreshold: config?.workers?.promotionThreshold,
20467
- totalThreshold: config?.workers?.totalPromotionThreshold
21174
+ validateWorkerConfig(config?.workers, config?.lifecycle);
21175
+ const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
21176
+ const executor = createDirectExecutor({
21177
+ vectorWorkerCopies: {
21178
+ enabled: (config?.workers?.enabled ?? workersEnabledByDefault()) && vectorWorkerCount > 0,
21179
+ count: vectorWorkerCount
21180
+ }
20468
21181
  });
20469
21182
  const pluginRegistry = createPluginRegistry();
20470
21183
  if (config?.plugins) {
@@ -20483,11 +21196,11 @@ function createEngineCore(config, hooks) {
20483
21196
  const shutdownState = { isShutdown: false };
20484
21197
  const abortController = new AbortController();
20485
21198
  const rebalancingIndexes = /* @__PURE__ */ new Set();
20486
- const orchestrator = createWorkerOrchestrator(config, executor, promoter, indexRegistry, {
20487
- shouldDeferPromotion() {
21199
+ const orchestrator = createWorkerOrchestrator(config, executor, indexRegistry, {
21200
+ shouldDeferCopies() {
20488
21201
  return rebalancingIndexes.size > 0;
20489
21202
  },
20490
- onPromotion(workerCount, reason) {
21203
+ onCopiesLoaded(workerCount, reason) {
20491
21204
  const handlers = eventHandlers.get("workerPromote");
20492
21205
  if (handlers) {
20493
21206
  for (const handler of handlers) handler({ workerCount, reason });
@@ -20496,10 +21209,10 @@ function createEngineCore(config, hooks) {
20496
21209
  console.warn("onWorkerPromote plugin hook failed:", err instanceof Error ? err.message : String(err));
20497
21210
  });
20498
21211
  },
20499
- onPromotionFailure(reason, error, retryable) {
21212
+ onCopyLoadFailure(reason, error, retryable) {
20500
21213
  const handlers = eventHandlers.get("workerPromoteFailure");
20501
21214
  if (!handlers || handlers.size === 0) {
20502
- console.warn(`Worker promotion failed (${reason}):`, error);
21215
+ console.warn(`Loading worker copies failed (${reason}):`, error);
20503
21216
  return;
20504
21217
  }
20505
21218
  for (const handler of handlers) handler({ reason, error, retryable });
@@ -20688,7 +21401,8 @@ function createEngineCore(config, hooks) {
20688
21401
  rebalancingIndexes,
20689
21402
  requireManager,
20690
21403
  onClose: hooks?.onIndexClose,
20691
- onOpen: hooks?.onIndexOpen
21404
+ onOpen: hooks?.onIndexOpen,
21405
+ onAccess: (indexName) => orchestrator.noteAccess(indexName)
20692
21406
  });
20693
21407
  const mutationCtx = {
20694
21408
  executor,
@@ -20722,7 +21436,6 @@ function createEngineCore(config, hooks) {
20722
21436
  };
20723
21437
  return {
20724
21438
  executor,
20725
- promoter,
20726
21439
  pluginRegistry,
20727
21440
  durability,
20728
21441
  invalidation,
@@ -20748,97 +21461,6 @@ function createEngineCore(config, hooks) {
20748
21461
  rebalanceCtx
20749
21462
  };
20750
21463
  }
20751
- var MAX_INDEX_NAME_LENGTH = 255;
20752
-
20753
- // src/engine/validation.ts
20754
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
20755
- var MAX_DOC_ID_LENGTH = 512;
20756
- var BATCH_CHUNK_SIZE = 1e3;
20757
- var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
20758
- var DEFAULT_OFFSET = 0;
20759
- function now() {
20760
- if (typeof performance !== "undefined" && typeof performance.now === "function") {
20761
- return performance.now();
20762
- }
20763
- return Date.now();
20764
- }
20765
- function validateIndexName(name) {
20766
- if (!name || name.length === 0) {
20767
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
20768
- }
20769
- if (name.length > MAX_INDEX_NAME_LENGTH) {
20770
- throw new NarsilError(
20771
- ErrorCodes.INDEX_NOT_FOUND,
20772
- `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
20773
- {
20774
- indexName: name,
20775
- length: name.length
20776
- }
20777
- );
20778
- }
20779
- if (!INDEX_NAME_PATTERN.test(name)) {
20780
- throw new NarsilError(
20781
- ErrorCodes.INDEX_NOT_FOUND,
20782
- `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
20783
- { indexName: name }
20784
- );
20785
- }
20786
- if (name.includes("..")) {
20787
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
20788
- }
20789
- }
20790
- function validateDocId(docId) {
20791
- if (!docId || docId.length === 0) {
20792
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
20793
- }
20794
- if (docId.length > MAX_DOC_ID_LENGTH) {
20795
- throw new NarsilError(
20796
- ErrorCodes.DOC_VALIDATION_FAILED,
20797
- `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
20798
- {
20799
- docId,
20800
- length: docId.length
20801
- }
20802
- );
20803
- }
20804
- if (docId.includes("\0")) {
20805
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
20806
- }
20807
- }
20808
- function validatePartitionConfig(partitions) {
20809
- const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
20810
- if (maxDocsPerPartition !== void 0 && (!Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
20811
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxDocsPerPartition must be a positive integer", {
20812
- maxDocsPerPartition
20813
- });
20814
- }
20815
- if (maxPartitions !== void 0 && (!Number.isInteger(maxPartitions) || maxPartitions < 1)) {
20816
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.maxPartitions must be a positive integer", {
20817
- maxPartitions
20818
- });
20819
- }
20820
- if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
20821
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "partitions.watermark must be above 0 and at most 1", {
20822
- watermark
20823
- });
20824
- }
20825
- }
20826
- function validateBM25Params(bm25) {
20827
- if (bm25 === void 0) return;
20828
- const { k1, b } = bm25;
20829
- if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
20830
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
20831
- }
20832
- if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
20833
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
20834
- }
20835
- }
20836
- function clampLimit(limit) {
20837
- return clampRowCount(limit, DEFAULT_LIMIT);
20838
- }
20839
- function clampOffset(offset) {
20840
- return clampRowCount(offset, DEFAULT_OFFSET);
20841
- }
20842
21464
 
20843
21465
  // src/engine/index-lifecycle.ts
20844
21466
  async function createEngineIndex(core, config, name, indexConfig, indexUuid) {
@@ -21450,7 +22072,7 @@ async function insertDocument(ctx, indexName, document, docId, options) {
21450
22072
  }
21451
22073
  }
21452
22074
  ctx.checkWatermark(indexName);
21453
- await ctx.orchestrator.checkPromotion();
22075
+ await ctx.orchestrator.scaleOutReadyIndexes();
21454
22076
  return resolvedDocId;
21455
22077
  }
21456
22078
 
@@ -21604,7 +22226,6 @@ async function applyAdmittedDocuments(ctx, indexName, admitted, options, failed)
21604
22226
  }
21605
22227
 
21606
22228
  // src/engine/mutations/segment-replication.ts
21607
- var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
21608
22229
  function shardCount(documentCount, workers) {
21609
22230
  if (workers <= 1) return 1;
21610
22231
  return Math.max(1, Math.min(workers, Math.ceil(documentCount / MIN_DOCUMENTS_FOR_SEGMENTS)));
@@ -21864,7 +22485,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
21864
22485
  vecIndexes.get(fieldPath)?.scheduleBuild();
21865
22486
  }
21866
22487
  ctx.checkWatermark(indexName);
21867
- await ctx.orchestrator.checkPromotion();
22488
+ await ctx.orchestrator.scaleOutReadyIndexes();
21868
22489
  return { succeeded: outcome.succeeded, failed };
21869
22490
  }
21870
22491
 
@@ -21872,7 +22493,7 @@ async function insertBatchViaSegments(ctx, indexName, documents, options) {
21872
22493
  async function insertDocumentBatch(ctx, indexName, documents, options) {
21873
22494
  ctx.guardShutdown();
21874
22495
  const entry = ctx.requireIndex(indexName);
21875
- await ctx.orchestrator.promoteBeforeBatch(indexName, documents.length);
22496
+ await ctx.orchestrator.scaleOutBeforeBatch(indexName, documents.length);
21876
22497
  if (documents.length >= MIN_DOCUMENTS_FOR_SEGMENTS && !ctx.isRebalancing(indexName) && ctx.orchestrator.segmentBuildConcurrency(indexName) > 0) {
21877
22498
  return insertBatchViaSegments(ctx, indexName, documents, options);
21878
22499
  }
@@ -22026,7 +22647,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
22026
22647
  }
22027
22648
  }
22028
22649
  ctx.checkWatermark(indexName);
22029
- await ctx.orchestrator.checkPromotion();
22650
+ await ctx.orchestrator.scaleOutReadyIndexes();
22030
22651
  return { succeeded, failed };
22031
22652
  }
22032
22653
 
@@ -22301,7 +22922,7 @@ async function updateDocumentBatch(ctx, indexName, updates) {
22301
22922
  }
22302
22923
 
22303
22924
  // src/partitioning/write-ahead-queue.ts
22304
- function createWriteAheadQueue(maxSize = 1e4) {
22925
+ function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
22305
22926
  const entries = [];
22306
22927
  const docStates = /* @__PURE__ */ new Map();
22307
22928
  let nextSequence = 1;
@@ -22734,8 +23355,8 @@ async function createSnapshot(manager, entry) {
22734
23355
  }
22735
23356
  const config = entry.config;
22736
23357
  const bm25 = config.bm25;
22737
- const { encode: encode14 } = await import('@msgpack/msgpack');
22738
- const payload = encode14({
23358
+ const { encode: encode13 } = await import('@msgpack/msgpack');
23359
+ const payload = encode13({
22739
23360
  version: 2,
22740
23361
  schema: config.schema,
22741
23362
  language: entry.language.name,
@@ -22772,10 +23393,10 @@ async function restoreFromSnapshot(indexName, data, deps) {
22772
23393
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data must be a Uint8Array");
22773
23394
  }
22774
23395
  const payloadBytes = hasNrslMagic(data) ? await unpackIndexSnapshotEnvelope(data) : data;
22775
- const { decode: decode14 } = await import('@msgpack/msgpack');
23396
+ const { decode: decode13 } = await import('@msgpack/msgpack');
22776
23397
  let decoded;
22777
23398
  try {
22778
- decoded = decode14(payloadBytes);
23399
+ decoded = decode13(payloadBytes);
22779
23400
  } catch (err) {
22780
23401
  throw new NarsilError(
22781
23402
  ErrorCodes.DOC_VALIDATION_FAILED,
@@ -24136,8 +24757,6 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
24136
24757
  }
24137
24758
 
24138
24759
  // src/engine/suggest.ts
24139
- var DEFAULT_SUGGEST_LIMIT = 10;
24140
- var MAX_SUGGEST_LIMIT = 100;
24141
24760
  function executeSuggest(manager, language, params, partitionIds) {
24142
24761
  const t0 = performance.now();
24143
24762
  const ceiling = MAX_SUGGEST_LIMIT ;
@@ -24210,7 +24829,7 @@ async function runEngineQuery(core, indexName, params, options) {
24210
24829
  entry.embeddingAdapterName
24211
24830
  );
24212
24831
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
24213
- const workerSearch = core.orchestrator.isPromoted() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24832
+ const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24214
24833
  const result = await executeQuery(resolvedParams, {
24215
24834
  manager,
24216
24835
  language: entry.language,
@@ -24250,7 +24869,7 @@ async function runEnginePreflight(core, indexName, params, options) {
24250
24869
  core.abortController.signal,
24251
24870
  entry.embeddingAdapterName
24252
24871
  );
24253
- const workerSearch = core.orchestrator.isPromoted() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24872
+ const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
24254
24873
  const result = await executePreflight(resolvedParams, {
24255
24874
  manager,
24256
24875
  language: entry.language,
@@ -24541,6 +25160,7 @@ function createNarsilFromCore(core, config) {
24541
25160
  openIndexCount: lifecycleCounts.open,
24542
25161
  closedIndexCount: lifecycleCounts.closed,
24543
25162
  reopenCount: lifecycleCounts.reopens,
25163
+ workerCopies: orchestrator.workerCopies(),
24544
25164
  workers
24545
25165
  };
24546
25166
  },