@delali/narsil 0.3.1-next.111 → 0.3.1-next.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/browser/{chunk-6YUUEMQ4.mjs → chunk-3YPLBK4C.mjs} +2 -2
  2. package/dist/browser/{chunk-5BQPBG3G.mjs → chunk-4FVSBSXY.mjs} +1 -1
  3. package/dist/browser/{chunk-HQOYKRMB.mjs → chunk-DC2SZO4D.mjs} +183 -44
  4. package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
  5. package/dist/browser/{durability-wiring-CYAJBYCO.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
  6. package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
  7. package/dist/browser/{restore-queue-RTZGJT4D.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
  8. package/dist/browser/{snapshot-BU6I25CW.mjs → snapshot-BE3F2PNC.mjs} +6 -5
  9. package/dist/browser/{write-RUPYHPBZ.mjs → write-MV7K7OCU.mjs} +2 -2
  10. package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
  11. package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
  12. package/dist/{chunk-6S7YRUPQ.mjs → chunk-27ZEY73B.mjs} +4 -4
  13. package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
  14. package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
  15. package/dist/{chunk-R2MRMH6M.mjs → chunk-5QIRLJH4.mjs} +3 -3
  16. package/dist/{chunk-7JOONQ24.mjs → chunk-5WEBVYOA.mjs} +176 -38
  17. package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
  18. package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
  19. package/dist/{chunk-JBPYFYIU.mjs → chunk-DZXGNLK5.mjs} +3 -3
  20. package/dist/chunk-EBIHRL7B.mjs +21 -0
  21. package/dist/{chunk-UGKO3RG2.mjs → chunk-GLZI636G.mjs} +1 -1
  22. package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
  23. package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
  24. package/dist/chunk-IWVEG5CX.mjs +67 -0
  25. package/dist/{chunk-SXVDBHYJ.mjs → chunk-JUAFWJON.mjs} +3 -3
  26. package/dist/{chunk-PKE2PCTS.mjs → chunk-LVNBY2TS.mjs} +1 -1
  27. package/dist/{chunk-YLE75NQ5.mjs → chunk-OABTAUCW.mjs} +2 -2
  28. package/dist/{chunk-YSR5L3XJ.mjs → chunk-OM5G2J6X.mjs} +1 -1
  29. package/dist/{chunk-IW6UR3AF.mjs → chunk-PABYSTKO.mjs} +13 -10
  30. package/dist/chunk-PELHVTO7.mjs +24 -0
  31. package/dist/{chunk-CUVTRLA4.mjs → chunk-QGVMEMCG.mjs} +4 -4
  32. package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
  33. package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
  34. package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
  35. package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
  36. package/dist/client.d.ts +6 -6
  37. package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
  38. package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
  39. package/dist/{direct-executor-QFAUKTCQ.mjs → direct-executor-R4HOSVPE.mjs} +6 -5
  40. package/dist/distribution.d.ts +5 -5
  41. package/dist/distribution.mjs +74 -37
  42. package/dist/{durability-wiring-NUHTYXHA.mjs → durability-wiring-XZI6355I.mjs} +15 -14
  43. package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
  44. package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
  45. package/dist/index.browser.mjs +1 -1
  46. package/dist/index.d.ts +5 -5
  47. package/dist/index.mjs +8 -7
  48. package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
  49. package/dist/persistence/durability/checkpoint-worker.mjs +12 -11
  50. package/dist/react.d.ts +6 -6
  51. package/dist/{restore-queue-QSLGXZJB.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
  52. package/dist/server.d.ts +8 -8
  53. package/dist/server.mjs +20 -62
  54. package/dist/{snapshot-IV3S2TMI.mjs → snapshot-PDUGHQ25.mjs} +16 -15
  55. package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
  56. package/dist/{thread-THLCKMEB.mjs → thread-S4SUCV4Y.mjs} +10 -7
  57. package/dist/vector/search-worker.mjs +2 -2
  58. package/dist/workers/entry.mjs +6 -6
  59. package/dist/{write-VGRTOS7N.mjs → write-KVJ7YJJ6.mjs} +9 -8
  60. package/package.json +7 -7
@@ -1,5 +1,5 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-5BQPBG3G.mjs';
2
- import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-HQOYKRMB.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-4FVSBSXY.mjs';
2
+ import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-DC2SZO4D.mjs';
3
3
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
4
4
  import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
@@ -1,4 +1,4 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-HQOYKRMB.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-DC2SZO4D.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
@@ -3245,6 +3245,8 @@ function createDocumentStore() {
3245
3245
  // src/core/constants.ts
3246
3246
  var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
3247
3247
  var MAX_TERM_FREQUENCY = 65535;
3248
+ var PENDING_REMOVAL_MERGE_FLOOR = 64;
3249
+ var PENDING_REMOVAL_MERGE_DIVISOR = 256;
3248
3250
 
3249
3251
  // src/core/fuzzy.ts
3250
3252
  var FIRST_SURROGATE = 55296;
@@ -4944,6 +4946,7 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
4944
4946
  }
4945
4947
 
4946
4948
  // src/geo/geo-index.ts
4949
+ var REMOVED_DOC_ID = -1;
4947
4950
  function firstAtOrAfter(entries, code) {
4948
4951
  let lo = 0;
4949
4952
  let hi = entries.length;
@@ -4954,25 +4957,58 @@ function firstAtOrAfter(entries, code) {
4954
4957
  }
4955
4958
  return lo;
4956
4959
  }
4960
+ function dropRemoved(entries) {
4961
+ let write = 0;
4962
+ for (let read = 0; read < entries.length; read++) {
4963
+ if (entries[read].docId !== REMOVED_DOC_ID) entries[write++] = entries[read];
4964
+ }
4965
+ entries.length = write;
4966
+ }
4967
+ function mergeSortedPending(entries, pending) {
4968
+ pending.sort((left2, right2) => left2.code - right2.code || left2.docId - right2.docId);
4969
+ const mainLength = entries.length;
4970
+ for (const entry of pending) entries.push(entry);
4971
+ let left = mainLength - 1;
4972
+ let right = pending.length - 1;
4973
+ let write = entries.length - 1;
4974
+ while (right >= 0) {
4975
+ const later = left >= 0 ? entries[left] : void 0;
4976
+ const next = pending[right];
4977
+ if (later !== void 0 && (later.code > next.code || later.code === next.code && later.docId > next.docId)) {
4978
+ entries[write--] = later;
4979
+ left--;
4980
+ } else {
4981
+ entries[write--] = next;
4982
+ right--;
4983
+ }
4984
+ }
4985
+ }
4957
4986
  function createGeoIndex() {
4958
- const entries = [];
4987
+ let entries = [];
4988
+ let pending = [];
4959
4989
  const byDocument = /* @__PURE__ */ new Map();
4960
- let sorted = true;
4990
+ let removedCount = 0;
4961
4991
  function ensureSorted() {
4962
- if (sorted) return;
4963
- entries.sort((left, right) => left.code - right.code || left.docId - right.docId);
4964
- sorted = true;
4965
- }
4966
- function removeEntry(internalId) {
4967
- const entry = byDocument.get(internalId);
4968
- if (entry === void 0) return;
4969
- byDocument.delete(internalId);
4970
- ensureSorted();
4971
- for (let at = firstAtOrAfter(entries, entry.code); at < entries.length; at++) {
4972
- if (entries[at] !== entry) continue;
4973
- entries.splice(at, 1);
4974
- return;
4992
+ if (removedCount > 0) {
4993
+ dropRemoved(entries);
4994
+ dropRemoved(pending);
4995
+ removedCount = 0;
4996
+ }
4997
+ if (pending.length === 0) return;
4998
+ if (entries.length === 0) {
4999
+ pending.sort((left, right) => left.code - right.code || left.docId - right.docId);
5000
+ entries = pending;
5001
+ } else {
5002
+ mergeSortedPending(entries, pending);
4975
5003
  }
5004
+ pending = [];
5005
+ }
5006
+ function removeEntry(entry) {
5007
+ byDocument.delete(entry.docId);
5008
+ entry.docId = REMOVED_DOC_ID;
5009
+ removedCount++;
5010
+ const held = entries.length + pending.length;
5011
+ if (removedCount >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) ensureSorted();
4976
5012
  }
4977
5013
  function matching(boxes, passes, inside) {
4978
5014
  ensureSorted();
@@ -4991,14 +5027,15 @@ function createGeoIndex() {
4991
5027
  }
4992
5028
  const index = {
4993
5029
  insert(internalId, lat, lon) {
4994
- removeEntry(internalId);
5030
+ const existing = byDocument.get(internalId);
5031
+ if (existing !== void 0) removeEntry(existing);
4995
5032
  const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
4996
- entries.push(entry);
5033
+ pending.push(entry);
4997
5034
  byDocument.set(internalId, entry);
4998
- sorted = false;
4999
5035
  },
5000
5036
  remove(internalId) {
5001
- removeEntry(internalId);
5037
+ const existing = byDocument.get(internalId);
5038
+ if (existing !== void 0) removeEntry(existing);
5002
5039
  },
5003
5040
  radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
5004
5041
  const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
@@ -5014,13 +5051,14 @@ function createGeoIndex() {
5014
5051
  },
5015
5052
  documentsBitset(capacity) {
5016
5053
  const bits = createBitSet(capacity);
5017
- for (const entry of entries) bitsetSet(bits, entry.docId);
5054
+ for (const docId of byDocument.keys()) bitsetSet(bits, docId);
5018
5055
  return bits;
5019
5056
  },
5020
5057
  clear() {
5021
- entries.length = 0;
5058
+ entries = [];
5059
+ pending = [];
5022
5060
  byDocument.clear();
5023
- sorted = true;
5061
+ removedCount = 0;
5024
5062
  },
5025
5063
  serialize() {
5026
5064
  ensureSorted();
@@ -5348,29 +5386,126 @@ function collectDocIdsBitset(entries, from, to, capacity) {
5348
5386
  }
5349
5387
  return bs;
5350
5388
  }
5389
+ function removalsFor(byDocument, docId, value) {
5390
+ const keys = byDocument.get(docId);
5391
+ if (keys === void 0) return void 0;
5392
+ for (const key of keys) {
5393
+ if (key.value === value) return key;
5394
+ }
5395
+ return void 0;
5396
+ }
5397
+ function groupRemovals(docIds, values, limits) {
5398
+ const byDocument = /* @__PURE__ */ new Map();
5399
+ for (let at = 0; at < docIds.length; at++) {
5400
+ const existing = removalsFor(byDocument, docIds[at], values[at]);
5401
+ if (existing !== void 0) {
5402
+ existing.limits.push(limits[at]);
5403
+ continue;
5404
+ }
5405
+ const key = { value: values[at], limits: [limits[at]], next: 0 };
5406
+ const keys = byDocument.get(docIds[at]);
5407
+ if (keys === void 0) byDocument.set(docIds[at], [key]);
5408
+ else keys.push(key);
5409
+ }
5410
+ return byDocument;
5411
+ }
5412
+ function markedDocuments(docIds) {
5413
+ let highest = 0;
5414
+ for (const docId of docIds) if (docId > highest) highest = docId;
5415
+ const marked = new Uint8Array(highest + 1);
5416
+ for (const docId of docIds) marked[docId] = 1;
5417
+ return marked;
5418
+ }
5419
+ function removeFromSorted(entries, from, marked, byDocument) {
5420
+ let write = from;
5421
+ for (let read = from; read < entries.length; read++) {
5422
+ const entry = entries[read];
5423
+ if (entry.docId < marked.length && marked[entry.docId] === 1) {
5424
+ const key = removalsFor(byDocument, entry.docId, entry.value);
5425
+ if (key !== void 0 && key.next < key.limits.length) {
5426
+ key.next++;
5427
+ continue;
5428
+ }
5429
+ }
5430
+ entries[write++] = entry;
5431
+ }
5432
+ entries.length = write;
5433
+ }
5434
+ function removeFromPending(pending, marked, byDocument) {
5435
+ let write = 0;
5436
+ for (let read = 0; read < pending.length; read++) {
5437
+ const entry = pending[read];
5438
+ if (entry.docId < marked.length && marked[entry.docId] === 1) {
5439
+ const key = removalsFor(byDocument, entry.docId, entry.value);
5440
+ if (key !== void 0) {
5441
+ while (key.next < key.limits.length && key.limits[key.next] <= read) key.next++;
5442
+ if (key.next < key.limits.length) {
5443
+ key.next++;
5444
+ continue;
5445
+ }
5446
+ }
5447
+ }
5448
+ pending[write++] = entry;
5449
+ }
5450
+ pending.length = write;
5451
+ }
5452
+ function mergeSortedPending2(entries, pending) {
5453
+ pending.sort((a, b) => a.value - b.value);
5454
+ const mainLength = entries.length;
5455
+ for (const entry of pending) entries.push(entry);
5456
+ let left = mainLength - 1;
5457
+ let right = pending.length - 1;
5458
+ let write = entries.length - 1;
5459
+ while (right >= 0) {
5460
+ if (left >= 0 && entries[left].value > pending[right].value) entries[write--] = entries[left--];
5461
+ else entries[write--] = pending[right--];
5462
+ }
5463
+ }
5351
5464
  function createNumericIndex() {
5352
5465
  let entries = [];
5353
- let sorted = true;
5466
+ let pending = [];
5467
+ const removedDocIds = [];
5468
+ const removedValues = [];
5469
+ const removedLimits = [];
5470
+ function applyRemovals() {
5471
+ const byDocument = groupRemovals(removedDocIds, removedValues, removedLimits);
5472
+ const marked = markedDocuments(removedDocIds);
5473
+ let lowest = removedValues[0];
5474
+ for (const value of removedValues) if (value < lowest) lowest = value;
5475
+ removeFromSorted(entries, lowerBound(entries, lowest), marked, byDocument);
5476
+ if (pending.length > 0) removeFromPending(pending, marked, byDocument);
5477
+ removedDocIds.length = 0;
5478
+ removedValues.length = 0;
5479
+ removedLimits.length = 0;
5480
+ }
5354
5481
  function ensureSorted() {
5355
- if (!sorted) {
5356
- entries.sort((a, b) => a.value - b.value);
5357
- sorted = true;
5482
+ if (removedDocIds.length > 0) applyRemovals();
5483
+ if (pending.length === 0) return;
5484
+ if (entries.length === 0) {
5485
+ pending.sort((a, b) => a.value - b.value);
5486
+ entries = pending;
5487
+ } else {
5488
+ mergeSortedPending2(entries, pending);
5358
5489
  }
5490
+ pending = [];
5491
+ }
5492
+ function discardPending() {
5493
+ pending = [];
5494
+ removedDocIds.length = 0;
5495
+ removedValues.length = 0;
5496
+ removedLimits.length = 0;
5359
5497
  }
5360
5498
  return {
5361
5499
  insert(internalId, value) {
5362
- entries.push({ value, docId: internalId });
5363
- sorted = false;
5500
+ pending.push({ value, docId: internalId });
5364
5501
  },
5365
5502
  remove(internalId, value) {
5366
- ensureSorted();
5367
- const start = lowerBound(entries, value);
5368
- const end = upperBound(entries, value);
5369
- for (let i = start; i < end; i++) {
5370
- if (entries[i].docId === internalId) {
5371
- entries.splice(i, 1);
5372
- return;
5373
- }
5503
+ removedDocIds.push(internalId);
5504
+ removedValues.push(value);
5505
+ removedLimits.push(pending.length);
5506
+ const held = entries.length + pending.length;
5507
+ if (removedDocIds.length >= Math.max(PENDING_REMOVAL_MERGE_FLOOR, held / PENDING_REMOVAL_MERGE_DIVISOR)) {
5508
+ ensureSorted();
5374
5509
  }
5375
5510
  },
5376
5511
  queryEq(value) {
@@ -5407,6 +5542,7 @@ function createNumericIndex() {
5407
5542
  return collectDocIds(entries, lowerBound(entries, min), upperBound(entries, max));
5408
5543
  },
5409
5544
  getAllDocIds() {
5545
+ ensureSorted();
5410
5546
  return collectDocIds(entries, 0, entries.length);
5411
5547
  },
5412
5548
  queryEqBitset(value, capacity) {
@@ -5434,9 +5570,11 @@ function createNumericIndex() {
5434
5570
  return collectDocIdsBitset(entries, lowerBound(entries, min), upperBound(entries, max), capacity);
5435
5571
  },
5436
5572
  getAllDocIdsBitset(capacity) {
5573
+ ensureSorted();
5437
5574
  return collectDocIdsBitset(entries, 0, entries.length, capacity);
5438
5575
  },
5439
5576
  count() {
5577
+ ensureSorted();
5440
5578
  return entries.length;
5441
5579
  },
5442
5580
  facetValueCounts(matched) {
@@ -5474,7 +5612,7 @@ function createNumericIndex() {
5474
5612
  },
5475
5613
  clear() {
5476
5614
  entries = [];
5477
- sorted = true;
5615
+ discardPending();
5478
5616
  },
5479
5617
  serialize() {
5480
5618
  ensureSorted();
@@ -5482,7 +5620,7 @@ function createNumericIndex() {
5482
5620
  },
5483
5621
  deserialize(data) {
5484
5622
  entries = data.slice();
5485
- sorted = true;
5623
+ discardPending();
5486
5624
  }
5487
5625
  };
5488
5626
  }
@@ -14949,7 +15087,7 @@ function requireLoadedVectorCode() {
14949
15087
  function loadVectorCode() {
14950
15088
  if (loaded !== null) return Promise.resolve(loaded);
14951
15089
  if (loading === null) {
14952
- loading = import('./module-DYKR4INA.mjs').then(
15090
+ loading = import('./module-5EWHBKHZ.mjs').then(
14953
15091
  (code) => {
14954
15092
  loaded = code;
14955
15093
  return code;
@@ -19037,7 +19175,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19037
19175
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19038
19176
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19039
19177
  }
19040
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-CYAJBYCO.mjs');
19178
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-2RG22WQJ.mjs');
19041
19179
  const filesystemDurability = durabilityTier?.kind === "wal";
19042
19180
  const executor = createDirectExecutor({
19043
19181
  vectorWorkerCopies: vectorCopyPolicy,
@@ -21781,6 +21919,7 @@ function getVectorMaintenanceStatus(manager) {
21781
21919
  graphCount: status.graphCount,
21782
21920
  bufferSize: status.bufferSize,
21783
21921
  building: status.building,
21922
+ waitingForMemory: status.waitingForMemory,
21784
21923
  estimatedCompactMs: status.estimatedCompactMs,
21785
21924
  estimatedOptimizeMs: status.estimatedOptimizeMs
21786
21925
  });
@@ -22470,14 +22609,14 @@ function portabilityOperations(core, config, engine) {
22470
22609
  return {
22471
22610
  snapshot(indexName) {
22472
22611
  return withOpenIndex(indexName, async () => {
22473
- const { createSnapshot } = await import('./snapshot-BU6I25CW.mjs');
22612
+ const { createSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
22474
22613
  return createSnapshot(snapshotDepsOf(indexName));
22475
22614
  });
22476
22615
  },
22477
22616
  async restore(indexName, data) {
22478
22617
  core.guardShutdown();
22479
22618
  validateIndexName(indexName);
22480
- const restored = import('./restore-queue-RTZGJT4D.mjs').then(
22619
+ const restored = import('./restore-queue-WCFUB4NS.mjs').then(
22481
22620
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22482
22621
  );
22483
22622
  await emitWriteEventAfter(core, indexName, restored);
@@ -22486,8 +22625,8 @@ function portabilityOperations(core, config, engine) {
22486
22625
  exportDocuments(indexName) {
22487
22626
  return withOpenIndex(indexName, async () => {
22488
22627
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22489
- import('./snapshot-BU6I25CW.mjs'),
22490
- import('./write-RUPYHPBZ.mjs')
22628
+ import('./snapshot-BE3F2PNC.mjs'),
22629
+ import('./write-MV7K7OCU.mjs')
22491
22630
  ]);
22492
22631
  const header = indexConfigurationOf(core.requireIndex(indexName));
22493
22632
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -19,6 +19,15 @@ function createFixedBuffer(bytes) {
19
19
  function nextGrowthTarget(have, needed, max) {
20
20
  return Math.min(max, Math.max(needed, have * 2));
21
21
  }
22
+ function grownByteLength(have, needed, max, step = 1) {
23
+ let size = have;
24
+ while (size < needed) {
25
+ const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
26
+ if (next <= size) return size;
27
+ size = next;
28
+ }
29
+ return size;
30
+ }
22
31
  function growBufferTo(buffer, bytes) {
23
32
  const held = buffer.byteLength;
24
33
  if (held >= bytes) return;
@@ -1115,6 +1124,25 @@ function ensureBlockSlots(handle, slots) {
1115
1124
  }
1116
1125
  growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
1117
1126
  }
1127
+ function blockGrowthBytes(handle, layout, slots) {
1128
+ const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
1129
+ if (handle !== null && handle.storage.kind === "bytes") {
1130
+ const buffer = handle.storage.buffer;
1131
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
1132
+ }
1133
+ const have = handle === null ? 0 : blockBuffer(handle).byteLength;
1134
+ const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
1135
+ return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
1136
+ }
1137
+ function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
1138
+ let bytes = 0;
1139
+ const lastBlock = Math.floor(lastOrdinal / layout.capacity);
1140
+ for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
1141
+ const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
1142
+ bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
1143
+ }
1144
+ return bytes;
1145
+ }
1118
1146
  function blockBuffer(handle) {
1119
1147
  const storage = handle.storage;
1120
1148
  if (storage.kind === "bytes") return storage.buffer;
@@ -3458,6 +3486,7 @@ var BUILD_CHUNK_SIZE = 100;
3458
3486
  var BUILD_TURN_HANDOVER_AFTER_MS = 50;
3459
3487
  var BUILD_RETRY_FIRST_DELAY_MS = 250;
3460
3488
  var BUILD_RETRY_MAX_DELAY_MS = 3e4;
3489
+ var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
3461
3490
  var REBUILD_REMOVED_RATIO = 0.2;
3462
3491
  var DRIFT_CHECK_GROWTH_FACTOR = 2;
3463
3492
  var RECALIBRATION_DRIFT_RATIO = 2.5;
@@ -4480,4 +4509,9 @@ function recordCheckpoint(state, plan, written) {
4480
4509
  state.savedSignature = plan.signature;
4481
4510
  }
4482
4511
 
4483
- export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
4512
+ // src/runtime/idle-heap-cleanup.browser.ts
4513
+ function collectAndReturnPages() {
4514
+ return false;
4515
+ }
4516
+
4517
+ export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-6YUUEMQ4.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-5BQPBG3G.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-HQOYKRMB.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-3YPLBK4C.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-4FVSBSXY.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-DC2SZO4D.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
6
6
  import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';