@delali/narsil 0.3.1-next.112 → 0.3.1-next.115

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/browser/{chunk-UR5MNN53.mjs → chunk-2TMWIATH.mjs} +108 -67
  2. package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-73U5DWOZ.mjs} +39 -4
  3. package/dist/browser/{chunk-O5IKTA27.mjs → chunk-FODCC3MK.mjs} +1 -1
  4. package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-I45FAGOF.mjs} +2 -2
  5. package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-CSMTKEYX.mjs} +3 -3
  6. package/dist/browser/{module-DYKR4INA.mjs → module-G2PPZV3B.mjs} +127 -33
  7. package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-N7N42PBS.mjs} +1 -1
  8. package/dist/browser/{runtime-FL5K4JOH.mjs → runtime-BX6XRFYR.mjs} +2 -2
  9. package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-EBH6RXQO.mjs} +6 -5
  10. package/dist/browser/{write-IABOLE4N.mjs → write-5ZKFINX3.mjs} +2 -2
  11. package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
  12. package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
  13. package/dist/{chunk-R2MRGYU5.mjs → chunk-4ZKBKPBV.mjs} +22 -2
  14. package/dist/{chunk-GLZI636G.mjs → chunk-5VVSHCXT.mjs} +27 -12
  15. package/dist/{chunk-3COL24MT.mjs → chunk-7EWQZQDT.mjs} +1 -1
  16. package/dist/{chunk-QI6JW4PM.mjs → chunk-BCNUJVTT.mjs} +1 -1
  17. package/dist/{chunk-3RRATITJ.mjs → chunk-BCSWODXP.mjs} +75 -55
  18. package/dist/{thread-WGBCVJII.mjs → chunk-DEXID4ZT.mjs} +301 -200
  19. package/dist/chunk-EBIHRL7B.mjs +21 -0
  20. package/dist/{chunk-YIDSHBQS.mjs → chunk-HC5NURS6.mjs} +33 -4
  21. package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
  22. package/dist/{chunk-63HFWQEA.mjs → chunk-IE43SRL3.mjs} +276 -102
  23. package/dist/chunk-IWVEG5CX.mjs +67 -0
  24. package/dist/{chunk-2IFU7SXG.mjs → chunk-KGVC6KJ3.mjs} +4 -4
  25. package/dist/{chunk-7CI34JAP.mjs → chunk-KVLX65KD.mjs} +4 -4
  26. package/dist/{chunk-5WEBVYOA.mjs → chunk-N53HN5VA.mjs} +16 -9
  27. package/dist/chunk-N6J6EJW3.mjs +198 -0
  28. package/dist/{chunk-PT7VRWS7.mjs → chunk-NX7KU77V.mjs} +3 -2
  29. package/dist/{chunk-QIFOXLWF.mjs → chunk-OGFSUXYY.mjs} +2 -2
  30. package/dist/chunk-PELHVTO7.mjs +24 -0
  31. package/dist/{chunk-V7ISFBFC.mjs → chunk-Q62NDKEC.mjs} +5 -4
  32. package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
  33. package/dist/{chunk-BVOUSX3U.mjs → chunk-UDZYIWDP.mjs} +3 -3
  34. package/dist/{chunk-HNAP2KTG.mjs → chunk-VFAXPDBH.mjs} +1 -0
  35. package/dist/{chunk-IBTKJIV7.mjs → chunk-VOKEYJHU.mjs} +1 -1
  36. package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
  37. package/dist/{chunk-IM76DLA7.mjs → chunk-YCFQ5754.mjs} +2 -2
  38. package/dist/{chunk-VLRT47GU.mjs → chunk-YCSRB3IN.mjs} +3 -3
  39. package/dist/{chunk-CQUMNIIG.mjs → chunk-Z4DZ4KK2.mjs} +1 -0
  40. package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
  41. package/dist/{chunk-2RAQKITH.mjs → chunk-ZR63OGT2.mjs} +1 -1
  42. package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
  43. package/dist/client.d.ts +6 -6
  44. package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
  45. package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
  46. package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-OUY4RPTU.mjs} +7 -6
  47. package/dist/distribution.d.ts +5 -5
  48. package/dist/distribution.mjs +79 -42
  49. package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-5V6DVHFK.mjs} +17 -16
  50. package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
  51. package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
  52. package/dist/index.browser.mjs +1 -1
  53. package/dist/index.d.ts +5 -5
  54. package/dist/index.mjs +10 -9
  55. package/dist/long-query-thread-HMQM2FWL.mjs +118 -0
  56. package/dist/{module-WP4VVGV2.mjs → module-3IMTXNPW.mjs} +92 -29
  57. package/dist/pattern.mjs +1 -1
  58. package/dist/persistence/durability/checkpoint-worker.mjs +14 -13
  59. package/dist/react.d.ts +6 -6
  60. package/dist/{rerank-3POMPQNX.mjs → rerank-NYX5S7OG.mjs} +3 -3
  61. package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-J6PCG4ON.mjs} +1 -1
  62. package/dist/{runtime-C2435MPK.mjs → runtime-XNYW6AWX.mjs} +2 -2
  63. package/dist/server.d.ts +8 -8
  64. package/dist/server.mjs +110 -258
  65. package/dist/{snapshot-EU6NNB2P.mjs → snapshot-57WF53MK.mjs} +18 -17
  66. package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
  67. package/dist/thread-FDEHU5G3.mjs +31 -0
  68. package/dist/vector/search-worker.mjs +2 -2
  69. package/dist/workers/entry.mjs +19 -7
  70. package/dist/{write-CKRWS3P5.mjs → write-YEI7A5OR.mjs} +11 -10
  71. package/package.json +7 -7
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-YRYBTRWB.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-O5IKTA27.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UR5MNN53.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-I45FAGOF.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-FODCC3MK.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-2TMWIATH.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
6
6
  import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
@@ -1,6 +1,6 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OVY4RPDS.mjs';
3
- export { releaseLocksHeldBy } from './chunk-OVY4RPDS.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-73U5DWOZ.mjs';
3
+ export { releaseLocksHeldBy } from './chunk-73U5DWOZ.mjs';
4
4
  import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
5
5
  import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
6
6
  import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
@@ -173,12 +173,13 @@ function readStoreFlags(store) {
173
173
  }
174
174
 
175
175
  // src/vector/shared-field/searcher.ts
176
- function distancesInTypeScript(store, query, metric, ordinals) {
176
+ function distancesInTypeScript(store, query, metric, ordinals, clock) {
177
177
  noteFallback("score");
178
178
  const arenaQuery = store.prepareQueryArena(query);
179
179
  const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
180
180
  const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
181
181
  for (let i = 0; i < ordinals.length; i++) {
182
+ if (clock !== void 0) clock.tick(query.length);
182
183
  if (arenaQuery) {
183
184
  distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
184
185
  continue;
@@ -188,10 +189,12 @@ function distancesInTypeScript(store, query, metric, ordinals) {
188
189
  }
189
190
  return distances;
190
191
  }
191
- function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
192
+ function scoreCandidates(store, candidates, query, k, metric, minSimilarity, clock) {
192
193
  const { ordinals, docIds } = candidates;
193
194
  if (ordinals.length === 0) return { results: [], matched: 0 };
194
- const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
195
+ clock?.check();
196
+ const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals, clock);
197
+ clock?.check();
195
198
  const heap = createBoundedMaxHeap(highScoreFirst, k);
196
199
  let matched = 0;
197
200
  for (let i = 0; i < ordinals.length; i++) {
@@ -262,6 +265,7 @@ function createSharedVectorSearcher(options) {
262
265
  }
263
266
  function search2(query, k, searchOptions, filter) {
264
267
  const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
268
+ const clock = searchOptions.clock;
265
269
  const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
266
270
  const liveDocuments = flags.liveDocuments();
267
271
  if (liveDocuments <= 0) return empty;
@@ -277,23 +281,27 @@ function createSharedVectorSearcher(options) {
277
281
  const graph = view.graph;
278
282
  if (graph === null) {
279
283
  const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
280
- return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
284
+ return scanned(
285
+ scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity, clock)
286
+ );
281
287
  }
282
288
  if (filter !== void 0) {
283
289
  const graphLiveSize = view.liveSize;
284
290
  const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
285
291
  if (selectivity < view.handles.filterThreshold) {
286
292
  const candidates = candidatesOf(ordinalFilterValues(filter), filter);
287
- return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
293
+ return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity, clock));
288
294
  }
289
295
  }
296
+ const unplaced = flags.unplacedOrdinals();
290
297
  let resolvedEveryHit = true;
291
298
  const graphResults = [];
292
299
  for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
293
300
  filter,
294
301
  efSearch,
295
302
  oversample,
296
- rescoreWindow
303
+ rescoreWindow,
304
+ clock
297
305
  })) {
298
306
  const docId = view.docIdOf(hit.ord);
299
307
  if (docId === void 0) {
@@ -302,9 +310,8 @@ function createSharedVectorSearcher(options) {
302
310
  }
303
311
  graphResults.push({ docId, score: hit.score });
304
312
  }
305
- const unplaced = flags.unplacedOrdinals();
306
313
  if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
307
- const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
314
+ const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity, clock);
308
315
  return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
309
316
  }
310
317
  return {
@@ -887,7 +894,8 @@ function openHandles(dimension, codeBits, blockBytes) {
887
894
  present: new Uint8Array(handles.present),
888
895
  partitions: new Int32Array(handles.partitions),
889
896
  diskFile: new Int32Array(handles.diskFile),
890
- diskOffset: new Uint32Array(handles.diskOffset)
897
+ diskOffset: new Uint32Array(handles.diskOffset),
898
+ longestDocIdBytes: 0
891
899
  };
892
900
  }
893
901
  function relayout(store) {
@@ -898,7 +906,10 @@ function claimSlot(store, docId, cold) {
898
906
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
899
907
  ensureOrdinal(store, ordinal, cold);
900
908
  store.diskFile[ordinal] = IN_MEMORY;
909
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
901
910
  appendDocId(store.handles, ordinal, docId);
911
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
912
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
902
913
  return ordinal;
903
914
  }
904
915
  function publishOrdinal(store, ordinal) {
@@ -974,6 +985,19 @@ function ensureOrdinal(store, ordinal, cold) {
974
985
  if (!cold) ensureFloatSlot(store, ordinal);
975
986
  ensureCodeSlot(store, ordinal);
976
987
  }
988
+ function bufferGrowthBytes(buffer, bytes) {
989
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
990
+ }
991
+ function growthBytesFor(store, vectors) {
992
+ const { handles } = store;
993
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
994
+ const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
995
+ if (last < slots) return 0;
996
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
997
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
998
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
999
+ return bytes;
1000
+ }
977
1001
  function writeVector(store, ordinal, vector) {
978
1002
  const block = store.view.blockOf(ordinal);
979
1003
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -1310,6 +1334,9 @@ function createVectorStore(options) {
1310
1334
  },
1311
1335
  memoryBytes() {
1312
1336
  return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1337
+ },
1338
+ growthBytesFor(vectors) {
1339
+ return open === null ? 0 : growthBytesFor(open, vectors);
1313
1340
  }
1314
1341
  };
1315
1342
  }
@@ -1372,9 +1399,9 @@ function releaseSlotLocks(slot) {
1372
1399
  for (const waiter of [...slot.idleWaiters]) waiter();
1373
1400
  }
1374
1401
  function rejectPending(slot, err) {
1375
- for (const [, waiting] of slot.pending) {
1376
- clearTimeout(waiting.timer);
1377
- waiting.reject(err);
1402
+ for (const [, waiting2] of slot.pending) {
1403
+ clearTimeout(waiting2.timer);
1404
+ waiting2.reject(err);
1378
1405
  }
1379
1406
  slot.pending.clear();
1380
1407
  }
@@ -1453,11 +1480,11 @@ async function createVectorSearchPool(requestedCount) {
1453
1480
  const key = message.requestId;
1454
1481
  if (typeof key !== "string") return;
1455
1482
  noteAnswered(slot, key, sharedFields);
1456
- const waiting = slot.pending.get(key);
1457
- if (!waiting) return;
1483
+ const waiting2 = slot.pending.get(key);
1484
+ if (!waiting2) return;
1458
1485
  slot.pending.delete(key);
1459
- clearTimeout(waiting.timer);
1460
- waiting.resolve(message);
1486
+ clearTimeout(waiting2.timer);
1487
+ waiting2.resolve(message);
1461
1488
  });
1462
1489
  listenForFailure(worker, (err) => {
1463
1490
  slot.alive = false;
@@ -2166,6 +2193,51 @@ async function releasePendingLocations(state) {
2166
2193
  await releaseInTurn(state, () => releaseLocations(state, paths, pending));
2167
2194
  }
2168
2195
 
2196
+ // src/runtime/background-memory.ts
2197
+ var admissions = /* @__PURE__ */ new Set();
2198
+ var waiting = /* @__PURE__ */ new Map();
2199
+ function admitsBackgroundBytes(bytes) {
2200
+ for (const admission of admissions) {
2201
+ if (!admission(bytes)) return false;
2202
+ }
2203
+ return true;
2204
+ }
2205
+ function noteWaitingForMemory(owner, work) {
2206
+ if (work === null) waiting.delete(owner);
2207
+ else waiting.set(owner, work);
2208
+ }
2209
+ function chargeHeapBytes(bytes) {
2210
+ }
2211
+
2212
+ // src/vector/vector-index/memory-charge.ts
2213
+ function admitShuffledRebuild(state) {
2214
+ const bytes = state.hnsw?.graphBytes ?? 0;
2215
+ if (!admitsBackgroundBytes(bytes)) {
2216
+ state.waitingForMemory = true;
2217
+ noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
2218
+ return false;
2219
+ }
2220
+ stopWaitingForMemory(state);
2221
+ chargeHeapBytes(bytes - state.rebuildChargeBytes);
2222
+ state.rebuildChargeBytes = bytes;
2223
+ return true;
2224
+ }
2225
+ function stopWaitingForMemory(state) {
2226
+ if (!state.waitingForMemory) return;
2227
+ state.waitingForMemory = false;
2228
+ noteWaitingForMemory(state, null);
2229
+ }
2230
+ function releaseRebuildCharge(state) {
2231
+ chargeHeapBytes(-state.rebuildChargeBytes);
2232
+ state.rebuildChargeBytes = 0;
2233
+ }
2234
+ function releaseMemoryCharges(state) {
2235
+ releaseRebuildCharge(state);
2236
+ stopWaitingForMemory(state);
2237
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2238
+ state.rebuildWaitTimer = null;
2239
+ }
2240
+
2169
2241
  // src/vector/vector-index/build.ts
2170
2242
  function bufferWhatTheGraphLacks(state, graph) {
2171
2243
  for (const docId of allLiveDocIds(state)) {
@@ -2248,8 +2320,25 @@ async function rebuildInShuffledOrder(state) {
2248
2320
  const adopted = await buildGraphFromStore(state);
2249
2321
  if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
2250
2322
  }
2323
+ function retryWhenMemoryAllows(state) {
2324
+ if (state.rebuildWaitTimer !== null || state.disposed) return;
2325
+ const timer = setTimeout(() => {
2326
+ state.rebuildWaitTimer = null;
2327
+ startShuffledRebuild(state);
2328
+ }, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
2329
+ timer.unref?.();
2330
+ state.rebuildWaitTimer = timer;
2331
+ }
2251
2332
  function startShuffledRebuild(state) {
2252
- if (state.shuffledRebuild !== null || state.disposed || !shuffledRebuildDue(state)) return;
2333
+ if (state.shuffledRebuild !== null || state.disposed) return;
2334
+ if (!shuffledRebuildDue(state)) {
2335
+ stopWaitingForMemory(state);
2336
+ return;
2337
+ }
2338
+ if (!admitShuffledRebuild(state)) {
2339
+ retryWhenMemoryAllows(state);
2340
+ return;
2341
+ }
2253
2342
  state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
2254
2343
  console.warn(
2255
2344
  `Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
@@ -2257,6 +2346,7 @@ function startShuffledRebuild(state) {
2257
2346
  );
2258
2347
  }).finally(() => {
2259
2348
  state.shuffledRebuild = null;
2349
+ releaseRebuildCharge(state);
2260
2350
  scheduleBuild(state);
2261
2351
  scheduleWorkerCopyLoad(state);
2262
2352
  });
@@ -2382,9 +2472,12 @@ function releaseHeldMemory(state) {
2382
2472
  state.savedFiles = [];
2383
2473
  state.savedSignature = null;
2384
2474
  state.store.release();
2475
+ releaseMemoryCharges(state);
2385
2476
  }
2386
2477
  function disposeIndex(state) {
2387
2478
  state.disposed = true;
2479
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2480
+ state.rebuildWaitTimer = null;
2388
2481
  if (state.buildTimer !== null) clearTimeout(state.buildTimer);
2389
2482
  state.buildTimer = null;
2390
2483
  invalidateWorkerCopies(state);
@@ -2723,6 +2816,7 @@ function maintenanceStatus(state) {
2723
2816
  graphCount,
2724
2817
  bufferSize: state.buffer.size,
2725
2818
  building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
2819
+ waitingForMemory: state.waitingForMemory,
2726
2820
  estimatedCompactMs,
2727
2821
  estimatedOptimizeMs
2728
2822
  };
@@ -2737,8 +2831,8 @@ function estimateMemoryBytes(state) {
2737
2831
  }
2738
2832
 
2739
2833
  // src/vector/vector-index/search.ts
2740
- function* bufferCandidates(state, filter) {
2741
- for (const docId of state.buffer) {
2834
+ function* bufferCandidates(state, buffered, filter) {
2835
+ for (const docId of buffered) {
2742
2836
  if (state.tombstones.has(docId)) continue;
2743
2837
  if (filter) {
2744
2838
  const ordinal = state.store.getOrdinal(docId);
@@ -2863,18 +2957,13 @@ function searchWithFilter(state, query, k, options, filter) {
2863
2957
  }
2864
2958
  }
2865
2959
  const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
2866
- return fetchedOutcome(
2867
- state,
2868
- withBufferedVectors(state, query, k, options, filter, hnswResults),
2869
- minSimilarity,
2870
- filter
2871
- );
2960
+ const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
2961
+ return fetchedOutcome(state, results, minSimilarity, filter);
2872
2962
  }
2873
- function withBufferedVectors(state, query, k, options, filter, graphResults) {
2874
- if (state.buffer.size === 0) return graphResults;
2875
- const candidates = bufferCandidates(state, filter);
2876
- const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
2877
- return mergeResults(graphResults, buffered.results, k);
2963
+ function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
2964
+ const candidates = bufferCandidates(state, buffered, filter);
2965
+ const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
2966
+ return mergeResults(graphResults, scan.results, k);
2878
2967
  }
2879
2968
 
2880
2969
  // src/vector/vector-index/index.ts
@@ -2925,6 +3014,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2925
3014
  hnsw: null,
2926
3015
  freshGraph: null,
2927
3016
  ...noBuildHistory(),
3017
+ rebuildChargeBytes: 0,
3018
+ waitingForMemory: false,
3019
+ rebuildWaitTimer: null,
2928
3020
  onLoadedGraphRepaired,
2929
3021
  compactedNodeCount: 0,
2930
3022
  building: false,
@@ -3027,6 +3119,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3027
3119
  }
3028
3120
  }
3029
3121
  const filterRevision = state.revision;
3122
+ const buffered = state.buffer.size === 0 ? null : [...state.buffer];
3030
3123
  scheduleWorkerCopyLoad(state);
3031
3124
  const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
3032
3125
  filter,
@@ -3038,7 +3131,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3038
3131
  filter = filterForOptions(state, options);
3039
3132
  }
3040
3133
  if (viaWorkerCopy !== null) {
3041
- const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
3134
+ const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
3042
3135
  return fetchedOutcome(state, results, options.minSimilarity, filter);
3043
3136
  }
3044
3137
  return searchWithFilter(state, query, k, options, filter);
@@ -3082,6 +3175,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3082
3175
  },
3083
3176
  maintenanceStatus: () => maintenanceStatus(state),
3084
3177
  estimateMemoryBytes: () => estimateMemoryBytes(state),
3178
+ growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3085
3179
  deserialize: (parts) => {
3086
3180
  invalidateWorkerCopies(state);
3087
3181
  deserialize(state, parts);
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-EBH6RXQO.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
@@ -49,7 +49,7 @@ function pageByPosition(placed, organic, { start, limit, pinnedIds, binding, enc
49
49
  sortKey: null,
50
50
  sortSignature: null,
51
51
  depth,
52
- binding
52
+ binding: binding()
53
53
  });
54
54
  return { page, cursor };
55
55
  }
@@ -229,7 +229,7 @@ function rerankTextOf(values) {
229
229
  // src/rerank/pipeline.ts
230
230
  function rerankCursorDepth(cursor, binding, host) {
231
231
  const decoded = host.decodePageCursor(cursor);
232
- host.requireMatchingCursor(decoded, cursor, null, true, binding);
232
+ host.requireMatchingCursor(decoded, cursor, null, true, binding());
233
233
  return decoded.depth ?? 0;
234
234
  }
235
235
  function requireWithinReach(cursorDepth, offset, limit) {
@@ -1,7 +1,7 @@
1
- import { vectorsPerFile } from './chunk-OVY4RPDS.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-YRYBTRWB.mjs';
3
- import './chunk-O5IKTA27.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-UR5MNN53.mjs';
1
+ import { collectAndReturnPages, vectorsPerFile } from './chunk-73U5DWOZ.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-I45FAGOF.mjs';
3
+ import './chunk-FODCC3MK.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2TMWIATH.mjs';
5
5
  import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
@@ -663,7 +663,8 @@ function refuse(indexName, replacesIndex, shortfall) {
663
663
  function requireRestoreMemory(indexName, footprint, replacesIndex) {
664
664
  const processBytes = footprint.heapBytes + footprint.externalBytes;
665
665
  if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
666
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, replacesIndex);
666
+ const collected = collectAndReturnPages();
667
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
667
668
  if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
668
669
  }
669
670
 
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-O5IKTA27.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-UR5MNN53.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-FODCC3MK.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2TMWIATH.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
4
4
  import './chunk-2CBQ2PCC.mjs';
5
5
  import './chunk-JPTQZWJQ.mjs';
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-ujZESkJz.js';
3
- import { h as TaskStore } from './tasks-CI3Fvkxw.js';
2
+ import { C as ClusterNamespace } from './config-CS5ADSrx.js';
3
+ import { h as TaskStore } from './tasks-5_LCFEzF.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -10,5 +10,9 @@ var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS = 10;
10
10
  var HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.75;
11
11
  var HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.65;
12
12
  var PROCESS_LIMIT_REFRESH_INTERVAL_MS = 1e3;
13
+ var V8_YOUNG_GENERATION_BYTES_BY_MAJOR = {
14
+ 12: 48 * 1024 * 1024,
15
+ 13: 192 * 1024 * 1024
16
+ };
13
17
 
14
- export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS };
18
+ export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR };
@@ -1,4 +1,4 @@
1
- import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-YIDSHBQS.mjs';
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-HC5NURS6.mjs';
2
2
  import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
3
3
  import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
4
4
  import { readSync, closeSync, openSync } from 'fs';
@@ -581,7 +581,8 @@ function openHandles(dimension, codeBits, blockBytes) {
581
581
  present: new Uint8Array(handles.present),
582
582
  partitions: new Int32Array(handles.partitions),
583
583
  diskFile: new Int32Array(handles.diskFile),
584
- diskOffset: new Uint32Array(handles.diskOffset)
584
+ diskOffset: new Uint32Array(handles.diskOffset),
585
+ longestDocIdBytes: 0
585
586
  };
586
587
  }
587
588
  function relayout(store) {
@@ -592,7 +593,10 @@ function claimSlot(store, docId, cold) {
592
593
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
594
  ensureOrdinal(store, ordinal, cold);
594
595
  store.diskFile[ordinal] = IN_MEMORY;
596
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
595
597
  appendDocId(store.handles, ordinal, docId);
598
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
599
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
596
600
  return ordinal;
597
601
  }
598
602
  function publishOrdinal(store, ordinal) {
@@ -668,6 +672,19 @@ function ensureOrdinal(store, ordinal, cold) {
668
672
  if (!cold) ensureFloatSlot(store, ordinal);
669
673
  ensureCodeSlot(store, ordinal);
670
674
  }
675
+ function bufferGrowthBytes(buffer, bytes) {
676
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
677
+ }
678
+ function growthBytesFor(store, vectors) {
679
+ const { handles } = store;
680
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
681
+ const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
682
+ if (last < slots) return 0;
683
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
684
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
685
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
686
+ return bytes;
687
+ }
671
688
  function writeVector(store, ordinal, vector) {
672
689
  const block = store.view.blockOf(ordinal);
673
690
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -1004,6 +1021,9 @@ function createVectorStore(options) {
1004
1021
  },
1005
1022
  memoryBytes() {
1006
1023
  return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1024
+ },
1025
+ growthBytesFor(vectors) {
1026
+ return open === null ? 0 : growthBytesFor(open, vectors);
1007
1027
  }
1008
1028
  };
1009
1029
  }