@delali/narsil 0.3.1-next.112 → 0.3.1-next.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/browser/{chunk-YRYBTRWB.mjs → chunk-3YPLBK4C.mjs} +2 -2
  2. package/dist/browser/{chunk-O5IKTA27.mjs → chunk-4FVSBSXY.mjs} +1 -1
  3. package/dist/browser/{chunk-UR5MNN53.mjs → chunk-DC2SZO4D.mjs} +7 -6
  4. package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
  5. package/dist/browser/{durability-wiring-CPEMDDK7.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
  6. package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
  7. package/dist/browser/{restore-queue-2H4QDQV4.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
  8. package/dist/browser/{snapshot-ISZMBDCN.mjs → snapshot-BE3F2PNC.mjs} +6 -5
  9. package/dist/browser/{write-IABOLE4N.mjs → write-MV7K7OCU.mjs} +2 -2
  10. package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
  11. package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
  12. package/dist/{chunk-2IFU7SXG.mjs → chunk-27ZEY73B.mjs} +3 -3
  13. package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
  14. package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
  15. package/dist/{chunk-V7ISFBFC.mjs → chunk-5QIRLJH4.mjs} +2 -2
  16. package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
  17. package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
  18. package/dist/{chunk-BVOUSX3U.mjs → chunk-DZXGNLK5.mjs} +3 -3
  19. package/dist/chunk-EBIHRL7B.mjs +21 -0
  20. package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
  21. package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
  22. package/dist/chunk-IWVEG5CX.mjs +67 -0
  23. package/dist/{chunk-VLRT47GU.mjs → chunk-JUAFWJON.mjs} +3 -3
  24. package/dist/{chunk-3COL24MT.mjs → chunk-LVNBY2TS.mjs} +1 -1
  25. package/dist/{chunk-IM76DLA7.mjs → chunk-OABTAUCW.mjs} +2 -2
  26. package/dist/{chunk-IBTKJIV7.mjs → chunk-OM5G2J6X.mjs} +1 -1
  27. package/dist/{chunk-3RRATITJ.mjs → chunk-PABYSTKO.mjs} +11 -8
  28. package/dist/chunk-PELHVTO7.mjs +24 -0
  29. package/dist/{chunk-7CI34JAP.mjs → chunk-QGVMEMCG.mjs} +4 -4
  30. package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
  31. package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
  32. package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
  33. package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
  34. package/dist/client.d.ts +6 -6
  35. package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
  36. package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
  37. package/dist/{direct-executor-ZSTKRA6Z.mjs → direct-executor-R4HOSVPE.mjs} +5 -4
  38. package/dist/distribution.d.ts +5 -5
  39. package/dist/distribution.mjs +72 -35
  40. package/dist/{durability-wiring-EQTFL3B4.mjs → durability-wiring-XZI6355I.mjs} +13 -12
  41. package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
  42. package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
  43. package/dist/index.browser.mjs +1 -1
  44. package/dist/index.d.ts +5 -5
  45. package/dist/index.mjs +6 -5
  46. package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
  47. package/dist/persistence/durability/checkpoint-worker.mjs +10 -9
  48. package/dist/react.d.ts +6 -6
  49. package/dist/{restore-queue-WZTMTJP2.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
  50. package/dist/server.d.ts +8 -8
  51. package/dist/server.mjs +20 -62
  52. package/dist/{snapshot-EU6NNB2P.mjs → snapshot-PDUGHQ25.mjs} +14 -13
  53. package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
  54. package/dist/{thread-WGBCVJII.mjs → thread-S4SUCV4Y.mjs} +8 -5
  55. package/dist/vector/search-worker.mjs +2 -2
  56. package/dist/workers/entry.mjs +6 -6
  57. package/dist/{write-CKRWS3P5.mjs → write-KVJ7YJJ6.mjs} +7 -6
  58. package/package.json +7 -7
@@ -1,5 +1,5 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-O5IKTA27.mjs';
2
- import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-UR5MNN53.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-4FVSBSXY.mjs';
2
+ import { VERSION, rulesOfStoredSynonyms, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-DC2SZO4D.mjs';
3
3
  import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-P7C2S5JX.mjs';
4
4
  import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-2CBQ2PCC.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
@@ -1,4 +1,4 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-UR5MNN53.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-DC2SZO4D.mjs';
2
2
  import { NarsilError, ErrorCodes } from './chunk-LJS6PREA.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
@@ -15087,7 +15087,7 @@ function requireLoadedVectorCode() {
15087
15087
  function loadVectorCode() {
15088
15088
  if (loaded !== null) return Promise.resolve(loaded);
15089
15089
  if (loading === null) {
15090
- loading = import('./module-DYKR4INA.mjs').then(
15090
+ loading = import('./module-5EWHBKHZ.mjs').then(
15091
15091
  (code) => {
15092
15092
  loaded = code;
15093
15093
  return code;
@@ -19175,7 +19175,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
19175
19175
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
19176
19176
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
19177
19177
  }
19178
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-CPEMDDK7.mjs');
19178
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-2RG22WQJ.mjs');
19179
19179
  const filesystemDurability = durabilityTier?.kind === "wal";
19180
19180
  const executor = createDirectExecutor({
19181
19181
  vectorWorkerCopies: vectorCopyPolicy,
@@ -21919,6 +21919,7 @@ function getVectorMaintenanceStatus(manager) {
21919
21919
  graphCount: status.graphCount,
21920
21920
  bufferSize: status.bufferSize,
21921
21921
  building: status.building,
21922
+ waitingForMemory: status.waitingForMemory,
21922
21923
  estimatedCompactMs: status.estimatedCompactMs,
21923
21924
  estimatedOptimizeMs: status.estimatedOptimizeMs
21924
21925
  });
@@ -22608,14 +22609,14 @@ function portabilityOperations(core, config, engine) {
22608
22609
  return {
22609
22610
  snapshot(indexName) {
22610
22611
  return withOpenIndex(indexName, async () => {
22611
- const { createSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
22612
+ const { createSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
22612
22613
  return createSnapshot(snapshotDepsOf(indexName));
22613
22614
  });
22614
22615
  },
22615
22616
  async restore(indexName, data) {
22616
22617
  core.guardShutdown();
22617
22618
  validateIndexName(indexName);
22618
- const restored = import('./restore-queue-2H4QDQV4.mjs').then(
22619
+ const restored = import('./restore-queue-WCFUB4NS.mjs').then(
22619
22620
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
22620
22621
  );
22621
22622
  await emitWriteEventAfter(core, indexName, restored);
@@ -22624,8 +22625,8 @@ function portabilityOperations(core, config, engine) {
22624
22625
  exportDocuments(indexName) {
22625
22626
  return withOpenIndex(indexName, async () => {
22626
22627
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
22627
- import('./snapshot-ISZMBDCN.mjs'),
22628
- import('./write-IABOLE4N.mjs')
22628
+ import('./snapshot-BE3F2PNC.mjs'),
22629
+ import('./write-MV7K7OCU.mjs')
22629
22630
  ]);
22630
22631
  const header = indexConfigurationOf(core.requireIndex(indexName));
22631
22632
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -19,6 +19,15 @@ function createFixedBuffer(bytes) {
19
19
  function nextGrowthTarget(have, needed, max) {
20
20
  return Math.min(max, Math.max(needed, have * 2));
21
21
  }
22
+ function grownByteLength(have, needed, max, step = 1) {
23
+ let size = have;
24
+ while (size < needed) {
25
+ const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
26
+ if (next <= size) return size;
27
+ size = next;
28
+ }
29
+ return size;
30
+ }
22
31
  function growBufferTo(buffer, bytes) {
23
32
  const held = buffer.byteLength;
24
33
  if (held >= bytes) return;
@@ -1115,6 +1124,25 @@ function ensureBlockSlots(handle, slots) {
1115
1124
  }
1116
1125
  growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
1117
1126
  }
1127
+ function blockGrowthBytes(handle, layout, slots) {
1128
+ const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
1129
+ if (handle !== null && handle.storage.kind === "bytes") {
1130
+ const buffer = handle.storage.buffer;
1131
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
1132
+ }
1133
+ const have = handle === null ? 0 : blockBuffer(handle).byteLength;
1134
+ const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
1135
+ return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
1136
+ }
1137
+ function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
1138
+ let bytes = 0;
1139
+ const lastBlock = Math.floor(lastOrdinal / layout.capacity);
1140
+ for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
1141
+ const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
1142
+ bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
1143
+ }
1144
+ return bytes;
1145
+ }
1118
1146
  function blockBuffer(handle) {
1119
1147
  const storage = handle.storage;
1120
1148
  if (storage.kind === "bytes") return storage.buffer;
@@ -3458,6 +3486,7 @@ var BUILD_CHUNK_SIZE = 100;
3458
3486
  var BUILD_TURN_HANDOVER_AFTER_MS = 50;
3459
3487
  var BUILD_RETRY_FIRST_DELAY_MS = 250;
3460
3488
  var BUILD_RETRY_MAX_DELAY_MS = 3e4;
3489
+ var SHUFFLED_REBUILD_MEMORY_RETRY_MS = 1e3;
3461
3490
  var REBUILD_REMOVED_RATIO = 0.2;
3462
3491
  var DRIFT_CHECK_GROWTH_FACTOR = 2;
3463
3492
  var RECALIBRATION_DRIFT_RATIO = 2.5;
@@ -4480,4 +4509,9 @@ function recordCheckpoint(state, plan, written) {
4480
4509
  state.savedSignature = plan.signature;
4481
4510
  }
4482
4511
 
4483
- export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
4512
+ // src/runtime/idle-heap-cleanup.browser.ts
4513
+ function collectAndReturnPages() {
4514
+ return false;
4515
+ }
4516
+
4517
+ export { BLOCK_MAX_BYTES, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, OSQ4_MIN_DIMENSION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assignStorePartitions, blocksGrowthBytes, centroidOfCentroidSums, collectAndReturnPages, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createSharedVectorStoreHandles, createVectorBlock, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, growBufferTo, grownByteLength, handOverBuildTurns, insertNode, liveOrdinals, liveSize, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, noBuildHistory, nodeCountOf, noteFallback, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, planCheckpoint, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releaseReplacedVectors, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, signatureOf, slotByteOffset, squaredLengthsOfCentroidSums, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, yieldBuildTurn };
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-YRYBTRWB.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-O5IKTA27.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UR5MNN53.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-3YPLBK4C.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-4FVSBSXY.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-DC2SZO4D.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-P7C2S5JX.mjs';
6
6
  import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
@@ -1,6 +1,6 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OVY4RPDS.mjs';
3
- export { releaseLocksHeldBy } from './chunk-OVY4RPDS.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-YZPOBLAS.mjs';
3
+ export { releaseLocksHeldBy } from './chunk-YZPOBLAS.mjs';
4
4
  import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
5
5
  import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
6
6
  import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
@@ -887,7 +887,8 @@ function openHandles(dimension, codeBits, blockBytes) {
887
887
  present: new Uint8Array(handles.present),
888
888
  partitions: new Int32Array(handles.partitions),
889
889
  diskFile: new Int32Array(handles.diskFile),
890
- diskOffset: new Uint32Array(handles.diskOffset)
890
+ diskOffset: new Uint32Array(handles.diskOffset),
891
+ longestDocIdBytes: 0
891
892
  };
892
893
  }
893
894
  function relayout(store) {
@@ -898,7 +899,10 @@ function claimSlot(store, docId, cold) {
898
899
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
899
900
  ensureOrdinal(store, ordinal, cold);
900
901
  store.diskFile[ordinal] = IN_MEMORY;
902
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
901
903
  appendDocId(store.handles, ordinal, docId);
904
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
905
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
902
906
  return ordinal;
903
907
  }
904
908
  function publishOrdinal(store, ordinal) {
@@ -974,6 +978,19 @@ function ensureOrdinal(store, ordinal, cold) {
974
978
  if (!cold) ensureFloatSlot(store, ordinal);
975
979
  ensureCodeSlot(store, ordinal);
976
980
  }
981
+ function bufferGrowthBytes(buffer, bytes) {
982
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
983
+ }
984
+ function growthBytesFor(store, vectors) {
985
+ const { handles } = store;
986
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
987
+ const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
988
+ if (last < slots) return 0;
989
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
990
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
991
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
992
+ return bytes;
993
+ }
977
994
  function writeVector(store, ordinal, vector) {
978
995
  const block = store.view.blockOf(ordinal);
979
996
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -1310,6 +1327,9 @@ function createVectorStore(options) {
1310
1327
  },
1311
1328
  memoryBytes() {
1312
1329
  return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1330
+ },
1331
+ growthBytesFor(vectors) {
1332
+ return open === null ? 0 : growthBytesFor(open, vectors);
1313
1333
  }
1314
1334
  };
1315
1335
  }
@@ -1372,9 +1392,9 @@ function releaseSlotLocks(slot) {
1372
1392
  for (const waiter of [...slot.idleWaiters]) waiter();
1373
1393
  }
1374
1394
  function rejectPending(slot, err) {
1375
- for (const [, waiting] of slot.pending) {
1376
- clearTimeout(waiting.timer);
1377
- waiting.reject(err);
1395
+ for (const [, waiting2] of slot.pending) {
1396
+ clearTimeout(waiting2.timer);
1397
+ waiting2.reject(err);
1378
1398
  }
1379
1399
  slot.pending.clear();
1380
1400
  }
@@ -1453,11 +1473,11 @@ async function createVectorSearchPool(requestedCount) {
1453
1473
  const key = message.requestId;
1454
1474
  if (typeof key !== "string") return;
1455
1475
  noteAnswered(slot, key, sharedFields);
1456
- const waiting = slot.pending.get(key);
1457
- if (!waiting) return;
1476
+ const waiting2 = slot.pending.get(key);
1477
+ if (!waiting2) return;
1458
1478
  slot.pending.delete(key);
1459
- clearTimeout(waiting.timer);
1460
- waiting.resolve(message);
1479
+ clearTimeout(waiting2.timer);
1480
+ waiting2.resolve(message);
1461
1481
  });
1462
1482
  listenForFailure(worker, (err) => {
1463
1483
  slot.alive = false;
@@ -2166,6 +2186,51 @@ async function releasePendingLocations(state) {
2166
2186
  await releaseInTurn(state, () => releaseLocations(state, paths, pending));
2167
2187
  }
2168
2188
 
2189
+ // src/runtime/background-memory.ts
2190
+ var admissions = /* @__PURE__ */ new Set();
2191
+ var waiting = /* @__PURE__ */ new Map();
2192
+ function admitsBackgroundBytes(bytes) {
2193
+ for (const admission of admissions) {
2194
+ if (!admission(bytes)) return false;
2195
+ }
2196
+ return true;
2197
+ }
2198
+ function noteWaitingForMemory(owner, work) {
2199
+ if (work === null) waiting.delete(owner);
2200
+ else waiting.set(owner, work);
2201
+ }
2202
+ function chargeHeapBytes(bytes) {
2203
+ }
2204
+
2205
+ // src/vector/vector-index/memory-charge.ts
2206
+ function admitShuffledRebuild(state) {
2207
+ const bytes = state.hnsw?.graphBytes ?? 0;
2208
+ if (!admitsBackgroundBytes(bytes)) {
2209
+ state.waitingForMemory = true;
2210
+ noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
2211
+ return false;
2212
+ }
2213
+ stopWaitingForMemory(state);
2214
+ chargeHeapBytes(bytes - state.rebuildChargeBytes);
2215
+ state.rebuildChargeBytes = bytes;
2216
+ return true;
2217
+ }
2218
+ function stopWaitingForMemory(state) {
2219
+ if (!state.waitingForMemory) return;
2220
+ state.waitingForMemory = false;
2221
+ noteWaitingForMemory(state, null);
2222
+ }
2223
+ function releaseRebuildCharge(state) {
2224
+ chargeHeapBytes(-state.rebuildChargeBytes);
2225
+ state.rebuildChargeBytes = 0;
2226
+ }
2227
+ function releaseMemoryCharges(state) {
2228
+ releaseRebuildCharge(state);
2229
+ stopWaitingForMemory(state);
2230
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2231
+ state.rebuildWaitTimer = null;
2232
+ }
2233
+
2169
2234
  // src/vector/vector-index/build.ts
2170
2235
  function bufferWhatTheGraphLacks(state, graph) {
2171
2236
  for (const docId of allLiveDocIds(state)) {
@@ -2248,8 +2313,25 @@ async function rebuildInShuffledOrder(state) {
2248
2313
  const adopted = await buildGraphFromStore(state);
2249
2314
  if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
2250
2315
  }
2316
+ function retryWhenMemoryAllows(state) {
2317
+ if (state.rebuildWaitTimer !== null || state.disposed) return;
2318
+ const timer = setTimeout(() => {
2319
+ state.rebuildWaitTimer = null;
2320
+ startShuffledRebuild(state);
2321
+ }, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
2322
+ timer.unref?.();
2323
+ state.rebuildWaitTimer = timer;
2324
+ }
2251
2325
  function startShuffledRebuild(state) {
2252
- if (state.shuffledRebuild !== null || state.disposed || !shuffledRebuildDue(state)) return;
2326
+ if (state.shuffledRebuild !== null || state.disposed) return;
2327
+ if (!shuffledRebuildDue(state)) {
2328
+ stopWaitingForMemory(state);
2329
+ return;
2330
+ }
2331
+ if (!admitShuffledRebuild(state)) {
2332
+ retryWhenMemoryAllows(state);
2333
+ return;
2334
+ }
2253
2335
  state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
2254
2336
  console.warn(
2255
2337
  `Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
@@ -2257,6 +2339,7 @@ function startShuffledRebuild(state) {
2257
2339
  );
2258
2340
  }).finally(() => {
2259
2341
  state.shuffledRebuild = null;
2342
+ releaseRebuildCharge(state);
2260
2343
  scheduleBuild(state);
2261
2344
  scheduleWorkerCopyLoad(state);
2262
2345
  });
@@ -2382,9 +2465,12 @@ function releaseHeldMemory(state) {
2382
2465
  state.savedFiles = [];
2383
2466
  state.savedSignature = null;
2384
2467
  state.store.release();
2468
+ releaseMemoryCharges(state);
2385
2469
  }
2386
2470
  function disposeIndex(state) {
2387
2471
  state.disposed = true;
2472
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2473
+ state.rebuildWaitTimer = null;
2388
2474
  if (state.buildTimer !== null) clearTimeout(state.buildTimer);
2389
2475
  state.buildTimer = null;
2390
2476
  invalidateWorkerCopies(state);
@@ -2723,6 +2809,7 @@ function maintenanceStatus(state) {
2723
2809
  graphCount,
2724
2810
  bufferSize: state.buffer.size,
2725
2811
  building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
2812
+ waitingForMemory: state.waitingForMemory,
2726
2813
  estimatedCompactMs,
2727
2814
  estimatedOptimizeMs
2728
2815
  };
@@ -2925,6 +3012,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2925
3012
  hnsw: null,
2926
3013
  freshGraph: null,
2927
3014
  ...noBuildHistory(),
3015
+ rebuildChargeBytes: 0,
3016
+ waitingForMemory: false,
3017
+ rebuildWaitTimer: null,
2928
3018
  onLoadedGraphRepaired,
2929
3019
  compactedNodeCount: 0,
2930
3020
  building: false,
@@ -3082,6 +3172,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3082
3172
  },
3083
3173
  maintenanceStatus: () => maintenanceStatus(state),
3084
3174
  estimateMemoryBytes: () => estimateMemoryBytes(state),
3175
+ growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3085
3176
  deserialize: (parts) => {
3086
3177
  invalidateWorkerCopies(state);
3087
3178
  deserialize(state, parts);
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-ISZMBDCN.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
@@ -1,7 +1,7 @@
1
- import { vectorsPerFile } from './chunk-OVY4RPDS.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-YRYBTRWB.mjs';
3
- import './chunk-O5IKTA27.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-UR5MNN53.mjs';
1
+ import { collectAndReturnPages, vectorsPerFile } from './chunk-YZPOBLAS.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-3YPLBK4C.mjs';
3
+ import './chunk-4FVSBSXY.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-DC2SZO4D.mjs';
5
5
  import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
@@ -663,7 +663,8 @@ function refuse(indexName, replacesIndex, shortfall) {
663
663
  function requireRestoreMemory(indexName, footprint, replacesIndex) {
664
664
  const processBytes = footprint.heapBytes + footprint.externalBytes;
665
665
  if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
666
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, replacesIndex);
666
+ const collected = collectAndReturnPages();
667
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
667
668
  if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
668
669
  }
669
670
 
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-O5IKTA27.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-UR5MNN53.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-4FVSBSXY.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-DC2SZO4D.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
4
4
  import './chunk-2CBQ2PCC.mjs';
5
5
  import './chunk-JPTQZWJQ.mjs';
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-ujZESkJz.js';
3
- import { h as TaskStore } from './tasks-CI3Fvkxw.js';
2
+ import { C as ClusterNamespace } from './config-CS5ADSrx.js';
3
+ import { h as TaskStore } from './tasks-5_LCFEzF.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -10,5 +10,9 @@ var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS = 10;
10
10
  var HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.75;
11
11
  var HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.65;
12
12
  var PROCESS_LIMIT_REFRESH_INTERVAL_MS = 1e3;
13
+ var V8_YOUNG_GENERATION_BYTES_BY_MAJOR = {
14
+ 12: 48 * 1024 * 1024,
15
+ 13: 192 * 1024 * 1024
16
+ };
13
17
 
14
- export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS };
18
+ export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR };
@@ -1,8 +1,8 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-3COL24MT.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-LVNBY2TS.mjs';
2
2
  import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
3
3
  import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-3RRATITJ.mjs';
5
- import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-V7ISFBFC.mjs';
4
+ import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-PABYSTKO.mjs';
5
+ import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-5QIRLJH4.mjs';
6
6
  import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-5WEBVYOA.mjs';
7
7
  import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
8
8
  import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-DG62H7OD.mjs';
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
14
14
  function loadVectorCode() {
15
15
  if (loaded !== null) return Promise.resolve(loaded);
16
16
  if (loading === null) {
17
- loading = import('./module-WP4VVGV2.mjs').then(
17
+ loading = import('./module-WVKSJL74.mjs').then(
18
18
  (code) => {
19
19
  loaded = code;
20
20
  return code;
@@ -25,6 +25,15 @@ function createFixedBuffer(bytes) {
25
25
  function nextGrowthTarget(have, needed, max) {
26
26
  return Math.min(max, Math.max(needed, have * 2));
27
27
  }
28
+ function grownByteLength(have, needed, max, step = 1) {
29
+ let size = have;
30
+ while (size < needed) {
31
+ const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
32
+ if (next <= size) return size;
33
+ size = next;
34
+ }
35
+ return size;
36
+ }
28
37
  function growBufferTo(buffer, bytes) {
29
38
  const held = buffer.byteLength;
30
39
  if (held >= bytes) return;
@@ -1038,6 +1047,25 @@ function ensureBlockSlots(handle, slots) {
1038
1047
  }
1039
1048
  growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
1040
1049
  }
1050
+ function blockGrowthBytes(handle, layout, slots) {
1051
+ const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
1052
+ if (handle !== null && handle.storage.kind === "bytes") {
1053
+ const buffer = handle.storage.buffer;
1054
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
1055
+ }
1056
+ const have = handle === null ? 0 : blockBuffer(handle).byteLength;
1057
+ const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
1058
+ return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
1059
+ }
1060
+ function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
1061
+ let bytes = 0;
1062
+ const lastBlock = Math.floor(lastOrdinal / layout.capacity);
1063
+ for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
1064
+ const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
1065
+ bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
1066
+ }
1067
+ return bytes;
1068
+ }
1041
1069
  function blockBuffer(handle) {
1042
1070
  const storage = handle.storage;
1043
1071
  if (storage.kind === "bytes") return storage.buffer;
@@ -3980,4 +4008,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
3980
4008
  };
3981
4009
  }
3982
4010
 
3983
- export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
4011
+ export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, blocksGrowthBytes, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, grownByteLength, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
@@ -1,6 +1,6 @@
1
1
  import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-5WEBVYOA.mjs';
2
2
  import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
3
- import { readHeapStatistics, readResidentMemory } from './chunk-SOADQINJ.mjs';
3
+ import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
4
4
  import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-HNAP2KTG.mjs';
5
5
  import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-KCQG2PJT.mjs';
6
6
  import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-DG62H7OD.mjs';
@@ -9,7 +9,7 @@ import { isWordIndexedBase } from './chunk-6ODIY6ZB.mjs';
9
9
  import { resolveIndexAnalysis, synonymSearchModule } from './chunk-IR4KQ4WB.mjs';
10
10
  import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-OFYKBKKV.mjs';
11
11
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
12
- import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-QI6JW4PM.mjs';
12
+ import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-4PFOOQFQ.mjs';
13
13
  import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
14
14
  import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
15
15
  import { Decoder, decode } from '@msgpack/msgpack';