@delali/narsil 0.3.1-next.111 → 0.3.1-next.113

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/browser/{chunk-6YUUEMQ4.mjs → chunk-3YPLBK4C.mjs} +2 -2
  2. package/dist/browser/{chunk-5BQPBG3G.mjs → chunk-4FVSBSXY.mjs} +1 -1
  3. package/dist/browser/{chunk-HQOYKRMB.mjs → chunk-DC2SZO4D.mjs} +183 -44
  4. package/dist/browser/{chunk-OVY4RPDS.mjs → chunk-YZPOBLAS.mjs} +35 -1
  5. package/dist/browser/{durability-wiring-CYAJBYCO.mjs → durability-wiring-2RG22WQJ.mjs} +3 -3
  6. package/dist/browser/{module-DYKR4INA.mjs → module-5EWHBKHZ.mjs} +102 -11
  7. package/dist/browser/{restore-queue-RTZGJT4D.mjs → restore-queue-WCFUB4NS.mjs} +1 -1
  8. package/dist/browser/{snapshot-BU6I25CW.mjs → snapshot-BE3F2PNC.mjs} +6 -5
  9. package/dist/browser/{write-RUPYHPBZ.mjs → write-MV7K7OCU.mjs} +2 -2
  10. package/dist/{capabilities-ByaX8Arf.d.ts → capabilities-DOgB3pPf.d.ts} +2 -2
  11. package/dist/{chunk-ZXZ26WSO.mjs → chunk-22YPZVMG.mjs} +5 -1
  12. package/dist/{chunk-6S7YRUPQ.mjs → chunk-27ZEY73B.mjs} +4 -4
  13. package/dist/{chunk-QI6JW4PM.mjs → chunk-4PFOOQFQ.mjs} +1 -1
  14. package/dist/{chunk-YIDSHBQS.mjs → chunk-4XWIW3F3.mjs} +29 -1
  15. package/dist/{chunk-R2MRMH6M.mjs → chunk-5QIRLJH4.mjs} +3 -3
  16. package/dist/{chunk-7JOONQ24.mjs → chunk-5WEBVYOA.mjs} +176 -38
  17. package/dist/{chunk-63HFWQEA.mjs → chunk-AOLD3LPV.mjs} +273 -102
  18. package/dist/{chunk-PT7VRWS7.mjs → chunk-CU4OB64V.mjs} +3 -2
  19. package/dist/{chunk-JBPYFYIU.mjs → chunk-DZXGNLK5.mjs} +3 -3
  20. package/dist/chunk-EBIHRL7B.mjs +21 -0
  21. package/dist/{chunk-UGKO3RG2.mjs → chunk-GLZI636G.mjs} +1 -1
  22. package/dist/{chunk-R2MRGYU5.mjs → chunk-H2GNWBLB.mjs} +22 -2
  23. package/dist/{chunk-V7G627TA.mjs → chunk-HFJXEFFV.mjs} +13 -62
  24. package/dist/chunk-IWVEG5CX.mjs +67 -0
  25. package/dist/{chunk-SXVDBHYJ.mjs → chunk-JUAFWJON.mjs} +3 -3
  26. package/dist/{chunk-PKE2PCTS.mjs → chunk-LVNBY2TS.mjs} +1 -1
  27. package/dist/{chunk-YLE75NQ5.mjs → chunk-OABTAUCW.mjs} +2 -2
  28. package/dist/{chunk-YSR5L3XJ.mjs → chunk-OM5G2J6X.mjs} +1 -1
  29. package/dist/{chunk-IW6UR3AF.mjs → chunk-PABYSTKO.mjs} +13 -10
  30. package/dist/chunk-PELHVTO7.mjs +24 -0
  31. package/dist/{chunk-CUVTRLA4.mjs → chunk-QGVMEMCG.mjs} +4 -4
  32. package/dist/{chunk-6OXNHLNZ.mjs → chunk-RIIUZJQ3.mjs} +1 -1
  33. package/dist/{chunk-C7SLA6CM.mjs → chunk-W55X2BYN.mjs} +5 -1
  34. package/dist/{chunk-SOADQINJ.mjs → chunk-ZATRDAYU.mjs} +31 -3
  35. package/dist/{client-BdnzDFtO.d.ts → client-BGzLNt78.d.ts} +2 -2
  36. package/dist/client.d.ts +6 -6
  37. package/dist/{config-ujZESkJz.d.ts → config-CS5ADSrx.d.ts} +18 -2
  38. package/dist/{config-CSgvApt0.d.ts → config-CSYMTTm1.d.ts} +1 -1
  39. package/dist/{direct-executor-QFAUKTCQ.mjs → direct-executor-R4HOSVPE.mjs} +6 -5
  40. package/dist/distribution.d.ts +5 -5
  41. package/dist/distribution.mjs +74 -37
  42. package/dist/{durability-wiring-NUHTYXHA.mjs → durability-wiring-XZI6355I.mjs} +15 -14
  43. package/dist/{engine-C_HSy3iz.d.ts → engine-CCBBGmMR.d.ts} +1 -1
  44. package/dist/{events-CJsXF1F9.d.ts → events-D8GPB1-H.d.ts} +5 -0
  45. package/dist/index.browser.mjs +1 -1
  46. package/dist/index.d.ts +5 -5
  47. package/dist/index.mjs +8 -7
  48. package/dist/{module-WP4VVGV2.mjs → module-WVKSJL74.mjs} +67 -7
  49. package/dist/persistence/durability/checkpoint-worker.mjs +12 -11
  50. package/dist/react.d.ts +6 -6
  51. package/dist/{restore-queue-QSLGXZJB.mjs → restore-queue-JAZAB5IQ.mjs} +1 -1
  52. package/dist/server.d.ts +8 -8
  53. package/dist/server.mjs +20 -62
  54. package/dist/{snapshot-IV3S2TMI.mjs → snapshot-PDUGHQ25.mjs} +16 -15
  55. package/dist/{tasks-CI3Fvkxw.d.ts → tasks-5_LCFEzF.d.ts} +1 -1
  56. package/dist/{thread-THLCKMEB.mjs → thread-S4SUCV4Y.mjs} +10 -7
  57. package/dist/vector/search-worker.mjs +2 -2
  58. package/dist/workers/entry.mjs +6 -6
  59. package/dist/{write-VGRTOS7N.mjs → write-KVJ7YJJ6.mjs} +9 -8
  60. package/package.json +7 -7
@@ -1,6 +1,6 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-OVY4RPDS.mjs';
3
- export { releaseLocksHeldBy } from './chunk-OVY4RPDS.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-YZPOBLAS.mjs';
3
+ export { releaseLocksHeldBy } from './chunk-YZPOBLAS.mjs';
4
4
  import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
5
5
  import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-2CBQ2PCC.mjs';
6
6
  import { NarsilError, ErrorCodes, isNarsilError } from './chunk-LJS6PREA.mjs';
@@ -887,7 +887,8 @@ function openHandles(dimension, codeBits, blockBytes) {
887
887
  present: new Uint8Array(handles.present),
888
888
  partitions: new Int32Array(handles.partitions),
889
889
  diskFile: new Int32Array(handles.diskFile),
890
- diskOffset: new Uint32Array(handles.diskOffset)
890
+ diskOffset: new Uint32Array(handles.diskOffset),
891
+ longestDocIdBytes: 0
891
892
  };
892
893
  }
893
894
  function relayout(store) {
@@ -898,7 +899,10 @@ function claimSlot(store, docId, cold) {
898
899
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
899
900
  ensureOrdinal(store, ordinal, cold);
900
901
  store.diskFile[ordinal] = IN_MEMORY;
902
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
901
903
  appendDocId(store.handles, ordinal, docId);
904
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
905
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
902
906
  return ordinal;
903
907
  }
904
908
  function publishOrdinal(store, ordinal) {
@@ -974,6 +978,19 @@ function ensureOrdinal(store, ordinal, cold) {
974
978
  if (!cold) ensureFloatSlot(store, ordinal);
975
979
  ensureCodeSlot(store, ordinal);
976
980
  }
981
+ function bufferGrowthBytes(buffer, bytes) {
982
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
983
+ }
984
+ function growthBytesFor(store, vectors) {
985
+ const { handles } = store;
986
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
987
+ const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
988
+ if (last < slots) return 0;
989
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
990
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
991
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
992
+ return bytes;
993
+ }
977
994
  function writeVector(store, ordinal, vector) {
978
995
  const block = store.view.blockOf(ordinal);
979
996
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -1310,6 +1327,9 @@ function createVectorStore(options) {
1310
1327
  },
1311
1328
  memoryBytes() {
1312
1329
  return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1330
+ },
1331
+ growthBytesFor(vectors) {
1332
+ return open === null ? 0 : growthBytesFor(open, vectors);
1313
1333
  }
1314
1334
  };
1315
1335
  }
@@ -1372,9 +1392,9 @@ function releaseSlotLocks(slot) {
1372
1392
  for (const waiter of [...slot.idleWaiters]) waiter();
1373
1393
  }
1374
1394
  function rejectPending(slot, err) {
1375
- for (const [, waiting] of slot.pending) {
1376
- clearTimeout(waiting.timer);
1377
- waiting.reject(err);
1395
+ for (const [, waiting2] of slot.pending) {
1396
+ clearTimeout(waiting2.timer);
1397
+ waiting2.reject(err);
1378
1398
  }
1379
1399
  slot.pending.clear();
1380
1400
  }
@@ -1453,11 +1473,11 @@ async function createVectorSearchPool(requestedCount) {
1453
1473
  const key = message.requestId;
1454
1474
  if (typeof key !== "string") return;
1455
1475
  noteAnswered(slot, key, sharedFields);
1456
- const waiting = slot.pending.get(key);
1457
- if (!waiting) return;
1476
+ const waiting2 = slot.pending.get(key);
1477
+ if (!waiting2) return;
1458
1478
  slot.pending.delete(key);
1459
- clearTimeout(waiting.timer);
1460
- waiting.resolve(message);
1479
+ clearTimeout(waiting2.timer);
1480
+ waiting2.resolve(message);
1461
1481
  });
1462
1482
  listenForFailure(worker, (err) => {
1463
1483
  slot.alive = false;
@@ -2166,6 +2186,51 @@ async function releasePendingLocations(state) {
2166
2186
  await releaseInTurn(state, () => releaseLocations(state, paths, pending));
2167
2187
  }
2168
2188
 
2189
+ // src/runtime/background-memory.ts
2190
+ var admissions = /* @__PURE__ */ new Set();
2191
+ var waiting = /* @__PURE__ */ new Map();
2192
+ function admitsBackgroundBytes(bytes) {
2193
+ for (const admission of admissions) {
2194
+ if (!admission(bytes)) return false;
2195
+ }
2196
+ return true;
2197
+ }
2198
+ function noteWaitingForMemory(owner, work) {
2199
+ if (work === null) waiting.delete(owner);
2200
+ else waiting.set(owner, work);
2201
+ }
2202
+ function chargeHeapBytes(bytes) {
2203
+ }
2204
+
2205
+ // src/vector/vector-index/memory-charge.ts
2206
+ function admitShuffledRebuild(state) {
2207
+ const bytes = state.hnsw?.graphBytes ?? 0;
2208
+ if (!admitsBackgroundBytes(bytes)) {
2209
+ state.waitingForMemory = true;
2210
+ noteWaitingForMemory(state, { indexName: state.indexName, fieldName: state.fieldName });
2211
+ return false;
2212
+ }
2213
+ stopWaitingForMemory(state);
2214
+ chargeHeapBytes(bytes - state.rebuildChargeBytes);
2215
+ state.rebuildChargeBytes = bytes;
2216
+ return true;
2217
+ }
2218
+ function stopWaitingForMemory(state) {
2219
+ if (!state.waitingForMemory) return;
2220
+ state.waitingForMemory = false;
2221
+ noteWaitingForMemory(state, null);
2222
+ }
2223
+ function releaseRebuildCharge(state) {
2224
+ chargeHeapBytes(-state.rebuildChargeBytes);
2225
+ state.rebuildChargeBytes = 0;
2226
+ }
2227
+ function releaseMemoryCharges(state) {
2228
+ releaseRebuildCharge(state);
2229
+ stopWaitingForMemory(state);
2230
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2231
+ state.rebuildWaitTimer = null;
2232
+ }
2233
+
2169
2234
  // src/vector/vector-index/build.ts
2170
2235
  function bufferWhatTheGraphLacks(state, graph) {
2171
2236
  for (const docId of allLiveDocIds(state)) {
@@ -2248,8 +2313,25 @@ async function rebuildInShuffledOrder(state) {
2248
2313
  const adopted = await buildGraphFromStore(state);
2249
2314
  if (adopted && replacing.loadedWithoutCount) state.onLoadedGraphRepaired();
2250
2315
  }
2316
+ function retryWhenMemoryAllows(state) {
2317
+ if (state.rebuildWaitTimer !== null || state.disposed) return;
2318
+ const timer = setTimeout(() => {
2319
+ state.rebuildWaitTimer = null;
2320
+ startShuffledRebuild(state);
2321
+ }, SHUFFLED_REBUILD_MEMORY_RETRY_MS);
2322
+ timer.unref?.();
2323
+ state.rebuildWaitTimer = timer;
2324
+ }
2251
2325
  function startShuffledRebuild(state) {
2252
- if (state.shuffledRebuild !== null || state.disposed || !shuffledRebuildDue(state)) return;
2326
+ if (state.shuffledRebuild !== null || state.disposed) return;
2327
+ if (!shuffledRebuildDue(state)) {
2328
+ stopWaitingForMemory(state);
2329
+ return;
2330
+ }
2331
+ if (!admitShuffledRebuild(state)) {
2332
+ retryWhenMemoryAllows(state);
2333
+ return;
2334
+ }
2253
2335
  state.shuffledRebuild = rebuildInShuffledOrder(state).catch((error) => {
2254
2336
  console.warn(
2255
2337
  `Rebuilding the graph of "${state.indexName}/${state.fieldName}" in a shuffled order failed, so the field keeps the graph it holds:`,
@@ -2257,6 +2339,7 @@ function startShuffledRebuild(state) {
2257
2339
  );
2258
2340
  }).finally(() => {
2259
2341
  state.shuffledRebuild = null;
2342
+ releaseRebuildCharge(state);
2260
2343
  scheduleBuild(state);
2261
2344
  scheduleWorkerCopyLoad(state);
2262
2345
  });
@@ -2382,9 +2465,12 @@ function releaseHeldMemory(state) {
2382
2465
  state.savedFiles = [];
2383
2466
  state.savedSignature = null;
2384
2467
  state.store.release();
2468
+ releaseMemoryCharges(state);
2385
2469
  }
2386
2470
  function disposeIndex(state) {
2387
2471
  state.disposed = true;
2472
+ if (state.rebuildWaitTimer !== null) clearTimeout(state.rebuildWaitTimer);
2473
+ state.rebuildWaitTimer = null;
2388
2474
  if (state.buildTimer !== null) clearTimeout(state.buildTimer);
2389
2475
  state.buildTimer = null;
2390
2476
  invalidateWorkerCopies(state);
@@ -2723,6 +2809,7 @@ function maintenanceStatus(state) {
2723
2809
  graphCount,
2724
2810
  bufferSize: state.buffer.size,
2725
2811
  building: state.building || state.buildScheduled || state.buildTimer !== null || state.shuffledRebuild !== null,
2812
+ waitingForMemory: state.waitingForMemory,
2726
2813
  estimatedCompactMs,
2727
2814
  estimatedOptimizeMs
2728
2815
  };
@@ -2925,6 +3012,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2925
3012
  hnsw: null,
2926
3013
  freshGraph: null,
2927
3014
  ...noBuildHistory(),
3015
+ rebuildChargeBytes: 0,
3016
+ waitingForMemory: false,
3017
+ rebuildWaitTimer: null,
2928
3018
  onLoadedGraphRepaired,
2929
3019
  compactedNodeCount: 0,
2930
3020
  building: false,
@@ -3082,6 +3172,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3082
3172
  },
3083
3173
  maintenanceStatus: () => maintenanceStatus(state),
3084
3174
  estimateMemoryBytes: () => estimateMemoryBytes(state),
3175
+ growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3085
3176
  deserialize: (parts) => {
3086
3177
  invalidateWorkerCopies(state);
3087
3178
  deserialize(state, parts);
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
18
18
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
19
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
20
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-BU6I25CW.mjs');
21
+ const { restoreFromSnapshot } = await import('./snapshot-BE3F2PNC.mjs');
22
22
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
23
  });
24
24
  const settled = restored.then(
@@ -1,7 +1,7 @@
1
- import { vectorsPerFile } from './chunk-OVY4RPDS.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-6YUUEMQ4.mjs';
3
- import './chunk-5BQPBG3G.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-HQOYKRMB.mjs';
1
+ import { collectAndReturnPages, vectorsPerFile } from './chunk-YZPOBLAS.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-3YPLBK4C.mjs';
3
+ import './chunk-4FVSBSXY.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-DC2SZO4D.mjs';
5
5
  import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-P7C2S5JX.mjs';
6
6
  import { MAX_FILE_BYTES } from './chunk-2CBQ2PCC.mjs';
7
7
  import './chunk-JPTQZWJQ.mjs';
@@ -663,7 +663,8 @@ function refuse(indexName, replacesIndex, shortfall) {
663
663
  function requireRestoreMemory(indexName, footprint, replacesIndex) {
664
664
  const processBytes = footprint.heapBytes + footprint.externalBytes;
665
665
  if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
666
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, replacesIndex);
666
+ const collected = collectAndReturnPages();
667
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
667
668
  if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
668
669
  }
669
670
 
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-5BQPBG3G.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-HQOYKRMB.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-4FVSBSXY.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-DC2SZO4D.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-P7C2S5JX.mjs';
4
4
  import './chunk-2CBQ2PCC.mjs';
5
5
  import './chunk-JPTQZWJQ.mjs';
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-ujZESkJz.js';
3
- import { h as TaskStore } from './tasks-CI3Fvkxw.js';
2
+ import { C as ClusterNamespace } from './config-CS5ADSrx.js';
3
+ import { h as TaskStore } from './tasks-5_LCFEzF.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -10,5 +10,9 @@ var REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS = 10;
10
10
  var HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.75;
11
11
  var HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE = 0.65;
12
12
  var PROCESS_LIMIT_REFRESH_INTERVAL_MS = 1e3;
13
+ var V8_YOUNG_GENERATION_BYTES_BY_MAJOR = {
14
+ 12: 48 * 1024 * 1024,
15
+ 13: 192 * 1024 * 1024
16
+ };
13
17
 
14
- export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS };
18
+ export { HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, IDLE_HEAP_CLEANUP_CHECK_MS, IDLE_HEAP_CLEANUP_MAX_GROWTH_BYTES, IDLE_HEAP_CLEANUP_MAX_UTILISATION, IDLE_HEAP_CLEANUP_MIN_RECLAIMABLE_BYTES, IDLE_HEAP_CLEANUP_QUIET_CHECKS, PROCESS_LIMIT_REFRESH_INTERVAL_MS, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, V8_YOUNG_GENERATION_BYTES_BY_MAJOR };
@@ -1,9 +1,9 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-PKE2PCTS.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-LVNBY2TS.mjs';
2
2
  import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
3
3
  import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-IW6UR3AF.mjs';
5
- import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-R2MRMH6M.mjs';
6
- import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-7JOONQ24.mjs';
4
+ import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-PABYSTKO.mjs';
5
+ import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-5QIRLJH4.mjs';
6
+ import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-5WEBVYOA.mjs';
7
7
  import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
8
8
  import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-DG62H7OD.mjs';
9
9
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
14
14
  function loadVectorCode() {
15
15
  if (loaded !== null) return Promise.resolve(loaded);
16
16
  if (loading === null) {
17
- loading = import('./module-WP4VVGV2.mjs').then(
17
+ loading = import('./module-WVKSJL74.mjs').then(
18
18
  (code) => {
19
19
  loaded = code;
20
20
  return code;
@@ -25,6 +25,15 @@ function createFixedBuffer(bytes) {
25
25
  function nextGrowthTarget(have, needed, max) {
26
26
  return Math.min(max, Math.max(needed, have * 2));
27
27
  }
28
+ function grownByteLength(have, needed, max, step = 1) {
29
+ let size = have;
30
+ while (size < needed) {
31
+ const next = Math.ceil(nextGrowthTarget(size, Math.min(needed, size + 1), max) / step) * step;
32
+ if (next <= size) return size;
33
+ size = next;
34
+ }
35
+ return size;
36
+ }
28
37
  function growBufferTo(buffer, bytes) {
29
38
  const held = buffer.byteLength;
30
39
  if (held >= bytes) return;
@@ -1038,6 +1047,25 @@ function ensureBlockSlots(handle, slots) {
1038
1047
  }
1039
1048
  growMemoryTo(storage.kind === "shared-memory" ? storage.memory : storage.simd.memory, bytes, handle.layout);
1040
1049
  }
1050
+ function blockGrowthBytes(handle, layout, slots) {
1051
+ const bytes = layout.slotsOffset + Math.min(slots, layout.capacity) * layout.slotStride;
1052
+ if (handle !== null && handle.storage.kind === "bytes") {
1053
+ const buffer = handle.storage.buffer;
1054
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
1055
+ }
1056
+ const have = handle === null ? 0 : blockBuffer(handle).byteLength;
1057
+ const start = handle === null ? pagesFor(initialBlockBytes(layout)) * WASM_PAGE_BYTES : have;
1058
+ return grownByteLength(start, bytes, fullBlockBytes(layout), WASM_PAGE_BYTES) - have;
1059
+ }
1060
+ function blocksGrowthBytes(blocks, layout, firstOrdinal, lastOrdinal) {
1061
+ let bytes = 0;
1062
+ const lastBlock = Math.floor(lastOrdinal / layout.capacity);
1063
+ for (let block = Math.floor(firstOrdinal / layout.capacity); block <= lastBlock; block++) {
1064
+ const slots = block === lastBlock ? lastOrdinal % layout.capacity + 1 : layout.capacity;
1065
+ bytes += blockGrowthBytes(blocks[block] ?? null, layout, slots);
1066
+ }
1067
+ return bytes;
1068
+ }
1041
1069
  function blockBuffer(handle) {
1042
1070
  const storage = handle.storage;
1043
1071
  if (storage.kind === "bytes") return storage.buffer;
@@ -3980,4 +4008,4 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
3980
4008
  };
3981
4009
  }
3982
4010
 
3983
- export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
4011
+ export { BLOCK_MAX_BYTES, GRAPH_RETIRED, IN_MEMORY, NOTHING_STAGED, QUERY_STAGED, STORE_BLOCK_COUNT, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, addToOrdinalFilter, arenaFloat32Distance, blocksGrowthBytes, centroidOfCentroidSums, cosineSimilarityWithMagnitudes, countOfCentroidSums, createCentroidSums, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, detachNativeStore, dotProduct, ensureBlockSlots, euclideanDistance, fixedView, graphWithoutNodes, growBufferTo, grownByteLength, insertNode, magnitude, meanOfCentroidSums, nativeScoresOf, nativeStoreScratchBytes, nodeCountOf, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqQuantize, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, resetStoreHeader, searchOrdinals, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, slotByteOffset, squaredLengthsOfCentroidSums, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount };
@@ -1,6 +1,6 @@
1
- import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-7JOONQ24.mjs';
1
+ import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-5WEBVYOA.mjs';
2
2
  import { generateId, getLanguage } from './chunk-FUUP45IS.mjs';
3
- import { readHeapStatistics, readResidentMemory } from './chunk-SOADQINJ.mjs';
3
+ import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
4
4
  import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-HNAP2KTG.mjs';
5
5
  import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-KCQG2PJT.mjs';
6
6
  import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-DG62H7OD.mjs';
@@ -9,7 +9,7 @@ import { isWordIndexedBase } from './chunk-6ODIY6ZB.mjs';
9
9
  import { resolveIndexAnalysis, synonymSearchModule } from './chunk-IR4KQ4WB.mjs';
10
10
  import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-OFYKBKKV.mjs';
11
11
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
12
- import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-QI6JW4PM.mjs';
12
+ import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-4PFOOQFQ.mjs';
13
13
  import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
14
14
  import { NarsilError, ErrorCodes } from './chunk-OBWUWZRA.mjs';
15
15
  import { Decoder, decode } from '@msgpack/msgpack';