@delali/narsil 0.3.1-next.140 → 0.3.1-next.143

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,9 +1,11 @@
1
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-3EYDURFD.mjs';
2
- export { releaseLocksHeldBy } from './chunk-3EYDURFD.mjs';
3
- import './chunk-OIZS5KIX.mjs';
4
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
5
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-5LYWZJE2.mjs';
6
- import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-4MFH226Q.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-6RHTAVZX.mjs';
3
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
4
+ export { releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
5
+ import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-LVY2LI35.mjs';
6
+ import { createVectorFileReader } from './chunk-3ZBJYAS4.mjs';
7
+ import './chunk-OIZS5KIX.mjs';
8
+ import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-D33A5VMU.mjs';
7
9
  import './chunk-MZI3TEOW.mjs';
8
10
  import { NarsilError, ErrorCodes, isNarsilError } from './chunk-35B42UFM.mjs';
9
11
 
@@ -352,21 +354,6 @@ function createSharedVectorSearcher(options) {
352
354
  };
353
355
  }
354
356
 
355
- // src/vector/vector-store/vector-file.browser.ts
356
- function filesystemUnavailable() {
357
- throw new NarsilError(
358
- ErrorCodes.CONFIG_INVALID,
359
- "Reading vectors from a checkpoint file requires a Node.js runtime; keep the field in memory in the browser"
360
- );
361
- }
362
- function createVectorFileReader() {
363
- return {
364
- readInto: () => filesystemUnavailable(),
365
- retainOnly: () => void 0,
366
- close: () => void 0
367
- };
368
- }
369
-
370
357
  // src/vector/vector-store/hot-block-distance.ts
371
358
  function jsDistance(a, b, magA, magB, metric) {
372
359
  switch (metric) {
@@ -378,6 +365,11 @@ function jsDistance(a, b, magA, magB, metric) {
378
365
  return euclideanDistance(a, b);
379
366
  }
380
367
  }
368
+ function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
369
+ if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
370
+ const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
371
+ return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
372
+ }
381
373
  function inMemory(reads, ordinal) {
382
374
  return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
383
375
  }
@@ -474,9 +466,7 @@ function openSharedVectorStore(initial, threadSlot) {
474
466
  }
475
467
  return diskFile[ordinal];
476
468
  }
477
- function isCold(ordinal) {
478
- return fileOf(ordinal) !== IN_MEMORY;
479
- }
469
+ const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
480
470
  function holdState(ordinal) {
481
471
  if (ordinal < 0) return NOT_HELD;
482
472
  if (ordinal >= present.length) {
@@ -488,9 +478,7 @@ function openSharedVectorStore(initial, threadSlot) {
488
478
  const index = Math.floor(ordinal / capacity);
489
479
  return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
490
480
  }
491
- function holds(ordinal) {
492
- return holdState(ordinal) !== NOT_HELD;
493
- }
481
+ const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
494
482
  function readFromFile(file, offset, target) {
495
483
  const path = handles.vectorFiles[file];
496
484
  if (path === void 0 || path.length === 0) return false;
@@ -698,6 +686,7 @@ function openSharedVectorStore(initial, threadSlot) {
698
686
  }
699
687
  return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
700
688
  },
689
+ distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
701
690
  hotBlockReads,
702
691
  queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
703
692
  ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
@@ -841,6 +830,11 @@ function createOrdinalReads(openStore, readable) {
841
830
  if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
842
831
  return open.view.distanceFromArena(prepared, ordinal, metric);
843
832
  },
833
+ distanceFromQuery(query, queryMagnitude, ordinal, metric) {
834
+ const open = openStore();
835
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
836
+ return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
837
+ },
844
838
  queryDistance(prepared, metric) {
845
839
  const open = openStore();
846
840
  if (open === null) return () => Number.POSITIVE_INFINITY;
@@ -1017,13 +1011,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
1017
1011
  }
1018
1012
  return true;
1019
1013
  }
1020
- function fileHoldsSameBytes(store, ordinal, location) {
1021
- const fromFile = new Float32Array(store.handles.dimension);
1014
+ function fileHoldsSameBytes(store, ordinal, location, fromFile) {
1022
1015
  if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
1023
1016
  const held = store.view.vectorAt(ordinal);
1024
- const a = new Uint8Array(fromFile.buffer);
1025
- const b = new Uint8Array(held.buffer, held.byteOffset, held.byteLength);
1026
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
1017
+ const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
1018
+ const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
1019
+ for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
1027
1020
  return true;
1028
1021
  }
1029
1022
  function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
@@ -1074,6 +1067,7 @@ function createVectorStore(options) {
1074
1067
  const replacedOrdinals = /* @__PURE__ */ new Set();
1075
1068
  let unknownPartitions = 0;
1076
1069
  let liveCount = 0;
1070
+ let readBackVector = null;
1077
1071
  let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
1078
1072
  function forgetDocuments() {
1079
1073
  docToOrd.clear();
@@ -1218,7 +1212,10 @@ function createVectorStore(options) {
1218
1212
  releaseToFile(ordinal, location) {
1219
1213
  const store = requireOpen();
1220
1214
  if (ordToDoc[ordinal] === void 0) return false;
1221
- if (store.diskFile[ordinal] === IN_MEMORY && !fileHoldsSameBytes(store, ordinal, location)) return false;
1215
+ if (store.diskFile[ordinal] === IN_MEMORY) {
1216
+ readBackVector ??= new Float32Array(store.handles.dimension);
1217
+ if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
1218
+ }
1222
1219
  Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
1223
1220
  store.diskOffset[ordinal] = location.offset;
1224
1221
  store.diskFile[ordinal] = location.fileIndex;
@@ -2600,19 +2597,25 @@ function beginCheckpointRestore(state, shape) {
2600
2597
  const dead = isDead(file.dead, i);
2601
2598
  if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
2602
2599
  const docId = payload.docIds[i];
2603
- const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
2604
2600
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
2601
+ const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
2605
2602
  if (dead) {
2606
- ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
2603
+ ordinals[i] = state.store.insertReplaced(
2604
+ docId,
2605
+ vectorHere(),
2606
+ fileIndex === null ? void 0 : { fileIndex, offset }
2607
+ );
2607
2608
  tombstoned.add(ordinals[i]);
2608
2609
  continue;
2609
2610
  }
2610
2611
  const replaced = state.store.getOrdinal(docId);
2611
2612
  if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
2612
2613
  if (fileIndex !== null) {
2613
- ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
2614
+ const known = file.byParts?.magnitudes[i];
2615
+ ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
2614
2616
  continue;
2615
2617
  }
2618
+ const vector = vectorHere();
2616
2619
  ordinals[i] = state.store.insert(docId, vector);
2617
2620
  if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
2618
2621
  }
@@ -2654,6 +2657,223 @@ function beginCheckpointRestore(state, shape) {
2654
2657
  return { addFile, finish };
2655
2658
  }
2656
2659
 
2660
+ // src/vector/vector-index/cold.ts
2661
+ function beginColdHold(state, field, loaded, searchAnswered) {
2662
+ const hold = {
2663
+ field,
2664
+ loaded,
2665
+ searchAnswered,
2666
+ written: /* @__PURE__ */ new Map(),
2667
+ removedNumbers: /* @__PURE__ */ new Set(),
2668
+ removedIds: /* @__PURE__ */ new Set(),
2669
+ writes: [],
2670
+ live: field.liveCount
2671
+ };
2672
+ state.cold = hold;
2673
+ return hold;
2674
+ }
2675
+ function endColdHold(state) {
2676
+ const hold = state.cold;
2677
+ if (hold === null) return;
2678
+ state.cold = null;
2679
+ for (const write of hold.writes) write();
2680
+ hold.field.close();
2681
+ }
2682
+ function cancelColdHold(state) {
2683
+ const hold = state.cold;
2684
+ if (hold === null) return;
2685
+ state.cold = null;
2686
+ hold.field.close();
2687
+ }
2688
+ function openColdField(state, field, load) {
2689
+ cancelColdHold(state);
2690
+ let finished = () => void 0;
2691
+ let failed = () => void 0;
2692
+ const loaded = new Promise((resolve, reject) => {
2693
+ finished = resolve;
2694
+ failed = reject;
2695
+ });
2696
+ let searchAnswered = () => void 0;
2697
+ const firstSearchAnswered = new Promise((resolve) => {
2698
+ searchAnswered = resolve;
2699
+ });
2700
+ const hold = beginColdHold(state, field, loaded, searchAnswered);
2701
+ void load(
2702
+ (shape) => beginCheckpointRestore(state, shape),
2703
+ () => state.cold === hold,
2704
+ firstSearchAnswered
2705
+ ).then(
2706
+ () => {
2707
+ if (state.cold === hold) endColdHold(state);
2708
+ finished();
2709
+ },
2710
+ (error) => failed(error)
2711
+ );
2712
+ loaded.catch(() => void 0);
2713
+ }
2714
+ function retireOlderCopy(hold, docId) {
2715
+ if (hold.written.delete(docId)) return true;
2716
+ if (hold.removedIds.has(docId)) return false;
2717
+ const number = hold.field.numberOf(docId);
2718
+ if (number === void 0) return false;
2719
+ hold.removedIds.add(docId);
2720
+ hold.removedNumbers.add(number);
2721
+ return true;
2722
+ }
2723
+ function coldInsert(hold, docId, vector, partitionId) {
2724
+ const replaced = retireOlderCopy(hold, docId);
2725
+ if (!replaced) hold.live += 1;
2726
+ hold.written.set(docId, { vector: vector.slice(), partitionId });
2727
+ return replaced;
2728
+ }
2729
+ function coldRemove(hold, docId) {
2730
+ const removed = retireOlderCopy(hold, docId);
2731
+ if (removed) hold.live -= 1;
2732
+ return removed;
2733
+ }
2734
+ function coldHas(hold, docId) {
2735
+ if (hold.written.has(docId)) return true;
2736
+ if (hold.removedIds.has(docId)) return false;
2737
+ return hold.field.numberOf(docId) !== void 0;
2738
+ }
2739
+ function coldVectorOf(hold, docId) {
2740
+ const written = hold.written.get(docId);
2741
+ if (written !== void 0) return written.vector.slice();
2742
+ if (hold.removedIds.has(docId)) return null;
2743
+ const number = hold.field.numberOf(docId);
2744
+ return number === void 0 ? null : hold.field.vectorOfNumber(number);
2745
+ }
2746
+ function isAdmitted(subsets, docId) {
2747
+ for (const subset of subsets) {
2748
+ const { docStore } = subset.owner;
2749
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
2750
+ const internalId = docStore.getInternalId(docId);
2751
+ if (internalId === void 0) continue;
2752
+ const word = subset.bits[internalId >>> 5];
2753
+ if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
2754
+ return true;
2755
+ }
2756
+ }
2757
+ return false;
2758
+ }
2759
+ function admissionOf(options, hold) {
2760
+ const { filterSubsets, filterPartitions } = options;
2761
+ if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
2762
+ if (filterPartitions !== void 0) {
2763
+ return (docId) => {
2764
+ const partition = hold.field.partitionOf(docId);
2765
+ return partition !== void 0 && filterPartitions.has(partition);
2766
+ };
2767
+ }
2768
+ return null;
2769
+ }
2770
+ function bitCount(word) {
2771
+ let value = word - (word >>> 1 & 1431655765);
2772
+ value = (value & 858993459) + (value >>> 2 & 858993459);
2773
+ return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
2774
+ }
2775
+ function admittedBySubsets(subsets) {
2776
+ let admitted = 0;
2777
+ for (const subset of subsets) {
2778
+ for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
2779
+ }
2780
+ return admitted;
2781
+ }
2782
+ function* idsOfSubsets(subsets) {
2783
+ for (const subset of subsets) {
2784
+ const { bits } = subset;
2785
+ const { docStore } = subset.owner;
2786
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
2787
+ for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
2788
+ let word = bits[wordIndex];
2789
+ while (word !== 0) {
2790
+ const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
2791
+ word &= word - 1;
2792
+ if (!docStore.resolver().isLive(internalId)) continue;
2793
+ const docId = docStore.getExternalId(internalId);
2794
+ if (docId !== void 0) yield docId;
2795
+ }
2796
+ }
2797
+ }
2798
+ }
2799
+ function scoreOf(query, qMag, vector, metric) {
2800
+ const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
2801
+ return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
2802
+ }
2803
+ function scanWritten(hold, query, k, options, admits) {
2804
+ if (hold.written.size === 0) return [];
2805
+ const qMag = magnitude(query);
2806
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
2807
+ for (const [docId, { vector }] of hold.written) {
2808
+ if (admits !== null && !admits(docId)) continue;
2809
+ const score = scoreOf(query, qMag, vector, options.metric);
2810
+ if (score >= options.minSimilarity) heap.push({ docId, score });
2811
+ }
2812
+ return heap.toSortedArray().reverse();
2813
+ }
2814
+ function scanSubsets(hold, query, k, options, subsets) {
2815
+ const qMag = magnitude(query);
2816
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
2817
+ let matched = 0;
2818
+ for (const docId of idsOfSubsets(subsets)) {
2819
+ const written = hold.written.get(docId);
2820
+ let score;
2821
+ if (written !== void 0) {
2822
+ score = scoreOf(query, qMag, written.vector, options.metric);
2823
+ } else {
2824
+ if (hold.removedIds.has(docId)) continue;
2825
+ const number = hold.field.numberOf(docId);
2826
+ if (number === void 0) continue;
2827
+ score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
2828
+ }
2829
+ if (score < options.minSimilarity) continue;
2830
+ matched += 1;
2831
+ heap.push({ docId, score });
2832
+ }
2833
+ return { results: heap.toSortedArray().reverse(), matched };
2834
+ }
2835
+ function outcomeWithoutGraphWalk(state, hold, query, k, options) {
2836
+ if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
2837
+ const subsets = options.filterSubsets;
2838
+ let admitted = hold.live;
2839
+ if (subsets !== void 0) {
2840
+ admitted = admittedBySubsets(subsets);
2841
+ if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
2842
+ if (admitted / hold.live < state.filterThreshold) {
2843
+ const { results, matched } = scanSubsets(hold, query, k, options, subsets);
2844
+ return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
2845
+ }
2846
+ }
2847
+ if (k <= 0) {
2848
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
2849
+ }
2850
+ return { admits: admissionOf(options, hold), admitted };
2851
+ }
2852
+ function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
2853
+ const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
2854
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
2855
+ }
2856
+ function coldSearch(state, hold, query, k, options) {
2857
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
2858
+ if ("results" in walk) {
2859
+ hold.searchAnswered();
2860
+ return walk;
2861
+ }
2862
+ const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
2863
+ hold.searchAnswered();
2864
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
2865
+ }
2866
+ async function coldSearchConcurrently(state, hold, query, k, options) {
2867
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
2868
+ if ("results" in walk) {
2869
+ hold.searchAnswered();
2870
+ return walk;
2871
+ }
2872
+ const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
2873
+ hold.searchAnswered();
2874
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
2875
+ }
2876
+
2657
2877
  // src/vector/vector-index/flagged-documents.ts
2658
2878
  var FlaggedDocuments = class extends Set {
2659
2879
  constructor(store, layout) {
@@ -2805,7 +3025,7 @@ function completeGraph(state) {
2805
3025
  }
2806
3026
  function maintenanceStatus(state) {
2807
3027
  const storeSize = state.store.size;
2808
- const graphCount = state.hnsw ? 1 : 0;
3028
+ const graphCount = state.hnsw || state.cold ? 1 : 0;
2809
3029
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
2810
3030
  const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
2811
3031
  const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
@@ -2968,6 +3188,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
2968
3188
  return mergeResults(graphResults, scan.results, k);
2969
3189
  }
2970
3190
 
3191
+ // src/vector/vector-index/writes.ts
3192
+ function createVectorWrites(state) {
3193
+ function validateDimension(vector) {
3194
+ if (vector.length !== state.dimension) {
3195
+ throw new NarsilError(
3196
+ ErrorCodes.VECTOR_DIMENSION_MISMATCH,
3197
+ `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
3198
+ { expected: state.dimension, received: vector.length }
3199
+ );
3200
+ }
3201
+ }
3202
+ function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
3203
+ try {
3204
+ return state.store.entryForOrdinal(ordinal)?.vector;
3205
+ } catch {
3206
+ return void 0;
3207
+ }
3208
+ }
3209
+ function holdsThisVectorAlready(docId, ordinal, vector) {
3210
+ if (state.tombstones.has(docId)) return false;
3211
+ const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
3212
+ if (held === void 0) return false;
3213
+ for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
3214
+ return true;
3215
+ }
3216
+ function applyInsert(docId, vector, partitionId) {
3217
+ handOverBuildTurns(state);
3218
+ const previous = state.store.getOrdinal(docId);
3219
+ if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
3220
+ if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
3221
+ noteWrite(state);
3222
+ state.store.setPartition(docId, partitionId);
3223
+ }
3224
+ return;
3225
+ }
3226
+ noteWrite(state);
3227
+ state.tombstones.delete(docId);
3228
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
3229
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
3230
+ if (previous !== void 0) {
3231
+ state.hnsw?.markTombstoneOrdinal(previous);
3232
+ state.freshGraph?.markTombstoneOrdinal(previous);
3233
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
3234
+ }
3235
+ state.buffer.add(docId);
3236
+ state.pendingLocations.delete(docId);
3237
+ if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
3238
+ }
3239
+ function applyRemove(docId) {
3240
+ handOverBuildTurns(state);
3241
+ if (!state.store.has(docId)) return;
3242
+ noteWrite(state);
3243
+ state.tombstones.add(docId);
3244
+ state.buffer.delete(docId);
3245
+ state.pendingLocations.delete(docId);
3246
+ if (state.hnsw) {
3247
+ state.hnsw.markTombstone(docId);
3248
+ }
3249
+ }
3250
+ return {
3251
+ insert(docId, vector, partitionId) {
3252
+ validateDimension(vector);
3253
+ const hold = state.cold;
3254
+ if (hold === null) {
3255
+ applyInsert(docId, vector, partitionId);
3256
+ return;
3257
+ }
3258
+ noteWrite(state);
3259
+ coldInsert(hold, docId, vector, partitionId);
3260
+ const held = vector.slice();
3261
+ hold.writes.push(() => applyInsert(docId, held, partitionId));
3262
+ },
3263
+ remove(docId) {
3264
+ const hold = state.cold;
3265
+ if (hold === null) {
3266
+ applyRemove(docId);
3267
+ return;
3268
+ }
3269
+ if (!coldRemove(hold, docId)) return;
3270
+ noteWrite(state);
3271
+ hold.writes.push(() => applyRemove(docId));
3272
+ }
3273
+ };
3274
+ }
3275
+
2971
3276
  // src/vector/vector-index/index.ts
2972
3277
  function defaultQuantizationFor(dimension) {
2973
3278
  if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
@@ -3038,76 +3343,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3038
3343
  sharedHandles: /* @__PURE__ */ new Map(),
3039
3344
  sharedLayoutRevision: 0,
3040
3345
  sharing: Promise.resolve(),
3041
- releaseToFilesInFlight: Promise.resolve()
3346
+ releaseToFilesInFlight: Promise.resolve(),
3347
+ cold: null
3042
3348
  };
3043
- function validateDimension(vector) {
3044
- if (vector.length !== state.dimension) {
3045
- throw new NarsilError(
3046
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
3047
- `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
3048
- { expected: state.dimension, received: vector.length }
3049
- );
3050
- }
3051
- }
3052
- function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
3053
- try {
3054
- return state.store.entryForOrdinal(ordinal)?.vector;
3055
- } catch {
3056
- return void 0;
3057
- }
3058
- }
3059
- function holdsThisVectorAlready(docId, ordinal, vector) {
3060
- if (state.tombstones.has(docId)) return false;
3061
- const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
3062
- if (held === void 0) return false;
3063
- for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
3064
- return true;
3065
- }
3066
- function insert(docId, vector, partitionId) {
3067
- validateDimension(vector);
3068
- handOverBuildTurns(state);
3069
- const previous = state.store.getOrdinal(docId);
3070
- if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
3071
- if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
3072
- noteWrite(state);
3073
- state.store.setPartition(docId, partitionId);
3074
- }
3075
- return;
3076
- }
3077
- noteWrite(state);
3078
- state.tombstones.delete(docId);
3079
- const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
3080
- state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
3081
- if (previous !== void 0) {
3082
- state.hnsw?.markTombstoneOrdinal(previous);
3083
- state.freshGraph?.markTombstoneOrdinal(previous);
3084
- if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
3085
- }
3086
- state.buffer.add(docId);
3087
- state.pendingLocations.delete(docId);
3088
- if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
3089
- }
3090
- function remove(docId) {
3091
- handOverBuildTurns(state);
3092
- if (!state.store.has(docId)) return;
3093
- noteWrite(state);
3094
- state.tombstones.add(docId);
3095
- state.buffer.delete(docId);
3096
- state.pendingLocations.delete(docId);
3097
- if (state.hnsw) {
3098
- state.hnsw.markTombstone(docId);
3099
- }
3100
- }
3349
+ const { insert, remove } = createVectorWrites(state);
3101
3350
  function getVector(docId) {
3351
+ if (state.cold !== null) return coldVectorOf(state.cold, docId);
3102
3352
  if (state.tombstones.has(docId)) return null;
3103
3353
  const entry = state.store.get(docId);
3104
3354
  if (!entry) return null;
3105
3355
  return new Float32Array(entry.vector);
3106
3356
  }
3107
3357
  function has(docId) {
3358
+ if (state.cold !== null) return coldHas(state.cold, docId);
3108
3359
  return state.store.has(docId) && !state.tombstones.has(docId);
3109
3360
  }
3361
+ async function whenLoaded() {
3362
+ while (state.cold !== null) await state.cold.loaded;
3363
+ }
3110
3364
  async function searchParallel(query, k, options) {
3365
+ if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
3111
3366
  handOverBuildTurns(state);
3112
3367
  const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
3113
3368
  let filter = filterForOptions(state, options);
@@ -3154,44 +3409,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3154
3409
  get storage() {
3155
3410
  return state.storage;
3156
3411
  },
3412
+ get metric() {
3413
+ return state.metric;
3414
+ },
3157
3415
  insert,
3158
3416
  remove,
3159
- partitionsKnown: () => state.store.partitionsKnown,
3160
- assignPartitions: (resolve) => assignStorePartitions(state, resolve),
3161
- scheduleBuild: () => scheduleBuild(state),
3162
- awaitPendingBuild: () => awaitBuilds(state),
3163
- completeGraph: () => completeGraph(state),
3164
- dispose: () => disposeIndex(state),
3165
- search: (query, k, options) => search(state, query, k, options),
3417
+ partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
3418
+ assignPartitions: (resolve) => {
3419
+ if (state.cold === null) assignStorePartitions(state, resolve);
3420
+ },
3421
+ scheduleBuild: () => {
3422
+ if (state.cold === null) scheduleBuild(state);
3423
+ },
3424
+ awaitPendingBuild: async () => {
3425
+ await whenLoaded();
3426
+ await awaitBuilds(state);
3427
+ },
3428
+ completeGraph: async () => {
3429
+ await whenLoaded();
3430
+ await completeGraph(state);
3431
+ },
3432
+ dispose: () => {
3433
+ cancelColdHold(state);
3434
+ disposeIndex(state);
3435
+ },
3436
+ search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
3166
3437
  searchParallel,
3167
- refreshWorkerCopies: () => refreshWorkerCopies(state),
3438
+ refreshWorkerCopies: () => {
3439
+ if (state.cold === null) refreshWorkerCopies(state);
3440
+ },
3168
3441
  getVector,
3169
3442
  has,
3170
3443
  compact: () => {
3444
+ if (state.cold !== null) return;
3171
3445
  noteWrite(state);
3172
3446
  compact(state);
3173
3447
  },
3174
3448
  optimize: async () => {
3449
+ await whenLoaded();
3175
3450
  noteWrite(state);
3176
3451
  await optimize(state);
3177
3452
  },
3178
- maintenanceStatus: () => maintenanceStatus(state),
3179
- estimateMemoryBytes: () => estimateMemoryBytes(state),
3453
+ maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
3454
+ estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
3180
3455
  growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3181
3456
  deserialize: (parts) => {
3457
+ cancelColdHold(state);
3182
3458
  invalidateWorkerCopies(state);
3183
3459
  deserialize(state, parts);
3184
3460
  },
3185
3461
  planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
3186
3462
  recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
3187
3463
  restoreCheckpoint: (shape) => {
3464
+ cancelColdHold(state);
3188
3465
  invalidateWorkerCopies(state);
3189
3466
  return beginCheckpointRestore(state, shape);
3190
3467
  },
3191
- startShuffledRebuildWhenDue: () => startShuffledRebuild(state),
3192
- vectorFilesInUse: () => state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== ""),
3468
+ openColdField: (field, load) => openColdField(state, field, load),
3469
+ startShuffledRebuildWhenDue: () => {
3470
+ if (state.cold === null) startShuffledRebuild(state);
3471
+ },
3472
+ vectorFilesInUse: () => {
3473
+ const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
3474
+ return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
3475
+ },
3193
3476
  adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
3194
3477
  releaseVectorFiles: async () => {
3478
+ await whenLoaded();
3195
3479
  await withdrawWorkerCopies(state);
3196
3480
  state.store.closeFiles();
3197
3481
  },
@@ -3201,7 +3485,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3201
3485
  state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
3202
3486
  }
3203
3487
  },
3204
- holdWorkerCopies: () => holdWorkerCopies(state),
3488
+ holdWorkerCopies: async () => {
3489
+ await whenLoaded();
3490
+ await holdWorkerCopies(state);
3491
+ },
3205
3492
  resumeWorkerCopies: () => {
3206
3493
  state.workerCopiesHeld = false;
3207
3494
  }
@@ -1,4 +1,4 @@
1
- import { abandonChangeOf } from './chunk-HMDT3I63.mjs';
1
+ import { abandonChangeOf } from './chunk-XNWPP5NS.mjs';
2
2
 
3
3
  // src/narsil/restore-queue.ts
4
4
  function restoreDepsOf(core, host) {
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
24
24
  function queueRestore(core, host, restoresInFlight, indexName, data) {
25
25
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
26
26
  const restored = previous.then(async () => {
27
- const { restoreFromSnapshot } = await import('./snapshot-BTRZ3MZK.mjs');
27
+ const { restoreFromSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
28
28
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
29
29
  });
30
30
  const settled = restored.then(