@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,10 +1,13 @@
1
1
  import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, osqBitsOf, noBuildHistory, createOsqQuantizer, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, ordinalFilterValues, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, OSQ4_MIN_DIMENSION, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, WORKER_COPY_MIN_VECTORS, allLiveDocIds, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, adoptGraph, topLayerInParts, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-VBBLTD7O.mjs';
3
- export { releaseLocksHeldBy } from './chunk-VBBLTD7O.mjs';
4
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-D7WQF6DV.mjs';
5
- import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-AIIJDWMG.mjs';
6
- import './chunk-VEGHFP2P.mjs';
7
- import { NarsilError, ErrorCodes, isNarsilError } from './chunk-UDGB2MWT.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-6RHTAVZX.mjs';
3
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
4
+ export { releaseLocksHeldBy } from './chunk-HKX2G6WH.mjs';
5
+ import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-LVY2LI35.mjs';
6
+ import { createVectorFileReader } from './chunk-3ZBJYAS4.mjs';
7
+ import './chunk-OIZS5KIX.mjs';
8
+ import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-D33A5VMU.mjs';
9
+ import './chunk-MZI3TEOW.mjs';
10
+ import { NarsilError, ErrorCodes, isNarsilError } from './chunk-35B42UFM.mjs';
8
11
 
9
12
  // src/core/heap.ts
10
13
  function createMaxHeap(compare) {
@@ -351,21 +354,6 @@ function createSharedVectorSearcher(options) {
351
354
  };
352
355
  }
353
356
 
354
- // src/vector/vector-store/vector-file.browser.ts
355
- function filesystemUnavailable() {
356
- throw new NarsilError(
357
- ErrorCodes.CONFIG_INVALID,
358
- "Reading vectors from a checkpoint file requires a Node.js runtime; keep the field in memory in the browser"
359
- );
360
- }
361
- function createVectorFileReader() {
362
- return {
363
- readInto: () => filesystemUnavailable(),
364
- retainOnly: () => void 0,
365
- close: () => void 0
366
- };
367
- }
368
-
369
357
  // src/vector/vector-store/hot-block-distance.ts
370
358
  function jsDistance(a, b, magA, magB, metric) {
371
359
  switch (metric) {
@@ -377,6 +365,11 @@ function jsDistance(a, b, magA, magB, metric) {
377
365
  return euclideanDistance(a, b);
378
366
  }
379
367
  }
368
+ function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
369
+ if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
370
+ const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
371
+ return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
372
+ }
380
373
  function inMemory(reads, ordinal) {
381
374
  return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
382
375
  }
@@ -473,9 +466,7 @@ function openSharedVectorStore(initial, threadSlot) {
473
466
  }
474
467
  return diskFile[ordinal];
475
468
  }
476
- function isCold(ordinal) {
477
- return fileOf(ordinal) !== IN_MEMORY;
478
- }
469
+ const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
479
470
  function holdState(ordinal) {
480
471
  if (ordinal < 0) return NOT_HELD;
481
472
  if (ordinal >= present.length) {
@@ -487,9 +478,7 @@ function openSharedVectorStore(initial, threadSlot) {
487
478
  const index = Math.floor(ordinal / capacity);
488
479
  return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
489
480
  }
490
- function holds(ordinal) {
491
- return holdState(ordinal) !== NOT_HELD;
492
- }
481
+ const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
493
482
  function readFromFile(file, offset, target) {
494
483
  const path = handles.vectorFiles[file];
495
484
  if (path === void 0 || path.length === 0) return false;
@@ -697,6 +686,7 @@ function openSharedVectorStore(initial, threadSlot) {
697
686
  }
698
687
  return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
699
688
  },
689
+ distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
700
690
  hotBlockReads,
701
691
  queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
702
692
  ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
@@ -840,6 +830,11 @@ function createOrdinalReads(openStore, readable) {
840
830
  if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
841
831
  return open.view.distanceFromArena(prepared, ordinal, metric);
842
832
  },
833
+ distanceFromQuery(query, queryMagnitude, ordinal, metric) {
834
+ const open = openStore();
835
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
836
+ return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
837
+ },
843
838
  queryDistance(prepared, metric) {
844
839
  const open = openStore();
845
840
  if (open === null) return () => Number.POSITIVE_INFINITY;
@@ -1016,13 +1011,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
1016
1011
  }
1017
1012
  return true;
1018
1013
  }
1019
- function fileHoldsSameBytes(store, ordinal, location) {
1020
- const fromFile = new Float32Array(store.handles.dimension);
1014
+ function fileHoldsSameBytes(store, ordinal, location, fromFile) {
1021
1015
  if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
1022
1016
  const held = store.view.vectorAt(ordinal);
1023
- const a = new Uint8Array(fromFile.buffer);
1024
- const b = new Uint8Array(held.buffer, held.byteOffset, held.byteLength);
1025
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
1017
+ const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
1018
+ const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
1019
+ for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
1026
1020
  return true;
1027
1021
  }
1028
1022
  function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
@@ -1073,6 +1067,7 @@ function createVectorStore(options) {
1073
1067
  const replacedOrdinals = /* @__PURE__ */ new Set();
1074
1068
  let unknownPartitions = 0;
1075
1069
  let liveCount = 0;
1070
+ let readBackVector = null;
1076
1071
  let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
1077
1072
  function forgetDocuments() {
1078
1073
  docToOrd.clear();
@@ -1217,7 +1212,10 @@ function createVectorStore(options) {
1217
1212
  releaseToFile(ordinal, location) {
1218
1213
  const store = requireOpen();
1219
1214
  if (ordToDoc[ordinal] === void 0) return false;
1220
- if (store.diskFile[ordinal] === IN_MEMORY && !fileHoldsSameBytes(store, ordinal, location)) return false;
1215
+ if (store.diskFile[ordinal] === IN_MEMORY) {
1216
+ readBackVector ??= new Float32Array(store.handles.dimension);
1217
+ if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
1218
+ }
1221
1219
  Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
1222
1220
  store.diskOffset[ordinal] = location.offset;
1223
1221
  store.diskFile[ordinal] = location.fileIndex;
@@ -2599,19 +2597,25 @@ function beginCheckpointRestore(state, shape) {
2599
2597
  const dead = isDead(file.dead, i);
2600
2598
  if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
2601
2599
  const docId = payload.docIds[i];
2602
- const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
2603
2600
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
2601
+ const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
2604
2602
  if (dead) {
2605
- ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
2603
+ ordinals[i] = state.store.insertReplaced(
2604
+ docId,
2605
+ vectorHere(),
2606
+ fileIndex === null ? void 0 : { fileIndex, offset }
2607
+ );
2606
2608
  tombstoned.add(ordinals[i]);
2607
2609
  continue;
2608
2610
  }
2609
2611
  const replaced = state.store.getOrdinal(docId);
2610
2612
  if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
2611
2613
  if (fileIndex !== null) {
2612
- ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
2614
+ const known = file.byParts?.magnitudes[i];
2615
+ ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
2613
2616
  continue;
2614
2617
  }
2618
+ const vector = vectorHere();
2615
2619
  ordinals[i] = state.store.insert(docId, vector);
2616
2620
  if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
2617
2621
  }
@@ -2653,6 +2657,223 @@ function beginCheckpointRestore(state, shape) {
2653
2657
  return { addFile, finish };
2654
2658
  }
2655
2659
 
2660
+ // src/vector/vector-index/cold.ts
2661
+ function beginColdHold(state, field, loaded, searchAnswered) {
2662
+ const hold = {
2663
+ field,
2664
+ loaded,
2665
+ searchAnswered,
2666
+ written: /* @__PURE__ */ new Map(),
2667
+ removedNumbers: /* @__PURE__ */ new Set(),
2668
+ removedIds: /* @__PURE__ */ new Set(),
2669
+ writes: [],
2670
+ live: field.liveCount
2671
+ };
2672
+ state.cold = hold;
2673
+ return hold;
2674
+ }
2675
+ function endColdHold(state) {
2676
+ const hold = state.cold;
2677
+ if (hold === null) return;
2678
+ state.cold = null;
2679
+ for (const write of hold.writes) write();
2680
+ hold.field.close();
2681
+ }
2682
+ function cancelColdHold(state) {
2683
+ const hold = state.cold;
2684
+ if (hold === null) return;
2685
+ state.cold = null;
2686
+ hold.field.close();
2687
+ }
2688
+ function openColdField(state, field, load) {
2689
+ cancelColdHold(state);
2690
+ let finished = () => void 0;
2691
+ let failed = () => void 0;
2692
+ const loaded = new Promise((resolve, reject) => {
2693
+ finished = resolve;
2694
+ failed = reject;
2695
+ });
2696
+ let searchAnswered = () => void 0;
2697
+ const firstSearchAnswered = new Promise((resolve) => {
2698
+ searchAnswered = resolve;
2699
+ });
2700
+ const hold = beginColdHold(state, field, loaded, searchAnswered);
2701
+ void load(
2702
+ (shape) => beginCheckpointRestore(state, shape),
2703
+ () => state.cold === hold,
2704
+ firstSearchAnswered
2705
+ ).then(
2706
+ () => {
2707
+ if (state.cold === hold) endColdHold(state);
2708
+ finished();
2709
+ },
2710
+ (error) => failed(error)
2711
+ );
2712
+ loaded.catch(() => void 0);
2713
+ }
2714
+ function retireOlderCopy(hold, docId) {
2715
+ if (hold.written.delete(docId)) return true;
2716
+ if (hold.removedIds.has(docId)) return false;
2717
+ const number = hold.field.numberOf(docId);
2718
+ if (number === void 0) return false;
2719
+ hold.removedIds.add(docId);
2720
+ hold.removedNumbers.add(number);
2721
+ return true;
2722
+ }
2723
+ function coldInsert(hold, docId, vector, partitionId) {
2724
+ const replaced = retireOlderCopy(hold, docId);
2725
+ if (!replaced) hold.live += 1;
2726
+ hold.written.set(docId, { vector: vector.slice(), partitionId });
2727
+ return replaced;
2728
+ }
2729
+ function coldRemove(hold, docId) {
2730
+ const removed = retireOlderCopy(hold, docId);
2731
+ if (removed) hold.live -= 1;
2732
+ return removed;
2733
+ }
2734
+ function coldHas(hold, docId) {
2735
+ if (hold.written.has(docId)) return true;
2736
+ if (hold.removedIds.has(docId)) return false;
2737
+ return hold.field.numberOf(docId) !== void 0;
2738
+ }
2739
+ function coldVectorOf(hold, docId) {
2740
+ const written = hold.written.get(docId);
2741
+ if (written !== void 0) return written.vector.slice();
2742
+ if (hold.removedIds.has(docId)) return null;
2743
+ const number = hold.field.numberOf(docId);
2744
+ return number === void 0 ? null : hold.field.vectorOfNumber(number);
2745
+ }
2746
+ function isAdmitted(subsets, docId) {
2747
+ for (const subset of subsets) {
2748
+ const { docStore } = subset.owner;
2749
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
2750
+ const internalId = docStore.getInternalId(docId);
2751
+ if (internalId === void 0) continue;
2752
+ const word = subset.bits[internalId >>> 5];
2753
+ if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
2754
+ return true;
2755
+ }
2756
+ }
2757
+ return false;
2758
+ }
2759
+ function admissionOf(options, hold) {
2760
+ const { filterSubsets, filterPartitions } = options;
2761
+ if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
2762
+ if (filterPartitions !== void 0) {
2763
+ return (docId) => {
2764
+ const partition = hold.field.partitionOf(docId);
2765
+ return partition !== void 0 && filterPartitions.has(partition);
2766
+ };
2767
+ }
2768
+ return null;
2769
+ }
2770
+ function bitCount(word) {
2771
+ let value = word - (word >>> 1 & 1431655765);
2772
+ value = (value & 858993459) + (value >>> 2 & 858993459);
2773
+ return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
2774
+ }
2775
+ function admittedBySubsets(subsets) {
2776
+ let admitted = 0;
2777
+ for (const subset of subsets) {
2778
+ for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
2779
+ }
2780
+ return admitted;
2781
+ }
2782
+ function* idsOfSubsets(subsets) {
2783
+ for (const subset of subsets) {
2784
+ const { bits } = subset;
2785
+ const { docStore } = subset.owner;
2786
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
2787
+ for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
2788
+ let word = bits[wordIndex];
2789
+ while (word !== 0) {
2790
+ const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
2791
+ word &= word - 1;
2792
+ if (!docStore.resolver().isLive(internalId)) continue;
2793
+ const docId = docStore.getExternalId(internalId);
2794
+ if (docId !== void 0) yield docId;
2795
+ }
2796
+ }
2797
+ }
2798
+ }
2799
+ function scoreOf(query, qMag, vector, metric) {
2800
+ const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
2801
+ return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
2802
+ }
2803
+ function scanWritten(hold, query, k, options, admits) {
2804
+ if (hold.written.size === 0) return [];
2805
+ const qMag = magnitude(query);
2806
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
2807
+ for (const [docId, { vector }] of hold.written) {
2808
+ if (admits !== null && !admits(docId)) continue;
2809
+ const score = scoreOf(query, qMag, vector, options.metric);
2810
+ if (score >= options.minSimilarity) heap.push({ docId, score });
2811
+ }
2812
+ return heap.toSortedArray().reverse();
2813
+ }
2814
+ function scanSubsets(hold, query, k, options, subsets) {
2815
+ const qMag = magnitude(query);
2816
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
2817
+ let matched = 0;
2818
+ for (const docId of idsOfSubsets(subsets)) {
2819
+ const written = hold.written.get(docId);
2820
+ let score;
2821
+ if (written !== void 0) {
2822
+ score = scoreOf(query, qMag, written.vector, options.metric);
2823
+ } else {
2824
+ if (hold.removedIds.has(docId)) continue;
2825
+ const number = hold.field.numberOf(docId);
2826
+ if (number === void 0) continue;
2827
+ score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
2828
+ }
2829
+ if (score < options.minSimilarity) continue;
2830
+ matched += 1;
2831
+ heap.push({ docId, score });
2832
+ }
2833
+ return { results: heap.toSortedArray().reverse(), matched };
2834
+ }
2835
+ function outcomeWithoutGraphWalk(state, hold, query, k, options) {
2836
+ if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
2837
+ const subsets = options.filterSubsets;
2838
+ let admitted = hold.live;
2839
+ if (subsets !== void 0) {
2840
+ admitted = admittedBySubsets(subsets);
2841
+ if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
2842
+ if (admitted / hold.live < state.filterThreshold) {
2843
+ const { results, matched } = scanSubsets(hold, query, k, options, subsets);
2844
+ return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
2845
+ }
2846
+ }
2847
+ if (k <= 0) {
2848
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
2849
+ }
2850
+ return { admits: admissionOf(options, hold), admitted };
2851
+ }
2852
+ function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
2853
+ const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
2854
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
2855
+ }
2856
+ function coldSearch(state, hold, query, k, options) {
2857
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
2858
+ if ("results" in walk) {
2859
+ hold.searchAnswered();
2860
+ return walk;
2861
+ }
2862
+ const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
2863
+ hold.searchAnswered();
2864
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
2865
+ }
2866
+ async function coldSearchConcurrently(state, hold, query, k, options) {
2867
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
2868
+ if ("results" in walk) {
2869
+ hold.searchAnswered();
2870
+ return walk;
2871
+ }
2872
+ const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
2873
+ hold.searchAnswered();
2874
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
2875
+ }
2876
+
2656
2877
  // src/vector/vector-index/flagged-documents.ts
2657
2878
  var FlaggedDocuments = class extends Set {
2658
2879
  constructor(store, layout) {
@@ -2804,7 +3025,7 @@ function completeGraph(state) {
2804
3025
  }
2805
3026
  function maintenanceStatus(state) {
2806
3027
  const storeSize = state.store.size;
2807
- const graphCount = state.hnsw ? 1 : 0;
3028
+ const graphCount = state.hnsw || state.cold ? 1 : 0;
2808
3029
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
2809
3030
  const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
2810
3031
  const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
@@ -2967,6 +3188,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
2967
3188
  return mergeResults(graphResults, scan.results, k);
2968
3189
  }
2969
3190
 
3191
+ // src/vector/vector-index/writes.ts
3192
+ function createVectorWrites(state) {
3193
+ function validateDimension(vector) {
3194
+ if (vector.length !== state.dimension) {
3195
+ throw new NarsilError(
3196
+ ErrorCodes.VECTOR_DIMENSION_MISMATCH,
3197
+ `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
3198
+ { expected: state.dimension, received: vector.length }
3199
+ );
3200
+ }
3201
+ }
3202
+ function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
3203
+ try {
3204
+ return state.store.entryForOrdinal(ordinal)?.vector;
3205
+ } catch {
3206
+ return void 0;
3207
+ }
3208
+ }
3209
+ function holdsThisVectorAlready(docId, ordinal, vector) {
3210
+ if (state.tombstones.has(docId)) return false;
3211
+ const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
3212
+ if (held === void 0) return false;
3213
+ for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
3214
+ return true;
3215
+ }
3216
+ function applyInsert(docId, vector, partitionId) {
3217
+ handOverBuildTurns(state);
3218
+ const previous = state.store.getOrdinal(docId);
3219
+ if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
3220
+ if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
3221
+ noteWrite(state);
3222
+ state.store.setPartition(docId, partitionId);
3223
+ }
3224
+ return;
3225
+ }
3226
+ noteWrite(state);
3227
+ state.tombstones.delete(docId);
3228
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
3229
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
3230
+ if (previous !== void 0) {
3231
+ state.hnsw?.markTombstoneOrdinal(previous);
3232
+ state.freshGraph?.markTombstoneOrdinal(previous);
3233
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
3234
+ }
3235
+ state.buffer.add(docId);
3236
+ state.pendingLocations.delete(docId);
3237
+ if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
3238
+ }
3239
+ function applyRemove(docId) {
3240
+ handOverBuildTurns(state);
3241
+ if (!state.store.has(docId)) return;
3242
+ noteWrite(state);
3243
+ state.tombstones.add(docId);
3244
+ state.buffer.delete(docId);
3245
+ state.pendingLocations.delete(docId);
3246
+ if (state.hnsw) {
3247
+ state.hnsw.markTombstone(docId);
3248
+ }
3249
+ }
3250
+ return {
3251
+ insert(docId, vector, partitionId) {
3252
+ validateDimension(vector);
3253
+ const hold = state.cold;
3254
+ if (hold === null) {
3255
+ applyInsert(docId, vector, partitionId);
3256
+ return;
3257
+ }
3258
+ noteWrite(state);
3259
+ coldInsert(hold, docId, vector, partitionId);
3260
+ const held = vector.slice();
3261
+ hold.writes.push(() => applyInsert(docId, held, partitionId));
3262
+ },
3263
+ remove(docId) {
3264
+ const hold = state.cold;
3265
+ if (hold === null) {
3266
+ applyRemove(docId);
3267
+ return;
3268
+ }
3269
+ if (!coldRemove(hold, docId)) return;
3270
+ noteWrite(state);
3271
+ hold.writes.push(() => applyRemove(docId));
3272
+ }
3273
+ };
3274
+ }
3275
+
2970
3276
  // src/vector/vector-index/index.ts
2971
3277
  function defaultQuantizationFor(dimension) {
2972
3278
  if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
@@ -3037,76 +3343,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3037
3343
  sharedHandles: /* @__PURE__ */ new Map(),
3038
3344
  sharedLayoutRevision: 0,
3039
3345
  sharing: Promise.resolve(),
3040
- releaseToFilesInFlight: Promise.resolve()
3346
+ releaseToFilesInFlight: Promise.resolve(),
3347
+ cold: null
3041
3348
  };
3042
- function validateDimension(vector) {
3043
- if (vector.length !== state.dimension) {
3044
- throw new NarsilError(
3045
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
3046
- `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
3047
- { expected: state.dimension, received: vector.length }
3048
- );
3049
- }
3050
- }
3051
- function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
3052
- try {
3053
- return state.store.entryForOrdinal(ordinal)?.vector;
3054
- } catch {
3055
- return void 0;
3056
- }
3057
- }
3058
- function holdsThisVectorAlready(docId, ordinal, vector) {
3059
- if (state.tombstones.has(docId)) return false;
3060
- const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
3061
- if (held === void 0) return false;
3062
- for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
3063
- return true;
3064
- }
3065
- function insert(docId, vector, partitionId) {
3066
- validateDimension(vector);
3067
- handOverBuildTurns(state);
3068
- const previous = state.store.getOrdinal(docId);
3069
- if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
3070
- if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
3071
- noteWrite(state);
3072
- state.store.setPartition(docId, partitionId);
3073
- }
3074
- return;
3075
- }
3076
- noteWrite(state);
3077
- state.tombstones.delete(docId);
3078
- const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
3079
- state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
3080
- if (previous !== void 0) {
3081
- state.hnsw?.markTombstoneOrdinal(previous);
3082
- state.freshGraph?.markTombstoneOrdinal(previous);
3083
- if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
3084
- }
3085
- state.buffer.add(docId);
3086
- state.pendingLocations.delete(docId);
3087
- if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
3088
- }
3089
- function remove(docId) {
3090
- handOverBuildTurns(state);
3091
- if (!state.store.has(docId)) return;
3092
- noteWrite(state);
3093
- state.tombstones.add(docId);
3094
- state.buffer.delete(docId);
3095
- state.pendingLocations.delete(docId);
3096
- if (state.hnsw) {
3097
- state.hnsw.markTombstone(docId);
3098
- }
3099
- }
3349
+ const { insert, remove } = createVectorWrites(state);
3100
3350
  function getVector(docId) {
3351
+ if (state.cold !== null) return coldVectorOf(state.cold, docId);
3101
3352
  if (state.tombstones.has(docId)) return null;
3102
3353
  const entry = state.store.get(docId);
3103
3354
  if (!entry) return null;
3104
3355
  return new Float32Array(entry.vector);
3105
3356
  }
3106
3357
  function has(docId) {
3358
+ if (state.cold !== null) return coldHas(state.cold, docId);
3107
3359
  return state.store.has(docId) && !state.tombstones.has(docId);
3108
3360
  }
3361
+ async function whenLoaded() {
3362
+ while (state.cold !== null) await state.cold.loaded;
3363
+ }
3109
3364
  async function searchParallel(query, k, options) {
3365
+ if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
3110
3366
  handOverBuildTurns(state);
3111
3367
  const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
3112
3368
  let filter = filterForOptions(state, options);
@@ -3153,44 +3409,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3153
3409
  get storage() {
3154
3410
  return state.storage;
3155
3411
  },
3412
+ get metric() {
3413
+ return state.metric;
3414
+ },
3156
3415
  insert,
3157
3416
  remove,
3158
- partitionsKnown: () => state.store.partitionsKnown,
3159
- assignPartitions: (resolve) => assignStorePartitions(state, resolve),
3160
- scheduleBuild: () => scheduleBuild(state),
3161
- awaitPendingBuild: () => awaitBuilds(state),
3162
- completeGraph: () => completeGraph(state),
3163
- dispose: () => disposeIndex(state),
3164
- search: (query, k, options) => search(state, query, k, options),
3417
+ partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
3418
+ assignPartitions: (resolve) => {
3419
+ if (state.cold === null) assignStorePartitions(state, resolve);
3420
+ },
3421
+ scheduleBuild: () => {
3422
+ if (state.cold === null) scheduleBuild(state);
3423
+ },
3424
+ awaitPendingBuild: async () => {
3425
+ await whenLoaded();
3426
+ await awaitBuilds(state);
3427
+ },
3428
+ completeGraph: async () => {
3429
+ await whenLoaded();
3430
+ await completeGraph(state);
3431
+ },
3432
+ dispose: () => {
3433
+ cancelColdHold(state);
3434
+ disposeIndex(state);
3435
+ },
3436
+ search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
3165
3437
  searchParallel,
3166
- refreshWorkerCopies: () => refreshWorkerCopies(state),
3438
+ refreshWorkerCopies: () => {
3439
+ if (state.cold === null) refreshWorkerCopies(state);
3440
+ },
3167
3441
  getVector,
3168
3442
  has,
3169
3443
  compact: () => {
3444
+ if (state.cold !== null) return;
3170
3445
  noteWrite(state);
3171
3446
  compact(state);
3172
3447
  },
3173
3448
  optimize: async () => {
3449
+ await whenLoaded();
3174
3450
  noteWrite(state);
3175
3451
  await optimize(state);
3176
3452
  },
3177
- maintenanceStatus: () => maintenanceStatus(state),
3178
- estimateMemoryBytes: () => estimateMemoryBytes(state),
3453
+ maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
3454
+ estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
3179
3455
  growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3180
3456
  deserialize: (parts) => {
3457
+ cancelColdHold(state);
3181
3458
  invalidateWorkerCopies(state);
3182
3459
  deserialize(state, parts);
3183
3460
  },
3184
3461
  planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
3185
3462
  recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
3186
3463
  restoreCheckpoint: (shape) => {
3464
+ cancelColdHold(state);
3187
3465
  invalidateWorkerCopies(state);
3188
3466
  return beginCheckpointRestore(state, shape);
3189
3467
  },
3190
- startShuffledRebuildWhenDue: () => startShuffledRebuild(state),
3191
- vectorFilesInUse: () => state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== ""),
3468
+ openColdField: (field, load) => openColdField(state, field, load),
3469
+ startShuffledRebuildWhenDue: () => {
3470
+ if (state.cold === null) startShuffledRebuild(state);
3471
+ },
3472
+ vectorFilesInUse: () => {
3473
+ const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
3474
+ return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
3475
+ },
3192
3476
  adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
3193
3477
  releaseVectorFiles: async () => {
3478
+ await whenLoaded();
3194
3479
  await withdrawWorkerCopies(state);
3195
3480
  state.store.closeFiles();
3196
3481
  },
@@ -3200,7 +3485,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3200
3485
  state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
3201
3486
  }
3202
3487
  },
3203
- holdWorkerCopies: () => holdWorkerCopies(state),
3488
+ holdWorkerCopies: async () => {
3489
+ await whenLoaded();
3490
+ await holdWorkerCopies(state);
3491
+ },
3204
3492
  resumeWorkerCopies: () => {
3205
3493
  state.workerCopiesHeld = false;
3206
3494
  }
@@ -1,5 +1,5 @@
1
1
  import { createBitSet, bitsetIsEmpty, GrowableUint32, PHRASE_ROWS_START, bitsetHas, bitsetSet, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-26A5JFT2.mjs';
2
- import { NarsilError, ErrorCodes, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-UDGB2MWT.mjs';
2
+ import { NarsilError, ErrorCodes, MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-35B42UFM.mjs';
3
3
 
4
4
  // src/core/partition/phrase-matching.ts
5
5
  var UINT32_BITS = 32;