@delali/narsil 0.3.1-next.145 → 0.3.1-next.149

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/dist/adapters/filesystem.mjs +2 -2
  2. package/dist/adapters/gcs.mjs +2 -2
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +2 -2
  5. package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
  6. package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
  7. package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
  8. package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
  9. package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
  10. package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
  11. package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
  12. package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
  13. package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
  14. package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
  15. package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
  16. package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
  17. package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
  18. package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
  19. package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
  20. package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
  21. package/dist/browser/chunk-MPK76GVS.mjs +63 -0
  22. package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
  23. package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
  24. package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
  25. package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
  26. package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
  27. package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
  28. package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
  29. package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
  30. package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
  31. package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
  32. package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
  33. package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
  34. package/dist/browser/import-PTT63AGK.mjs +12 -0
  35. package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
  36. package/dist/browser/module-X5PGOE47.mjs +4288 -0
  37. package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
  38. package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
  39. package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
  40. package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
  41. package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
  42. package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
  43. package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
  44. package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
  45. package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
  46. package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
  47. package/dist/chunk-3MXV3VHR.mjs +46 -0
  48. package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
  49. package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
  50. package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
  51. package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
  52. package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
  53. package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
  54. package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
  55. package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
  56. package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
  57. package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
  58. package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
  59. package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
  60. package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
  61. package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
  62. package/dist/chunk-BTPGQC6Z.mjs +27 -0
  63. package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
  64. package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
  65. package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
  66. package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
  67. package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
  68. package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
  69. package/dist/chunk-KAKQ74LS.mjs +53 -0
  70. package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
  71. package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
  72. package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
  73. package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
  74. package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
  75. package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
  76. package/dist/chunk-LQG5G2LY.mjs +39 -0
  77. package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
  78. package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
  79. package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
  80. package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
  81. package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
  82. package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
  83. package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
  84. package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
  85. package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
  86. package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
  87. package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
  88. package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
  89. package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
  90. package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
  91. package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
  92. package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
  93. package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
  94. package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
  95. package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
  96. package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
  97. package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
  98. package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
  99. package/dist/chunk-ZXNNR73R.mjs +31 -0
  100. package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
  101. package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
  102. package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
  103. package/dist/client.d.ts +5 -5
  104. package/dist/client.mjs +2 -0
  105. package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
  106. package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
  107. package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
  108. package/dist/direct-executor-QLVHMELJ.mjs +24 -0
  109. package/dist/distribution/coordinator/etcd.mjs +1 -1
  110. package/dist/distribution/transport/grpc.mjs +1 -1
  111. package/dist/distribution.d.ts +4 -4
  112. package/dist/distribution.mjs +150 -255
  113. package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
  114. package/dist/embeddings/openai.mjs +2 -2
  115. package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
  116. package/dist/gzip-F4HMIEEQ.mjs +1 -0
  117. package/dist/import-DX55PN4G.mjs +41 -0
  118. package/dist/index.browser.mjs +7 -4
  119. package/dist/index.d.ts +539 -5
  120. package/dist/index.mjs +22 -16
  121. package/dist/index.react-native.mjs +19 -0
  122. package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
  123. package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
  124. package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
  125. package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
  126. package/dist/partial-words.mjs +5 -5
  127. package/dist/pattern.mjs +3 -3
  128. package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
  129. package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
  130. package/dist/pool-UPV5PVPY.mjs +298 -0
  131. package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
  132. package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
  133. package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
  134. package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
  135. package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
  136. package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
  137. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
  138. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
  139. package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
  140. package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
  141. package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
  142. package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
  143. package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
  144. package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
  145. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
  146. package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
  147. package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
  148. package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
  149. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
  150. package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
  151. package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
  152. package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
  153. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
  154. package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
  155. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
  156. package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
  157. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
  158. package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
  159. package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
  160. package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
  161. package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
  162. package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
  163. package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
  164. package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
  165. package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
  166. package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
  167. package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
  168. package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
  169. package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
  170. package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
  171. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
  172. package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
  173. package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
  174. package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
  175. package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
  176. package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
  177. package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
  178. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  179. package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
  180. package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
  181. package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
  182. package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
  183. package/dist/react-native/partial-words.mjs +733 -0
  184. package/dist/react-native/pattern.mjs +2442 -0
  185. package/dist/react-native/synonyms.mjs +460 -0
  186. package/dist/react.d.ts +7 -7
  187. package/dist/react.mjs +2 -0
  188. package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
  189. package/dist/rerankers/http.mjs +2 -2
  190. package/dist/restore-queue-BQWWNNQI.mjs +41 -0
  191. package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
  192. package/dist/server.d.ts +8 -8
  193. package/dist/server.mjs +32 -28
  194. package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
  195. package/dist/synonyms.mjs +3 -3
  196. package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
  197. package/dist/thread-I7DVHHW7.mjs +31 -0
  198. package/dist/vector/search-worker.mjs +7 -5
  199. package/dist/workers/entry.mjs +9 -9
  200. package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
  201. package/package.json +88 -7
  202. package/dist/analysis-change-D5APH4Z3.mjs +0 -34
  203. package/dist/browser/import-FTMLTPGQ.mjs +0 -9
  204. package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
  205. package/dist/import-6KXFY6XI.mjs +0 -35
  206. package/dist/thread-OV6T555H.mjs +0 -28
  207. /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
@@ -1,13 +1,65 @@
1
- import { spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-K7UFXEJS.mjs';
3
- import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, BLOCK_MAX_BYTES, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, QUERY_STAGED, grownByteLength, createVectorBlock, ensureBlockSlots, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-U77XQJOY.mjs';
4
- export { releaseLocksHeldBy } from './chunk-U77XQJOY.mjs';
5
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-BTMRV5MY.mjs';
6
- import { createVectorFileReader } from './chunk-FHO5N7FD.mjs';
7
- import './chunk-OIZS5KIX.mjs';
8
- import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, compareCodePoints, UNPLACED_FLAG_CHUNK_ORDINALS, MAX_VECTOR_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, resolveWorkerCount, resolveWorkerEntry, detectRuntime, noteThreadSlotLive, noteThreadSlotGone, WORKER_YOUNG_GENERATION_MB, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-QNDWMPV2.mjs';
9
- import './chunk-75PMIFQA.mjs';
10
- import { NarsilError, ErrorCodes, isNarsilError } from './chunk-IFGTRRPU.mjs';
1
+ import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, yieldBuildTurn, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, emptyFieldBeforeRestore, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, recalibrateFromStore, signatureOf, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, allLiveDocIds, fieldHandlesOf, releaseReplacedVectors, adoptGraph, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, DRIFT_CHECK_GROWTH_FACTOR, adoptLoadedCalibration, planCheckpoint, deserialize, RECALIBRATION_DRIFT_RATIO, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-GV5KFQ6M.mjs';
2
+ import './chunk-PSWKQVVT.mjs';
3
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, osqBitsOf, createOsqQuantizer, ordinalFilterValues, openGraphState, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, fixedView, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_SLOTS, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, threadedNativeCore, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, nativeStoreScratchBytes, noteFallback, toDistance, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, grownByteLength, createVectorBlock, ensureBlockSlots, sharedMemoryAvailable, createHNSWIndex, topLayerInParts, shuffleInPlace, requireWholeNeighbourLists, createCentroidSums, shuffleStep, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, nativeStoreFor, withNativeThreadSlot, nativeMetricCode, NATIVE_OK, releaseLocksHeldBy } from './chunk-HURV3ASY.mjs';
4
+ export { releaseLocksHeldBy } from './chunk-HURV3ASY.mjs';
5
+ import { NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, isDead } from './chunk-LQ6Q4L5B.mjs';
6
+ import { createVectorFileReader } from './chunk-UFNCLEQM.mjs';
7
+ import { createDocIdMap, turnsAreBudgeted, yieldBackgroundTurn, chargeHeapBytes, holdWritesAndBackgroundWork, backgroundTurnIsSpent, turnIsSpent, yieldTurn, noteThreadSlotLive, noteThreadSlotGone } from './chunk-LQAKO5QV.mjs';
8
+ import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, resolveWorkerCount, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-L4M7QJNP.mjs';
9
+ import './chunk-AM7RSBPF.mjs';
10
+ import './chunk-TJMSKBEM.mjs';
11
+ import './chunk-KWRW5U65.mjs';
12
+ import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
13
+ import './chunk-AAAETEJK.mjs';
14
+ import './chunk-NFGBOI46.mjs';
15
+ import { NarsilError, ErrorCodes, isNarsilError } from './chunk-YH7W556M.mjs';
16
+
17
+ // src/engine/vector-maintenance.ts
18
+ function resolveVectorIndexes(manager, indexName, fieldName) {
19
+ const all = manager.getVectorIndexes();
20
+ if (fieldName === void 0) return all;
21
+ const idx = all.get(fieldName);
22
+ if (!idx) {
23
+ throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Vector field "${fieldName}" not found in index "${indexName}"`, {
24
+ indexName,
25
+ fieldName
26
+ });
27
+ }
28
+ const single = /* @__PURE__ */ new Map();
29
+ single.set(fieldName, idx);
30
+ return single;
31
+ }
32
+ function compactVectors(manager, indexName, fieldName) {
33
+ const indexes = resolveVectorIndexes(manager, indexName, fieldName);
34
+ for (const [, idx] of indexes) {
35
+ idx.compact();
36
+ }
37
+ }
38
+ async function optimizeVectors(manager, indexName, fieldName) {
39
+ const indexes = resolveVectorIndexes(manager, indexName, fieldName);
40
+ for (const [, idx] of indexes) {
41
+ await idx.optimize();
42
+ }
43
+ }
44
+ function getVectorMaintenanceStatus(manager) {
45
+ const indexes = manager.getVectorIndexes();
46
+ const results = [];
47
+ for (const [field, idx] of indexes) {
48
+ const status = idx.maintenanceStatus();
49
+ results.push({
50
+ fieldName: field,
51
+ tombstoneRatio: status.tombstoneRatio,
52
+ graphCount: status.graphCount,
53
+ bufferSize: status.bufferSize,
54
+ building: status.building,
55
+ waitingForMemory: status.waitingForMemory,
56
+ loadingFromFiles: status.loadingFromFiles === true,
57
+ estimatedCompactMs: status.estimatedCompactMs,
58
+ estimatedOptimizeMs: status.estimatedOptimizeMs
59
+ });
60
+ }
61
+ return results;
62
+ }
11
63
 
12
64
  // src/core/heap.ts
13
65
  function createMaxHeap(compare) {
@@ -218,10 +270,10 @@ function createSharedVectorSearcher(options) {
218
270
  }
219
271
  function ordinalSpace() {
220
272
  const store = view.store;
221
- const layoutGeneration = ordinalLayoutGeneration();
273
+ const layoutGeneration2 = ordinalLayoutGeneration();
222
274
  return {
223
275
  fieldName,
224
- layoutGeneration,
276
+ layoutGeneration: layoutGeneration2,
225
277
  slots: store.slots,
226
278
  header: store.handles.header,
227
279
  ordinalOf: (docId) => view.ordinalOf(docId),
@@ -332,7 +384,7 @@ function createSharedVectorSearcher(options) {
332
384
  );
333
385
  }
334
386
  for (let attempt = 1; ; attempt++) {
335
- const layoutGeneration = ordinalLayoutGeneration();
387
+ const layoutGeneration2 = ordinalLayoutGeneration();
336
388
  const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
337
389
  if (view.graphRetired) {
338
390
  throw new NarsilError(
@@ -341,7 +393,7 @@ function createSharedVectorSearcher(options) {
341
393
  { fieldName }
342
394
  );
343
395
  }
344
- if (ordinalLayoutGeneration() === layoutGeneration) return Promise.resolve(answerOf(outcome));
396
+ if (ordinalLayoutGeneration() === layoutGeneration2) return Promise.resolve(answerOf(outcome));
345
397
  if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
346
398
  throw new NarsilError(
347
399
  ErrorCodes.WORKER_BUSY,
@@ -719,11 +771,11 @@ function openSharedVectorField(initial, threadSlot) {
719
771
  return store.docIdAt(ordinal);
720
772
  }
721
773
  function ordinalOf(docId) {
722
- const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
723
- if (layoutGeneration !== scannedLayoutGeneration) {
774
+ const layoutGeneration2 = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
775
+ if (layoutGeneration2 !== scannedLayoutGeneration) {
724
776
  ordinals = null;
725
777
  scanned = 0;
726
- scannedLayoutGeneration = layoutGeneration;
778
+ scannedLayoutGeneration = layoutGeneration2;
727
779
  }
728
780
  const slots = store.slots;
729
781
  if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
@@ -799,7 +851,7 @@ function appendDocId(handles, ordinal, docId) {
799
851
  Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
800
852
  }
801
853
  function resetDocIds(handles) {
802
- new Uint32Array(handles.docIdOffsets).fill(0);
854
+ new Uint32Array(handles.docIdOffsets, 0, handles.docIdOffsets.byteLength / 4).fill(0);
803
855
  Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
804
856
  }
805
857
 
@@ -957,11 +1009,12 @@ function ensureFloatSlot(store, ordinal) {
957
1009
  if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
958
1010
  }
959
1011
  function ensureOrdinal(store, ordinal, cold) {
960
- if (ordinal >= MAX_VECTOR_ORDINALS) {
1012
+ const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
1013
+ if (ordinal >= maxOrdinals) {
961
1014
  throw new NarsilError(
962
1015
  ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
963
- `A vector field holds at most ${MAX_VECTOR_ORDINALS} vectors, deleted ones included, until it is rebuilt`,
964
- { ordinal, maxOrdinals: MAX_VECTOR_ORDINALS }
1016
+ `A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
1017
+ { ordinal, maxOrdinals }
965
1018
  );
966
1019
  }
967
1020
  const { handles } = store;
@@ -975,7 +1028,7 @@ function ensureOrdinal(store, ordinal, cold) {
975
1028
  const diskFileBytes = handles.diskFile.byteLength;
976
1029
  growBufferTo(handles.diskFile, (ordinal + 1) * 4);
977
1030
  if (handles.diskFile.byteLength > diskFileBytes) {
978
- new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
1031
+ new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY, diskFileBytes / 4);
979
1032
  }
980
1033
  growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
981
1034
  if (!cold) ensureFloatSlot(store, ordinal);
@@ -987,7 +1040,7 @@ function bufferGrowthBytes(buffer, bytes) {
987
1040
  function growthBytesFor(store, vectors) {
988
1041
  const { handles } = store;
989
1042
  const slots = Atomics.load(handles.header, STORE_SLOTS);
990
- const last = Math.min(slots + vectors, MAX_VECTOR_ORDINALS) - 1;
1043
+ const last = Math.min(slots + vectors, VECTOR_MEMORY_LIMITS.maxOrdinals) - 1;
991
1044
  if (last < slots) return 0;
992
1045
  const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
993
1046
  let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
@@ -1115,8 +1168,8 @@ var UNKNOWN_PARTITION = -1;
1115
1168
  var NO_PARTITION = -2;
1116
1169
  function createVectorStore(options) {
1117
1170
  const codeBits = options?.codeBits ?? null;
1118
- const blockBytes = options?.blockBytes ?? BLOCK_MAX_BYTES;
1119
- const docToOrd = /* @__PURE__ */ new Map();
1171
+ const blockBytes = Math.min(options?.blockBytes ?? VECTOR_MEMORY_LIMITS.blockBytes, VECTOR_MEMORY_LIMITS.blockBytes);
1172
+ const docToOrd = createDocIdMap();
1120
1173
  const ordToDoc = [];
1121
1174
  const replacedOrdinals = /* @__PURE__ */ new Set();
1122
1175
  const pins = createRetiredVectorPins();
@@ -1338,9 +1391,9 @@ function createVectorStore(options) {
1338
1391
  clear() {
1339
1392
  forgetDocuments();
1340
1393
  if (open === null) return;
1341
- open.present.fill(0);
1342
- new Uint8Array(open.handles.codePresent).fill(0);
1343
- new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
1394
+ open.present.fill(0, 0, open.handles.present.byteLength);
1395
+ new Uint8Array(open.handles.codePresent, 0, open.handles.codePresent.byteLength).fill(0);
1396
+ new Int32Array(open.handles.diskFile, 0, open.handles.diskFile.byteLength / 4).fill(IN_MEMORY);
1344
1397
  open.handles.centroid.fill(0);
1345
1398
  if (open.handles.vectorFiles.length > 0) {
1346
1399
  open.handles.vectorFiles.length = 0;
@@ -1400,6 +1453,108 @@ function createVectorStore(options) {
1400
1453
  };
1401
1454
  }
1402
1455
 
1456
+ // src/vector/vector-index/graph-pass.ts
1457
+ function holdGraph(state, graph) {
1458
+ const current = state.graphsHeldOffThread;
1459
+ if (current !== null) {
1460
+ current.graphs.push(graph);
1461
+ return current;
1462
+ }
1463
+ let release = () => void 0;
1464
+ const released = new Promise((resolve) => {
1465
+ release = resolve;
1466
+ });
1467
+ const held = { graphs: [graph], released, release, writesAfter: [] };
1468
+ state.graphsHeldOffThread = held;
1469
+ return held;
1470
+ }
1471
+ function applyWritesHeldBack(held) {
1472
+ const writes = held.writesAfter.splice(0);
1473
+ let firstError = null;
1474
+ for (const write of writes) {
1475
+ try {
1476
+ write();
1477
+ } catch (error) {
1478
+ firstError ??= error;
1479
+ }
1480
+ }
1481
+ if (firstError !== null) throw firstError;
1482
+ }
1483
+ function releaseGraph(state, held, graph) {
1484
+ const at = held.graphs.indexOf(graph);
1485
+ if (at !== -1) held.graphs.splice(at, 1);
1486
+ if (held.graphs.length > 0) return;
1487
+ if (state.graphsHeldOffThread === held) state.graphsHeldOffThread = null;
1488
+ try {
1489
+ applyWritesHeldBack(held);
1490
+ } finally {
1491
+ held.release();
1492
+ }
1493
+ }
1494
+ async function connectGraph(state, graph) {
1495
+ const offThread = graph.connectOffThread();
1496
+ if (offThread === null) {
1497
+ await graph.connectOnThisThread(turnsAreBudgeted);
1498
+ return;
1499
+ }
1500
+ const held = holdGraph(state, graph);
1501
+ let connected = false;
1502
+ try {
1503
+ connected = await offThread;
1504
+ } finally {
1505
+ releaseGraph(state, held, graph);
1506
+ }
1507
+ if (!connected && !state.disposed) await graph.connectOnThisThread(true);
1508
+ }
1509
+ async function untilGraphReleased(state) {
1510
+ for (; ; ) {
1511
+ const held = state.graphsHeldOffThread;
1512
+ if (held === null) return;
1513
+ await held.released;
1514
+ }
1515
+ }
1516
+ function waitOutGraphPass(state) {
1517
+ const held = state.graphsHeldOffThread;
1518
+ if (held === null) return;
1519
+ for (const graph of held.graphs) graph.exclusively(() => void 0);
1520
+ applyWritesHeldBack(held);
1521
+ }
1522
+ function withGraphFree(state, operation) {
1523
+ return (...args) => {
1524
+ waitOutGraphPass(state);
1525
+ return operation(...args);
1526
+ };
1527
+ }
1528
+ function onceGraphFree(state, operation) {
1529
+ return async (...args) => {
1530
+ await untilGraphReleased(state);
1531
+ return operation(...args);
1532
+ };
1533
+ }
1534
+ function afterGraphPass(state, write) {
1535
+ const held = state.graphsHeldOffThread;
1536
+ if (held === null) write();
1537
+ else held.writesAfter.push(write);
1538
+ }
1539
+
1540
+ // src/vector/vector-index/search-writes.ts
1541
+ function noteSearchedDocWritten(state, docId) {
1542
+ for (const watch of state.searchWatches) watch.written.add(docId);
1543
+ }
1544
+ function moveOutOfBuffer(state, docId) {
1545
+ if (!state.buffer.delete(docId)) return;
1546
+ for (const watch of state.searchWatches) watch.leftBuffer.add(docId);
1547
+ }
1548
+ async function watchingSearch(state, search2) {
1549
+ const watch = { written: /* @__PURE__ */ new Set(), leftBuffer: /* @__PURE__ */ new Set() };
1550
+ state.searchWatches.add(watch);
1551
+ try {
1552
+ return await search2(watch);
1553
+ } finally {
1554
+ state.searchWatches.delete(watch);
1555
+ }
1556
+ }
1557
+
1403
1558
  // src/vector/search-pool/constants.ts
1404
1559
  var VECTOR_POOL_DRAIN_TIMEOUT_MS = 1e4;
1405
1560
 
@@ -1465,34 +1620,8 @@ function rejectPending(slot, err) {
1465
1620
  slot.pending.clear();
1466
1621
  }
1467
1622
 
1468
- // src/workers/resource-limits.ts
1469
- function workerResourceLimits() {
1470
- return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
1471
- }
1472
-
1473
1623
  // src/vector/search-pool/spawn.ts
1474
- function resolveWorkerEntryPoint() {
1475
- return resolveWorkerEntry(import.meta.url, /\/src\/vector\/search-pool\/[^/]+$/, "vector/search-worker.mjs");
1476
- }
1477
1624
  async function spawnWorker(entryPoint) {
1478
- if (entryPoint === null) return null;
1479
- const runtime = detectRuntime();
1480
- if (runtime.supportsWorkerThreads) {
1481
- try {
1482
- return await spawnNodeWorker(new URL(entryPoint), workerResourceLimits());
1483
- } catch {
1484
- return null;
1485
- }
1486
- }
1487
- if (runtime.supportsWebWorkers) {
1488
- try {
1489
- const WorkerCtor = globalThis.Worker;
1490
- if (typeof WorkerCtor !== "function") return null;
1491
- return new WorkerCtor(entryPoint, { type: "module" });
1492
- } catch {
1493
- return null;
1494
- }
1495
- }
1496
1625
  return null;
1497
1626
  }
1498
1627
  function listen(worker, handler) {
@@ -1526,12 +1655,11 @@ function graphSearchOptionsOf(options) {
1526
1655
  };
1527
1656
  }
1528
1657
  async function createVectorSearchPool(requestedCount) {
1529
- const entryPoint = resolveWorkerEntryPoint();
1530
1658
  const count = resolveWorkerCount(requestedCount);
1531
1659
  const slots = [];
1532
1660
  const sharedFields = /* @__PURE__ */ new Map();
1533
1661
  for (let i = 0; i < count; i++) {
1534
- const worker = await spawnWorker(entryPoint);
1662
+ const worker = await spawnWorker();
1535
1663
  if (worker === null) break;
1536
1664
  const slot = createSlot(worker, searchPoolScratchSlot(i));
1537
1665
  listen(worker, (raw) => {
@@ -2023,7 +2151,9 @@ async function insertIntoGraph(state, graph, docIds, admit) {
2023
2151
  await flush();
2024
2152
  await Promise.all(pending);
2025
2153
  if (state.disposed) return false;
2026
- graph.connectUnreached();
2154
+ if (turnsAreBudgeted) await connectGraph(state, graph);
2155
+ else graph.connectUnreached();
2156
+ if (state.disposed) return false;
2027
2157
  noteWrite(state);
2028
2158
  return true;
2029
2159
  }
@@ -2053,7 +2183,7 @@ async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fa
2053
2183
  continue;
2054
2184
  }
2055
2185
  if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
2056
- if (graph === state.hnsw) state.buffer.delete(docId);
2186
+ if (graph === state.hnsw) moveOutOfBuffer(state, docId);
2057
2187
  }
2058
2188
  if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
2059
2189
  fallback.shortAnswerWarned = true;
@@ -2069,22 +2199,81 @@ async function insertLocally(state, graph, docIds, admit) {
2069
2199
  if (!admit(docId)) continue;
2070
2200
  const ordinal = state.store.getOrdinal(docId);
2071
2201
  if (ordinal === void 0) continue;
2072
- graph.insertOrdinal(ordinal);
2073
- if (graph === state.hnsw) state.buffer.delete(docId);
2202
+ if (turnsAreBudgeted) {
2203
+ graph.insertOrdinal(ordinal, true);
2204
+ if (graph.holdsUnlinkedNode) await connectGraph(state, graph);
2205
+ if (state.disposed) return false;
2206
+ } else {
2207
+ graph.insertOrdinal(ordinal);
2208
+ }
2209
+ if (graph === state.hnsw) moveOutOfBuffer(state, docId);
2074
2210
  count += 1;
2075
- if (count % BUILD_CHUNK_SIZE === 0) {
2211
+ if (turnsAreBudgeted) {
2212
+ if (!backgroundTurnIsSpent()) continue;
2213
+ noteWrite(state);
2214
+ await yieldBackgroundTurn();
2215
+ } else if (count % BUILD_CHUNK_SIZE === 0) {
2076
2216
  noteWrite(state);
2077
2217
  await yieldBuildTurn(state);
2078
2218
  }
2079
2219
  }
2080
2220
  if (state.disposed) return false;
2081
2221
  if (count > 0) {
2082
- graph.connectUnreached();
2222
+ if (turnsAreBudgeted) await connectGraph(state, graph);
2223
+ else graph.connectUnreached();
2083
2224
  noteWrite(state);
2084
2225
  }
2085
2226
  return true;
2086
2227
  }
2087
2228
 
2229
+ // src/vector/vector-index/turns.ts
2230
+ function turnDue(steps) {
2231
+ return steps % BUILD_STEPS_BETWEEN_CLOCK_CHECKS === 0 && backgroundTurnIsSpent();
2232
+ }
2233
+ async function forEachInTurns(items, visit, stillWanted) {
2234
+ let steps = 0;
2235
+ for (const item of items) {
2236
+ steps += 1;
2237
+ if (turnDue(steps)) {
2238
+ await yieldBackgroundTurn();
2239
+ if (!stillWanted()) return false;
2240
+ }
2241
+ visit(item);
2242
+ }
2243
+ return true;
2244
+ }
2245
+ async function copyInTurns(items, stillWanted) {
2246
+ const copied = [];
2247
+ const finished = await forEachInTurns(items, (item) => copied.push(item), stillWanted);
2248
+ return finished ? copied : null;
2249
+ }
2250
+ async function shuffledLiveDocIdsInTurns(state) {
2251
+ const stillWanted = () => !state.disposed;
2252
+ const docIds = await copyInTurns(allLiveDocIds(state), stillWanted);
2253
+ if (docIds === null) return null;
2254
+ let stream = docIds.length;
2255
+ for (let i = docIds.length - 1; i > 0; i--) {
2256
+ stream = shuffleStep(docIds, i, stream);
2257
+ if (turnDue(i)) {
2258
+ await yieldBackgroundTurn();
2259
+ if (!stillWanted()) return null;
2260
+ }
2261
+ }
2262
+ return docIds;
2263
+ }
2264
+ async function liveOrdinalsInTurns(state) {
2265
+ const ordinals = [];
2266
+ for (let ordinal = 0; ordinal < state.store.slots; ordinal++) {
2267
+ if (turnDue(ordinal + 1)) {
2268
+ await yieldBackgroundTurn();
2269
+ if (state.disposed) return null;
2270
+ }
2271
+ const docId = state.store.docIdForOrdinal(ordinal);
2272
+ if (docId !== void 0 && !state.tombstones.has(docId)) ordinals.push(ordinal);
2273
+ }
2274
+ return Int32Array.from(ordinals);
2275
+ }
2276
+
2088
2277
  // src/vector/vector-index/drift.ts
2089
2278
  function liveVectorMeanOf(sums) {
2090
2279
  const centroid = centroidOfCentroidSums(sums);
@@ -2103,7 +2292,8 @@ function liveVectorMeanOf(sums) {
2103
2292
  }
2104
2293
  async function measureLiveVectors(state, quantizer) {
2105
2294
  const sums = createCentroidSums(state.dimension, state.metric);
2106
- const ordinals = liveOrdinals(state);
2295
+ const ordinals = turnsAreBudgeted ? await liveOrdinalsInTurns(state) : liveOrdinals(state);
2296
+ if (ordinals === null) return null;
2107
2297
  for (let start = 0; start < ordinals.length; start += RECALIBRATION_CHUNK_VECTORS) {
2108
2298
  if (start > 0) await yieldBuildTurn(state);
2109
2299
  if (state.disposed || state.osq !== quantizer) return null;
@@ -2210,11 +2400,13 @@ async function shareFilesWithEveryThread(state, paths) {
2210
2400
  }
2211
2401
  async function releaseLocations(state, paths, locations) {
2212
2402
  const fileIndexes = await shareFilesWithEveryThread(state, paths);
2403
+ await untilGraphReleased(state);
2213
2404
  if (state.disposed) return;
2214
2405
  let count = 0;
2215
2406
  for (const [docId, location] of locations) {
2216
2407
  if (count > 0 && count % DISK_ADOPTION_YIELD_INTERVAL === 0) {
2217
2408
  await yieldBuildTurn(state);
2409
+ await untilGraphReleased(state);
2218
2410
  if (state.disposed) return;
2219
2411
  }
2220
2412
  count += 1;
@@ -2265,8 +2457,6 @@ function noteWaitingForMemory(owner, work) {
2265
2457
  if (work === null) waiting.delete(owner);
2266
2458
  else waiting.set(owner, work);
2267
2459
  }
2268
- function chargeHeapBytes(bytes) {
2269
- }
2270
2460
 
2271
2461
  // src/vector/vector-index/memory-charge.ts
2272
2462
  function admitShuffledRebuild(state) {
@@ -2298,20 +2488,27 @@ function releaseMemoryCharges(state) {
2298
2488
  }
2299
2489
 
2300
2490
  // src/vector/vector-index/build.ts
2491
+ function bufferIfTheGraphLacks(state, graph, docId) {
2492
+ const ordinal = state.store.getOrdinal(docId);
2493
+ if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) moveOutOfBuffer(state, docId);
2494
+ else state.buffer.add(docId);
2495
+ }
2301
2496
  function bufferWhatTheGraphLacks(state, graph) {
2302
- for (const docId of allLiveDocIds(state)) {
2303
- const ordinal = state.store.getOrdinal(docId);
2304
- if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
2305
- else state.buffer.add(docId);
2306
- }
2497
+ for (const docId of allLiveDocIds(state)) bufferIfTheGraphLacks(state, graph, docId);
2498
+ }
2499
+ async function bufferWhatTheGraphLacksInTurns(state, graph) {
2500
+ const visit = (docId) => bufferIfTheGraphLacks(state, graph, docId);
2501
+ await forEachInTurns(allLiveDocIds(state), visit, () => !state.disposed && state.hnsw === graph);
2307
2502
  }
2308
2503
  async function buildGraphFromStore(state) {
2309
2504
  const previous = state.hnsw;
2505
+ const order = turnsAreBudgeted ? await shuffledLiveDocIdsInTurns(state) : shuffleInPlace([...allLiveDocIds(state)]);
2506
+ if (order === null || state.hnsw !== previous) return false;
2310
2507
  const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.osq ?? void 0);
2311
2508
  state.freshGraph = graph;
2312
2509
  let completed = false;
2313
2510
  try {
2314
- completed = await insertIntoGraph(state, graph, shuffleInPlace([...allLiveDocIds(state)]), () => true);
2511
+ completed = await insertIntoGraph(state, graph, order, () => true);
2315
2512
  } finally {
2316
2513
  state.freshGraph = null;
2317
2514
  }
@@ -2322,7 +2519,8 @@ async function buildGraphFromStore(state) {
2322
2519
  adoptGraph(state, graph);
2323
2520
  state.shuffledBuildCount = graph.size;
2324
2521
  noteWrite(state);
2325
- bufferWhatTheGraphLacks(state, graph);
2522
+ if (turnsAreBudgeted) await bufferWhatTheGraphLacksInTurns(state, graph);
2523
+ else bufferWhatTheGraphLacks(state, graph);
2326
2524
  if (previous !== null) dropSharedGraph(state, previous);
2327
2525
  await releasePendingLocations(state);
2328
2526
  return true;
@@ -2426,7 +2624,7 @@ async function growGraph(state, graph, bufferSnapshot) {
2426
2624
  function triggerBuild(state) {
2427
2625
  if (state.building) return;
2428
2626
  state.building = true;
2429
- const bufferSnapshot = new Set(state.buffer);
2627
+ const bufferSnapshot = turnsAreBudgeted ? null : new Set(state.buffer);
2430
2628
  const existingGraph = state.hnsw;
2431
2629
  let settled = false;
2432
2630
  let timedOut = false;
@@ -2434,7 +2632,9 @@ function triggerBuild(state) {
2434
2632
  try {
2435
2633
  if (state.disposed) return;
2436
2634
  if (existingGraph !== null) {
2437
- await growGraph(state, existingGraph, bufferSnapshot);
2635
+ const snapshot = bufferSnapshot ?? await copyInTurns(state.buffer, () => !state.disposed && state.hnsw === existingGraph);
2636
+ if (snapshot === null) return;
2637
+ await growGraph(state, existingGraph, snapshot);
2438
2638
  return;
2439
2639
  }
2440
2640
  await promoteToGraph(state);
@@ -2509,7 +2709,8 @@ function scheduleBuild(state) {
2509
2709
  if (state.building || state.buildScheduled || state.disposed) return;
2510
2710
  if (!buildDue(state)) return;
2511
2711
  state.buildScheduled = true;
2512
- void yieldBuildTurn(state).then(() => {
2712
+ const turn = turnsAreBudgeted ? yieldBackgroundTurn() : yieldBuildTurn(state);
2713
+ void turn.then(() => {
2513
2714
  state.buildScheduled = false;
2514
2715
  triggerBuild(state);
2515
2716
  });
@@ -2935,26 +3136,35 @@ async function coldSearchConcurrently(state, hold, query, k, options) {
2935
3136
  }
2936
3137
 
2937
3138
  // src/vector/vector-index/flagged-documents.ts
2938
- var FlaggedDocuments = class extends Set {
3139
+ var FlaggedDocuments = class {
2939
3140
  constructor(store, layout) {
2940
- super();
2941
3141
  this.store = store;
2942
3142
  this.layout = layout;
2943
3143
  }
2944
- ordinals = /* @__PURE__ */ new Map();
3144
+ members = createDocIdMap();
3145
+ ordinals = createDocIdMap();
2945
3146
  handles = null;
2946
3147
  flags = new Uint8Array(0);
2947
3148
  chunks = null;
3149
+ get size() {
3150
+ return this.members.size;
3151
+ }
3152
+ has(docId) {
3153
+ return this.members.has(docId);
3154
+ }
3155
+ [Symbol.iterator]() {
3156
+ return this.members.keys()[Symbol.iterator]();
3157
+ }
2948
3158
  add(docId) {
2949
- super.add(docId);
3159
+ this.members.set(docId, true);
2950
3160
  const handles = this.openHandles();
2951
3161
  if (handles === null) return this;
2952
3162
  this.flagCurrentOrdinal(docId);
2953
- Atomics.store(handles.header, this.layout.countWord, this.size);
3163
+ Atomics.store(handles.header, this.layout.countWord, this.members.size);
2954
3164
  return this;
2955
3165
  }
2956
3166
  delete(docId) {
2957
- const held = super.delete(docId);
3167
+ const held = this.members.delete(docId);
2958
3168
  const handles = this.openHandles();
2959
3169
  if (handles === null) return held;
2960
3170
  const ordinal = this.ordinals.get(docId);
@@ -2962,11 +3172,11 @@ var FlaggedDocuments = class extends Set {
2962
3172
  this.ordinals.delete(docId);
2963
3173
  this.unflag(ordinal);
2964
3174
  }
2965
- Atomics.store(handles.header, this.layout.countWord, this.size);
3175
+ Atomics.store(handles.header, this.layout.countWord, this.members.size);
2966
3176
  return held;
2967
3177
  }
2968
3178
  clear() {
2969
- super.clear();
3179
+ this.members.clear();
2970
3180
  const handles = this.openHandles();
2971
3181
  if (handles !== null) {
2972
3182
  for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
@@ -2996,7 +3206,7 @@ var FlaggedDocuments = class extends Set {
2996
3206
  const chunksOf = this.layout.chunksOf;
2997
3207
  this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
2998
3208
  this.ordinals.clear();
2999
- for (const docId of this) this.flagCurrentOrdinal(docId);
3209
+ for (const docId of this.members.keys()) this.flagCurrentOrdinal(docId);
3000
3210
  return handles;
3001
3211
  }
3002
3212
  };
@@ -3141,46 +3351,64 @@ function fetchedOutcome(state, results, minSimilarity, filter) {
3141
3351
  function scannedOutcome(scan) {
3142
3352
  return { results: scan.results, matched: scan.matched, matchedExact: true };
3143
3353
  }
3144
- function liveCandidates(state, candidates) {
3354
+ function liveOrdinalOf(state, docId, filter) {
3355
+ if (state.tombstones.has(docId)) return void 0;
3356
+ const ordinal = state.store.getOrdinal(docId);
3357
+ if (ordinal === void 0 || filter !== void 0 && !ordinalFilterHas(filter, ordinal)) return void 0;
3358
+ return ordinal;
3359
+ }
3360
+ function liveCandidates(state, candidates, filter) {
3145
3361
  const docIds = [];
3146
3362
  const ordinals = [];
3147
3363
  for (const docId of candidates) {
3148
- if (state.tombstones.has(docId)) continue;
3149
- const ordinal = state.store.getOrdinal(docId);
3364
+ const ordinal = liveOrdinalOf(state, docId, filter);
3150
3365
  if (ordinal === void 0) continue;
3151
3366
  docIds.push(docId);
3152
3367
  ordinals.push(ordinal);
3153
3368
  }
3154
3369
  return { docIds, ordinals };
3155
3370
  }
3156
- function scanThroughTheCore(state, query, k, metric, minSimilarity, scanned) {
3157
- const distances = nativeScoresOf(state.store.handles, query, metric, scanned.ordinals);
3158
- if (distances === null) return null;
3159
- const heap = createBoundedMaxHeap(highScoreFirst, k);
3160
- let matched = 0;
3161
- for (let i = 0; i < scanned.ordinals.length; i++) {
3162
- if (distances[i] === Number.POSITIVE_INFINITY) continue;
3163
- const score = toScore(distances[i], metric);
3164
- if (score < minSimilarity) continue;
3165
- matched++;
3166
- heap.push({ docId: scanned.docIds[i], score });
3167
- }
3168
- return { results: heap.toSortedArray().reverse(), matched };
3371
+ function createScanTally(capacity) {
3372
+ return {
3373
+ best: createBoundedMaxHeap(highScoreFirst, capacity),
3374
+ capacity,
3375
+ matched: 0,
3376
+ bestDiscardedScore: Number.NEGATIVE_INFINITY
3377
+ };
3169
3378
  }
3170
- function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
3171
- const scanned = liveCandidates(state, candidates);
3172
- const fromTheCore = scanThroughTheCore(state, query, k, metric, minSimilarity, scanned);
3173
- if (fromTheCore !== null) return fromTheCore;
3379
+ function admitScore(tally, docId, score) {
3380
+ tally.matched++;
3381
+ const worst = tally.best.size < tally.capacity ? void 0 : tally.best.peek();
3382
+ if (worst !== void 0 && score < worst.score) {
3383
+ if (score > tally.bestDiscardedScore) tally.bestDiscardedScore = score;
3384
+ return;
3385
+ }
3386
+ const result = { docId, score };
3387
+ tally.best.push(result);
3388
+ if (worst === void 0) return;
3389
+ const discarded = highScoreFirst(result, worst) < 0 ? worst.score : score;
3390
+ if (discarded > tally.bestDiscardedScore) tally.bestDiscardedScore = discarded;
3391
+ }
3392
+ function scoreCandidates2(state, query, metric, minSimilarity, docIds, ordinals, tally) {
3393
+ const count = ordinals.length;
3394
+ const distances = nativeScoresOf(state.store.handles, query, metric, ordinals);
3395
+ if (distances !== null) {
3396
+ for (let i = 0; i < count; i++) {
3397
+ if (distances[i] === Number.POSITIVE_INFINITY) continue;
3398
+ const score = toScore(distances[i], metric);
3399
+ if (score < minSimilarity) continue;
3400
+ admitScore(tally, docIds[i], score);
3401
+ }
3402
+ return;
3403
+ }
3174
3404
  noteFallback("score");
3175
3405
  const arenaQuery = state.store.prepareQueryArena(query);
3176
3406
  const queryMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
3177
- const heap = createBoundedMaxHeap(highScoreFirst, k);
3178
- let matched = 0;
3179
- for (let i = 0; i < scanned.docIds.length; i++) {
3180
- const docId = scanned.docIds[i];
3407
+ for (let i = 0; i < count; i++) {
3408
+ const docId = docIds[i];
3181
3409
  let score;
3182
3410
  if (arenaQuery) {
3183
- const distance = state.store.distanceFromArena(arenaQuery, scanned.ordinals[i], metric);
3411
+ const distance = state.store.distanceFromArena(arenaQuery, ordinals[i], metric);
3184
3412
  if (distance === Number.POSITIVE_INFINITY) continue;
3185
3413
  score = toScore(distance, metric);
3186
3414
  } else {
@@ -3201,16 +3429,29 @@ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
3201
3429
  }
3202
3430
  }
3203
3431
  if (score < minSimilarity) continue;
3204
- matched++;
3205
- heap.push({ docId, score });
3432
+ admitScore(tally, docId, score);
3206
3433
  }
3207
- return { results: heap.toSortedArray().reverse(), matched };
3434
+ }
3435
+ function bruteForceSearch(state, query, k, metric, minSimilarity, candidates, filter) {
3436
+ const scanned = liveCandidates(state, candidates, filter);
3437
+ const tally = createScanTally(k);
3438
+ scoreCandidates2(state, query, metric, minSimilarity, scanned.docIds, scanned.ordinals, tally);
3439
+ return { results: tally.best.toSortedArray().reverse(), matched: tally.matched };
3208
3440
  }
3209
3441
  function search(state, query, k, options) {
3210
3442
  handOverBuildTurns(state);
3211
3443
  return searchWithFilter(state, query, k, options, filterForOptions(state, options));
3212
3444
  }
3213
3445
  function searchWithFilter(state, query, k, options, filter) {
3446
+ const steps = searchSteps(state, query, k, options, filter);
3447
+ let step = steps.next();
3448
+ while (step.done !== true) {
3449
+ const { candidates, filter: remaining } = step.value;
3450
+ step = steps.next(bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates, remaining));
3451
+ }
3452
+ return step.value;
3453
+ }
3454
+ function* searchSteps(state, query, k, options, filter) {
3214
3455
  if (query.length !== state.dimension) {
3215
3456
  throw new NarsilError(
3216
3457
  ErrorCodes.VECTOR_DIMENSION_MISMATCH,
@@ -3219,7 +3460,7 @@ function searchWithFilter(state, query, k, options, filter) {
3219
3460
  );
3220
3461
  }
3221
3462
  const currentLiveSize = liveSize(state);
3222
- const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
3463
+ const { minSimilarity } = options;
3223
3464
  if (currentLiveSize === 0) return { results: [], matched: 0, matchedExact: true };
3224
3465
  if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
3225
3466
  if (k <= 0) return fetchedOutcome(state, [], minSimilarity, filter);
@@ -3228,26 +3469,233 @@ function searchWithFilter(state, query, k, options, filter) {
3228
3469
  }
3229
3470
  if (!state.hnsw) {
3230
3471
  const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
3231
- return scannedOutcome(bruteForceSearch(state, query, k, metric, minSimilarity, candidates));
3472
+ return scannedOutcome(yield { candidates, filter: void 0, graphResults: null });
3232
3473
  }
3233
3474
  if (filter) {
3234
3475
  const hnswLiveSize = state.hnsw.size;
3235
3476
  const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
3236
3477
  if (selectivity < state.filterThreshold) {
3237
- const scan = bruteForceSearch(state, query, k, metric, minSimilarity, filteredDocIds(state, filter));
3238
- return scannedOutcome(scan);
3478
+ return scannedOutcome(yield { candidates: filteredDocIds(state, filter), filter: void 0, graphResults: null });
3239
3479
  }
3240
3480
  }
3241
- const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
3242
- const results = state.buffer.size === 0 ? hnswResults : withBufferedVectors(state, query, k, options, filter, hnswResults, state.buffer);
3243
- return fetchedOutcome(state, results, minSimilarity, filter);
3481
+ const hnswResults = graphResultsOf(state, query, k, options, filter);
3482
+ if (state.buffer.size === 0) return fetchedOutcome(state, hnswResults, minSimilarity, filter);
3483
+ const scan = yield { candidates: state.buffer, filter, graphResults: hnswResults };
3484
+ return fetchedOutcome(state, mergeResults(hnswResults, scan.results, k), minSimilarity, filter);
3485
+ }
3486
+ function graphResultsOf(state, query, k, options, filter) {
3487
+ if (state.hnsw === null) return [];
3488
+ const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
3489
+ return state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
3244
3490
  }
3245
3491
  function withBufferedVectors(state, query, k, options, filter, graphResults, buffered) {
3246
- const candidates = bufferCandidates(state, buffered, filter);
3247
- const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
3492
+ const scan = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, buffered, filter);
3248
3493
  return mergeResults(graphResults, scan.results, k);
3249
3494
  }
3250
3495
 
3496
+ // src/vector/vector-index/search-in-turns.ts
3497
+ function layoutGeneration(state) {
3498
+ return Atomics.load(state.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
3499
+ }
3500
+ async function scanInTurns(state, query, capacity, options, request, chunkOfThisSearch) {
3501
+ const tally = createScanTally(capacity);
3502
+ const { docIds, ordinals } = chunkOfThisSearch;
3503
+ let length = 0;
3504
+ const scoreChunk = () => {
3505
+ const chunk = length === SEARCH_SCAN_CHUNK_VECTORS ? ordinals : ordinals.subarray(0, length);
3506
+ scoreCandidates2(state, query, options.metric, options.minSimilarity, docIds, chunk, tally);
3507
+ length = 0;
3508
+ };
3509
+ for (const docId of request.candidates) {
3510
+ const ordinal = liveOrdinalOf(state, docId, request.filter);
3511
+ if (ordinal === void 0) continue;
3512
+ docIds[length] = docId;
3513
+ ordinals[length] = ordinal;
3514
+ length++;
3515
+ if (length < SEARCH_SCAN_CHUNK_VECTORS) continue;
3516
+ scoreChunk();
3517
+ if (turnIsSpent()) await yieldTurn();
3518
+ }
3519
+ if (length > 0) scoreChunk();
3520
+ return {
3521
+ kept: tally.best.toSortedArray().reverse(),
3522
+ matched: tally.matched,
3523
+ bestDiscardedScore: tally.bestDiscardedScore
3524
+ };
3525
+ }
3526
+ async function passInTurns(state, query, k, options, filter, chunk) {
3527
+ const steps = searchSteps(state, query, k, options, filter);
3528
+ let graphResults = null;
3529
+ let scan = null;
3530
+ let step = steps.next();
3531
+ while (step.done !== true) {
3532
+ graphResults = step.value.graphResults;
3533
+ scan = await scanInTurns(state, query, k + SEARCH_REFILL_RESERVE, options, step.value, chunk);
3534
+ step = steps.next({ results: scan.kept.slice(0, k), matched: scan.matched });
3535
+ }
3536
+ return { outcome: step.value, graphResults, scan };
3537
+ }
3538
+ function rescoredSinceTheScan(state, query, k, options, filter, watch) {
3539
+ const touched = new Set(watch.written);
3540
+ for (const docId of watch.leftBuffer) touched.add(docId);
3541
+ return bruteForceSearch(state, query, k, options.metric, options.minSimilarity, touched, filter).results;
3542
+ }
3543
+ function outrunsEveryDiscard(results, k, bestDiscardedScore) {
3544
+ if (bestDiscardedScore === Number.NEGATIVE_INFINITY) return true;
3545
+ const lowest = results[k - 1];
3546
+ return lowest !== void 0 && lowest.score > bestDiscardedScore;
3547
+ }
3548
+ function settledOutcome(state, query, k, options, pass, watch) {
3549
+ const { written, leftBuffer } = watch;
3550
+ const { scan, graphResults } = pass;
3551
+ if (scan === null || written.size === 0 && leftBuffer.size === 0) return pass.outcome;
3552
+ if (graphResults === null) return written.size === 0 ? pass.outcome : null;
3553
+ const filter = filterForOptions(state, options);
3554
+ const graphTouched = graphResults.some((result) => written.has(result.docId));
3555
+ const graphPart = graphTouched ? graphResultsOf(state, query, k, options, filter) : graphResults;
3556
+ const survivors = scan.kept.filter((result) => !written.has(result.docId));
3557
+ const rescored = rescoredSinceTheScan(state, query, k, options, filter, watch);
3558
+ const results = mergeResults(graphPart, mergeResults(survivors, rescored, k), k);
3559
+ const refilled = survivors.length < scan.kept.length;
3560
+ if (refilled && !outrunsEveryDiscard(results, k, scan.bestDiscardedScore)) return null;
3561
+ return fetchedOutcome(state, results, options.minSimilarity, filter);
3562
+ }
3563
+ function attemptInTurns(state, query, k, options, filter, chunk) {
3564
+ const generation = liveSize(state) === 0 ? null : layoutGeneration(state);
3565
+ return watchingSearch(state, async (watch) => {
3566
+ const pass = await passInTurns(state, query, k, options, filter, chunk);
3567
+ if (state.disposed) return pass.outcome;
3568
+ if (generation !== null && layoutGeneration(state) !== generation) return null;
3569
+ return settledOutcome(state, query, k, options, pass, watch);
3570
+ });
3571
+ }
3572
+ async function searchWithFilterInTurns(state, query, k, options, filter) {
3573
+ let current = filter;
3574
+ const chunk = { docIds: [], ordinals: new Int32Array(SEARCH_SCAN_CHUNK_VECTORS) };
3575
+ for (let attempt = 1; ; attempt++) {
3576
+ const search2 = () => attemptInTurns(state, query, k, options, current, chunk);
3577
+ const outcome = attempt <= SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES ? await search2() : await holdWritesAndBackgroundWork(search2);
3578
+ if (outcome !== null) return outcome;
3579
+ current = filterForOptions(state, options);
3580
+ }
3581
+ }
3582
+ function searchFiltered(state, query, k, options, filter) {
3583
+ if (turnsAreBudgeted) return searchWithFilterInTurns(state, query, k, options, filter);
3584
+ return searchWithFilter(state, query, k, options, filter);
3585
+ }
3586
+
3587
+ // src/vector/vector-index/search-off-thread.ts
3588
+ function currentFilter(search2) {
3589
+ const { state, options, filter, confinedAtRevision } = search2;
3590
+ if (confinedAtRevision === null || state.revision === confinedAtRevision) return filter;
3591
+ return filterForOptions(state, options);
3592
+ }
3593
+ async function scoredChunk(search2, metric, ordinals, count) {
3594
+ const store = nativeStoreFor(search2.state.store.handles);
3595
+ if (store === null) return null;
3596
+ try {
3597
+ const scored = await withNativeThreadSlot(
3598
+ (slot) => search2.core.scoreAsync(store.handle, search2.query, nativeMetricCode(metric), ordinals, count, slot)
3599
+ );
3600
+ return scored.status === NATIVE_OK ? scored.distances : null;
3601
+ } catch {
3602
+ return null;
3603
+ }
3604
+ }
3605
+ async function scanOffThread(search2, candidates) {
3606
+ const { state, k, options } = search2;
3607
+ const { metric, minSimilarity } = options;
3608
+ const heap = createBoundedMaxHeap(highScoreFirst, k);
3609
+ let matched = 0;
3610
+ const docIds = [];
3611
+ const ordinals = new Int32Array(NATIVE_THREAD_SCAN_CHUNK_ORDINALS);
3612
+ const remaining = candidates[Symbol.iterator]();
3613
+ let exhausted = false;
3614
+ while (!exhausted) {
3615
+ docIds.length = 0;
3616
+ while (docIds.length < NATIVE_THREAD_SCAN_CHUNK_ORDINALS) {
3617
+ if (turnIsSpent()) await yieldTurn();
3618
+ const next = remaining.next();
3619
+ if (next.done === true) {
3620
+ exhausted = true;
3621
+ break;
3622
+ }
3623
+ if (state.tombstones.has(next.value)) continue;
3624
+ const ordinal = state.store.getOrdinal(next.value);
3625
+ if (ordinal === void 0) continue;
3626
+ ordinals[docIds.length] = ordinal;
3627
+ docIds.push(next.value);
3628
+ }
3629
+ if (docIds.length === 0) break;
3630
+ const distances = await scoredChunk(search2, metric, ordinals, docIds.length);
3631
+ if (distances === null) return null;
3632
+ for (let i = 0; i < docIds.length; i++) {
3633
+ if (distances[i] === Number.POSITIVE_INFINITY) continue;
3634
+ const score = toScore(distances[i], metric);
3635
+ if (score < minSimilarity) continue;
3636
+ matched++;
3637
+ heap.push({ docId: docIds[i], score });
3638
+ }
3639
+ }
3640
+ return { results: heap.toSortedArray().reverse(), matched };
3641
+ }
3642
+ function scansEveryCandidate(state, filter) {
3643
+ if (state.hnsw === null) return true;
3644
+ if (filter === void 0) return false;
3645
+ const graphSize = state.hnsw.size;
3646
+ const selectivity = graphSize > 0 ? filter.count / graphSize : 1;
3647
+ return selectivity < state.filterThreshold;
3648
+ }
3649
+ async function outcomeOffThread(search2) {
3650
+ const { state, query, k, options, filter } = search2;
3651
+ const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
3652
+ const graph = state.hnsw;
3653
+ if (graph === null || scansEveryCandidate(state, filter)) {
3654
+ const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
3655
+ const scan2 = await scanOffThread(search2, candidates);
3656
+ return scan2 === null ? null : { results: scan2.results, matched: scan2.matched, matchedExact: true };
3657
+ }
3658
+ const buffered = state.buffer.size === 0 ? null : [...state.buffer];
3659
+ const hits = await graph.searchOffThread(query, k, metric, minSimilarity, {
3660
+ filter,
3661
+ efSearch,
3662
+ oversample,
3663
+ rescoreWindow
3664
+ });
3665
+ if (hits === null) return null;
3666
+ const graphResults = hits.map((hit) => ({ docId: hit.docId, score: hit.score }));
3667
+ if (buffered === null) return fetchedOutcome(state, graphResults, minSimilarity, currentFilter(search2));
3668
+ const scan = await scanOffThread(search2, bufferCandidates(state, buffered, filter));
3669
+ if (scan === null) return null;
3670
+ return fetchedOutcome(state, mergeResults(graphResults, scan.results, k), minSimilarity, currentFilter(search2));
3671
+ }
3672
+ async function searchOnNativeThreads(search2) {
3673
+ const { state, query, k, options, filter } = search2;
3674
+ if (query.length !== state.dimension) {
3675
+ throw new NarsilError(
3676
+ ErrorCodes.VECTOR_DIMENSION_MISMATCH,
3677
+ `Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
3678
+ { expected: state.dimension, received: query.length }
3679
+ );
3680
+ }
3681
+ if (liveSize(state) === 0) return { results: [], matched: 0, matchedExact: true };
3682
+ if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
3683
+ if (k <= 0) return fetchedOutcome(state, [], options.minSimilarity, filter);
3684
+ if (state.buffer.size > 0 && !state.building && !state.buildScheduled) scheduleBuild(state);
3685
+ const generation = layoutGeneration(state);
3686
+ const settled = await watchingSearch(state, async (watch) => {
3687
+ const outcome = await outcomeOffThread(search2);
3688
+ if (outcome === null || watch.written.size > 0 || layoutGeneration(state) !== generation) return null;
3689
+ return outcome;
3690
+ });
3691
+ return settled ?? searchFiltered(state, query, k, options, filterForOptions(state, options));
3692
+ }
3693
+ function searchOffThread(state, query, k, options, filter, confinedAtRevision) {
3694
+ const core = threadedNativeCore();
3695
+ if (core === null) return null;
3696
+ return searchOnNativeThreads({ state, core, query, k, options, filter, confinedAtRevision });
3697
+ }
3698
+
3251
3699
  // src/vector/vector-index/writes.ts
3252
3700
  function createVectorWrites(state) {
3253
3701
  function validateDimension(vector) {
@@ -3279,17 +3727,23 @@ function createVectorWrites(state) {
3279
3727
  if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
3280
3728
  if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
3281
3729
  noteWrite(state);
3730
+ noteSearchedDocWritten(state, docId);
3282
3731
  state.store.setPartition(docId, partitionId);
3283
3732
  }
3284
3733
  return;
3285
3734
  }
3286
3735
  noteWrite(state);
3736
+ noteSearchedDocWritten(state, docId);
3287
3737
  state.tombstones.delete(docId);
3288
3738
  const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
3289
3739
  state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
3290
3740
  if (previous !== void 0) {
3291
- state.hnsw?.markTombstoneOrdinal(previous);
3292
- state.freshGraph?.markTombstoneOrdinal(previous);
3741
+ const graph = state.hnsw;
3742
+ const freshGraph = state.freshGraph;
3743
+ afterGraphPass(state, () => {
3744
+ graph?.markTombstoneOrdinal(previous);
3745
+ freshGraph?.markTombstoneOrdinal(previous);
3746
+ });
3293
3747
  if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
3294
3748
  }
3295
3749
  state.buffer.add(docId);
@@ -3300,12 +3754,13 @@ function createVectorWrites(state) {
3300
3754
  handOverBuildTurns(state);
3301
3755
  if (!state.store.has(docId)) return;
3302
3756
  noteWrite(state);
3757
+ noteSearchedDocWritten(state, docId);
3303
3758
  state.tombstones.add(docId);
3304
3759
  state.buffer.delete(docId);
3305
3760
  state.pendingLocations.delete(docId);
3306
- if (state.hnsw) {
3307
- state.hnsw.markTombstone(docId);
3308
- }
3761
+ const graph = state.hnsw;
3762
+ const ordinal = state.store.getOrdinal(docId);
3763
+ if (graph !== null && ordinal !== void 0) afterGraphPass(state, () => graph.markTombstoneOrdinal(ordinal));
3309
3764
  }
3310
3765
  return {
3311
3766
  insert(docId, vector, partitionId) {
@@ -3404,7 +3859,9 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3404
3859
  sharedLayoutRevision: 0,
3405
3860
  sharing: Promise.resolve(),
3406
3861
  releaseToFilesInFlight: Promise.resolve(),
3407
- cold: null
3862
+ cold: null,
3863
+ graphsHeldOffThread: null,
3864
+ searchWatches: /* @__PURE__ */ new Set()
3408
3865
  };
3409
3866
  const { insert, remove } = createVectorWrites(state);
3410
3867
  function getVector(docId) {
@@ -3422,22 +3879,24 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3422
3879
  while (state.cold !== null) await state.cold.loaded;
3423
3880
  }
3424
3881
  async function searchParallel(query, k, options) {
3882
+ if (state.graphsHeldOffThread !== null) await untilGraphReleased(state);
3425
3883
  if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
3426
3884
  handOverBuildTurns(state);
3427
3885
  const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
3428
3886
  let filter = filterForOptions(state, options);
3887
+ const offThread = searchOffThread(state, query, k, options, filter, confined ? state.revision : null);
3888
+ if (offThread !== null) return offThread;
3429
3889
  if (filter !== void 0) {
3430
3890
  if (state.hnsw) {
3431
3891
  const hnswLiveSize = state.hnsw.size;
3432
3892
  const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
3433
- if (selectivity < state.filterThreshold) {
3434
- return searchWithFilter(state, query, k, options, filter);
3435
- }
3893
+ if (selectivity < state.filterThreshold) return searchFiltered(state, query, k, options, filter);
3436
3894
  }
3437
3895
  }
3438
3896
  const filterRevision = state.revision;
3439
- const buffered = state.buffer.size === 0 ? null : [...state.buffer];
3440
3897
  scheduleWorkerCopyLoad(state);
3898
+ const searchesWorkerCopies = state.workerCopyPool !== null && state.workerCopyHandle !== null;
3899
+ const buffered = state.buffer.size === 0 || !searchesWorkerCopies ? null : [...state.buffer];
3441
3900
  const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
3442
3901
  filter,
3443
3902
  efSearch: options.efSearch,
@@ -3451,7 +3910,7 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3451
3910
  const results = buffered === null ? viaWorkerCopy : withBufferedVectors(state, query, k, options, filter, viaWorkerCopy, buffered);
3452
3911
  return fetchedOutcome(state, results, options.minSimilarity, filter);
3453
3912
  }
3454
- return searchWithFilter(state, query, k, options, filter);
3913
+ return searchFiltered(state, query, k, options, filter);
3455
3914
  }
3456
3915
  return {
3457
3916
  get size() {
@@ -3493,18 +3952,21 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3493
3952
  cancelColdHold(state);
3494
3953
  disposeIndex(state);
3495
3954
  },
3496
- search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
3955
+ search: withGraphFree(
3956
+ state,
3957
+ (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options)
3958
+ ),
3497
3959
  searchParallel,
3498
3960
  refreshWorkerCopies: () => {
3499
3961
  if (state.cold === null) refreshWorkerCopies(state);
3500
3962
  },
3501
3963
  getVector,
3502
3964
  has,
3503
- compact: () => {
3965
+ compact: withGraphFree(state, () => {
3504
3966
  if (state.cold !== null) return;
3505
3967
  noteWrite(state);
3506
3968
  compact(state);
3507
- },
3969
+ }),
3508
3970
  optimize: async () => {
3509
3971
  await whenLoaded();
3510
3972
  noteWrite(state);
@@ -3513,18 +3975,21 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3513
3975
  maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
3514
3976
  estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
3515
3977
  growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
3516
- deserialize: (parts) => {
3978
+ deserialize: withGraphFree(state, (parts) => {
3517
3979
  cancelColdHold(state);
3518
3980
  invalidateWorkerCopies(state);
3519
3981
  deserialize(state, parts);
3520
- },
3521
- planCheckpoint: (listedKeys, rewritesGraph, pinsRetiredVectors) => planCheckpoint(state, listedKeys, rewritesGraph, pinsRetiredVectors),
3982
+ }),
3983
+ planCheckpoint: withGraphFree(
3984
+ state,
3985
+ (listedKeys, rewritesGraph, pinsRetiredVectors) => planCheckpoint(state, listedKeys, rewritesGraph, pinsRetiredVectors)
3986
+ ),
3522
3987
  recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
3523
- restoreCheckpoint: (shape) => {
3988
+ restoreCheckpoint: withGraphFree(state, (shape) => {
3524
3989
  cancelColdHold(state);
3525
3990
  invalidateWorkerCopies(state);
3526
3991
  return beginCheckpointRestore(state, shape);
3527
- },
3992
+ }),
3528
3993
  openColdField: (field, load) => openColdField(state, field, load),
3529
3994
  startShuffledRebuildWhenDue: () => {
3530
3995
  if (state.cold === null) startShuffledRebuild(state);
@@ -3533,12 +3998,12 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3533
3998
  const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
3534
3999
  return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
3535
4000
  },
3536
- adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
3537
- releaseVectorFiles: async () => {
4001
+ adoptDiskLayout: onceGraphFree(state, (layout) => adoptDiskLayout(state, layout)),
4002
+ releaseVectorFiles: onceGraphFree(state, async () => {
3538
4003
  await whenLoaded();
3539
4004
  await withdrawWorkerCopies(state);
3540
4005
  state.store.closeFiles();
3541
- },
4006
+ }),
3542
4007
  relocateVectorFiles: (rewrite) => {
3543
4008
  state.store.relocateVectorFiles(rewrite);
3544
4009
  for (const [docId, location] of state.pendingLocations) {
@@ -3555,4 +4020,4 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
3555
4020
  };
3556
4021
  }
3557
4022
 
3558
- export { createSharedVectorSearcher, createVectorIndex, openSharedVectorField };
4023
+ export { compactVectors, createSharedVectorSearcher, createVectorIndex, getVectorMaintenanceStatus, openSharedVectorField, optimizeVectors };