@delali/narsil 0.3.1-next.99 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +45 -0
  2. package/README.md +27 -8
  3. package/dist/adapters/filesystem.d.ts +1 -1
  4. package/dist/adapters/filesystem.mjs +101 -13
  5. package/dist/adapters/gcs.d.ts +49 -0
  6. package/dist/adapters/gcs.mjs +213 -0
  7. package/dist/adapters/indexeddb.d.ts +1 -1
  8. package/dist/adapters/indexeddb.mjs +2 -7
  9. package/dist/adapters/memory.d.ts +1 -1
  10. package/dist/adapters/memory.mjs +5 -0
  11. package/dist/adapters/s3.d.ts +53 -0
  12. package/dist/adapters/s3.mjs +263 -0
  13. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  14. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  15. package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
  16. package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
  17. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  18. package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
  19. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  20. package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
  21. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  22. package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
  23. package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
  24. package/dist/browser/embed-GG6PS3EA.mjs +264 -0
  25. package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
  26. package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
  27. package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
  28. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  29. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  30. package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
  31. package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
  32. package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
  33. package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
  34. package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
  35. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  36. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  37. package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
  38. package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
  39. package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
  40. package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
  41. package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
  42. package/dist/chunk-2AFZE7Y3.mjs +72 -0
  43. package/dist/chunk-2CKDCC65.mjs +47 -0
  44. package/dist/chunk-3ECIRFS5.mjs +61 -0
  45. package/dist/chunk-44W3CBNG.mjs +7 -0
  46. package/dist/chunk-4H4RG2FR.mjs +13 -0
  47. package/dist/chunk-4PY4JYMY.mjs +11 -0
  48. package/dist/chunk-4ZPVAHWF.mjs +1289 -0
  49. package/dist/chunk-5EI5CXO6.mjs +197 -0
  50. package/dist/chunk-5F5MXWS5.mjs +283 -0
  51. package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
  52. package/dist/chunk-6FOENO4E.mjs +273 -0
  53. package/dist/chunk-6GHX35ZM.mjs +108 -0
  54. package/dist/chunk-6MSZ543Y.mjs +32 -0
  55. package/dist/chunk-6N6HHBCQ.mjs +4596 -0
  56. package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
  57. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  58. package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
  59. package/dist/chunk-AVBPB3MI.mjs +602 -0
  60. package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
  61. package/dist/chunk-BUMREPLU.mjs +15 -0
  62. package/dist/chunk-BY6NALRK.mjs +31921 -0
  63. package/dist/chunk-C4ZMXXTL.mjs +92 -0
  64. package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
  65. package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
  66. package/dist/chunk-DXZ6JDWG.mjs +797 -0
  67. package/dist/chunk-FJXJZUWF.mjs +209 -0
  68. package/dist/chunk-FS3HPPSC.mjs +212 -0
  69. package/dist/chunk-FT4KLZXW.mjs +1164 -0
  70. package/dist/chunk-GR5GMFJJ.mjs +104 -0
  71. package/dist/chunk-GXCDWLQI.mjs +193 -0
  72. package/dist/chunk-GXED6R3Q.mjs +271 -0
  73. package/dist/chunk-HLPRHOSO.mjs +17 -0
  74. package/dist/chunk-IL5DRCNU.mjs +3041 -0
  75. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  76. package/dist/chunk-K5HSUPYS.mjs +11 -0
  77. package/dist/chunk-KYIRXTCA.mjs +100 -0
  78. package/dist/chunk-LFXS6OF7.mjs +289 -0
  79. package/dist/chunk-LNGE7SLY.mjs +151 -0
  80. package/dist/chunk-M2Z5WVWW.mjs +194 -0
  81. package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
  82. package/dist/chunk-M3OW2XSP.mjs +40 -0
  83. package/dist/chunk-MFPSQZ6P.mjs +127 -0
  84. package/dist/chunk-MRCJBWJZ.mjs +1292 -0
  85. package/dist/chunk-MSIN4NM3.mjs +25 -0
  86. package/dist/chunk-MW3CJVGP.mjs +219 -0
  87. package/dist/chunk-MYM6CAVK.mjs +76 -0
  88. package/dist/chunk-NJJN4FMM.mjs +84 -0
  89. package/dist/chunk-NM7I3BM7.mjs +59 -0
  90. package/dist/chunk-OIJ64LGT.mjs +19 -0
  91. package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
  92. package/dist/chunk-Q2AP2FSN.mjs +15 -0
  93. package/dist/chunk-R2PJKSNL.mjs +54 -0
  94. package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
  95. package/dist/chunk-S2POFTLN.mjs +10 -0
  96. package/dist/chunk-S5QMWECL.mjs +7 -0
  97. package/dist/chunk-SLAC5PAB.mjs +5008 -0
  98. package/dist/chunk-T4HMSZCX.mjs +5 -0
  99. package/dist/chunk-TACP6VK6.mjs +111 -0
  100. package/dist/chunk-TGPKS2UM.mjs +125 -0
  101. package/dist/chunk-TKKXPKEO.mjs +979 -0
  102. package/dist/chunk-V6EUBAIT.mjs +19 -0
  103. package/dist/chunk-XL56L4IR.mjs +228 -0
  104. package/dist/chunk-XQIAMDAG.mjs +3076 -0
  105. package/dist/chunk-YTUZ4R22.mjs +1063 -0
  106. package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
  107. package/dist/chunk-ZPJFGGGI.mjs +638 -0
  108. package/dist/chunk-ZXX5GZEX.mjs +17 -0
  109. package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
  110. package/dist/client.d.ts +11 -10
  111. package/dist/client.mjs +8 -898
  112. package/dist/config-Bnl7L-0H.d.ts +608 -0
  113. package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
  114. package/dist/core-leaf-6JVHEVFB.mjs +8 -0
  115. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  116. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  117. package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
  118. package/dist/distribution/coordinator/etcd.d.ts +63 -7
  119. package/dist/distribution/coordinator/etcd.mjs +725 -81
  120. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  121. package/dist/distribution/coordinator/in-memory.mjs +65 -10
  122. package/dist/distribution/transport/grpc.d.ts +2 -2
  123. package/dist/distribution/transport/grpc.mjs +39 -10
  124. package/dist/distribution/transport/in-memory.d.ts +2 -2
  125. package/dist/distribution/transport/in-memory.mjs +9 -6
  126. package/dist/distribution/transport/tcp.d.ts +2 -2
  127. package/dist/distribution/transport/tcp.mjs +20 -15
  128. package/dist/distribution.d.ts +267 -56
  129. package/dist/distribution.mjs +23156 -11362
  130. package/dist/durability-config-VWS2K24E.mjs +96 -0
  131. package/dist/embed-4PHHKGBK.mjs +263 -0
  132. package/dist/embeddings/openai.d.ts +1 -1
  133. package/dist/embeddings/openai.mjs +2 -2
  134. package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
  135. package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
  136. package/dist/gzip-2UCZW2MM.mjs +31 -0
  137. package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
  138. package/dist/index.browser.mjs +4 -4
  139. package/dist/index.d.ts +2133 -57
  140. package/dist/index.mjs +16 -32
  141. package/dist/index.react-native.mjs +12 -0
  142. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  143. package/dist/invalidation/filesystem.d.ts +1 -1
  144. package/dist/invalidation/filesystem.mjs +1 -2
  145. package/dist/invalidation/noop.d.ts +1 -1
  146. package/dist/languages/albanian.mjs +1 -1
  147. package/dist/languages/amharic.mjs +1 -1
  148. package/dist/languages/arabic.mjs +1 -1
  149. package/dist/languages/armenian.mjs +1 -1
  150. package/dist/languages/azerbaijani.mjs +1 -1
  151. package/dist/languages/bambara.mjs +1 -1
  152. package/dist/languages/basque.mjs +1 -1
  153. package/dist/languages/belarusian.mjs +1 -1
  154. package/dist/languages/bengali.mjs +1 -1
  155. package/dist/languages/bosnian.mjs +1 -1
  156. package/dist/languages/breton.mjs +1 -1
  157. package/dist/languages/bulgarian.mjs +1 -1
  158. package/dist/languages/burmese.mjs +1 -1
  159. package/dist/languages/catalan.mjs +1 -1
  160. package/dist/languages/chinese.mjs +1 -1
  161. package/dist/languages/croatian.mjs +1 -1
  162. package/dist/languages/czech.mjs +1 -1
  163. package/dist/languages/dagbani.mjs +1 -1
  164. package/dist/languages/danish.mjs +1 -1
  165. package/dist/languages/dutch.mjs +1 -1
  166. package/dist/languages/english.mjs +1 -1
  167. package/dist/languages/esperanto.mjs +1 -1
  168. package/dist/languages/estonian.mjs +1 -1
  169. package/dist/languages/ewe.mjs +1 -1
  170. package/dist/languages/faroese.mjs +1 -1
  171. package/dist/languages/fijian.mjs +1 -1
  172. package/dist/languages/finnish.mjs +1 -1
  173. package/dist/languages/french.mjs +1 -1
  174. package/dist/languages/ga.mjs +1 -1
  175. package/dist/languages/galician.mjs +1 -1
  176. package/dist/languages/georgian.mjs +1 -1
  177. package/dist/languages/german.mjs +1 -1
  178. package/dist/languages/greek.mjs +1 -1
  179. package/dist/languages/guarani.mjs +1 -1
  180. package/dist/languages/gujarati.mjs +1 -1
  181. package/dist/languages/haitian-creole.mjs +1 -1
  182. package/dist/languages/hausa.mjs +1 -1
  183. package/dist/languages/hawaiian.mjs +1 -1
  184. package/dist/languages/hebrew.mjs +1 -1
  185. package/dist/languages/hindi.mjs +1 -1
  186. package/dist/languages/hungarian.mjs +1 -1
  187. package/dist/languages/icelandic.mjs +1 -1
  188. package/dist/languages/igbo.mjs +1 -1
  189. package/dist/languages/indonesian.mjs +1 -1
  190. package/dist/languages/irish.mjs +1 -1
  191. package/dist/languages/italian.mjs +1 -1
  192. package/dist/languages/japanese.mjs +1 -1
  193. package/dist/languages/kannada.mjs +1 -1
  194. package/dist/languages/kazakh.mjs +1 -1
  195. package/dist/languages/khmer.mjs +1 -1
  196. package/dist/languages/kinyarwanda.mjs +1 -1
  197. package/dist/languages/kirundi.mjs +1 -1
  198. package/dist/languages/korean.mjs +1 -1
  199. package/dist/languages/kurmanji.mjs +1 -1
  200. package/dist/languages/kyrgyz.mjs +1 -1
  201. package/dist/languages/lao.mjs +1 -1
  202. package/dist/languages/latin.mjs +1 -1
  203. package/dist/languages/latvian.mjs +1 -1
  204. package/dist/languages/lingala.mjs +1 -1
  205. package/dist/languages/lithuanian.mjs +1 -1
  206. package/dist/languages/luxembourgish.mjs +1 -1
  207. package/dist/languages/macedonian.mjs +1 -1
  208. package/dist/languages/malagasy.mjs +1 -1
  209. package/dist/languages/malay.mjs +1 -1
  210. package/dist/languages/malayalam.mjs +1 -1
  211. package/dist/languages/maltese.mjs +1 -1
  212. package/dist/languages/maori.mjs +1 -1
  213. package/dist/languages/marathi.mjs +1 -1
  214. package/dist/languages/nepali.mjs +1 -1
  215. package/dist/languages/norwegian.mjs +1 -1
  216. package/dist/languages/oromo.mjs +1 -1
  217. package/dist/languages/persian.mjs +1 -1
  218. package/dist/languages/polish.mjs +1 -1
  219. package/dist/languages/portuguese.mjs +1 -1
  220. package/dist/languages/punjabi.mjs +1 -1
  221. package/dist/languages/romanian.mjs +1 -1
  222. package/dist/languages/russian.mjs +1 -1
  223. package/dist/languages/samoan.mjs +1 -1
  224. package/dist/languages/sanskrit.mjs +1 -1
  225. package/dist/languages/scottish-gaelic.mjs +1 -1
  226. package/dist/languages/serbian.mjs +1 -1
  227. package/dist/languages/shona.mjs +1 -1
  228. package/dist/languages/sinhala.mjs +1 -1
  229. package/dist/languages/slovak.mjs +1 -1
  230. package/dist/languages/slovenian.mjs +1 -1
  231. package/dist/languages/sorani.mjs +1 -1
  232. package/dist/languages/spanish.mjs +1 -1
  233. package/dist/languages/swahili.mjs +1 -1
  234. package/dist/languages/swedish.mjs +1 -1
  235. package/dist/languages/tagalog.mjs +1 -1
  236. package/dist/languages/tamil.mjs +1 -1
  237. package/dist/languages/tatar.mjs +1 -1
  238. package/dist/languages/telugu.mjs +1 -1
  239. package/dist/languages/thai.mjs +1 -1
  240. package/dist/languages/tibetan.mjs +1 -1
  241. package/dist/languages/tigrinya.mjs +1 -1
  242. package/dist/languages/tongan.mjs +1 -1
  243. package/dist/languages/turkish.mjs +1 -1
  244. package/dist/languages/twi.mjs +1 -1
  245. package/dist/languages/ukrainian.mjs +1 -1
  246. package/dist/languages/urdu.mjs +1 -1
  247. package/dist/languages/vietnamese.mjs +1 -1
  248. package/dist/languages/welsh.mjs +1 -1
  249. package/dist/languages/wolof.mjs +1 -1
  250. package/dist/languages/xhosa.mjs +1 -1
  251. package/dist/languages/yoruba.mjs +1 -1
  252. package/dist/languages/zulu.mjs +1 -1
  253. package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
  254. package/dist/module-5ZSLUQII.mjs +3729 -0
  255. package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
  256. package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
  257. package/dist/optional-writes-4UOQKETU.mjs +247 -0
  258. package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
  259. package/dist/partial-words.browser.mjs +1650 -0
  260. package/dist/partial-words.mjs +17 -11
  261. package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
  262. package/dist/pattern.mjs +6 -7
  263. package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
  264. package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
  265. package/dist/phrase-module-EFESWOCR.mjs +635 -0
  266. package/dist/query-actions-HMJUVWGT.mjs +119 -0
  267. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  268. package/dist/query-suggestions/merge-worker.mjs +53 -0
  269. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  270. package/dist/query-suggestions.browser.mjs +47 -0
  271. package/dist/query-suggestions.d.ts +20 -0
  272. package/dist/query-suggestions.mjs +19 -0
  273. package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
  274. package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
  275. package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
  276. package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
  277. package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
  278. package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
  279. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  280. package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
  281. package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
  282. package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
  283. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  284. package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
  285. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  286. package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
  287. package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
  288. package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
  289. package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
  290. package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
  291. package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
  292. package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
  293. package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
  294. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  295. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  296. package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
  297. package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
  298. package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
  299. package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
  300. package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
  301. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  302. package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
  303. package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
  304. package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
  305. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  306. package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
  307. package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
  308. package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
  309. package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
  310. package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
  311. package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
  312. package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
  313. package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
  314. package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
  315. package/dist/react-native/partial-words.mjs +1658 -0
  316. package/dist/react-native/pattern.mjs +2570 -0
  317. package/dist/react-native/query-suggestions.mjs +55 -0
  318. package/dist/react-native/sync.mjs +2759 -0
  319. package/dist/react-native/synonyms.mjs +676 -0
  320. package/dist/react.d.ts +12 -8
  321. package/dist/react.mjs +45 -1
  322. package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
  323. package/dist/rerankers/http.d.ts +1 -1
  324. package/dist/rerankers/http.mjs +2 -2
  325. package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
  326. package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
  327. package/dist/runtime-USQYKN3I.mjs +615 -0
  328. package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
  329. package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
  330. package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
  331. package/dist/server.d.ts +348 -22
  332. package/dist/server.mjs +2211 -327
  333. package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
  334. package/dist/store-leaf-6VMMSVBE.mjs +20 -0
  335. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  336. package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
  337. package/dist/sync/filesystem.d.ts +30 -0
  338. package/dist/sync/filesystem.mjs +288 -0
  339. package/dist/sync/indexeddb.browser.mjs +287 -0
  340. package/dist/sync/indexeddb.d.ts +30 -0
  341. package/dist/sync/indexeddb.mjs +242 -0
  342. package/dist/sync.browser.mjs +1271 -0
  343. package/dist/sync.d.ts +23 -0
  344. package/dist/sync.mjs +2712 -0
  345. package/dist/synonyms.browser.mjs +658 -0
  346. package/dist/synonyms.mjs +17 -7
  347. package/dist/tasks-65b6t6lk.d.ts +782 -0
  348. package/dist/thread-UJ3W5LVI.mjs +31 -0
  349. package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
  350. package/dist/types-CpCg-Gq-.d.ts +227 -0
  351. package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
  352. package/dist/vector/search-worker.d.ts +10 -2
  353. package/dist/vector/search-worker.mjs +11 -11
  354. package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
  355. package/dist/workers/entry.mjs +75 -15
  356. package/openapi.json +6840 -0
  357. package/package.json +406 -19
  358. package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
  359. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  360. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  361. package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
  362. package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
  363. package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
  364. package/dist/browser/chunk-QWES44AW.mjs +0 -473
  365. package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
  366. package/dist/browser/module-PDXYHT75.mjs +0 -2822
  367. package/dist/browser/release-76MXRPGV.mjs +0 -33
  368. package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
  369. package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
  370. package/dist/browser/write-L7FAQA47.mjs +0 -208
  371. package/dist/chunk-37CD3OKS.mjs +0 -87
  372. package/dist/chunk-5XNIRLFC.mjs +0 -96
  373. package/dist/chunk-7X5JST5L.mjs +0 -114
  374. package/dist/chunk-AV4IJKD2.mjs +0 -35
  375. package/dist/chunk-AY3AQ3UH.mjs +0 -73
  376. package/dist/chunk-BKB3XTMI.mjs +0 -707
  377. package/dist/chunk-C7SLA6CM.mjs +0 -19
  378. package/dist/chunk-COVCQ73F.mjs +0 -587
  379. package/dist/chunk-DJHHF2L5.mjs +0 -17
  380. package/dist/chunk-E74YLAIG.mjs +0 -24
  381. package/dist/chunk-FMWRCP2C.mjs +0 -57
  382. package/dist/chunk-GES7U2PM.mjs +0 -68
  383. package/dist/chunk-HJSN7UYG.mjs +0 -10
  384. package/dist/chunk-JFRNNBOM.mjs +0 -35
  385. package/dist/chunk-K25LZA54.mjs +0 -8484
  386. package/dist/chunk-KEHK2O6P.mjs +0 -13
  387. package/dist/chunk-KOKM4PQN.mjs +0 -52
  388. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  389. package/dist/chunk-L3L5YFXC.mjs +0 -614
  390. package/dist/chunk-L5ORED7V.mjs +0 -25
  391. package/dist/chunk-LH4NZNJJ.mjs +0 -1731
  392. package/dist/chunk-LU2CE5AE.mjs +0 -507
  393. package/dist/chunk-LXWDQLTW.mjs +0 -30
  394. package/dist/chunk-MHOSCQCQ.mjs +0 -948
  395. package/dist/chunk-NKRBEGJZ.mjs +0 -1705
  396. package/dist/chunk-NTRMFARG.mjs +0 -462
  397. package/dist/chunk-NW4XDSP2.mjs +0 -212
  398. package/dist/chunk-OOEXLUGR.mjs +0 -117
  399. package/dist/chunk-ORBLD3EL.mjs +0 -6
  400. package/dist/chunk-OWNZYMRE.mjs +0 -3591
  401. package/dist/chunk-PPY2GSIB.mjs +0 -44
  402. package/dist/chunk-QY7GC2MW.mjs +0 -33
  403. package/dist/chunk-R6A773WK.mjs +0 -226
  404. package/dist/chunk-R6PR4WYZ.mjs +0 -68
  405. package/dist/chunk-R7EEKAJE.mjs +0 -593
  406. package/dist/chunk-RONEOEON.mjs +0 -426
  407. package/dist/chunk-VWULPWRJ.mjs +0 -7469
  408. package/dist/chunk-WCJKUGHB.mjs +0 -262
  409. package/dist/chunk-WQBCBKY3.mjs +0 -8
  410. package/dist/chunk-X5CV2E2B.mjs +0 -1
  411. package/dist/chunk-YA7XLE2R.mjs +0 -1256
  412. package/dist/chunk-YTJCGKM4.mjs +0 -56
  413. package/dist/chunk-ZG477IPV.mjs +0 -526
  414. package/dist/chunk-ZYNAMIYX.mjs +0 -113
  415. package/dist/config-BGB_fDtq.d.ts +0 -237
  416. package/dist/direct-executor-CXZ66ACP.mjs +0 -22
  417. package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
  418. package/dist/engine-BGCEm9E_.d.ts +0 -577
  419. package/dist/module-3NONKBZX.mjs +0 -1860
  420. package/dist/release-BH3DKW3T.mjs +0 -33
  421. package/dist/restore-queue-JPRBUYII.mjs +0 -35
  422. package/dist/runtime-ZKYZFJJC.mjs +0 -177
  423. package/dist/snapshot-HOGXNO5W.mjs +0 -793
  424. package/dist/tasks-HG3rlxx6.d.ts +0 -302
  425. package/dist/thread-B657LQXK.mjs +0 -692
  426. package/dist/write-XLIGTXBU.mjs +0 -238
@@ -1,1860 +0,0 @@
1
- import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, WORKER_COPY_MIN_VECTORS, allLiveDocIds, recalibrateFromStore, signatureOf, adoptGraph, calibrateQuantizer, fieldHandlesOf, BUILD_CHUNK_SIZE, yieldToEventLoop, DISK_ADOPTION_YIELD_INTERVAL } from './chunk-COVCQ73F.mjs';
2
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-WCJKUGHB.mjs';
3
- import { createVectorStore } from './chunk-UCDF35PU.mjs';
4
- export { openSharedVectorField } from './chunk-UCDF35PU.mjs';
5
- import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, releaseLocksHeldBy } from './chunk-HXJBSHXF.mjs';
6
- export { releaseLocksHeldBy } from './chunk-HXJBSHXF.mjs';
7
- import { noteFallback } from './chunk-DJHHF2L5.mjs';
8
- import './chunk-37CD3OKS.mjs';
9
- import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
10
- import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-AY3AQ3UH.mjs';
11
- import { detectRuntime } from './chunk-AV4IJKD2.mjs';
12
- import './chunk-LXWDQLTW.mjs';
13
- import './chunk-KOKM4PQN.mjs';
14
- import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-QY7GC2MW.mjs';
15
- import './chunk-GGSH6KBQ.mjs';
16
- import './chunk-E74YLAIG.mjs';
17
- import './chunk-KEHK2O6P.mjs';
18
- import './chunk-R6A773WK.mjs';
19
- import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
20
- import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
21
-
22
- // src/core/heap.ts
23
- function createMaxHeap(compare) {
24
- return createHeap((a, b) => compare(b, a));
25
- }
26
- function createBoundedMaxHeap(compare, capacity) {
27
- const heap = createMaxHeap(compare);
28
- const innerPush = heap.push;
29
- return {
30
- push(item) {
31
- if (heap.size < capacity) {
32
- innerPush(item);
33
- return;
34
- }
35
- const top = heap.peek();
36
- if (top !== void 0 && compare(item, top) < 0) {
37
- heap.pop();
38
- innerPush(item);
39
- }
40
- },
41
- pop: heap.pop,
42
- peek: heap.peek,
43
- get size() {
44
- return heap.size;
45
- },
46
- toSortedArray: heap.toSortedArray
47
- };
48
- }
49
- function createHeap(compare) {
50
- const items = [];
51
- function siftUp(idx) {
52
- while (idx > 0) {
53
- const parent = idx - 1 >> 1;
54
- if (compare(items[idx], items[parent]) >= 0) break;
55
- const tmp = items[idx];
56
- items[idx] = items[parent];
57
- items[parent] = tmp;
58
- idx = parent;
59
- }
60
- }
61
- function siftDown(idx) {
62
- const len = items.length;
63
- while (true) {
64
- let smallest = idx;
65
- const left = 2 * idx + 1;
66
- const right = 2 * idx + 2;
67
- if (left < len && compare(items[left], items[smallest]) < 0) smallest = left;
68
- if (right < len && compare(items[right], items[smallest]) < 0) smallest = right;
69
- if (smallest === idx) break;
70
- const tmp = items[idx];
71
- items[idx] = items[smallest];
72
- items[smallest] = tmp;
73
- idx = smallest;
74
- }
75
- }
76
- return {
77
- push(item) {
78
- items.push(item);
79
- siftUp(items.length - 1);
80
- },
81
- pop() {
82
- if (items.length === 0) return void 0;
83
- const top = items[0];
84
- const last = items.pop();
85
- if (items.length > 0) {
86
- items[0] = last;
87
- siftDown(0);
88
- }
89
- return top;
90
- },
91
- peek() {
92
- return items[0];
93
- },
94
- get size() {
95
- return items.length;
96
- },
97
- toSortedArray() {
98
- return [...items].sort(compare);
99
- }
100
- };
101
- }
102
-
103
- // src/vector/vector-index/merge-results.ts
104
- function highScoreFirst(a, b) {
105
- return b.score - a.score || compareCodePoints(a.docId, b.docId);
106
- }
107
- function mergeResults(hnswResults, bufferResults, k) {
108
- const seen = /* @__PURE__ */ new Set();
109
- const merged = [];
110
- let hi = 0;
111
- let bi = 0;
112
- while (merged.length < k && (hi < hnswResults.length || bi < bufferResults.length)) {
113
- const h = hi < hnswResults.length ? hnswResults[hi] : void 0;
114
- const b = bi < bufferResults.length ? bufferResults[bi] : void 0;
115
- let pick;
116
- if (h && b) {
117
- if (h.score > b.score || h.score === b.score && compareCodePoints(h.docId, b.docId) < 0) {
118
- pick = h;
119
- hi++;
120
- } else {
121
- pick = b;
122
- bi++;
123
- }
124
- } else if (h) {
125
- pick = h;
126
- hi++;
127
- } else if (b) {
128
- pick = b;
129
- bi++;
130
- } else {
131
- break;
132
- }
133
- if (seen.has(pick.docId)) continue;
134
- seen.add(pick.docId);
135
- merged.push(pick);
136
- }
137
- return merged;
138
- }
139
-
140
- // src/vector/shared-field/store-flags.ts
141
- function bytesOf(flags, source) {
142
- if (flags.source !== source) {
143
- flags.source = source;
144
- flags.view = new Uint8Array(source);
145
- }
146
- return flags.view;
147
- }
148
- function countsOf(chunks, source) {
149
- if (chunks.source !== source) {
150
- chunks.source = source;
151
- chunks.view = new Int32Array(source);
152
- }
153
- return chunks.view;
154
- }
155
- function readStoreFlags(store) {
156
- const unplaced = { source: null, view: new Uint8Array(0) };
157
- const unplacedChunks = { source: null, view: new Int32Array(0) };
158
- const removed = { source: null, view: new Uint8Array(0) };
159
- return {
160
- liveDocuments() {
161
- const header = store.handles.header;
162
- return Atomics.load(header, STORE_LIVE_COUNT) - Atomics.load(header, STORE_REMOVED_COUNT);
163
- },
164
- isRemoved(ordinal) {
165
- const bytes = bytesOf(removed, store.handles.removed);
166
- return ordinal < bytes.length && Atomics.load(bytes, ordinal) === 1;
167
- },
168
- unplacedOrdinals() {
169
- const handles = store.handles;
170
- if (Atomics.load(handles.header, STORE_UNPLACED_COUNT) === 0) return [];
171
- const bytes = bytesOf(unplaced, handles.unplaced);
172
- const counts = countsOf(unplacedChunks, handles.unplacedChunks);
173
- const end = Math.min(store.slots, bytes.length);
174
- const ordinals = [];
175
- for (let chunk = 0; chunk < counts.length; chunk++) {
176
- const from = chunk * UNPLACED_FLAG_CHUNK_ORDINALS;
177
- if (from >= end) break;
178
- if (Atomics.load(counts, chunk) === 0) continue;
179
- const to = Math.min(end, from + UNPLACED_FLAG_CHUNK_ORDINALS);
180
- for (let ordinal = from; ordinal < to; ordinal++) {
181
- if (Atomics.load(bytes, ordinal) === 1) ordinals.push(ordinal);
182
- }
183
- }
184
- return ordinals;
185
- }
186
- };
187
- }
188
-
189
- // src/vector/shared-field/searcher.ts
190
- function distancesInTypeScript(store, query, metric, ordinals) {
191
- noteFallback("score");
192
- const arenaQuery = store.prepareQueryArena(query);
193
- const queryMagnitude = arenaQuery ? arenaQuery.magnitude : magnitude(query);
194
- const distances = new Float64Array(ordinals.length).fill(Number.POSITIVE_INFINITY);
195
- for (let i = 0; i < ordinals.length; i++) {
196
- if (arenaQuery) {
197
- distances[i] = store.distanceFromArena(arenaQuery, ordinals[i], metric);
198
- continue;
199
- }
200
- const entry = store.entryForOrdinal(ordinals[i]);
201
- if (entry) distances[i] = toDistance(query, entry.vector, queryMagnitude, entry.magnitude, metric);
202
- }
203
- return distances;
204
- }
205
- function scoreCandidates(store, candidates, query, k, metric, minSimilarity) {
206
- const { ordinals, docIds } = candidates;
207
- if (ordinals.length === 0) return { results: [], matched: 0 };
208
- const distances = nativeScoresOf(store.handles, query, metric, ordinals) ?? distancesInTypeScript(store, query, metric, ordinals);
209
- const heap = createBoundedMaxHeap(highScoreFirst, k);
210
- let matched = 0;
211
- for (let i = 0; i < ordinals.length; i++) {
212
- if (distances[i] === Number.POSITIVE_INFINITY) continue;
213
- const score = toScore(distances[i], metric);
214
- if (score < minSimilarity) continue;
215
- matched++;
216
- heap.push({ docId: docIds[i], score });
217
- }
218
- return { results: heap.toSortedArray().reverse(), matched };
219
- }
220
- function createSharedVectorSearcher(options) {
221
- const { fieldName, view, holdsDocument } = options;
222
- const flags = readStoreFlags(view.store);
223
- function ordinalLayoutGeneration() {
224
- return Atomics.load(view.store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
225
- }
226
- function ordinalSpace() {
227
- const store = view.store;
228
- const layoutGeneration = ordinalLayoutGeneration();
229
- return {
230
- fieldName,
231
- layoutGeneration,
232
- slots: store.slots,
233
- header: store.handles.header,
234
- ordinalOf: (docId) => view.ordinalOf(docId),
235
- holdsCurrentOrdinal: (ordinal) => store.holdsOrdinal(ordinal) && store.partitionAt(ordinal) !== void 0,
236
- isRemoved: (ordinal) => flags.isRemoved(ordinal)
237
- };
238
- }
239
- function filterForPartitions(allowed) {
240
- const slots = view.store.slots;
241
- const filter = createOrdinalFilter(slots);
242
- for (let ordinal = 0; ordinal < slots; ordinal++) {
243
- const partition = view.store.partitionAt(ordinal);
244
- if (partition === void 0 || !allowed.has(partition)) continue;
245
- if (!view.store.holdsOrdinal(ordinal) || flags.isRemoved(ordinal)) continue;
246
- addToOrdinalFilter(filter, ordinal);
247
- }
248
- return filter;
249
- }
250
- function filterFor(searchOptions) {
251
- if (searchOptions.filterSubsets !== void 0) {
252
- return ordinalFilterForSubsets(searchOptions.filterSubsets, ordinalSpace());
253
- }
254
- if (searchOptions.filterPartitions !== void 0) return filterForPartitions(searchOptions.filterPartitions);
255
- return void 0;
256
- }
257
- function candidatesOf(ordinals, filter) {
258
- const candidates = { ordinals: [], docIds: [] };
259
- for (const ordinal of ordinals) {
260
- if (filter !== void 0 && !ordinalFilterHas(filter, ordinal)) continue;
261
- if (flags.isRemoved(ordinal)) continue;
262
- const docId = view.docIdOf(ordinal);
263
- if (docId === void 0 || view.ordinalOf(docId) !== ordinal) continue;
264
- candidates.ordinals.push(ordinal);
265
- candidates.docIds.push(docId);
266
- }
267
- return candidates;
268
- }
269
- function answerOf(outcome) {
270
- const results = outcome.results.filter((result) => holdsDocument(result.docId));
271
- const holdsEveryHit = outcome.resolvedEveryHit && results.length === outcome.results.length;
272
- return { results, matched: outcome.matched, matchedExact: outcome.matchedExact && holdsEveryHit };
273
- }
274
- function scanned(scan) {
275
- return { results: scan.results, matched: scan.matched, matchedExact: true, resolvedEveryHit: true };
276
- }
277
- function search2(query, k, searchOptions, filter) {
278
- const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = searchOptions;
279
- const empty = { results: [], matched: 0, matchedExact: true, resolvedEveryHit: true };
280
- const liveDocuments = flags.liveDocuments();
281
- if (liveDocuments <= 0) return empty;
282
- if (filter !== void 0 && filter.count === 0) return empty;
283
- function fetched(results, resolvedEveryHit2) {
284
- if (minSimilarity === Number.NEGATIVE_INFINITY) {
285
- const matched = filter === void 0 ? liveDocuments : filter.count;
286
- return { results, matched, matchedExact: true, resolvedEveryHit: resolvedEveryHit2 };
287
- }
288
- return { results, matched: results.length, matchedExact: false, resolvedEveryHit: resolvedEveryHit2 };
289
- }
290
- if (k <= 0) return fetched([], true);
291
- const graph = view.graph;
292
- if (graph === null) {
293
- const ordinals = filter === void 0 ? flags.unplacedOrdinals() : ordinalFilterValues(filter);
294
- return scanned(scoreCandidates(view.store, candidatesOf(ordinals, filter), query, k, metric, minSimilarity));
295
- }
296
- if (filter !== void 0) {
297
- const graphLiveSize = view.liveSize;
298
- const selectivity = graphLiveSize > 0 ? filter.count / graphLiveSize : 1;
299
- if (selectivity < view.handles.filterThreshold) {
300
- const candidates = candidatesOf(ordinalFilterValues(filter), filter);
301
- return scanned(scoreCandidates(view.store, candidates, query, k, metric, minSimilarity));
302
- }
303
- }
304
- let resolvedEveryHit = true;
305
- const graphResults = [];
306
- for (const hit of view.searchOrdinals(query, k, metric, minSimilarity, {
307
- filter,
308
- efSearch,
309
- oversample,
310
- rescoreWindow
311
- })) {
312
- const docId = view.docIdOf(hit.ord);
313
- if (docId === void 0) {
314
- resolvedEveryHit = false;
315
- continue;
316
- }
317
- graphResults.push({ docId, score: hit.score });
318
- }
319
- const unplaced = flags.unplacedOrdinals();
320
- if (unplaced.length === 0) return fetched(graphResults, resolvedEveryHit);
321
- const scan = scoreCandidates(view.store, candidatesOf(unplaced, filter), query, k, metric, minSimilarity);
322
- return fetched(mergeResults(graphResults, scan.results, k), resolvedEveryHit);
323
- }
324
- return {
325
- fieldName,
326
- dimension: view.handles.dimension,
327
- partitionsKnown: () => true,
328
- assignPartitions: () => void 0,
329
- searchParallel(query, k, searchOptions) {
330
- if (query.length !== view.handles.dimension) {
331
- throw new NarsilError(
332
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
333
- `Vector dimension mismatch: expected ${view.handles.dimension}, got ${query.length}`,
334
- { expected: view.handles.dimension, received: query.length }
335
- );
336
- }
337
- for (let attempt = 1; ; attempt++) {
338
- const layoutGeneration = ordinalLayoutGeneration();
339
- const outcome = search2(query, k, searchOptions, filterFor(searchOptions));
340
- if (view.graphRetired) {
341
- throw new NarsilError(
342
- ErrorCodes.WORKER_BUSY,
343
- `The main thread replaced the graph of vector field "${fieldName}" while this thread searched it`,
344
- { fieldName }
345
- );
346
- }
347
- if (ordinalLayoutGeneration() === layoutGeneration) return Promise.resolve(answerOf(outcome));
348
- if (attempt >= MAX_RENUMBERED_SEARCH_ATTEMPTS) {
349
- throw new NarsilError(
350
- ErrorCodes.WORKER_BUSY,
351
- `The main thread renumbered the vectors of field "${fieldName}" during each of ${attempt} searches on this thread`,
352
- { fieldName, attempts: attempt }
353
- );
354
- }
355
- }
356
- }
357
- };
358
- }
359
-
360
- // src/vector/search-pool/spawn.ts
361
- function resolveWorkerEntryPoint() {
362
- return resolveWorkerEntry(import.meta.url, /\/src\/vector\/search-pool\/[^/]+$/, "vector/search-worker.mjs");
363
- }
364
- async function spawnWorker(entryPoint) {
365
- if (entryPoint === null) return null;
366
- const runtime = detectRuntime();
367
- if (runtime.supportsWorkerThreads) {
368
- try {
369
- return await spawnNodeWorker(new URL(entryPoint), workerResourceLimits());
370
- } catch {
371
- return null;
372
- }
373
- }
374
- if (runtime.supportsWebWorkers) {
375
- try {
376
- const WorkerCtor = globalThis.Worker;
377
- if (typeof WorkerCtor !== "function") return null;
378
- return new WorkerCtor(entryPoint, { type: "module" });
379
- } catch {
380
- return null;
381
- }
382
- }
383
- return null;
384
- }
385
- function listen(worker, handler) {
386
- if (typeof worker.on === "function") {
387
- worker.on("message", handler);
388
- return;
389
- }
390
- worker.addEventListener?.("message", (event) => {
391
- handler(event.data);
392
- });
393
- }
394
- function listenForFailure(worker, handler) {
395
- if (typeof worker.on === "function") {
396
- worker.on("error", (err) => handler(err instanceof Error ? err : new Error(String(err))));
397
- worker.on("exit", () => handler(new Error("Vector search worker exited")));
398
- return;
399
- }
400
- worker.addEventListener?.("error", () => handler(new Error("Vector search worker failed")));
401
- }
402
-
403
- // src/vector/search-pool/index.ts
404
- function searchPoolScratchSlot(index) {
405
- return VECTOR_SCRATCH_SLOTS - 1 - index;
406
- }
407
- function graphSearchOptionsOf(options) {
408
- return {
409
- ...options.filter !== void 0 ? { filter: options.filter } : {},
410
- ...options.efSearch !== void 0 ? { efSearch: options.efSearch } : {},
411
- ...options.oversample !== void 0 ? { oversample: options.oversample } : {},
412
- ...options.rescoreWindow !== void 0 ? { rescoreWindow: options.rescoreWindow } : {}
413
- };
414
- }
415
- async function createVectorSearchPool(requestedCount) {
416
- const entryPoint = resolveWorkerEntryPoint();
417
- const count = resolveWorkerCount(requestedCount);
418
- const slots = [];
419
- const sharedFields = /* @__PURE__ */ new Map();
420
- for (let i = 0; i < count; i++) {
421
- const worker = await spawnWorker(entryPoint);
422
- if (worker === null) break;
423
- const slot = {
424
- worker,
425
- scratchSlot: searchPoolScratchSlot(i),
426
- pending: /* @__PURE__ */ new Map(),
427
- alive: true,
428
- outstanding: 0
429
- };
430
- listen(worker, (raw) => {
431
- const message = raw;
432
- const key = message.requestId;
433
- if (typeof key !== "string") return;
434
- const waiting = slot.pending.get(key);
435
- if (!waiting) return;
436
- slot.pending.delete(key);
437
- clearTimeout(waiting.timer);
438
- waiting.resolve(message);
439
- });
440
- listenForFailure(worker, (err) => {
441
- slot.alive = false;
442
- for (const handles of sharedFields.values()) {
443
- if (handles.graph !== null) releaseLocksHeldBy(handles.graph, slot.scratchSlot);
444
- }
445
- for (const [, waiting] of slot.pending) {
446
- clearTimeout(waiting.timer);
447
- waiting.reject(err);
448
- }
449
- slot.pending.clear();
450
- });
451
- slots.push(slot);
452
- }
453
- if (slots.length === 0) return null;
454
- let requestCounter = 0;
455
- function send(slot, key, message, timeoutMs) {
456
- return new Promise((resolve, reject) => {
457
- if (!slot.alive) {
458
- reject(new Error("Vector search worker is not running"));
459
- return;
460
- }
461
- const timer = setTimeout(() => {
462
- slot.pending.delete(key);
463
- reject(new Error(`Vector search worker did not answer within ${timeoutMs}ms`));
464
- }, timeoutMs);
465
- slot.pending.set(key, { resolve, reject, timer });
466
- try {
467
- slot.worker.postMessage(message);
468
- } catch (err) {
469
- slot.pending.delete(key);
470
- clearTimeout(timer);
471
- reject(err instanceof Error ? err : new Error(String(err)));
472
- }
473
- });
474
- }
475
- function refusalOf(message) {
476
- return message.type === "error" ? message.message : `the worker answered "${message.type}"`;
477
- }
478
- function pickSlot() {
479
- let slot = null;
480
- for (const candidate of slots) {
481
- if (!candidate.alive) continue;
482
- if (slot === null || candidate.outstanding < slot.outstanding) slot = candidate;
483
- if (slot.outstanding === 0) break;
484
- }
485
- return slot;
486
- }
487
- async function sendBusy(slot, request, timeoutMs) {
488
- slot.outstanding += 1;
489
- try {
490
- return await send(slot, request.requestId, request, timeoutMs);
491
- } finally {
492
- slot.outstanding -= 1;
493
- }
494
- }
495
- async function broadcast(build, timeoutMs) {
496
- const outcomes = await Promise.allSettled(
497
- slots.map((slot) => {
498
- requestCounter += 1;
499
- const requestId = `${requestCounter}`;
500
- return send(slot, requestId, build(slot, requestId), timeoutMs);
501
- })
502
- );
503
- return outcomes.every(
504
- (outcome) => outcome.status === "fulfilled" && outcome.value.type === "ack"
505
- );
506
- }
507
- return {
508
- get workerCount() {
509
- return slots.filter((slot) => slot.alive).length;
510
- },
511
- load(handle, snapshot) {
512
- return broadcast(
513
- (_slot, requestId) => ({ type: "load", requestId, handle, snapshot }),
514
- VECTOR_SEARCH_LOAD_TIMEOUT_MS
515
- );
516
- },
517
- async loadShared(handle, handles) {
518
- sharedFields.set(handle, handles);
519
- const loaded = await broadcast(
520
- (slot, requestId) => ({ type: "loadShared", requestId, handle, scratchSlot: slot.scratchSlot, handles }),
521
- VECTOR_SEARCH_LOAD_TIMEOUT_MS
522
- );
523
- if (!loaded) sharedFields.delete(handle);
524
- return loaded;
525
- },
526
- async drop(handle) {
527
- sharedFields.delete(handle);
528
- await broadcast((_slot, requestId) => ({ type: "drop", requestId, handle }), VECTOR_SEARCH_TIMEOUT_MS);
529
- },
530
- async insertOrdinals(handle, ordinals) {
531
- const slot = pickSlot();
532
- if (slot === null) return null;
533
- requestCounter += 1;
534
- const request = { type: "insertOrdinals", requestId: `${requestCounter}`, handle, ordinals };
535
- const message = await sendBusy(slot, request, VECTOR_INSERT_TIMEOUT_MS);
536
- if (message.type !== "inserted") {
537
- console.warn("A vector worker placed no vectors in the shared graph:", refusalOf(message));
538
- return null;
539
- }
540
- return message.outcome;
541
- },
542
- async search(handle, query, k, metric, minSimilarity, options = {}) {
543
- const slot = pickSlot();
544
- if (slot === null) throw new Error("No vector search worker is running");
545
- requestCounter += 1;
546
- const request = {
547
- type: "search",
548
- requestId: `${requestCounter}`,
549
- handle,
550
- query,
551
- k,
552
- metric,
553
- minSimilarity,
554
- ...graphSearchOptionsOf(options)
555
- };
556
- const message = await sendBusy(slot, request, VECTOR_SEARCH_TIMEOUT_MS);
557
- if (message.type === "error") throw new Error(message.message);
558
- if (message.type !== "result") throw new Error("Vector search worker returned an unexpected message");
559
- const results = new Array(message.docIds.length);
560
- for (let i = 0; i < message.docIds.length; i++) {
561
- results[i] = { docId: message.docIds[i], score: message.scores[i] };
562
- }
563
- return results;
564
- },
565
- async searchOrdinals(handle, query, k, metric, minSimilarity, options = {}) {
566
- const slot = pickSlot();
567
- if (slot === null) throw new Error("No vector search worker is running");
568
- requestCounter += 1;
569
- const request = {
570
- type: "searchOrdinals",
571
- requestId: `${requestCounter}`,
572
- handle,
573
- query,
574
- k,
575
- metric,
576
- minSimilarity,
577
- ...graphSearchOptionsOf(options)
578
- };
579
- const message = await sendBusy(slot, request, VECTOR_SEARCH_TIMEOUT_MS);
580
- if (message.type === "error") throw new Error(message.message);
581
- if (message.type !== "ordinalResult") throw new Error("Vector search worker returned an unexpected message");
582
- return { ordinals: message.ordinals, scores: message.scores };
583
- },
584
- async shutdown() {
585
- for (const slot of slots) {
586
- slot.alive = false;
587
- for (const [, waiting] of slot.pending) {
588
- clearTimeout(waiting.timer);
589
- waiting.reject(new Error("Vector search pool has shut down"));
590
- }
591
- slot.pending.clear();
592
- try {
593
- await slot.worker.terminate();
594
- } catch {
595
- slot.alive = false;
596
- }
597
- }
598
- }
599
- };
600
- }
601
- var sharedPool = null;
602
- var poolHolders = 0;
603
- var workersUnavailable = false;
604
- function acquireVectorSearchPool(requestedCount) {
605
- poolHolders += 1;
606
- if (workersUnavailable) return Promise.resolve(null);
607
- if (sharedPool === null) {
608
- sharedPool = createVectorSearchPool(requestedCount).then(
609
- (pool) => {
610
- if (pool === null) workersUnavailable = true;
611
- return pool;
612
- },
613
- () => {
614
- workersUnavailable = true;
615
- return null;
616
- }
617
- );
618
- }
619
- return sharedPool;
620
- }
621
- async function releaseVectorSearchPool() {
622
- if (poolHolders === 0) return;
623
- poolHolders -= 1;
624
- if (poolHolders > 0 || sharedPool === null) return;
625
- const pending = sharedPool;
626
- sharedPool = null;
627
- const pool = await pending;
628
- await pool?.shutdown();
629
- }
630
-
631
- // src/vector/vector-index/worker-copies.ts
632
- var handleCounter = 0;
633
- function nextHandle(state) {
634
- handleCounter += 1;
635
- return `${state.indexName}/${state.fieldName}#${handleCounter}`;
636
- }
637
- async function dropHandle(state, handle) {
638
- const host = state.workerCopies.host;
639
- if (host !== void 0) {
640
- await host.drop(state.indexName, state.fieldName, handle).catch(() => void 0);
641
- return;
642
- }
643
- const pool = state.workerCopyPool;
644
- if (pool !== null) {
645
- await pool.drop(handle).catch(() => void 0);
646
- await releaseVectorSearchPool().catch(() => void 0);
647
- }
648
- }
649
- function dropSharedGraph(state, graph) {
650
- const shared = state.sharedHandles.get(graph);
651
- if (shared === void 0) return;
652
- state.sharedHandles.delete(graph);
653
- if (state.workerCopyHandle === shared.handle) {
654
- state.workerCopyHandle = null;
655
- state.workerCopyRevision = -1;
656
- state.workerCopyMode = null;
657
- }
658
- void dropHandle(state, shared.handle);
659
- }
660
- function invalidateWorkerCopies(state) {
661
- state.revision += 1;
662
- for (const graph of [...state.sharedHandles.keys()]) dropSharedGraph(state, graph);
663
- if (state.workerCopyHandle !== null) {
664
- const handle = state.workerCopyHandle;
665
- state.workerCopyHandle = null;
666
- state.workerCopyRevision = -1;
667
- state.workerCopyMode = null;
668
- void dropHandle(state, handle);
669
- }
670
- if (state.workerCopies.host === void 0 && state.workerCopyPool !== null && state.sharedHandles.size === 0) {
671
- state.workerCopyPool = null;
672
- }
673
- }
674
- async function withdrawWorkerCopies(state) {
675
- state.revision += 1;
676
- const handles = /* @__PURE__ */ new Set();
677
- for (const shared of state.sharedHandles.values()) handles.add(shared.handle);
678
- if (state.workerCopyHandle !== null) handles.add(state.workerCopyHandle);
679
- state.sharedHandles.clear();
680
- state.workerCopyHandle = null;
681
- state.workerCopyRevision = -1;
682
- state.workerCopyMode = null;
683
- await Promise.all([...handles].map((handle) => dropHandle(state, handle)));
684
- if (state.workerCopies.host === void 0 && state.sharedHandles.size === 0) state.workerCopyPool = null;
685
- }
686
- async function holdWorkerCopies(state) {
687
- state.workerCopiesHeld = true;
688
- await state.sharing;
689
- await withdrawWorkerCopies(state);
690
- }
691
- function noteWrite(state) {
692
- state.revision += 1;
693
- if (state.workerCopyMode === "clone") invalidateWorkerCopies(state);
694
- }
695
- function refreshWorkerCopies(state) {
696
- invalidateWorkerCopies(state);
697
- scheduleWorkerCopyLoad(state);
698
- }
699
- function recordSharedField(state, record) {
700
- const { graph, handle, searchable, mode, revision, layoutRevision } = record;
701
- state.sharedHandles.set(graph, { handle, searchable, mode });
702
- state.sharedLayoutRevision = layoutRevision;
703
- if (!searchable) return;
704
- state.workerCopyHandle = handle;
705
- state.workerCopyRevision = revision;
706
- state.workerCopyMode = mode;
707
- }
708
- async function loadOnHost(state, host, graph, searchable) {
709
- if (!host.holdsIndex(state.indexName)) return null;
710
- if (!state.store.partitionsKnown) {
711
- assignStorePartitions(state, (docId) => host.resolvePartition(state.indexName, docId));
712
- }
713
- const handle = state.sharedHandles.get(graph)?.handle ?? nextHandle(state);
714
- const layoutRevision = state.store.handles.layoutRevision;
715
- let loaded = false;
716
- try {
717
- loaded = await host.loadShared(state.indexName, state.fieldName, handle, fieldHandlesOf(state, graph, searchable));
718
- } catch {
719
- loaded = false;
720
- }
721
- if (!loaded || state.disposed) return null;
722
- recordSharedField(state, { graph, handle, searchable, mode: "hosted", revision: state.revision, layoutRevision });
723
- return handle;
724
- }
725
- function captureCloneCodes(state) {
726
- const quantizer = state.osq;
727
- const centroid = quantizer?.centroid ?? null;
728
- const layout = state.store.handles.codeLayout;
729
- if (quantizer === null || centroid === null || layout === null) return null;
730
- const slots = state.store.slots;
731
- const records = new Uint8Array(slots * layout.slotStride);
732
- const present = new Uint8Array(slots);
733
- for (let ordinal = 0; ordinal < slots; ordinal++) {
734
- const record = quantizer.recordAtOrdinal(ordinal);
735
- if (record === void 0) continue;
736
- records.set(record, ordinal * layout.slotStride);
737
- present[ordinal] = 1;
738
- }
739
- return { centroid: Float32Array.from(centroid), records, present };
740
- }
741
- function captureCloneSnapshot(state, graph) {
742
- return {
743
- dimension: state.dimension,
744
- quantization: state.quantizationMode,
745
- metric: state.metric,
746
- codes: captureCloneCodes(state),
747
- store: state.store.exportSnapshot(),
748
- graph: graph.exportSnapshot(),
749
- tombstones: Array.from(state.tombstones)
750
- };
751
- }
752
- async function loadOnPool(state, graph, searchable) {
753
- if (graph === null) return null;
754
- const revision = state.revision;
755
- const pool = state.workerCopyPool ?? await acquireVectorSearchPool(state.workerCopies.count);
756
- if (pool === null) {
757
- await releaseVectorSearchPool();
758
- return null;
759
- }
760
- if (state.disposed) {
761
- await releaseVectorSearchPool();
762
- return null;
763
- }
764
- state.workerCopyPool = pool;
765
- const handle = state.sharedHandles.get(graph)?.handle ?? nextHandle(state);
766
- const layoutRevision = state.store.handles.layoutRevision;
767
- let mode = "shared";
768
- let loaded = false;
769
- try {
770
- if (sharedMemoryAvailable()) {
771
- loaded = await pool.loadShared(handle, fieldHandlesOf(state, graph, searchable));
772
- }
773
- if (!loaded && searchable) {
774
- mode = "clone";
775
- loaded = await pool.load(handle, captureCloneSnapshot(state, graph));
776
- }
777
- } catch {
778
- loaded = false;
779
- }
780
- if (!loaded || state.disposed || mode === "clone" && state.revision !== revision) {
781
- await pool.drop(handle).catch(() => void 0);
782
- return null;
783
- }
784
- recordSharedField(state, { graph, handle, searchable, mode, revision, layoutRevision });
785
- return handle;
786
- }
787
- function shareGraph(state, graph, searchable) {
788
- const run = state.sharing.then(async () => {
789
- if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return null;
790
- const existing = state.sharedHandles.get(graph);
791
- const layoutUnchanged = threadsHoldCurrentLayout(state);
792
- if (existing !== void 0 && layoutUnchanged && (existing.searchable || !searchable)) return existing.handle;
793
- const host = state.workerCopies.host;
794
- return host !== void 0 ? loadOnHost(state, host, graph, searchable) : loadOnPool(state, graph, searchable);
795
- });
796
- state.sharing = run.then(
797
- () => void 0,
798
- () => void 0
799
- );
800
- return run;
801
- }
802
- function resendSharedHandles(state) {
803
- const run = state.sharing.then(async () => {
804
- if (!state.workerCopies.enabled || state.disposed || state.workerCopiesHeld) return;
805
- const host = state.workerCopies.host;
806
- for (const [graph, shared] of [...state.sharedHandles]) {
807
- if (host !== void 0) await loadOnHost(state, host, graph, shared.searchable);
808
- else await loadOnPool(state, graph, shared.searchable);
809
- }
810
- });
811
- state.sharing = run.then(
812
- () => void 0,
813
- () => void 0
814
- );
815
- return run;
816
- }
817
- function scheduleWorkerCopyLoad(state) {
818
- if (!state.workerCopies.enabled || state.workerCopiesHeld) return;
819
- if (state.disposed || state.workerCopyLoading || state.building) return;
820
- const graph = state.hnsw;
821
- const host = state.workerCopies.host;
822
- if (graph === null && host === void 0) return;
823
- const existing = state.sharedHandles.get(graph);
824
- if (existing?.searchable && threadsHoldCurrentLayout(state)) return;
825
- if (host === void 0 && liveSize(state) < WORKER_COPY_MIN_VECTORS) return;
826
- state.workerCopyLoading = true;
827
- void shareGraph(state, graph, true).then(
828
- () => {
829
- state.workerCopyLoading = false;
830
- },
831
- () => {
832
- state.workerCopyLoading = false;
833
- }
834
- );
835
- }
836
- async function searchViaWorkerCopies(state, query, k, metric, minSimilarity, options) {
837
- const pool = state.workerCopyPool;
838
- const handle = state.workerCopyHandle;
839
- if (pool === null || handle === null || state.workerCopyMode === "hosted") return null;
840
- if (state.workerCopyMode === "clone" && state.workerCopyRevision !== state.revision) return null;
841
- try {
842
- if (state.workerCopyMode === "shared") {
843
- const outcome = await pool.searchOrdinals(handle, query, k, metric, minSimilarity, options);
844
- const results = [];
845
- for (let i = 0; i < outcome.ordinals.length; i++) {
846
- const docId = state.store.docIdForOrdinal(outcome.ordinals[i]);
847
- if (docId === void 0) continue;
848
- results.push({ docId, score: outcome.scores[i] });
849
- }
850
- return results;
851
- }
852
- return await pool.search(handle, query, k, metric, minSimilarity, options);
853
- } catch {
854
- return null;
855
- }
856
- }
857
-
858
- // src/vector/vector-index/build-host.ts
859
- async function dispatcherFor(state, graph) {
860
- if (!state.workerCopies.enabled || state.disposed) return null;
861
- const host = state.workerCopies.host;
862
- if (host !== void 0) {
863
- if (!host.holdsIndex(state.indexName)) return null;
864
- const handle2 = await shareGraph(state, graph, graph === state.hnsw);
865
- if (handle2 === null) return null;
866
- return {
867
- send: (ordinals) => host.insertOrdinals(state.indexName, state.fieldName, handle2, ordinals),
868
- workerCount: Math.max(1, host.workerCount)
869
- };
870
- }
871
- if (liveSize(state) < WORKER_COPY_MIN_VECTORS || !sharedMemoryAvailable()) return null;
872
- const handle = await shareGraph(state, graph, graph === state.hnsw);
873
- const pool = state.workerCopyPool;
874
- if (handle === null || pool === null || state.sharedHandles.get(graph)?.mode !== "shared") return null;
875
- return {
876
- send: (ordinals) => pool.insertOrdinals(handle, ordinals),
877
- workerCount: Math.max(1, pool.workerCount)
878
- };
879
- }
880
- async function insertIntoGraph(state, graph, docIds, admit) {
881
- const found = await dispatcherFor(state, graph);
882
- if (found === null) return insertLocally(state, graph, docIds, admit);
883
- const dispatch = found;
884
- const pending = /* @__PURE__ */ new Set();
885
- const limit = dispatch.workerCount * GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER;
886
- const fallback = { noWorkerWarned: false, shortAnswerWarned: false };
887
- let batchDocIds = [];
888
- let batchOrdinals = [];
889
- async function flush() {
890
- if (batchOrdinals.length === 0) return;
891
- const chunkDocIds = batchDocIds;
892
- const chunkOrdinals = Int32Array.from(batchOrdinals);
893
- batchDocIds = [];
894
- batchOrdinals = [];
895
- if (!threadsHoldCurrentLayout(state)) await shareGraph(state, graph, graph === state.hnsw);
896
- const run = placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fallback).finally(() => {
897
- pending.delete(run);
898
- });
899
- pending.add(run);
900
- if (pending.size >= limit) await Promise.race(pending);
901
- }
902
- for (const docId of docIds) {
903
- if (state.disposed) break;
904
- if (!admit(docId)) continue;
905
- const ordinal = state.store.getOrdinal(docId);
906
- if (ordinal === void 0) continue;
907
- batchDocIds.push(docId);
908
- batchOrdinals.push(ordinal);
909
- if (batchOrdinals.length >= GRAPH_BUILD_DISPATCH_CHUNK) await flush();
910
- }
911
- await flush();
912
- await Promise.all(pending);
913
- return !state.disposed;
914
- }
915
- async function placeChunk(state, graph, dispatch, chunkDocIds, chunkOrdinals, fallback) {
916
- let outcome = null;
917
- try {
918
- outcome = await dispatch.send(chunkOrdinals);
919
- } catch {
920
- outcome = null;
921
- }
922
- if (state.disposed) return;
923
- if (outcome === null) {
924
- if (!fallback.noWorkerWarned) {
925
- fallback.noWorkerWarned = true;
926
- console.warn(
927
- `No worker thread placed the vectors of "${state.indexName}/${state.fieldName}", so this thread is building its graph itself`
928
- );
929
- }
930
- for (const ordinal of chunkOrdinals) graph.insertOrdinal(ordinal);
931
- }
932
- let placedHere = 0;
933
- for (let i = 0; i < chunkDocIds.length; i++) {
934
- const docId = chunkDocIds[i];
935
- const ordinal = chunkOrdinals[i];
936
- if (state.store.getOrdinal(docId) !== ordinal) {
937
- graph.markTombstoneOrdinal(ordinal);
938
- continue;
939
- }
940
- if (!state.tombstones.has(docId) && !graph.has(docId) && graph.insertOrdinal(ordinal)) placedHere += 1;
941
- if (graph === state.hnsw) state.buffer.delete(docId);
942
- }
943
- if (placedHere > 0 && outcome !== null && !fallback.shortAnswerWarned) {
944
- fallback.shortAnswerWarned = true;
945
- console.warn(
946
- `A worker thread answered a chunk of "${state.indexName}/${state.fieldName}" without placing ${placedHere} of its vectors, so this thread placed them itself`
947
- );
948
- }
949
- }
950
- async function insertLocally(state, graph, docIds, admit) {
951
- let count = 0;
952
- for (const docId of docIds) {
953
- if (state.disposed) return false;
954
- if (!admit(docId)) continue;
955
- const ordinal = state.store.getOrdinal(docId);
956
- if (ordinal === void 0) continue;
957
- graph.insertOrdinal(ordinal);
958
- if (graph === state.hnsw) state.buffer.delete(docId);
959
- count += 1;
960
- if (count % BUILD_CHUNK_SIZE === 0) {
961
- noteWrite(state);
962
- await yieldToEventLoop();
963
- }
964
- }
965
- if (count > 0) noteWrite(state);
966
- return !state.disposed;
967
- }
968
-
969
- // src/vector/vector-index/disk.ts
970
- function readsFromDisk(state) {
971
- return state.storage === "disk";
972
- }
973
- function holdsVectorsOnDisk(state) {
974
- return readsFromDisk(state) && state.hnsw !== null;
975
- }
976
- function* locationsOf(state, layout) {
977
- const stride = state.dimension * 4;
978
- for (let i = 0; i < layout.docIds.length; i++) {
979
- if (isDead(layout.dead, i)) continue;
980
- yield [layout.docIds[i], { path: layout.path, offset: layout.vectorsOffset + i * stride }];
981
- }
982
- }
983
- async function shareFilesWithEveryThread(state, paths) {
984
- const fileIndexes = /* @__PURE__ */ new Map();
985
- for (const path of paths) fileIndexes.set(path, state.store.addVectorFile(path));
986
- await resendSharedHandles(state);
987
- return fileIndexes;
988
- }
989
- async function releaseLocations(state, paths, locations) {
990
- const fileIndexes = await shareFilesWithEveryThread(state, paths);
991
- if (state.disposed) return;
992
- let count = 0;
993
- for (const [docId, location] of locations) {
994
- if (count > 0 && count % DISK_ADOPTION_YIELD_INTERVAL === 0) {
995
- await yieldToEventLoop();
996
- if (state.disposed) return;
997
- }
998
- count += 1;
999
- const ordinal = state.store.getOrdinal(docId);
1000
- const fileIndex = fileIndexes.get(location.path);
1001
- if (ordinal === void 0 || fileIndex === void 0) continue;
1002
- state.store.releaseToFile(ordinal, { fileIndex, offset: location.offset });
1003
- }
1004
- state.store.releaseColdBlocks();
1005
- await resendSharedHandles(state);
1006
- }
1007
- function releaseInTurn(state, release) {
1008
- const run = state.releaseToFilesInFlight.then(release);
1009
- state.releaseToFilesInFlight = run.then(
1010
- () => void 0,
1011
- () => void 0
1012
- );
1013
- return run;
1014
- }
1015
- async function adoptDiskLayout(state, layout) {
1016
- if (!readsFromDisk(state) || state.disposed || layout.docIds.length === 0) return;
1017
- if (!holdsVectorsOnDisk(state)) {
1018
- for (const [docId, location] of locationsOf(state, layout)) {
1019
- if (state.store.has(docId)) state.pendingLocations.set(docId, location);
1020
- }
1021
- return;
1022
- }
1023
- await releaseInTurn(state, () => releaseLocations(state, [layout.path], locationsOf(state, layout)));
1024
- }
1025
- async function releasePendingLocations(state) {
1026
- if (state.pendingLocations.size === 0 || !holdsVectorsOnDisk(state) || state.disposed) return;
1027
- const pending = [...state.pendingLocations];
1028
- state.pendingLocations.clear();
1029
- const paths = new Set(pending.map(([, location]) => location.path));
1030
- await releaseInTurn(state, () => releaseLocations(state, paths, pending));
1031
- }
1032
-
1033
- // src/vector/vector-index/build.ts
1034
- async function buildGraphFromStore(state) {
1035
- const graph = createHNSWIndex(state.dimension, state.store, state.hnswConfig, state.osq ?? void 0);
1036
- state.freshGraph = graph;
1037
- let completed = false;
1038
- try {
1039
- completed = await insertIntoGraph(state, graph, [...allLiveDocIds(state)], () => true);
1040
- } finally {
1041
- state.freshGraph = null;
1042
- }
1043
- if (!completed) {
1044
- dropSharedGraph(state, graph);
1045
- return;
1046
- }
1047
- const previous = state.hnsw;
1048
- adoptGraph(state, graph);
1049
- for (const docId of [...state.buffer]) {
1050
- const ordinal = state.store.getOrdinal(docId);
1051
- if (ordinal !== void 0 && graph.holdsNode(ordinal) && !graph.holdsTombstone(ordinal)) state.buffer.delete(docId);
1052
- }
1053
- if (previous !== null && previous !== graph) dropSharedGraph(state, previous);
1054
- await releasePendingLocations(state);
1055
- }
1056
- async function promoteToGraph(state) {
1057
- if (liveSize(state) === 0) return;
1058
- calibrateQuantizer(state);
1059
- await buildGraphFromStore(state);
1060
- }
1061
- async function growGraph(state, graph, bufferSnapshot) {
1062
- const admit = (docId) => {
1063
- if (state.hnsw !== graph) return false;
1064
- if (state.tombstones.has(docId) || !state.store.has(docId)) {
1065
- state.buffer.delete(docId);
1066
- return false;
1067
- }
1068
- return true;
1069
- };
1070
- if (state.osq && !state.osq.isCalibrated()) calibrateQuantizer(state);
1071
- await insertIntoGraph(state, graph, bufferSnapshot, admit);
1072
- }
1073
- function triggerBuild(state) {
1074
- if (state.building) return;
1075
- state.building = true;
1076
- const bufferSnapshot = new Set(state.buffer);
1077
- const existingGraph = state.hnsw;
1078
- const buildPromise = (async () => {
1079
- try {
1080
- if (state.disposed) return;
1081
- if (existingGraph !== null) {
1082
- await growGraph(state, existingGraph, bufferSnapshot);
1083
- return;
1084
- }
1085
- await promoteToGraph(state);
1086
- } finally {
1087
- state.building = false;
1088
- state.pendingBuild = null;
1089
- if (state.buffer.size > 0) {
1090
- scheduleBuild(state);
1091
- } else {
1092
- scheduleWorkerCopyLoad(state);
1093
- }
1094
- }
1095
- })();
1096
- state.pendingBuild = buildPromise;
1097
- }
1098
- function buildDue(state) {
1099
- if (state.hnsw === null) return liveSize(state) >= state.promotionThreshold;
1100
- if (state.buffer.size === 0) return false;
1101
- if (state.workerCopies.enabled && state.workerCopies.host !== void 0) return true;
1102
- return state.buffer.size >= state.promotionThreshold;
1103
- }
1104
- function scheduleBuild(state) {
1105
- if (state.building || state.buildScheduled || state.disposed) return;
1106
- if (!buildDue(state)) return;
1107
- state.buildScheduled = true;
1108
- setTimeout(() => {
1109
- state.buildScheduled = false;
1110
- triggerBuild(state);
1111
- }, 0);
1112
- }
1113
-
1114
- // src/vector/vector-index/checkpoint-restore.ts
1115
- var CALIBRATION_NEVER_CURRENT = -1;
1116
- function sameCentroid(a, b) {
1117
- if (a.length !== b.length) return false;
1118
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
1119
- return true;
1120
- }
1121
- function bufferEveryVectorOutsideTheGraph(state) {
1122
- for (const docId of state.store.docIds()) {
1123
- if (state.tombstones.has(docId)) continue;
1124
- if (state.hnsw === null || !state.hnsw.has(docId)) state.buffer.add(docId);
1125
- }
1126
- }
1127
- function insertNodesOfLaterGraph(state, graph, later, ordinals) {
1128
- for (const part of later.parts) requireWholeNeighbourLists(part);
1129
- for (let number = 0; number < ordinals.length; number++) {
1130
- const ordinal = ordinals[number];
1131
- if (ordinal === NO_LIVE_VECTOR_AT_POSITION || topLayerInParts(later, number) < 0) continue;
1132
- const docId = state.store.docIdForOrdinal(ordinal);
1133
- if (docId !== void 0 && !graph.has(docId)) graph.insertOrdinal(ordinal);
1134
- }
1135
- }
1136
- function restoreGraph(state, payload, ordinalOfNumber, tombstoned) {
1137
- const first = payload?.graphs[0]?.parts[0];
1138
- if (payload === null || first === void 0) {
1139
- releaseReplacedVectors(state);
1140
- bufferEveryVectorOutsideTheGraph(state);
1141
- return;
1142
- }
1143
- const graph = createHNSWIndex(
1144
- state.dimension,
1145
- state.store,
1146
- { m: first.m, efConstruction: first.efConstruction, metric: first.metric },
1147
- state.osq ?? void 0
1148
- );
1149
- graph.deserializeNumbered(payload.graphs[0], ordinalOfNumber);
1150
- for (const ordinal of tombstoned) graph.markTombstoneOrdinal(ordinal);
1151
- adoptGraph(state, graph);
1152
- for (const later of payload.graphs.slice(1)) insertNodesOfLaterGraph(state, graph, later, ordinalOfNumber);
1153
- bufferEveryVectorOutsideTheGraph(state);
1154
- if (state.buffer.size <= graph.size) return;
1155
- adoptGraph(state, null);
1156
- releaseReplacedVectors(state);
1157
- state.buffer.clear();
1158
- bufferEveryVectorOutsideTheGraph(state);
1159
- }
1160
- function beginCheckpointRestore(state, shape) {
1161
- emptyFieldBeforeRestore(state);
1162
- const cold = readsFromDisk(state) && (shape.holdsGraph || shape.liveVectors >= state.promotionThreshold);
1163
- const restoresCodes = state.osq !== null && osqBitsOf(state.quantizationMode) !== null && shape.holdsGraph;
1164
- const files = [];
1165
- const graphNodes = shape.graph?.graphs[0] ?? null;
1166
- const tombstoned = /* @__PURE__ */ new Set();
1167
- let nextNumber = 0;
1168
- let sharedCodes = null;
1169
- let codesAgree = true;
1170
- function restoreRecords(payload, ordinals) {
1171
- const quantizer = state.osq;
1172
- const layout = state.store.handles.codeLayout;
1173
- if (!restoresCodes || quantizer === null || layout === null || !codesAgree) return;
1174
- const codes = payload.codes;
1175
- if (codes === null || codes.bits !== quantizer.bits) {
1176
- codesAgree = false;
1177
- return;
1178
- }
1179
- if (sharedCodes === null) {
1180
- sharedCodes = codes;
1181
- quantizer.restoreCentroid(Float32Array.from(codes.centroid));
1182
- } else if (!sameCentroid(sharedCodes.centroid, codes.centroid)) {
1183
- codesAgree = false;
1184
- return;
1185
- }
1186
- const stride = layout.slotStride;
1187
- for (let i = 0; i < ordinals.length; i++) {
1188
- if (ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) continue;
1189
- const record = codes.records.subarray(i * stride, (i + 1) * stride);
1190
- if (tombstoned.has(ordinals[i])) quantizer.restoreRecordAtOrdinal(ordinals[i], record);
1191
- else quantizer.restoreRecord(payload.docIds[i], record);
1192
- }
1193
- }
1194
- function graphHoldsNodeFor(number) {
1195
- return graphNodes !== null && topLayerInParts(graphNodes, number) >= 0;
1196
- }
1197
- function addFile(file) {
1198
- const { payload, location } = file;
1199
- if (payload.dimension !== state.dimension) {
1200
- throw new NarsilError(
1201
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
1202
- `Payload dimension ${payload.dimension} does not match index dimension ${state.dimension}`,
1203
- { expected: state.dimension, received: payload.dimension }
1204
- );
1205
- }
1206
- const dimension = state.dimension;
1207
- const onDisk = readsFromDisk(state) && location !== null;
1208
- const vectors = bytesToVectors(payload.vectors);
1209
- const fileIndex = onDisk && cold ? state.store.addVectorFile(location.path) : null;
1210
- const ordinals = new Int32Array(payload.docIds.length).fill(NO_LIVE_VECTOR_AT_POSITION);
1211
- const firstNumber = nextNumber;
1212
- nextNumber += payload.docIds.length;
1213
- for (let i = 0; i < payload.docIds.length; i++) {
1214
- const dead = isDead(file.dead, i);
1215
- if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
1216
- const docId = payload.docIds[i];
1217
- const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
1218
- const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
1219
- if (dead) {
1220
- ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
1221
- tombstoned.add(ordinals[i]);
1222
- continue;
1223
- }
1224
- const replaced = state.store.getOrdinal(docId);
1225
- if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
1226
- if (fileIndex !== null) {
1227
- ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
1228
- continue;
1229
- }
1230
- ordinals[i] = state.store.insert(docId, vector);
1231
- if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
1232
- }
1233
- restoreRecords(payload, ordinals);
1234
- files.push({ id: file.id, key: file.key, ordinals, coded: payload.codes !== null });
1235
- }
1236
- function finish() {
1237
- const codesRestored = restoresCodes && codesAgree && sharedCodes !== null;
1238
- if (restoresCodes && !codesRestored) {
1239
- recalibrateFromStore(state);
1240
- for (const ordinal of tombstoned) state.osq?.writeCodes(ordinal);
1241
- }
1242
- let total = 0;
1243
- for (const file of files) total += file.ordinals.length;
1244
- const ordinalOfNumber = new Int32Array(total);
1245
- let number = 0;
1246
- for (const file of files) {
1247
- for (let i = 0; i < file.ordinals.length; i++) {
1248
- const ordinal = file.ordinals[i];
1249
- if (!state.store.holdsOrdinal(ordinal) && !tombstoned.has(ordinal)) {
1250
- file.ordinals[i] = NO_LIVE_VECTOR_AT_POSITION;
1251
- }
1252
- }
1253
- ordinalOfNumber.set(file.ordinals, number);
1254
- number += file.ordinals.length;
1255
- }
1256
- restoreGraph(state, shape.graph, ordinalOfNumber, tombstoned);
1257
- const current = signatureOf(state).calibration;
1258
- const codedUnder = codesRestored && current !== null ? current : CALIBRATION_NEVER_CURRENT;
1259
- const saved = files.map((file) => ({
1260
- id: file.id,
1261
- key: file.key,
1262
- ordinals: file.ordinals,
1263
- calibration: file.coded ? codedUnder : null
1264
- }));
1265
- state.savedFiles = saved;
1266
- state.savedSignature = signatureOf(state);
1267
- }
1268
- return { addFile, finish };
1269
- }
1270
-
1271
- // src/vector/vector-index/flagged-documents.ts
1272
- var FlaggedDocuments = class extends Set {
1273
- constructor(store, layout) {
1274
- super();
1275
- this.store = store;
1276
- this.layout = layout;
1277
- }
1278
- ordinals = /* @__PURE__ */ new Map();
1279
- handles = null;
1280
- flags = new Uint8Array(0);
1281
- chunks = null;
1282
- add(docId) {
1283
- super.add(docId);
1284
- const handles = this.openHandles();
1285
- if (handles === null) return this;
1286
- this.flagCurrentOrdinal(docId);
1287
- Atomics.store(handles.header, this.layout.countWord, this.size);
1288
- return this;
1289
- }
1290
- delete(docId) {
1291
- const held = super.delete(docId);
1292
- const handles = this.openHandles();
1293
- if (handles === null) return held;
1294
- const ordinal = this.ordinals.get(docId);
1295
- if (ordinal !== void 0) {
1296
- this.ordinals.delete(docId);
1297
- this.unflag(ordinal);
1298
- }
1299
- Atomics.store(handles.header, this.layout.countWord, this.size);
1300
- return held;
1301
- }
1302
- clear() {
1303
- super.clear();
1304
- const handles = this.openHandles();
1305
- if (handles !== null) {
1306
- for (const ordinal of this.ordinals.values()) this.unflag(ordinal);
1307
- Atomics.store(handles.header, this.layout.countWord, 0);
1308
- }
1309
- this.ordinals.clear();
1310
- }
1311
- flagCurrentOrdinal(docId) {
1312
- const ordinal = this.store.getOrdinal(docId);
1313
- const previous = this.ordinals.get(docId);
1314
- if (ordinal === void 0 || ordinal === previous) return;
1315
- if (this.chunks !== null) Atomics.add(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
1316
- Atomics.store(this.flags, ordinal, 1);
1317
- this.ordinals.set(docId, ordinal);
1318
- if (previous !== void 0) this.unflag(previous);
1319
- }
1320
- unflag(ordinal) {
1321
- Atomics.store(this.flags, ordinal, 0);
1322
- if (this.chunks !== null) Atomics.sub(this.chunks, Math.floor(ordinal / UNPLACED_FLAG_CHUNK_ORDINALS), 1);
1323
- }
1324
- openHandles() {
1325
- if (this.store.dimension === 0) return null;
1326
- const handles = this.store.handles;
1327
- if (handles === this.handles) return handles;
1328
- this.handles = handles;
1329
- this.flags = new Uint8Array(this.layout.flagsOf(handles));
1330
- const chunksOf = this.layout.chunksOf;
1331
- this.chunks = chunksOf === null ? null : new Int32Array(chunksOf(handles));
1332
- this.ordinals.clear();
1333
- for (const docId of this) this.flagCurrentOrdinal(docId);
1334
- return handles;
1335
- }
1336
- };
1337
- function createUnplacedVectors(store) {
1338
- return new FlaggedDocuments(store, {
1339
- flagsOf: (handles) => handles.unplaced,
1340
- chunksOf: (handles) => handles.unplacedChunks,
1341
- countWord: STORE_UNPLACED_COUNT
1342
- });
1343
- }
1344
- function createRemovedDocuments(store) {
1345
- return new FlaggedDocuments(store, {
1346
- flagsOf: (handles) => handles.removed,
1347
- chunksOf: null,
1348
- countWord: STORE_REMOVED_COUNT
1349
- });
1350
- }
1351
-
1352
- // src/vector/vector-index/maintenance.ts
1353
- function compact(state) {
1354
- if (state.tombstones.size === 0 && state.store.replacedVectorCount === 0) return;
1355
- if (state.hnsw) {
1356
- state.compactedNodeCount += state.hnsw.tombstoneCount;
1357
- state.hnsw.compactTombstones();
1358
- }
1359
- state.freshGraph?.compactTombstones();
1360
- for (const docId of state.tombstones) {
1361
- state.store.remove(docId);
1362
- state.buffer.delete(docId);
1363
- if (state.osq) {
1364
- state.osq.remove(docId);
1365
- }
1366
- }
1367
- releaseReplacedVectors(state);
1368
- state.tombstones.clear();
1369
- }
1370
- function recalibrateWhileNoThreadSearches(state, graph) {
1371
- if (graph === null) {
1372
- recalibrateFromStore(state);
1373
- return;
1374
- }
1375
- graph.exclusively(() => recalibrateFromStore(state));
1376
- }
1377
- async function insertMissing(state, graph) {
1378
- const missingOrReplaced = (docId) => !graph.has(docId) || state.buffer.has(docId);
1379
- const heldWhenTheCallStarted = [...allLiveDocIds(state)];
1380
- const stillLiveAndUnplaced = (docId) => !state.tombstones.has(docId) && missingOrReplaced(docId);
1381
- await insertIntoGraph(state, graph, heldWhenTheCallStarted, stillLiveAndUnplaced);
1382
- }
1383
- async function foldIntoGraph(state) {
1384
- const rebuildNeeded = graphNeedsRebuild(state);
1385
- compact(state);
1386
- if (liveSize(state) === 0) {
1387
- const previous = state.hnsw;
1388
- adoptGraph(state, null);
1389
- if (previous !== null) dropSharedGraph(state, previous);
1390
- state.buffer.clear();
1391
- if (state.osq) {
1392
- state.osq.clear();
1393
- }
1394
- return;
1395
- }
1396
- const graph = state.hnsw;
1397
- if (graph === null || rebuildNeeded) {
1398
- if (state.osq?.isCalibrated()) recalibrateWhileNoThreadSearches(state, graph);
1399
- else calibrateQuantizer(state);
1400
- await buildGraphFromStore(state);
1401
- } else {
1402
- await insertMissing(state, graph);
1403
- }
1404
- }
1405
- async function buildExclusively(state, work) {
1406
- while (state.pendingBuild) {
1407
- await state.pendingBuild;
1408
- }
1409
- if (state.disposed) return;
1410
- state.building = true;
1411
- const run = work();
1412
- state.pendingBuild = run;
1413
- try {
1414
- await run;
1415
- } finally {
1416
- state.building = false;
1417
- state.pendingBuild = null;
1418
- if (state.buffer.size > 0) {
1419
- scheduleBuild(state);
1420
- }
1421
- }
1422
- }
1423
- function optimize(state) {
1424
- return buildExclusively(state, () => foldIntoGraph(state));
1425
- }
1426
- async function fillGraph(state) {
1427
- const graph = state.hnsw;
1428
- if (graph === null) {
1429
- if (liveSize(state) >= state.promotionThreshold) await promoteToGraph(state);
1430
- return;
1431
- }
1432
- if (state.osq && !state.osq.isCalibrated()) calibrateQuantizer(state);
1433
- if (graphNeedsRebuild(state)) {
1434
- await buildGraphFromStore(state);
1435
- return;
1436
- }
1437
- await insertMissing(state, graph);
1438
- }
1439
- function completeGraph(state) {
1440
- return buildExclusively(state, () => fillGraph(state));
1441
- }
1442
- function maintenanceStatus(state) {
1443
- const storeSize = state.store.size;
1444
- const graphCount = state.hnsw ? 1 : 0;
1445
- const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
1446
- const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
1447
- const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
1448
- const estimatedCompactMs = Math.round(tombstoned * ESTIMATED_MS_PER_TOMBSTONE * state.dimensionScale);
1449
- const optimizeVectorCount = state.hnsw === null || graphNeedsRebuild(state) ? storeSize : state.buffer.size;
1450
- const estimatedOptimizeMs = Math.round(optimizeVectorCount * ESTIMATED_MS_PER_VECTOR_REBUILD * state.dimensionScale);
1451
- return {
1452
- tombstoneRatio,
1453
- graphCount,
1454
- bufferSize: state.buffer.size,
1455
- building: state.building || state.buildScheduled,
1456
- estimatedCompactMs,
1457
- estimatedOptimizeMs
1458
- };
1459
- }
1460
- function flatScanScratchBytes(state) {
1461
- return state.store.dimension === 0 ? 0 : nativeStoreScratchBytes(state.store.handles);
1462
- }
1463
- function estimateMemoryBytes(state) {
1464
- if (state.hnsw === null) return state.store.memoryBytes() + flatScanScratchBytes(state);
1465
- return state.store.memoryBytes() + state.hnsw.graphBytes + state.hnsw.searchScratchBytes;
1466
- }
1467
-
1468
- // src/vector/vector-index/search.ts
1469
- function* bufferCandidates(state, filter) {
1470
- for (const docId of state.buffer) {
1471
- if (state.tombstones.has(docId)) continue;
1472
- if (filter) {
1473
- const ordinal = state.store.getOrdinal(docId);
1474
- if (ordinal === void 0 || !ordinalFilterHas(filter, ordinal)) continue;
1475
- }
1476
- yield docId;
1477
- }
1478
- }
1479
- function* filteredDocIds(state, filter) {
1480
- for (const ordinal of ordinalFilterValues(filter)) {
1481
- const docId = state.store.docIdForOrdinal(ordinal);
1482
- if (docId !== void 0) yield docId;
1483
- }
1484
- }
1485
- function admittedTotal(state, filter) {
1486
- return filter === void 0 ? liveSize(state) : filter.count;
1487
- }
1488
- function fetchedOutcome(state, results, minSimilarity, filter) {
1489
- if (minSimilarity === Number.NEGATIVE_INFINITY) {
1490
- return { results, matched: admittedTotal(state, filter), matchedExact: true };
1491
- }
1492
- return { results, matched: results.length, matchedExact: false };
1493
- }
1494
- function scannedOutcome(scan) {
1495
- return { results: scan.results, matched: scan.matched, matchedExact: true };
1496
- }
1497
- function liveCandidates(state, candidates) {
1498
- const docIds = [];
1499
- const ordinals = [];
1500
- for (const docId of candidates) {
1501
- if (state.tombstones.has(docId)) continue;
1502
- const ordinal = state.store.getOrdinal(docId);
1503
- if (ordinal === void 0) continue;
1504
- docIds.push(docId);
1505
- ordinals.push(ordinal);
1506
- }
1507
- return { docIds, ordinals };
1508
- }
1509
- function scanThroughTheCore(state, query, k, metric, minSimilarity, scanned) {
1510
- const distances = nativeScoresOf(state.store.handles, query, metric, scanned.ordinals);
1511
- if (distances === null) return null;
1512
- const heap = createBoundedMaxHeap(highScoreFirst, k);
1513
- let matched = 0;
1514
- for (let i = 0; i < scanned.ordinals.length; i++) {
1515
- if (distances[i] === Number.POSITIVE_INFINITY) continue;
1516
- const score = toScore(distances[i], metric);
1517
- if (score < minSimilarity) continue;
1518
- matched++;
1519
- heap.push({ docId: scanned.docIds[i], score });
1520
- }
1521
- return { results: heap.toSortedArray().reverse(), matched };
1522
- }
1523
- function bruteForceSearch(state, query, k, metric, minSimilarity, candidates) {
1524
- const scanned = liveCandidates(state, candidates);
1525
- const fromTheCore = scanThroughTheCore(state, query, k, metric, minSimilarity, scanned);
1526
- if (fromTheCore !== null) return fromTheCore;
1527
- noteFallback("score");
1528
- const arenaQuery = state.store.prepareQueryArena(query);
1529
- const queryMag = arenaQuery ? arenaQuery.magnitude : magnitude(query);
1530
- const heap = createBoundedMaxHeap(highScoreFirst, k);
1531
- let matched = 0;
1532
- for (let i = 0; i < scanned.docIds.length; i++) {
1533
- const docId = scanned.docIds[i];
1534
- let score;
1535
- if (arenaQuery) {
1536
- const distance = state.store.distanceFromArena(arenaQuery, scanned.ordinals[i], metric);
1537
- if (distance === Number.POSITIVE_INFINITY) continue;
1538
- score = toScore(distance, metric);
1539
- } else {
1540
- const entry = state.store.get(docId);
1541
- if (!entry) continue;
1542
- switch (metric) {
1543
- case "cosine":
1544
- score = cosineSimilarityWithMagnitudes(query, entry.vector, queryMag, entry.magnitude);
1545
- break;
1546
- case "dotProduct":
1547
- score = dotProduct(query, entry.vector);
1548
- break;
1549
- case "euclidean": {
1550
- const dist = euclideanDistance(query, entry.vector);
1551
- score = 1 / (1 + dist);
1552
- break;
1553
- }
1554
- }
1555
- }
1556
- if (score < minSimilarity) continue;
1557
- matched++;
1558
- heap.push({ docId, score });
1559
- }
1560
- return { results: heap.toSortedArray().reverse(), matched };
1561
- }
1562
- function search(state, query, k, options) {
1563
- return searchWithFilter(state, query, k, options, filterForOptions(state, options));
1564
- }
1565
- function searchWithFilter(state, query, k, options, filter) {
1566
- if (query.length !== state.dimension) {
1567
- throw new NarsilError(
1568
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
1569
- `Vector dimension mismatch: expected ${state.dimension}, got ${query.length}`,
1570
- { expected: state.dimension, received: query.length }
1571
- );
1572
- }
1573
- const currentLiveSize = liveSize(state);
1574
- const { metric, minSimilarity, efSearch, oversample, rescoreWindow } = options;
1575
- if (currentLiveSize === 0) return { results: [], matched: 0, matchedExact: true };
1576
- if (filter && filter.count === 0) return { results: [], matched: 0, matchedExact: true };
1577
- if (k <= 0) return fetchedOutcome(state, [], minSimilarity, filter);
1578
- if (state.buffer.size > 0 && !state.building && !state.buildScheduled) {
1579
- scheduleBuild(state);
1580
- }
1581
- if (!state.hnsw) {
1582
- const candidates = filter ? filteredDocIds(state, filter) : allLiveDocIds(state);
1583
- return scannedOutcome(bruteForceSearch(state, query, k, metric, minSimilarity, candidates));
1584
- }
1585
- if (filter) {
1586
- const hnswLiveSize = state.hnsw.size;
1587
- const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
1588
- if (selectivity < state.filterThreshold) {
1589
- const scan = bruteForceSearch(state, query, k, metric, minSimilarity, filteredDocIds(state, filter));
1590
- return scannedOutcome(scan);
1591
- }
1592
- }
1593
- const hnswResults = state.hnsw.search(query, k, metric, minSimilarity, { filter, efSearch, oversample, rescoreWindow }).map((r) => ({ docId: r.docId, score: r.score }));
1594
- return fetchedOutcome(
1595
- state,
1596
- withBufferedVectors(state, query, k, options, filter, hnswResults),
1597
- minSimilarity,
1598
- filter
1599
- );
1600
- }
1601
- function withBufferedVectors(state, query, k, options, filter, graphResults) {
1602
- if (state.buffer.size === 0) return graphResults;
1603
- const candidates = bufferCandidates(state, filter);
1604
- const buffered = bruteForceSearch(state, query, k, options.metric, options.minSimilarity, candidates);
1605
- return mergeResults(graphResults, buffered.results, k);
1606
- }
1607
-
1608
- // src/vector/vector-index/index.ts
1609
- function defaultQuantizationFor(dimension) {
1610
- if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
1611
- return "osq8";
1612
- }
1613
- function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_WORKER_COPIES_ALLOWED, indexName = "", storage = "memory") {
1614
- if (!Number.isInteger(dimension) || dimension <= 0) {
1615
- throw new NarsilError(
1616
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
1617
- `Vector dimension must be a positive integer, got ${dimension}`,
1618
- { dimension }
1619
- );
1620
- }
1621
- const promotionThreshold = config?.threshold ?? DEFAULT_PROMOTION_THRESHOLD;
1622
- const filterThreshold = Math.max(0, Math.min(1, config?.filterThreshold ?? DEFAULT_FILTER_THRESHOLD));
1623
- const quantizationMode = config?.quantization ?? defaultQuantizationFor(dimension);
1624
- const codeBits = osqBitsOf(quantizationMode);
1625
- const metric = config?.hnswConfig?.metric ?? "cosine";
1626
- const rawHnswM = config?.hnswConfig?.m;
1627
- const hnswConfig = config?.hnswConfig ? { ...config.hnswConfig, m: rawHnswM !== void 0 ? Math.max(rawHnswM, 2) : void 0 } : void 0;
1628
- const dimensionScale = dimension / 256;
1629
- const store = createVectorStore({
1630
- dimension,
1631
- codeBits,
1632
- ...storage === "disk" ? { blockBytes: DISK_STORAGE_BLOCK_BYTES } : {}
1633
- });
1634
- const state = {
1635
- indexName,
1636
- fieldName,
1637
- dimension,
1638
- dimensionScale,
1639
- promotionThreshold,
1640
- filterThreshold,
1641
- quantizationMode,
1642
- storage,
1643
- metric,
1644
- hnswConfig,
1645
- workerCopies,
1646
- store,
1647
- tombstones: createRemovedDocuments(store),
1648
- buffer: createUnplacedVectors(store),
1649
- pendingLocations: /* @__PURE__ */ new Map(),
1650
- savedFiles: [],
1651
- savedSignature: null,
1652
- osq: codeBits === null ? null : createOsqQuantizer(dimension, codeBits, metric, store),
1653
- hnsw: null,
1654
- freshGraph: null,
1655
- compactedNodeCount: 0,
1656
- building: false,
1657
- buildScheduled: false,
1658
- pendingBuild: null,
1659
- disposed: false,
1660
- revision: 0,
1661
- workerCopyPool: null,
1662
- workerCopyHandle: null,
1663
- workerCopyRevision: -1,
1664
- workerCopyMode: null,
1665
- workerCopyLoading: false,
1666
- workerCopiesHeld: false,
1667
- sharedHandles: /* @__PURE__ */ new Map(),
1668
- sharedLayoutRevision: 0,
1669
- sharing: Promise.resolve(),
1670
- releaseToFilesInFlight: Promise.resolve()
1671
- };
1672
- function validateDimension(vector) {
1673
- if (vector.length !== state.dimension) {
1674
- throw new NarsilError(
1675
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
1676
- `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
1677
- { expected: state.dimension, received: vector.length }
1678
- );
1679
- }
1680
- }
1681
- function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
1682
- try {
1683
- return state.store.entryForOrdinal(ordinal)?.vector;
1684
- } catch {
1685
- return void 0;
1686
- }
1687
- }
1688
- function holdsThisVectorAlready(docId, ordinal, vector) {
1689
- if (state.tombstones.has(docId)) return false;
1690
- const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
1691
- if (held === void 0) return false;
1692
- for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
1693
- return true;
1694
- }
1695
- function insert(docId, vector, partitionId) {
1696
- validateDimension(vector);
1697
- const previous = state.store.getOrdinal(docId);
1698
- if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
1699
- if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
1700
- noteWrite(state);
1701
- state.store.setPartition(docId, partitionId);
1702
- }
1703
- return;
1704
- }
1705
- noteWrite(state);
1706
- state.tombstones.delete(docId);
1707
- const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
1708
- state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
1709
- if (previous !== void 0) {
1710
- state.hnsw?.markTombstoneOrdinal(previous);
1711
- state.freshGraph?.markTombstoneOrdinal(previous);
1712
- if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
1713
- }
1714
- state.buffer.add(docId);
1715
- state.pendingLocations.delete(docId);
1716
- if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
1717
- }
1718
- function remove(docId) {
1719
- if (!state.store.has(docId)) return;
1720
- noteWrite(state);
1721
- state.tombstones.add(docId);
1722
- state.buffer.delete(docId);
1723
- state.pendingLocations.delete(docId);
1724
- if (state.hnsw) {
1725
- state.hnsw.markTombstone(docId);
1726
- }
1727
- }
1728
- function getVector(docId) {
1729
- if (state.tombstones.has(docId)) return null;
1730
- const entry = state.store.get(docId);
1731
- if (!entry) return null;
1732
- return new Float32Array(entry.vector);
1733
- }
1734
- function has(docId) {
1735
- return state.store.has(docId) && !state.tombstones.has(docId);
1736
- }
1737
- async function awaitPendingBuild() {
1738
- while (state.pendingBuild) {
1739
- await state.pendingBuild;
1740
- }
1741
- }
1742
- function releaseHeldMemory() {
1743
- state.hnsw = null;
1744
- state.freshGraph = null;
1745
- state.osq = null;
1746
- state.tombstones.clear();
1747
- state.buffer.clear();
1748
- state.pendingLocations.clear();
1749
- state.savedFiles = [];
1750
- state.savedSignature = null;
1751
- state.store.release();
1752
- }
1753
- function dispose() {
1754
- state.disposed = true;
1755
- invalidateWorkerCopies(state);
1756
- const pending = state.pendingBuild;
1757
- if (pending !== null) {
1758
- void pending.then(releaseHeldMemory, releaseHeldMemory);
1759
- return;
1760
- }
1761
- releaseHeldMemory();
1762
- }
1763
- async function searchParallel(query, k, options) {
1764
- const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
1765
- let filter = filterForOptions(state, options);
1766
- if (filter !== void 0) {
1767
- if (state.hnsw) {
1768
- const hnswLiveSize = state.hnsw.size;
1769
- const selectivity = hnswLiveSize > 0 ? filter.count / hnswLiveSize : 1;
1770
- if (selectivity < state.filterThreshold) {
1771
- return searchWithFilter(state, query, k, options, filter);
1772
- }
1773
- }
1774
- }
1775
- const filterRevision = state.revision;
1776
- scheduleWorkerCopyLoad(state);
1777
- const viaWorkerCopy = await searchViaWorkerCopies(state, query, k, options.metric, options.minSimilarity, {
1778
- filter,
1779
- efSearch: options.efSearch,
1780
- oversample: options.oversample,
1781
- rescoreWindow: options.rescoreWindow
1782
- });
1783
- if (confined && state.revision !== filterRevision) {
1784
- filter = filterForOptions(state, options);
1785
- }
1786
- if (viaWorkerCopy !== null) {
1787
- const results = withBufferedVectors(state, query, k, options, filter, viaWorkerCopy);
1788
- return fetchedOutcome(state, results, options.minSimilarity, filter);
1789
- }
1790
- return searchWithFilter(state, query, k, options, filter);
1791
- }
1792
- return {
1793
- get size() {
1794
- return liveSize(state);
1795
- },
1796
- get dimension() {
1797
- return state.dimension;
1798
- },
1799
- get fieldName() {
1800
- return state.fieldName;
1801
- },
1802
- get quantization() {
1803
- return state.quantizationMode;
1804
- },
1805
- get storage() {
1806
- return state.storage;
1807
- },
1808
- insert,
1809
- remove,
1810
- partitionsKnown: () => state.store.partitionsKnown,
1811
- assignPartitions: (resolve) => assignStorePartitions(state, resolve),
1812
- scheduleBuild: () => scheduleBuild(state),
1813
- awaitPendingBuild,
1814
- completeGraph: () => completeGraph(state),
1815
- dispose,
1816
- search: (query, k, options) => search(state, query, k, options),
1817
- searchParallel,
1818
- refreshWorkerCopies: () => refreshWorkerCopies(state),
1819
- getVector,
1820
- has,
1821
- compact: () => {
1822
- noteWrite(state);
1823
- compact(state);
1824
- },
1825
- optimize: async () => {
1826
- noteWrite(state);
1827
- await optimize(state);
1828
- },
1829
- maintenanceStatus: () => maintenanceStatus(state),
1830
- estimateMemoryBytes: () => estimateMemoryBytes(state),
1831
- deserialize: (parts) => {
1832
- invalidateWorkerCopies(state);
1833
- deserialize(state, parts);
1834
- },
1835
- planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
1836
- recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
1837
- restoreCheckpoint: (shape) => {
1838
- invalidateWorkerCopies(state);
1839
- return beginCheckpointRestore(state, shape);
1840
- },
1841
- vectorFilesInUse: () => state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== ""),
1842
- adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
1843
- releaseVectorFiles: async () => {
1844
- await withdrawWorkerCopies(state);
1845
- state.store.closeFiles();
1846
- },
1847
- relocateVectorFiles: (rewrite) => {
1848
- state.store.relocateVectorFiles(rewrite);
1849
- for (const [docId, location] of state.pendingLocations) {
1850
- state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
1851
- }
1852
- },
1853
- holdWorkerCopies: () => holdWorkerCopies(state),
1854
- resumeWorkerCopies: () => {
1855
- state.workerCopiesHeld = false;
1856
- }
1857
- };
1858
- }
1859
-
1860
- export { createSharedVectorSearcher, createVectorIndex };