@delali/narsil 0.3.1-next.98 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +45 -0
  2. package/README.md +27 -8
  3. package/dist/adapters/filesystem.d.ts +1 -1
  4. package/dist/adapters/filesystem.mjs +101 -13
  5. package/dist/adapters/gcs.d.ts +49 -0
  6. package/dist/adapters/gcs.mjs +213 -0
  7. package/dist/adapters/indexeddb.d.ts +1 -1
  8. package/dist/adapters/indexeddb.mjs +2 -7
  9. package/dist/adapters/memory.d.ts +1 -1
  10. package/dist/adapters/memory.mjs +5 -0
  11. package/dist/adapters/s3.d.ts +53 -0
  12. package/dist/adapters/s3.mjs +263 -0
  13. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  14. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  15. package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
  16. package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
  17. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  18. package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
  19. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  20. package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
  21. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  22. package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
  23. package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
  24. package/dist/browser/embed-GG6PS3EA.mjs +264 -0
  25. package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
  26. package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
  27. package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
  28. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  29. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  30. package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
  31. package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
  32. package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
  33. package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
  34. package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
  35. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  36. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  37. package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
  38. package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
  39. package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
  40. package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
  41. package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
  42. package/dist/chunk-2AFZE7Y3.mjs +72 -0
  43. package/dist/chunk-2CKDCC65.mjs +47 -0
  44. package/dist/chunk-3ECIRFS5.mjs +61 -0
  45. package/dist/chunk-44W3CBNG.mjs +7 -0
  46. package/dist/chunk-4H4RG2FR.mjs +13 -0
  47. package/dist/chunk-4PY4JYMY.mjs +11 -0
  48. package/dist/chunk-4ZPVAHWF.mjs +1289 -0
  49. package/dist/chunk-5EI5CXO6.mjs +197 -0
  50. package/dist/chunk-5F5MXWS5.mjs +283 -0
  51. package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
  52. package/dist/chunk-6FOENO4E.mjs +273 -0
  53. package/dist/chunk-6GHX35ZM.mjs +108 -0
  54. package/dist/chunk-6MSZ543Y.mjs +32 -0
  55. package/dist/chunk-6N6HHBCQ.mjs +4596 -0
  56. package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
  57. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  58. package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
  59. package/dist/chunk-AVBPB3MI.mjs +602 -0
  60. package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
  61. package/dist/chunk-BUMREPLU.mjs +15 -0
  62. package/dist/chunk-BY6NALRK.mjs +31921 -0
  63. package/dist/chunk-C4ZMXXTL.mjs +92 -0
  64. package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
  65. package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
  66. package/dist/chunk-DXZ6JDWG.mjs +797 -0
  67. package/dist/chunk-FJXJZUWF.mjs +209 -0
  68. package/dist/chunk-FS3HPPSC.mjs +212 -0
  69. package/dist/chunk-FT4KLZXW.mjs +1164 -0
  70. package/dist/chunk-GR5GMFJJ.mjs +104 -0
  71. package/dist/chunk-GXCDWLQI.mjs +193 -0
  72. package/dist/chunk-GXED6R3Q.mjs +271 -0
  73. package/dist/chunk-HLPRHOSO.mjs +17 -0
  74. package/dist/chunk-IL5DRCNU.mjs +3041 -0
  75. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  76. package/dist/chunk-K5HSUPYS.mjs +11 -0
  77. package/dist/chunk-KYIRXTCA.mjs +100 -0
  78. package/dist/chunk-LFXS6OF7.mjs +289 -0
  79. package/dist/chunk-LNGE7SLY.mjs +151 -0
  80. package/dist/chunk-M2Z5WVWW.mjs +194 -0
  81. package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
  82. package/dist/chunk-M3OW2XSP.mjs +40 -0
  83. package/dist/chunk-MFPSQZ6P.mjs +127 -0
  84. package/dist/chunk-MRCJBWJZ.mjs +1292 -0
  85. package/dist/chunk-MSIN4NM3.mjs +25 -0
  86. package/dist/chunk-MW3CJVGP.mjs +219 -0
  87. package/dist/chunk-MYM6CAVK.mjs +76 -0
  88. package/dist/chunk-NJJN4FMM.mjs +84 -0
  89. package/dist/chunk-NM7I3BM7.mjs +59 -0
  90. package/dist/chunk-OIJ64LGT.mjs +19 -0
  91. package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
  92. package/dist/chunk-Q2AP2FSN.mjs +15 -0
  93. package/dist/chunk-R2PJKSNL.mjs +54 -0
  94. package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
  95. package/dist/chunk-S2POFTLN.mjs +10 -0
  96. package/dist/chunk-S5QMWECL.mjs +7 -0
  97. package/dist/chunk-SLAC5PAB.mjs +5008 -0
  98. package/dist/chunk-T4HMSZCX.mjs +5 -0
  99. package/dist/chunk-TACP6VK6.mjs +111 -0
  100. package/dist/chunk-TGPKS2UM.mjs +125 -0
  101. package/dist/chunk-TKKXPKEO.mjs +979 -0
  102. package/dist/chunk-V6EUBAIT.mjs +19 -0
  103. package/dist/chunk-XL56L4IR.mjs +228 -0
  104. package/dist/chunk-XQIAMDAG.mjs +3076 -0
  105. package/dist/chunk-YTUZ4R22.mjs +1063 -0
  106. package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
  107. package/dist/chunk-ZPJFGGGI.mjs +638 -0
  108. package/dist/chunk-ZXX5GZEX.mjs +17 -0
  109. package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
  110. package/dist/client.d.ts +11 -10
  111. package/dist/client.mjs +8 -898
  112. package/dist/config-Bnl7L-0H.d.ts +608 -0
  113. package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
  114. package/dist/core-leaf-6JVHEVFB.mjs +8 -0
  115. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  116. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  117. package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
  118. package/dist/distribution/coordinator/etcd.d.ts +63 -7
  119. package/dist/distribution/coordinator/etcd.mjs +725 -81
  120. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  121. package/dist/distribution/coordinator/in-memory.mjs +65 -10
  122. package/dist/distribution/transport/grpc.d.ts +2 -2
  123. package/dist/distribution/transport/grpc.mjs +39 -10
  124. package/dist/distribution/transport/in-memory.d.ts +2 -2
  125. package/dist/distribution/transport/in-memory.mjs +9 -6
  126. package/dist/distribution/transport/tcp.d.ts +2 -2
  127. package/dist/distribution/transport/tcp.mjs +20 -15
  128. package/dist/distribution.d.ts +267 -56
  129. package/dist/distribution.mjs +23156 -11362
  130. package/dist/durability-config-VWS2K24E.mjs +96 -0
  131. package/dist/embed-4PHHKGBK.mjs +263 -0
  132. package/dist/embeddings/openai.d.ts +1 -1
  133. package/dist/embeddings/openai.mjs +2 -2
  134. package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
  135. package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
  136. package/dist/gzip-2UCZW2MM.mjs +31 -0
  137. package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
  138. package/dist/index.browser.mjs +4 -4
  139. package/dist/index.d.ts +2133 -57
  140. package/dist/index.mjs +16 -32
  141. package/dist/index.react-native.mjs +12 -0
  142. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  143. package/dist/invalidation/filesystem.d.ts +1 -1
  144. package/dist/invalidation/filesystem.mjs +1 -2
  145. package/dist/invalidation/noop.d.ts +1 -1
  146. package/dist/languages/albanian.mjs +1 -1
  147. package/dist/languages/amharic.mjs +1 -1
  148. package/dist/languages/arabic.mjs +1 -1
  149. package/dist/languages/armenian.mjs +1 -1
  150. package/dist/languages/azerbaijani.mjs +1 -1
  151. package/dist/languages/bambara.mjs +1 -1
  152. package/dist/languages/basque.mjs +1 -1
  153. package/dist/languages/belarusian.mjs +1 -1
  154. package/dist/languages/bengali.mjs +1 -1
  155. package/dist/languages/bosnian.mjs +1 -1
  156. package/dist/languages/breton.mjs +1 -1
  157. package/dist/languages/bulgarian.mjs +1 -1
  158. package/dist/languages/burmese.mjs +1 -1
  159. package/dist/languages/catalan.mjs +1 -1
  160. package/dist/languages/chinese.mjs +1 -1
  161. package/dist/languages/croatian.mjs +1 -1
  162. package/dist/languages/czech.mjs +1 -1
  163. package/dist/languages/dagbani.mjs +1 -1
  164. package/dist/languages/danish.mjs +1 -1
  165. package/dist/languages/dutch.mjs +1 -1
  166. package/dist/languages/english.mjs +1 -1
  167. package/dist/languages/esperanto.mjs +1 -1
  168. package/dist/languages/estonian.mjs +1 -1
  169. package/dist/languages/ewe.mjs +1 -1
  170. package/dist/languages/faroese.mjs +1 -1
  171. package/dist/languages/fijian.mjs +1 -1
  172. package/dist/languages/finnish.mjs +1 -1
  173. package/dist/languages/french.mjs +1 -1
  174. package/dist/languages/ga.mjs +1 -1
  175. package/dist/languages/galician.mjs +1 -1
  176. package/dist/languages/georgian.mjs +1 -1
  177. package/dist/languages/german.mjs +1 -1
  178. package/dist/languages/greek.mjs +1 -1
  179. package/dist/languages/guarani.mjs +1 -1
  180. package/dist/languages/gujarati.mjs +1 -1
  181. package/dist/languages/haitian-creole.mjs +1 -1
  182. package/dist/languages/hausa.mjs +1 -1
  183. package/dist/languages/hawaiian.mjs +1 -1
  184. package/dist/languages/hebrew.mjs +1 -1
  185. package/dist/languages/hindi.mjs +1 -1
  186. package/dist/languages/hungarian.mjs +1 -1
  187. package/dist/languages/icelandic.mjs +1 -1
  188. package/dist/languages/igbo.mjs +1 -1
  189. package/dist/languages/indonesian.mjs +1 -1
  190. package/dist/languages/irish.mjs +1 -1
  191. package/dist/languages/italian.mjs +1 -1
  192. package/dist/languages/japanese.mjs +1 -1
  193. package/dist/languages/kannada.mjs +1 -1
  194. package/dist/languages/kazakh.mjs +1 -1
  195. package/dist/languages/khmer.mjs +1 -1
  196. package/dist/languages/kinyarwanda.mjs +1 -1
  197. package/dist/languages/kirundi.mjs +1 -1
  198. package/dist/languages/korean.mjs +1 -1
  199. package/dist/languages/kurmanji.mjs +1 -1
  200. package/dist/languages/kyrgyz.mjs +1 -1
  201. package/dist/languages/lao.mjs +1 -1
  202. package/dist/languages/latin.mjs +1 -1
  203. package/dist/languages/latvian.mjs +1 -1
  204. package/dist/languages/lingala.mjs +1 -1
  205. package/dist/languages/lithuanian.mjs +1 -1
  206. package/dist/languages/luxembourgish.mjs +1 -1
  207. package/dist/languages/macedonian.mjs +1 -1
  208. package/dist/languages/malagasy.mjs +1 -1
  209. package/dist/languages/malay.mjs +1 -1
  210. package/dist/languages/malayalam.mjs +1 -1
  211. package/dist/languages/maltese.mjs +1 -1
  212. package/dist/languages/maori.mjs +1 -1
  213. package/dist/languages/marathi.mjs +1 -1
  214. package/dist/languages/nepali.mjs +1 -1
  215. package/dist/languages/norwegian.mjs +1 -1
  216. package/dist/languages/oromo.mjs +1 -1
  217. package/dist/languages/persian.mjs +1 -1
  218. package/dist/languages/polish.mjs +1 -1
  219. package/dist/languages/portuguese.mjs +1 -1
  220. package/dist/languages/punjabi.mjs +1 -1
  221. package/dist/languages/romanian.mjs +1 -1
  222. package/dist/languages/russian.mjs +1 -1
  223. package/dist/languages/samoan.mjs +1 -1
  224. package/dist/languages/sanskrit.mjs +1 -1
  225. package/dist/languages/scottish-gaelic.mjs +1 -1
  226. package/dist/languages/serbian.mjs +1 -1
  227. package/dist/languages/shona.mjs +1 -1
  228. package/dist/languages/sinhala.mjs +1 -1
  229. package/dist/languages/slovak.mjs +1 -1
  230. package/dist/languages/slovenian.mjs +1 -1
  231. package/dist/languages/sorani.mjs +1 -1
  232. package/dist/languages/spanish.mjs +1 -1
  233. package/dist/languages/swahili.mjs +1 -1
  234. package/dist/languages/swedish.mjs +1 -1
  235. package/dist/languages/tagalog.mjs +1 -1
  236. package/dist/languages/tamil.mjs +1 -1
  237. package/dist/languages/tatar.mjs +1 -1
  238. package/dist/languages/telugu.mjs +1 -1
  239. package/dist/languages/thai.mjs +1 -1
  240. package/dist/languages/tibetan.mjs +1 -1
  241. package/dist/languages/tigrinya.mjs +1 -1
  242. package/dist/languages/tongan.mjs +1 -1
  243. package/dist/languages/turkish.mjs +1 -1
  244. package/dist/languages/twi.mjs +1 -1
  245. package/dist/languages/ukrainian.mjs +1 -1
  246. package/dist/languages/urdu.mjs +1 -1
  247. package/dist/languages/vietnamese.mjs +1 -1
  248. package/dist/languages/welsh.mjs +1 -1
  249. package/dist/languages/wolof.mjs +1 -1
  250. package/dist/languages/xhosa.mjs +1 -1
  251. package/dist/languages/yoruba.mjs +1 -1
  252. package/dist/languages/zulu.mjs +1 -1
  253. package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
  254. package/dist/module-5ZSLUQII.mjs +3729 -0
  255. package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
  256. package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
  257. package/dist/optional-writes-4UOQKETU.mjs +247 -0
  258. package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
  259. package/dist/partial-words.browser.mjs +1650 -0
  260. package/dist/partial-words.mjs +17 -11
  261. package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
  262. package/dist/pattern.mjs +6 -7
  263. package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
  264. package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
  265. package/dist/phrase-module-EFESWOCR.mjs +635 -0
  266. package/dist/query-actions-HMJUVWGT.mjs +119 -0
  267. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  268. package/dist/query-suggestions/merge-worker.mjs +53 -0
  269. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  270. package/dist/query-suggestions.browser.mjs +47 -0
  271. package/dist/query-suggestions.d.ts +20 -0
  272. package/dist/query-suggestions.mjs +19 -0
  273. package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
  274. package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
  275. package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
  276. package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
  277. package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
  278. package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
  279. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  280. package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
  281. package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
  282. package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
  283. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  284. package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
  285. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  286. package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
  287. package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
  288. package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
  289. package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
  290. package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
  291. package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
  292. package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
  293. package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
  294. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  295. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  296. package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
  297. package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
  298. package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
  299. package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
  300. package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
  301. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  302. package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
  303. package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
  304. package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
  305. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  306. package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
  307. package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
  308. package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
  309. package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
  310. package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
  311. package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
  312. package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
  313. package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
  314. package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
  315. package/dist/react-native/partial-words.mjs +1658 -0
  316. package/dist/react-native/pattern.mjs +2570 -0
  317. package/dist/react-native/query-suggestions.mjs +55 -0
  318. package/dist/react-native/sync.mjs +2759 -0
  319. package/dist/react-native/synonyms.mjs +676 -0
  320. package/dist/react.d.ts +12 -8
  321. package/dist/react.mjs +45 -1
  322. package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
  323. package/dist/rerankers/http.d.ts +1 -1
  324. package/dist/rerankers/http.mjs +2 -2
  325. package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
  326. package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
  327. package/dist/runtime-USQYKN3I.mjs +615 -0
  328. package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
  329. package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
  330. package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
  331. package/dist/server.d.ts +348 -22
  332. package/dist/server.mjs +2211 -327
  333. package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
  334. package/dist/store-leaf-6VMMSVBE.mjs +20 -0
  335. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  336. package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
  337. package/dist/sync/filesystem.d.ts +30 -0
  338. package/dist/sync/filesystem.mjs +288 -0
  339. package/dist/sync/indexeddb.browser.mjs +287 -0
  340. package/dist/sync/indexeddb.d.ts +30 -0
  341. package/dist/sync/indexeddb.mjs +242 -0
  342. package/dist/sync.browser.mjs +1271 -0
  343. package/dist/sync.d.ts +23 -0
  344. package/dist/sync.mjs +2712 -0
  345. package/dist/synonyms.browser.mjs +658 -0
  346. package/dist/synonyms.mjs +17 -7
  347. package/dist/tasks-65b6t6lk.d.ts +782 -0
  348. package/dist/thread-UJ3W5LVI.mjs +31 -0
  349. package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
  350. package/dist/types-CpCg-Gq-.d.ts +227 -0
  351. package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
  352. package/dist/vector/search-worker.d.ts +10 -2
  353. package/dist/vector/search-worker.mjs +11 -11
  354. package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
  355. package/dist/workers/entry.mjs +75 -15
  356. package/openapi.json +6840 -0
  357. package/package.json +406 -19
  358. package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
  359. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  360. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  361. package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
  362. package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
  363. package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
  364. package/dist/browser/chunk-QWES44AW.mjs +0 -473
  365. package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
  366. package/dist/browser/module-PDXYHT75.mjs +0 -2822
  367. package/dist/browser/release-76MXRPGV.mjs +0 -33
  368. package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
  369. package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
  370. package/dist/browser/write-L7FAQA47.mjs +0 -208
  371. package/dist/chunk-37CD3OKS.mjs +0 -87
  372. package/dist/chunk-5XNIRLFC.mjs +0 -96
  373. package/dist/chunk-7X5JST5L.mjs +0 -114
  374. package/dist/chunk-AV4IJKD2.mjs +0 -35
  375. package/dist/chunk-AY3AQ3UH.mjs +0 -73
  376. package/dist/chunk-BKB3XTMI.mjs +0 -707
  377. package/dist/chunk-C7SLA6CM.mjs +0 -19
  378. package/dist/chunk-COVCQ73F.mjs +0 -587
  379. package/dist/chunk-DJHHF2L5.mjs +0 -17
  380. package/dist/chunk-E74YLAIG.mjs +0 -24
  381. package/dist/chunk-FMWRCP2C.mjs +0 -57
  382. package/dist/chunk-GES7U2PM.mjs +0 -68
  383. package/dist/chunk-HJSN7UYG.mjs +0 -10
  384. package/dist/chunk-JFRNNBOM.mjs +0 -35
  385. package/dist/chunk-K25LZA54.mjs +0 -8484
  386. package/dist/chunk-KEHK2O6P.mjs +0 -13
  387. package/dist/chunk-KOKM4PQN.mjs +0 -52
  388. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  389. package/dist/chunk-L3L5YFXC.mjs +0 -614
  390. package/dist/chunk-L5ORED7V.mjs +0 -25
  391. package/dist/chunk-LH4NZNJJ.mjs +0 -1731
  392. package/dist/chunk-LU2CE5AE.mjs +0 -507
  393. package/dist/chunk-LXWDQLTW.mjs +0 -30
  394. package/dist/chunk-MHOSCQCQ.mjs +0 -948
  395. package/dist/chunk-NKRBEGJZ.mjs +0 -1705
  396. package/dist/chunk-NTRMFARG.mjs +0 -462
  397. package/dist/chunk-NW4XDSP2.mjs +0 -212
  398. package/dist/chunk-OOEXLUGR.mjs +0 -117
  399. package/dist/chunk-ORBLD3EL.mjs +0 -6
  400. package/dist/chunk-OWNZYMRE.mjs +0 -3591
  401. package/dist/chunk-PPY2GSIB.mjs +0 -44
  402. package/dist/chunk-QY7GC2MW.mjs +0 -33
  403. package/dist/chunk-R6A773WK.mjs +0 -226
  404. package/dist/chunk-R6PR4WYZ.mjs +0 -68
  405. package/dist/chunk-R7EEKAJE.mjs +0 -593
  406. package/dist/chunk-RONEOEON.mjs +0 -426
  407. package/dist/chunk-VWULPWRJ.mjs +0 -7469
  408. package/dist/chunk-WCJKUGHB.mjs +0 -262
  409. package/dist/chunk-WQBCBKY3.mjs +0 -8
  410. package/dist/chunk-X5CV2E2B.mjs +0 -1
  411. package/dist/chunk-YA7XLE2R.mjs +0 -1256
  412. package/dist/chunk-YTJCGKM4.mjs +0 -56
  413. package/dist/chunk-ZG477IPV.mjs +0 -526
  414. package/dist/chunk-ZYNAMIYX.mjs +0 -113
  415. package/dist/config-BGB_fDtq.d.ts +0 -237
  416. package/dist/direct-executor-CXZ66ACP.mjs +0 -22
  417. package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
  418. package/dist/engine-BGCEm9E_.d.ts +0 -577
  419. package/dist/module-3NONKBZX.mjs +0 -1860
  420. package/dist/release-BH3DKW3T.mjs +0 -33
  421. package/dist/restore-queue-JPRBUYII.mjs +0 -35
  422. package/dist/runtime-ZKYZFJJC.mjs +0 -177
  423. package/dist/snapshot-HOGXNO5W.mjs +0 -793
  424. package/dist/tasks-HG3rlxx6.d.ts +0 -302
  425. package/dist/thread-B657LQXK.mjs +0 -692
  426. package/dist/write-XLIGTXBU.mjs +0 -238
@@ -1,56 +1,6 @@
1
- import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-HXJBSHXF.mjs';
2
- import { MAX_VECTOR_ORDINALS } from './chunk-QY7GC2MW.mjs';
3
- import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
4
- import { readSync, closeSync, openSync } from 'fs';
5
-
6
- function createVectorFileReader() {
7
- const descriptors = /* @__PURE__ */ new Map();
8
- function descriptorOf(path) {
9
- const open = descriptors.get(path);
10
- if (open !== void 0) return open;
11
- try {
12
- const descriptor = openSync(path, "r");
13
- descriptors.set(path, descriptor);
14
- return descriptor;
15
- } catch {
16
- return null;
17
- }
18
- }
19
- function closeDescriptor(path, descriptor) {
20
- descriptors.delete(path);
21
- try {
22
- closeSync(descriptor);
23
- } catch {
24
- }
25
- }
26
- return {
27
- readInto(path, offset, target) {
28
- const descriptor = descriptorOf(path);
29
- if (descriptor === null) return false;
30
- let read = 0;
31
- while (read < target.byteLength) {
32
- let chunk;
33
- try {
34
- chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
35
- } catch {
36
- closeDescriptor(path, descriptor);
37
- return false;
38
- }
39
- if (chunk <= 0) return false;
40
- read += chunk;
41
- }
42
- return true;
43
- },
44
- retainOnly(paths) {
45
- for (const [path, descriptor] of descriptors) {
46
- if (!paths.has(path)) closeDescriptor(path, descriptor);
47
- }
48
- },
49
- close() {
50
- for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
51
- }
52
- };
53
- }
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, withTransientAdmission, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, VECTOR_MEMORY_LIMITS, sharedVectorStoreBytes, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_SLOTS, writeSequenceOf, openGraphState, fixedView, createSharedVectorStoreHandles, sequenceWritesOf, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, oldestHeldPin, coverOrdinal, nextSequence, publishSequence, createOrdinalFilter, addToOrdinalFilter, magnitude, stampBorn, mayFree, retiredAtOf, requireBytesWithinLimit, stampRetired, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, slotByteOffset, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorFileReader, grownByteLength, growBufferTo, resetStoreHeader, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createVectorBlock, ensureBlockSlots } from './chunk-R724DJSG.mjs';
2
+ import { createDocIdMap } from './chunk-FJXJZUWF.mjs';
3
+ import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
54
4
 
55
5
  // src/vector/vector-store/hot-block-distance.ts
56
6
  function jsDistance(a, b, magA, magB, metric) {
@@ -63,6 +13,11 @@ function jsDistance(a, b, magA, magB, metric) {
63
13
  return euclideanDistance(a, b);
64
14
  }
65
15
  }
16
+ function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
17
+ if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
18
+ const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
19
+ return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
20
+ }
66
21
  function inMemory(reads, ordinal) {
67
22
  return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
68
23
  }
@@ -159,9 +114,7 @@ function openSharedVectorStore(initial, threadSlot) {
159
114
  }
160
115
  return diskFile[ordinal];
161
116
  }
162
- function isCold(ordinal) {
163
- return fileOf(ordinal) !== IN_MEMORY;
164
- }
117
+ const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
165
118
  function holdState(ordinal) {
166
119
  if (ordinal < 0) return NOT_HELD;
167
120
  if (ordinal >= present.length) {
@@ -173,9 +126,7 @@ function openSharedVectorStore(initial, threadSlot) {
173
126
  const index = Math.floor(ordinal / capacity);
174
127
  return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
175
128
  }
176
- function holds(ordinal) {
177
- return holdState(ordinal) !== NOT_HELD;
178
- }
129
+ const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
179
130
  function readFromFile(file, offset, target) {
180
131
  const path = handles.vectorFiles[file];
181
132
  if (path === void 0 || path.length === 0) return false;
@@ -383,6 +334,7 @@ function openSharedVectorStore(initial, threadSlot) {
383
334
  }
384
335
  return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
385
336
  },
337
+ distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
386
338
  hotBlockReads,
387
339
  queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
388
340
  ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
@@ -474,31 +426,19 @@ function openSharedVectorField(initial, threadSlot) {
474
426
  return outcome;
475
427
  },
476
428
  searchOrdinals(query, k, metric, minSimilarity, options) {
477
- if (graph === null) return [];
478
- return searchOrdinals(graph, docIdOf, query, k, metric, minSimilarity, options);
429
+ const current = graph;
430
+ if (current === null) return [];
431
+ if (options.admission !== void 0) {
432
+ return searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, options);
433
+ }
434
+ return withTransientAdmission(
435
+ store,
436
+ (admission) => searchOrdinals(current, docIdOf, query, k, metric, minSimilarity, { ...options, admission })
437
+ );
479
438
  }
480
439
  };
481
440
  }
482
441
 
483
- // src/vector/vector-store/doc-ids.ts
484
- var encoder = new TextEncoder();
485
- function appendDocId(handles, ordinal, docId) {
486
- const bytes = encoder.encode(docId);
487
- const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
488
- const end = start + bytes.length;
489
- growBufferTo(handles.docIdBytes, end);
490
- growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
491
- new Uint8Array(handles.docIdBytes).set(bytes, start);
492
- const offsets = new Uint32Array(handles.docIdOffsets);
493
- offsets[ordinal] = start;
494
- offsets[ordinal + 1] = end;
495
- Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
496
- }
497
- function resetDocIds(handles) {
498
- new Uint32Array(handles.docIdOffsets).fill(0);
499
- Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
500
- }
501
-
502
442
  // src/vector/vector-store/ordinal-reads.ts
503
443
  function entryAt(store, ordinal) {
504
444
  return { vector: store.view.vectorAt(ordinal), magnitude: store.magnitudes[ordinal] };
@@ -526,6 +466,11 @@ function createOrdinalReads(openStore, readable) {
526
466
  if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
527
467
  return open.view.distanceFromArena(prepared, ordinal, metric);
528
468
  },
469
+ distanceFromQuery(query, queryMagnitude, ordinal, metric) {
470
+ const open = openStore();
471
+ if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
472
+ return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
473
+ },
529
474
  queryDistance(prepared, metric) {
530
475
  const open = openStore();
531
476
  if (open === null) return () => Number.POSITIVE_INFINITY;
@@ -571,6 +516,25 @@ function snapshotOf(open, documents, replacedOrdinals) {
571
516
  return { dimension, slots, vectors, magnitudes, docIds, replacedOrdinals: [...replacedOrdinals] };
572
517
  }
573
518
 
519
+ // src/vector/vector-store/doc-ids.ts
520
+ var encoder = new TextEncoder();
521
+ function appendDocId(handles, ordinal, docId) {
522
+ const bytes = encoder.encode(docId);
523
+ const start = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
524
+ const end = start + bytes.length;
525
+ growBufferTo(handles.docIdBytes, end);
526
+ growBufferTo(handles.docIdOffsets, (ordinal + 2) * 4);
527
+ new Uint8Array(handles.docIdBytes).set(bytes, start);
528
+ const offsets = new Uint32Array(handles.docIdOffsets);
529
+ offsets[ordinal] = start;
530
+ offsets[ordinal + 1] = end;
531
+ Atomics.store(handles.header, STORE_DOC_ID_BYTES, end);
532
+ }
533
+ function resetDocIds(handles) {
534
+ new Uint32Array(handles.docIdOffsets, 0, handles.docIdOffsets.byteLength / 4).fill(0);
535
+ Atomics.store(handles.header, STORE_DOC_ID_BYTES, 0);
536
+ }
537
+
574
538
  // src/vector/vector-store/open-store.ts
575
539
  function openHandles(dimension, codeBits, blockBytes) {
576
540
  const handles = createSharedVectorStoreHandles(dimension, codeBits, blockBytes);
@@ -581,7 +545,9 @@ function openHandles(dimension, codeBits, blockBytes) {
581
545
  present: new Uint8Array(handles.present),
582
546
  partitions: new Int32Array(handles.partitions),
583
547
  diskFile: new Int32Array(handles.diskFile),
584
- diskOffset: new Uint32Array(handles.diskOffset)
548
+ diskOffset: new Uint32Array(handles.diskOffset),
549
+ longestDocIdBytes: 0,
550
+ sequences: sequenceWritesOf(handles)
585
551
  };
586
552
  }
587
553
  function relayout(store) {
@@ -592,17 +558,23 @@ function claimSlot(store, docId, cold) {
592
558
  const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
559
  ensureOrdinal(store, ordinal, cold);
594
560
  store.diskFile[ordinal] = IN_MEMORY;
561
+ const idBytesBefore = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES);
595
562
  appendDocId(store.handles, ordinal, docId);
563
+ const idBytes = Atomics.load(store.handles.header, STORE_DOC_ID_BYTES) - idBytesBefore;
564
+ if (idBytes > store.longestDocIdBytes) store.longestDocIdBytes = idBytes;
596
565
  return ordinal;
597
566
  }
598
567
  function publishOrdinal(store, ordinal) {
599
568
  Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
600
569
  Atomics.store(store.present, ordinal, 1);
601
570
  }
602
- function publishOnDisk(store, ordinal, vectorMagnitude, location) {
571
+ function placeOnDisk(store, ordinal, vectorMagnitude, location) {
603
572
  store.magnitudes[ordinal] = vectorMagnitude;
604
573
  store.diskOffset[ordinal] = location.offset;
605
574
  store.diskFile[ordinal] = location.fileIndex;
575
+ }
576
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
577
+ placeOnDisk(store, ordinal, vectorMagnitude, location);
606
578
  publishOrdinal(store, ordinal);
607
579
  }
608
580
  function relocateFiles(store, rewrite) {
@@ -643,14 +615,26 @@ function ensureFloatSlot(store, ordinal) {
643
615
  const block = handles.blocks[blockIndex];
644
616
  if (block !== null) ensureBlockSlots(block, ordinal % handles.layout.capacity + 1);
645
617
  }
618
+ var LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT = 3;
619
+ var encoder2 = new TextEncoder();
620
+ function requireOrdinalWithinLimit(ordinal) {
621
+ const maxOrdinals = VECTOR_MEMORY_LIMITS.maxOrdinals;
622
+ if (ordinal < maxOrdinals) return;
623
+ throw new NarsilError(
624
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
625
+ `A vector field holds at most ${maxOrdinals} vectors, deleted ones included, until it is rebuilt`,
626
+ { ordinal, maxOrdinals }
627
+ );
628
+ }
629
+ function requireRoomForSlot(store, docId) {
630
+ const { header, docIdBytes } = store.handles;
631
+ requireOrdinalWithinLimit(Atomics.load(header, STORE_SLOTS));
632
+ const heldBytes = Atomics.load(header, STORE_DOC_ID_BYTES);
633
+ if (heldBytes + docId.length * LONGEST_UTF8_BYTES_OF_A_UTF16_UNIT <= docIdBytes.maxByteLength) return;
634
+ requireBytesWithinLimit(heldBytes + encoder2.encode(docId).length, docIdBytes.maxByteLength);
635
+ }
646
636
  function ensureOrdinal(store, ordinal, cold) {
647
- if (ordinal >= MAX_VECTOR_ORDINALS) {
648
- throw new NarsilError(
649
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
650
- `A vector field holds at most ${MAX_VECTOR_ORDINALS} vectors, deleted ones included, until it is rebuilt`,
651
- { ordinal, maxOrdinals: MAX_VECTOR_ORDINALS }
652
- );
653
- }
637
+ requireOrdinalWithinLimit(ordinal);
654
638
  const { handles } = store;
655
639
  growBufferTo(handles.magnitudes, (ordinal + 1) * 8);
656
640
  growBufferTo(handles.present, ordinal + 1);
@@ -662,12 +646,27 @@ function ensureOrdinal(store, ordinal, cold) {
662
646
  const diskFileBytes = handles.diskFile.byteLength;
663
647
  growBufferTo(handles.diskFile, (ordinal + 1) * 4);
664
648
  if (handles.diskFile.byteLength > diskFileBytes) {
665
- new Int32Array(handles.diskFile).fill(IN_MEMORY, diskFileBytes / 4);
649
+ new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY, diskFileBytes / 4);
666
650
  }
667
651
  growBufferTo(handles.diskOffset, (ordinal + 1) * 4);
652
+ growBufferTo(handles.bornAt, (ordinal + 1) * 8);
653
+ growBufferTo(handles.retiredAt, (ordinal + 1) * 8);
668
654
  if (!cold) ensureFloatSlot(store, ordinal);
669
655
  ensureCodeSlot(store, ordinal);
670
656
  }
657
+ function bufferGrowthBytes(buffer, bytes) {
658
+ return grownByteLength(buffer.byteLength, bytes, buffer.maxByteLength) - buffer.byteLength;
659
+ }
660
+ function growthBytesFor(store, vectors) {
661
+ const { handles } = store;
662
+ const slots = Atomics.load(handles.header, STORE_SLOTS);
663
+ const last = Math.min(slots + vectors, VECTOR_MEMORY_LIMITS.maxOrdinals) - 1;
664
+ if (last < slots) return 0;
665
+ const idBytes = Atomics.load(handles.header, STORE_DOC_ID_BYTES);
666
+ let bytes = bufferGrowthBytes(handles.magnitudes, (last + 1) * 8) + bufferGrowthBytes(handles.present, last + 1) + bufferGrowthBytes(handles.unplaced, last + 1) + bufferGrowthBytes(handles.unplacedChunks, unplacedChunkCount(last) * 4) + bufferGrowthBytes(handles.removed, last + 1) + bufferGrowthBytes(handles.partitions, (last + 1) * 4) + bufferGrowthBytes(handles.codePresent, last + 1) + bufferGrowthBytes(handles.diskFile, (last + 1) * 4) + bufferGrowthBytes(handles.diskOffset, (last + 1) * 4) + bufferGrowthBytes(handles.bornAt, (last + 1) * 8) + bufferGrowthBytes(handles.retiredAt, (last + 1) * 8) + bufferGrowthBytes(handles.docIdOffsets, (last + 2) * 4) + bufferGrowthBytes(handles.docIdBytes, idBytes + (last + 1 - slots) * store.longestDocIdBytes) + blocksGrowthBytes(handles.blocks, handles.layout, slots, last);
667
+ if (handles.codeLayout !== null) bytes += blocksGrowthBytes(handles.codeBlocks, handles.codeLayout, slots, last);
668
+ return bytes;
669
+ }
671
670
  function writeVector(store, ordinal, vector) {
672
671
  const block = store.view.blockOf(ordinal);
673
672
  const base = slotByteOffset(store.handles.layout, store.view.localOrdinal(ordinal)) / 4;
@@ -685,13 +684,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
685
684
  }
686
685
  return true;
687
686
  }
688
- function fileHoldsSameBytes(store, ordinal, location) {
689
- const fromFile = new Float32Array(store.handles.dimension);
687
+ function fileHoldsSameBytes(store, ordinal, location, fromFile) {
690
688
  if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
691
689
  const held = store.view.vectorAt(ordinal);
692
- const a = new Uint8Array(fromFile.buffer);
693
- const b = new Uint8Array(held.buffer, held.byteOffset, held.byteLength);
694
- for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
690
+ const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
691
+ const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
692
+ for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
695
693
  return true;
696
694
  }
697
695
  function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
@@ -731,89 +729,331 @@ function partitionFilterOf(store, documents, partitionIds) {
731
729
  return filter;
732
730
  }
733
731
 
734
- // src/vector/vector-store/index.ts
732
+ // src/vector/vector-store/retirement.ts
735
733
  var UNKNOWN_PARTITION = -1;
736
734
  var NO_PARTITION = -2;
735
+ var LOADED_SEQUENCE = 1;
736
+ var RETIRED_BEFORE_LOAD = 1;
737
+ function oldestPinOf(store) {
738
+ return oldestHeldPin(store.handles.pins);
739
+ }
740
+ function freeable(store, ordinal, oldestPin) {
741
+ return mayFree(retiredAtOf(store.sequences, ordinal), oldestPin);
742
+ }
743
+ function recordPartition(state, store, ordinal, partitionId) {
744
+ const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
745
+ store.partitions[ordinal] = given;
746
+ if (given === UNKNOWN_PARTITION) state.unknownPartitions += 1;
747
+ }
748
+ function freeRetired(state, store, ordinal) {
749
+ state.awaitingFree.delete(ordinal);
750
+ Atomics.store(store.present, ordinal, 0);
751
+ state.checkpointPins.note(ordinal);
752
+ }
753
+ function retireOrdinal(state, store, ordinal, oldestPin) {
754
+ state.ordToDoc[ordinal] = void 0;
755
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
756
+ if (freeable(store, ordinal, oldestPin)) {
757
+ freeRetired(state, store, ordinal);
758
+ return;
759
+ }
760
+ state.awaitingFree.add(ordinal);
761
+ }
762
+ function keepReplacedOrdinal(state, store, ordinal) {
763
+ state.ordToDoc[ordinal] = void 0;
764
+ if (store.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
765
+ state.replacedOrdinals.add(ordinal);
766
+ }
767
+ function freeAwaitingOrdinals(state) {
768
+ const store = state.open;
769
+ if (store === null || state.awaitingFree.size === 0) return;
770
+ const oldestPin = oldestPinOf(store);
771
+ for (const ordinal of state.awaitingFree) {
772
+ if (freeable(store, ordinal, oldestPin)) freeRetired(state, store, ordinal);
773
+ }
774
+ }
775
+ function claimOrdinal(state, store, docId, cold, partitionId) {
776
+ const ordinal = claimSlot(store, docId, cold);
777
+ recordPartition(state, store, ordinal, partitionId);
778
+ state.ordToDoc[ordinal] = docId;
779
+ state.docToOrd.set(docId, ordinal);
780
+ return ordinal;
781
+ }
782
+ function appendOrdinal(state, store, docId, vector, sequence, partitionId) {
783
+ const ordinal = claimOrdinal(state, store, docId, false, partitionId);
784
+ writeVector(store, ordinal, vector);
785
+ stampBorn(store.sequences, ordinal, sequence);
786
+ publishOrdinal(store, ordinal);
787
+ return ordinal;
788
+ }
789
+ function settleReplacement(state, store, previous, keepsReplacedVector) {
790
+ if (previous === void 0) {
791
+ state.liveCount++;
792
+ Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
793
+ return;
794
+ }
795
+ if (keepsReplacedVector) keepReplacedOrdinal(state, store, previous);
796
+ else retireOrdinal(state, store, previous, oldestPinOf(store));
797
+ }
798
+ function beginWrite(state, store) {
799
+ freeAwaitingOrdinals(state);
800
+ return nextSequence(store.sequences);
801
+ }
802
+ function retireInWrite(store, ordinal, sequence) {
803
+ stampRetired(store.sequences, ordinal, sequence);
804
+ }
805
+ function endWrite(store, sequence) {
806
+ publishSequence(store.sequences, sequence);
807
+ }
808
+ var NOT_RETIRED = 0n;
809
+ function clearRetirement(store, ordinal) {
810
+ const writes = store.sequences;
811
+ coverOrdinal(writes, ordinal);
812
+ Atomics.store(writes.retired, ordinal, NOT_RETIRED);
813
+ }
814
+ function reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector) {
815
+ const store = state.open;
816
+ if (store === null) return false;
817
+ const current = state.docToOrd.get(docId);
818
+ if (current === ordinal) {
819
+ clearRetirement(store, ordinal);
820
+ return true;
821
+ }
822
+ if (state.ordToDoc[ordinal] !== void 0) return false;
823
+ if (!state.replacedOrdinals.delete(ordinal) && !state.awaitingFree.delete(ordinal)) return false;
824
+ const sequence = beginWrite(state, store);
825
+ clearRetirement(store, ordinal);
826
+ recordPartition(state, store, ordinal, partitionId);
827
+ state.ordToDoc[ordinal] = docId;
828
+ state.docToOrd.set(docId, ordinal);
829
+ if (current !== void 0) retireInWrite(store, current, sequence);
830
+ endWrite(store, sequence);
831
+ settleReplacement(state, store, current, keepsReplacedVector);
832
+ return true;
833
+ }
834
+ function retireStoredOrdinal(state, ordinal) {
835
+ const store = state.open;
836
+ if (store === null) return false;
837
+ if (state.awaitingFree.size > 0) freeAwaitingOrdinals(state);
838
+ const writes = store.sequences;
839
+ if (ordinal >= writes.retired.length) coverOrdinal(writes, ordinal);
840
+ const retiredHalves = writes.retiredHalves;
841
+ const retiredLow = ordinal * 2;
842
+ if (retiredHalves[retiredLow] !== 0 || retiredHalves[retiredLow + 1] !== 0) return false;
843
+ const sequence = nextSequence(writes);
844
+ Atomics.store(writes.retired, ordinal, BigInt(sequence));
845
+ publishSequence(writes, sequence);
846
+ return true;
847
+ }
848
+
849
+ // src/vector/vector-store/load.ts
850
+ function retiredBeforeLoad(store, ordinal) {
851
+ stampRetired(store.sequences, ordinal, RETIRED_BEFORE_LOAD);
852
+ }
853
+ function settleLoadedReplacement(state, store, previous) {
854
+ if (previous === void 0) {
855
+ state.liveCount++;
856
+ Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
857
+ return;
858
+ }
859
+ stampRetired(store.sequences, previous, RETIRED_BEFORE_LOAD);
860
+ retireOrdinal(state, store, previous, Number.POSITIVE_INFINITY);
861
+ }
862
+ function loadColdVector(state, store, docId, vectorMagnitude, location, partitionId) {
863
+ const previous = state.docToOrd.get(docId);
864
+ Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
865
+ const ordinal = claimOrdinal(state, store, docId, true, partitionId);
866
+ placeOnDisk(store, ordinal, vectorMagnitude, location);
867
+ stampBorn(store.sequences, ordinal, LOADED_SEQUENCE);
868
+ publishOrdinal(store, ordinal);
869
+ settleLoadedReplacement(state, store, previous);
870
+ return ordinal;
871
+ }
872
+ function loadReplacedVector(state, store, docId, vector, location) {
873
+ if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
874
+ const ordinal = claimSlot(store, docId, location !== void 0);
875
+ store.partitions[ordinal] = NO_PARTITION;
876
+ state.ordToDoc[ordinal] = void 0;
877
+ state.replacedOrdinals.add(ordinal);
878
+ retiredBeforeLoad(store, ordinal);
879
+ if (location !== void 0) {
880
+ publishOnDisk(store, ordinal, magnitude(vector), location);
881
+ return ordinal;
882
+ }
883
+ writeVector(store, ordinal, vector);
884
+ publishOrdinal(store, ordinal);
885
+ return ordinal;
886
+ }
887
+ function restoreReplacedVector(state, store, ordinal, vector) {
888
+ store.diskFile[ordinal] = IN_MEMORY;
889
+ retiredBeforeLoad(store, ordinal);
890
+ writeVector(store, ordinal, vector);
891
+ Atomics.store(store.present, ordinal, 1);
892
+ state.replacedOrdinals.add(ordinal);
893
+ }
894
+ function loadSnapshot(state, store, snapshot) {
895
+ const replaced = new Set(snapshot.replacedOrdinals);
896
+ for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
897
+ const docId = snapshot.docIds[ordinal];
898
+ const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
899
+ if (docId === null || docId === void 0) {
900
+ ensureOrdinal(store, ordinal, false);
901
+ state.ordToDoc[ordinal] = void 0;
902
+ store.partitions[ordinal] = NO_PARTITION;
903
+ if (replaced.has(ordinal)) restoreReplacedVector(state, store, ordinal, vector);
904
+ else retiredBeforeLoad(store, ordinal);
905
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
906
+ continue;
907
+ }
908
+ appendOrdinal(state, store, docId, vector, LOADED_SEQUENCE);
909
+ state.liveCount++;
910
+ }
911
+ Atomics.store(store.handles.header, STORE_LIVE_COUNT, state.liveCount);
912
+ }
913
+
914
+ // src/vector/vector-store/retired-pins.ts
915
+ function createRetiredVectorPins() {
916
+ let pins = 0;
917
+ let lost = false;
918
+ const retired = /* @__PURE__ */ new Set();
919
+ return {
920
+ pin() {
921
+ pins += 1;
922
+ },
923
+ unpin() {
924
+ if (pins === 0) return;
925
+ pins -= 1;
926
+ if (pins > 0) return;
927
+ retired.clear();
928
+ lost = false;
929
+ },
930
+ note(ordinal) {
931
+ if (pins > 0) retired.add(ordinal);
932
+ },
933
+ holds(ordinal) {
934
+ return retired.has(ordinal);
935
+ },
936
+ keptWith(replaced) {
937
+ if (retired.size === 0) return replaced;
938
+ const kept = new Set(replaced);
939
+ for (const ordinal of retired) kept.add(ordinal);
940
+ return kept;
941
+ },
942
+ forget() {
943
+ retired.clear();
944
+ if (pins > 0) lost = true;
945
+ },
946
+ requireIntact() {
947
+ if (!lost) return;
948
+ throw new NarsilError(
949
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
950
+ "The vector store was emptied while a checkpoint read the vectors it captured, so the checkpoint stops"
951
+ );
952
+ }
953
+ };
954
+ }
955
+ function pinnedReads(pins, openStore, readable) {
956
+ return {
957
+ pinRetiredVectors: () => pins.pin(),
958
+ unpinRetiredVectors: () => pins.unpin(),
959
+ entryForPinnedOrdinal(ordinal) {
960
+ pins.requireIntact();
961
+ const open = openStore();
962
+ if (open === null || !(readable(ordinal) || pins.holds(ordinal))) return void 0;
963
+ return entryAt(open, ordinal);
964
+ }
965
+ };
966
+ }
967
+
968
+ // src/vector/vector-store/store-clear.ts
969
+ function retireEveryOrdinal(store) {
970
+ const sequences = store.sequences;
971
+ const sequence = nextSequence(sequences);
972
+ const slots = Atomics.load(store.handles.header, STORE_SLOTS);
973
+ for (let ordinal = 0; ordinal < slots; ordinal++) stampRetired(sequences, ordinal, sequence);
974
+ publishSequence(sequences, sequence);
975
+ }
976
+ function forgetCodes(store) {
977
+ const { header, codePresent } = store.handles;
978
+ new Uint8Array(codePresent, 0, codePresent.byteLength).fill(0);
979
+ Atomics.store(header, STORE_LIVE_COUNT, 0);
980
+ Atomics.store(header, STORE_CODE_COUNT, 0);
981
+ Atomics.store(header, STORE_CALIBRATED, 0);
982
+ Atomics.add(header, STORE_CALIBRATION_GENERATION, 1);
983
+ }
984
+ function resetInPlace(store) {
985
+ const { handles } = store;
986
+ store.present.fill(0, 0, handles.present.byteLength);
987
+ new Uint8Array(handles.codePresent, 0, handles.codePresent.byteLength).fill(0);
988
+ new Int32Array(handles.diskFile, 0, handles.diskFile.byteLength / 4).fill(IN_MEMORY);
989
+ handles.centroid.fill(0);
990
+ if (handles.vectorFiles.length > 0) {
991
+ handles.vectorFiles.length = 0;
992
+ relayout(store);
993
+ }
994
+ resetDocIds(handles);
995
+ resetStoreHeader(handles.header);
996
+ }
997
+ function clearStore(store) {
998
+ retireEveryOrdinal(store);
999
+ if (oldestHeldPin(store.handles.pins) === Number.POSITIVE_INFINITY) {
1000
+ resetInPlace(store);
1001
+ return;
1002
+ }
1003
+ forgetCodes(store);
1004
+ }
1005
+
1006
+ // src/vector/vector-store/index.ts
737
1007
  function createVectorStore(options) {
738
1008
  const codeBits = options?.codeBits ?? null;
739
- const blockBytes = options?.blockBytes ?? BLOCK_MAX_BYTES;
740
- const docToOrd = /* @__PURE__ */ new Map();
741
- const ordToDoc = [];
742
- const replacedOrdinals = /* @__PURE__ */ new Set();
743
- let unknownPartitions = 0;
744
- let liveCount = 0;
745
- let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
1009
+ const blockBytes = Math.min(options?.blockBytes ?? VECTOR_MEMORY_LIMITS.blockBytes, VECTOR_MEMORY_LIMITS.blockBytes);
1010
+ const state = {
1011
+ open: options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes),
1012
+ docToOrd: createDocIdMap(),
1013
+ ordToDoc: [],
1014
+ replacedOrdinals: /* @__PURE__ */ new Set(),
1015
+ awaitingFree: /* @__PURE__ */ new Set(),
1016
+ checkpointPins: createRetiredVectorPins(),
1017
+ unknownPartitions: 0,
1018
+ liveCount: 0
1019
+ };
1020
+ const { docToOrd, ordToDoc, replacedOrdinals, awaitingFree } = state;
1021
+ let readBackVector = null;
746
1022
  function forgetDocuments() {
747
1023
  docToOrd.clear();
748
1024
  ordToDoc.length = 0;
749
1025
  replacedOrdinals.clear();
750
- unknownPartitions = 0;
751
- liveCount = 0;
1026
+ state.awaitingFree.clear();
1027
+ state.checkpointPins.forget();
1028
+ state.unknownPartitions = 0;
1029
+ state.liveCount = 0;
752
1030
  }
753
- function readable(ordinal) {
754
- return ordToDoc[ordinal] !== void 0 || replacedOrdinals.has(ordinal);
1031
+ function heldAfterRemoval(ordinal) {
1032
+ return replacedOrdinals.has(ordinal) || awaitingFree.has(ordinal);
1033
+ }
1034
+ function isReadable(ordinal) {
1035
+ return ordToDoc[ordinal] !== void 0 || heldAfterRemoval(ordinal);
755
1036
  }
756
1037
  function requireOpen() {
757
- if (open === null) {
1038
+ if (state.open === null) {
758
1039
  throw new NarsilError(ErrorCodes.VECTOR_DIMENSION_MISMATCH, "The vector store holds no vector yet");
759
1040
  }
760
- return open;
761
- }
762
- function recordPartition(store, ordinal, partitionId) {
763
- const given = partitionId !== void 0 && partitionId >= 0 ? partitionId : UNKNOWN_PARTITION;
764
- store.partitions[ordinal] = given;
765
- if (given === UNKNOWN_PARTITION) unknownPartitions += 1;
766
- }
767
- function retireOrdinal(store, ordinal) {
768
- ordToDoc[ordinal] = void 0;
769
- Atomics.store(store.present, ordinal, 0);
770
- if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
771
- }
772
- function keepReplacedOrdinal(store, ordinal) {
773
- ordToDoc[ordinal] = void 0;
774
- if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
775
- store.partitions[ordinal] = NO_PARTITION;
776
- replacedOrdinals.add(ordinal);
777
- }
778
- function restoreReplacedVector(store, ordinal, vector) {
779
- store.diskFile[ordinal] = IN_MEMORY;
780
- writeVector(store, ordinal, vector);
781
- Atomics.store(store.present, ordinal, 1);
782
- replacedOrdinals.add(ordinal);
783
- }
784
- function claimOrdinal(store, docId, cold, partitionId) {
785
- const ordinal = claimSlot(store, docId, cold);
786
- recordPartition(store, ordinal, partitionId);
787
- ordToDoc[ordinal] = docId;
788
- docToOrd.set(docId, ordinal);
789
- return ordinal;
1041
+ return state.open;
790
1042
  }
791
- function appendOrdinal(store, docId, vector, partitionId) {
792
- const ordinal = claimOrdinal(store, docId, false, partitionId);
793
- writeVector(store, ordinal, vector);
794
- publishOrdinal(store, ordinal);
795
- return ordinal;
796
- }
797
- function settleReplacement(store, ordinal, previous, keepsReplacedVector) {
798
- if (previous !== void 0) {
799
- if (keepsReplacedVector) keepReplacedOrdinal(store, previous);
800
- else retireOrdinal(store, previous);
801
- return ordinal;
802
- }
803
- liveCount++;
804
- Atomics.add(store.handles.header, STORE_LIVE_COUNT, 1);
805
- return ordinal;
1043
+ function openFor(dimension) {
1044
+ state.open ??= openHandles(dimension, codeBits, blockBytes);
1045
+ return state.open;
806
1046
  }
807
1047
  return {
808
- ...createOrdinalReads(() => open, readable),
1048
+ ...createOrdinalReads(() => state.open, isReadable),
809
1049
  get size() {
810
- return liveCount;
1050
+ return state.liveCount;
811
1051
  },
812
1052
  get slots() {
813
1053
  return ordToDoc.length;
814
1054
  },
815
1055
  get dimension() {
816
- return open === null ? 0 : open.handles.dimension;
1056
+ return state.open === null ? 0 : state.open.handles.dimension;
817
1057
  },
818
1058
  get codeBits() {
819
1059
  return codeBits;
@@ -828,8 +1068,7 @@ function createVectorStore(options) {
828
1068
  return replacedOrdinals.size;
829
1069
  },
830
1070
  insert(docId, vector, partitionId, keepsReplacedVector = false) {
831
- if (open === null) open = openHandles(vector.length, codeBits, blockBytes);
832
- const store = open;
1071
+ const store = openFor(vector.length);
833
1072
  if (vector.length !== store.handles.dimension) {
834
1073
  throw new NarsilError(
835
1074
  ErrorCodes.VECTOR_DIMENSION_MISMATCH,
@@ -838,38 +1077,60 @@ function createVectorStore(options) {
838
1077
  );
839
1078
  }
840
1079
  const previous = docToOrd.get(docId);
841
- const ordinal = appendOrdinal(store, docId, vector, partitionId);
842
- return settleReplacement(store, ordinal, previous, keepsReplacedVector);
1080
+ const sequence = beginWrite(state, store);
1081
+ const ordinal = appendOrdinal(state, store, docId, vector, sequence, partitionId);
1082
+ if (previous !== void 0) retireInWrite(store, previous, sequence);
1083
+ endWrite(store, sequence);
1084
+ settleReplacement(state, store, previous, keepsReplacedVector);
1085
+ return ordinal;
1086
+ },
1087
+ requireRoomFor(docId) {
1088
+ if (state.open !== null) requireRoomForSlot(state.open, docId);
1089
+ },
1090
+ reclaim(docId, ordinal, partitionId, keepsReplacedVector = false) {
1091
+ return reclaimOrdinal(state, docId, ordinal, partitionId, keepsReplacedVector);
843
1092
  },
844
- releaseReplacedVectors() {
845
- const released = [...replacedOrdinals];
846
- replacedOrdinals.clear();
847
- if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
1093
+ retire(ordinal) {
1094
+ retireStoredOrdinal(state, ordinal);
1095
+ },
1096
+ freeRetiredVectors() {
1097
+ freeAwaitingOrdinals(state);
1098
+ },
1099
+ oldestPin() {
1100
+ return state.open === null ? Number.POSITIVE_INFINITY : oldestPinOf(state.open);
1101
+ },
1102
+ writeSequence() {
1103
+ return state.open === null ? 0 : writeSequenceOf(state.open.sequences.header);
1104
+ },
1105
+ retiredStampHalves() {
1106
+ return requireOpen().sequences.retiredHalves;
1107
+ },
1108
+ claimedSlots() {
1109
+ return state.open === null ? 0 : state.open.handles.header[STORE_SLOTS];
1110
+ },
1111
+ mayFreeOrdinal(ordinal, oldestPin) {
1112
+ return state.open !== null && freeable(state.open, ordinal, oldestPin);
1113
+ },
1114
+ releaseReplacedVectors(oldestPin) {
1115
+ const store = state.open;
1116
+ const released = [];
1117
+ const oldest = oldestPin ?? (store === null ? Number.POSITIVE_INFINITY : oldestPinOf(store));
1118
+ for (const ordinal of replacedOrdinals) {
1119
+ if (store !== null && !freeable(store, ordinal, oldest)) continue;
1120
+ released.push(ordinal);
1121
+ }
1122
+ for (const ordinal of released) {
1123
+ replacedOrdinals.delete(ordinal);
1124
+ if (store !== null) freeRetired(state, store, ordinal);
1125
+ }
848
1126
  return released;
849
1127
  },
1128
+ ...pinnedReads(state.checkpointPins, () => state.open, isReadable),
850
1129
  insertCold(docId, vectorMagnitude, location, partitionId) {
851
- const store = requireOpen();
852
- const previous = docToOrd.get(docId);
853
- Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
854
- const ordinal = claimOrdinal(store, docId, true, partitionId);
855
- publishOnDisk(store, ordinal, vectorMagnitude, location);
856
- return settleReplacement(store, ordinal, previous, false);
1130
+ return loadColdVector(state, requireOpen(), docId, vectorMagnitude, location, partitionId);
857
1131
  },
858
1132
  insertReplaced(docId, vector, location) {
859
- if (open === null) open = openHandles(vector.length, codeBits, blockBytes);
860
- const store = open;
861
- if (location !== void 0) Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
862
- const ordinal = claimSlot(store, docId, location !== void 0);
863
- store.partitions[ordinal] = NO_PARTITION;
864
- ordToDoc[ordinal] = void 0;
865
- replacedOrdinals.add(ordinal);
866
- if (location !== void 0) {
867
- publishOnDisk(store, ordinal, magnitude(vector), location);
868
- return ordinal;
869
- }
870
- writeVector(store, ordinal, vector);
871
- publishOrdinal(store, ordinal);
872
- return ordinal;
1133
+ return loadReplacedVector(state, openFor(vector.length), docId, vector, location);
873
1134
  },
874
1135
  addVectorFile(path) {
875
1136
  const store = requireOpen();
@@ -880,13 +1141,16 @@ function createVectorStore(options) {
880
1141
  return store.handles.vectorFiles.length - 1;
881
1142
  },
882
1143
  isCold(ordinal) {
883
- if (open === null) return false;
884
- return open.view.isCold(ordinal);
1144
+ if (state.open === null) return false;
1145
+ return state.open.view.isCold(ordinal);
885
1146
  },
886
1147
  releaseToFile(ordinal, location) {
887
1148
  const store = requireOpen();
888
1149
  if (ordToDoc[ordinal] === void 0) return false;
889
- if (store.diskFile[ordinal] === IN_MEMORY && !fileHoldsSameBytes(store, ordinal, location)) return false;
1150
+ if (store.diskFile[ordinal] === IN_MEMORY) {
1151
+ readBackVector ??= new Float32Array(store.handles.dimension);
1152
+ if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
1153
+ }
890
1154
  Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
891
1155
  store.diskOffset[ordinal] = location.offset;
892
1156
  store.diskFile[ordinal] = location.fileIndex;
@@ -894,49 +1158,54 @@ function createVectorStore(options) {
894
1158
  },
895
1159
  releaseColdBlocks() {
896
1160
  const store = requireOpen();
1161
+ const held = new Set(replacedOrdinals);
1162
+ for (const ordinal of state.awaitingFree) held.add(ordinal);
1163
+ const kept = state.checkpointPins.keptWith(held);
897
1164
  let released = 0;
898
1165
  for (let index = 0; index < store.handles.blocks.length; index++) {
899
- if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, replacedOrdinals)) continue;
1166
+ if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, kept)) continue;
900
1167
  store.handles.blocks[index] = null;
901
1168
  released += 1;
902
1169
  }
903
- if (released + forgetUnreferencedFiles(store, ordToDoc, replacedOrdinals) > 0) relayout(store);
1170
+ if (released + forgetUnreferencedFiles(store, ordToDoc, kept) > 0) relayout(store);
904
1171
  return released;
905
1172
  },
906
1173
  setPartition(docId, partitionId) {
907
1174
  const ordinal = docToOrd.get(docId);
908
- if (ordinal === void 0 || open === null || partitionId < 0) return;
909
- if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
910
- open.partitions[ordinal] = partitionId;
1175
+ if (ordinal === void 0 || state.open === null || partitionId < 0) return;
1176
+ if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
1177
+ state.open.partitions[ordinal] = partitionId;
911
1178
  },
912
1179
  forgetPartition(docId) {
913
1180
  const ordinal = docToOrd.get(docId);
914
- if (ordinal === void 0 || open === null) return;
915
- if (open.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
916
- open.partitions[ordinal] = NO_PARTITION;
1181
+ if (ordinal === void 0 || state.open === null) return;
1182
+ if (state.open.partitions[ordinal] === UNKNOWN_PARTITION) state.unknownPartitions -= 1;
1183
+ state.open.partitions[ordinal] = NO_PARTITION;
917
1184
  },
918
1185
  partitionOfOrdinal(ordinal) {
919
- if (open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
920
- const partitionId = open.partitions[ordinal];
1186
+ if (state.open === null || ordinal < 0 || ordinal >= ordToDoc.length) return void 0;
1187
+ const partitionId = state.open.partitions[ordinal];
921
1188
  return partitionId < 0 ? void 0 : partitionId;
922
1189
  },
923
1190
  get partitionsKnown() {
924
- return unknownPartitions === 0;
1191
+ return state.unknownPartitions === 0;
925
1192
  },
926
1193
  partitionFilter(partitionIds) {
927
- return partitionFilterOf(open, ordToDoc, partitionIds);
1194
+ return partitionFilterOf(state.open, ordToDoc, partitionIds);
928
1195
  },
929
- remove(docId) {
1196
+ remove(docId, oldestPin) {
930
1197
  const ordinal = docToOrd.get(docId);
931
- if (ordinal === void 0 || open === null) return;
1198
+ const store = state.open;
1199
+ if (ordinal === void 0 || store === null) return;
1200
+ const retiredNow = retireStoredOrdinal(state, ordinal);
932
1201
  docToOrd.delete(docId);
933
- retireOrdinal(open, ordinal);
934
- liveCount--;
935
- Atomics.sub(open.handles.header, STORE_LIVE_COUNT, 1);
1202
+ retireOrdinal(state, store, ordinal, retiredNow || oldestPin === void 0 ? oldestPinOf(store) : oldestPin);
1203
+ state.liveCount--;
1204
+ Atomics.sub(store.handles.header, STORE_LIVE_COUNT, 1);
936
1205
  },
937
1206
  get(docId) {
938
1207
  const ordinal = docToOrd.get(docId);
939
- return ordinal === void 0 || open === null ? void 0 : entryAt(open, ordinal);
1208
+ return ordinal === void 0 || state.open === null ? void 0 : entryAt(state.open, ordinal);
940
1209
  },
941
1210
  has(docId) {
942
1211
  return docToOrd.has(docId);
@@ -944,31 +1213,22 @@ function createVectorStore(options) {
944
1213
  holdsOrdinal(ordinal) {
945
1214
  return ordinal >= 0 && ordinal < ordToDoc.length && ordToDoc[ordinal] !== void 0;
946
1215
  },
947
- entries: () => entriesOf(open, ordToDoc),
948
- docIds: () => docIdsOf(open, ordToDoc),
1216
+ entries: () => entriesOf(state.open, ordToDoc),
1217
+ docIds: () => docIdsOf(state.open, ordToDoc),
949
1218
  clear() {
1219
+ const store = state.open;
950
1220
  forgetDocuments();
951
- if (open === null) return;
952
- open.present.fill(0);
953
- new Uint8Array(open.handles.codePresent).fill(0);
954
- new Int32Array(open.handles.diskFile).fill(IN_MEMORY);
955
- open.handles.centroid.fill(0);
956
- if (open.handles.vectorFiles.length > 0) {
957
- open.handles.vectorFiles.length = 0;
958
- relayout(open);
959
- }
960
- resetDocIds(open.handles);
961
- resetStoreHeader(open.handles.header);
1221
+ if (store !== null) clearStore(store);
962
1222
  },
963
1223
  release() {
964
1224
  forgetDocuments();
965
- if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
966
- open?.view.close();
967
- open = null;
1225
+ if (state.open !== null) Atomics.add(state.open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
1226
+ state.open?.view.close();
1227
+ state.open = null;
968
1228
  },
969
- closeFiles: () => open?.view.close(),
1229
+ closeFiles: () => state.open?.view.close(),
970
1230
  relocateVectorFiles(rewrite) {
971
- if (open !== null) relocateFiles(open, rewrite);
1231
+ if (state.open !== null) relocateFiles(state.open, rewrite);
972
1232
  },
973
1233
  getOrdinal(docId) {
974
1234
  return docToOrd.get(docId);
@@ -977,35 +1237,24 @@ function createVectorStore(options) {
977
1237
  return ordToDoc[ordinal];
978
1238
  },
979
1239
  prepareQueryArena(query) {
980
- return open === null ? null : open.view.prepareQueryArena(query);
1240
+ return state.open === null ? null : state.open.view.prepareQueryArena(query);
981
1241
  },
982
- exportSnapshot: () => snapshotOf(open, ordToDoc, replacedOrdinals),
1242
+ exportSnapshot: () => snapshotOf(state.open, ordToDoc, replacedOrdinals),
983
1243
  restoreSnapshot(snapshot) {
984
1244
  this.clear();
985
1245
  if (snapshot.dimension === 0 || snapshot.slots === 0) return;
986
- if (open === null) open = openHandles(snapshot.dimension, codeBits, blockBytes);
987
- const store = open;
988
- const replaced = new Set(snapshot.replacedOrdinals);
989
- for (let ordinal = 0; ordinal < snapshot.slots; ordinal++) {
990
- const docId = snapshot.docIds[ordinal];
991
- const vector = snapshot.vectors.subarray(ordinal * snapshot.dimension, (ordinal + 1) * snapshot.dimension);
992
- if (docId === null || docId === void 0) {
993
- ensureOrdinal(store, ordinal, false);
994
- ordToDoc[ordinal] = void 0;
995
- store.partitions[ordinal] = NO_PARTITION;
996
- if (replaced.has(ordinal)) restoreReplacedVector(store, ordinal, vector);
997
- Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
998
- continue;
999
- }
1000
- appendOrdinal(store, docId, vector);
1001
- liveCount++;
1002
- }
1003
- Atomics.store(store.handles.header, STORE_LIVE_COUNT, liveCount);
1246
+ loadSnapshot(state, openFor(snapshot.dimension), snapshot);
1004
1247
  },
1005
1248
  memoryBytes() {
1006
- return open === null ? 0 : sharedVectorStoreBytes(open.handles);
1249
+ return state.open === null ? 0 : sharedVectorStoreBytes(state.open.handles);
1250
+ },
1251
+ growthBytesFor(vectors) {
1252
+ return state.open === null ? 0 : growthBytesFor(state.open, vectors);
1253
+ },
1254
+ visitDocumentTables(visit) {
1255
+ visit("documents", docToOrd.size, true);
1007
1256
  }
1008
1257
  };
1009
1258
  }
1010
1259
 
1011
- export { createVectorStore, openSharedVectorField };
1260
+ export { createVectorStore, openSharedVectorField, requireOrdinalWithinLimit };