@delali/narsil 0.3.1-next.98 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +45 -0
  2. package/README.md +27 -8
  3. package/dist/adapters/filesystem.d.ts +1 -1
  4. package/dist/adapters/filesystem.mjs +101 -13
  5. package/dist/adapters/gcs.d.ts +49 -0
  6. package/dist/adapters/gcs.mjs +213 -0
  7. package/dist/adapters/indexeddb.d.ts +1 -1
  8. package/dist/adapters/indexeddb.mjs +2 -7
  9. package/dist/adapters/memory.d.ts +1 -1
  10. package/dist/adapters/memory.mjs +5 -0
  11. package/dist/adapters/s3.d.ts +53 -0
  12. package/dist/adapters/s3.mjs +263 -0
  13. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  14. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  15. package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
  16. package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
  17. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  18. package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
  19. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  20. package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
  21. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  22. package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
  23. package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
  24. package/dist/browser/embed-GG6PS3EA.mjs +264 -0
  25. package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
  26. package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
  27. package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
  28. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  29. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  30. package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
  31. package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
  32. package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
  33. package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
  34. package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
  35. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  36. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  37. package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
  38. package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
  39. package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
  40. package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
  41. package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
  42. package/dist/chunk-2AFZE7Y3.mjs +72 -0
  43. package/dist/chunk-2CKDCC65.mjs +47 -0
  44. package/dist/chunk-3ECIRFS5.mjs +61 -0
  45. package/dist/chunk-44W3CBNG.mjs +7 -0
  46. package/dist/chunk-4H4RG2FR.mjs +13 -0
  47. package/dist/chunk-4PY4JYMY.mjs +11 -0
  48. package/dist/chunk-4ZPVAHWF.mjs +1289 -0
  49. package/dist/chunk-5EI5CXO6.mjs +197 -0
  50. package/dist/chunk-5F5MXWS5.mjs +283 -0
  51. package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
  52. package/dist/chunk-6FOENO4E.mjs +273 -0
  53. package/dist/chunk-6GHX35ZM.mjs +108 -0
  54. package/dist/chunk-6MSZ543Y.mjs +32 -0
  55. package/dist/chunk-6N6HHBCQ.mjs +4596 -0
  56. package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
  57. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  58. package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
  59. package/dist/chunk-AVBPB3MI.mjs +602 -0
  60. package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
  61. package/dist/chunk-BUMREPLU.mjs +15 -0
  62. package/dist/chunk-BY6NALRK.mjs +31921 -0
  63. package/dist/chunk-C4ZMXXTL.mjs +92 -0
  64. package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
  65. package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
  66. package/dist/chunk-DXZ6JDWG.mjs +797 -0
  67. package/dist/chunk-FJXJZUWF.mjs +209 -0
  68. package/dist/chunk-FS3HPPSC.mjs +212 -0
  69. package/dist/chunk-FT4KLZXW.mjs +1164 -0
  70. package/dist/chunk-GR5GMFJJ.mjs +104 -0
  71. package/dist/chunk-GXCDWLQI.mjs +193 -0
  72. package/dist/chunk-GXED6R3Q.mjs +271 -0
  73. package/dist/chunk-HLPRHOSO.mjs +17 -0
  74. package/dist/chunk-IL5DRCNU.mjs +3041 -0
  75. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  76. package/dist/chunk-K5HSUPYS.mjs +11 -0
  77. package/dist/chunk-KYIRXTCA.mjs +100 -0
  78. package/dist/chunk-LFXS6OF7.mjs +289 -0
  79. package/dist/chunk-LNGE7SLY.mjs +151 -0
  80. package/dist/chunk-M2Z5WVWW.mjs +194 -0
  81. package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
  82. package/dist/chunk-M3OW2XSP.mjs +40 -0
  83. package/dist/chunk-MFPSQZ6P.mjs +127 -0
  84. package/dist/chunk-MRCJBWJZ.mjs +1292 -0
  85. package/dist/chunk-MSIN4NM3.mjs +25 -0
  86. package/dist/chunk-MW3CJVGP.mjs +219 -0
  87. package/dist/chunk-MYM6CAVK.mjs +76 -0
  88. package/dist/chunk-NJJN4FMM.mjs +84 -0
  89. package/dist/chunk-NM7I3BM7.mjs +59 -0
  90. package/dist/chunk-OIJ64LGT.mjs +19 -0
  91. package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
  92. package/dist/chunk-Q2AP2FSN.mjs +15 -0
  93. package/dist/chunk-R2PJKSNL.mjs +54 -0
  94. package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
  95. package/dist/chunk-S2POFTLN.mjs +10 -0
  96. package/dist/chunk-S5QMWECL.mjs +7 -0
  97. package/dist/chunk-SLAC5PAB.mjs +5008 -0
  98. package/dist/chunk-T4HMSZCX.mjs +5 -0
  99. package/dist/chunk-TACP6VK6.mjs +111 -0
  100. package/dist/chunk-TGPKS2UM.mjs +125 -0
  101. package/dist/chunk-TKKXPKEO.mjs +979 -0
  102. package/dist/chunk-V6EUBAIT.mjs +19 -0
  103. package/dist/chunk-XL56L4IR.mjs +228 -0
  104. package/dist/chunk-XQIAMDAG.mjs +3076 -0
  105. package/dist/chunk-YTUZ4R22.mjs +1063 -0
  106. package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
  107. package/dist/chunk-ZPJFGGGI.mjs +638 -0
  108. package/dist/chunk-ZXX5GZEX.mjs +17 -0
  109. package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
  110. package/dist/client.d.ts +11 -10
  111. package/dist/client.mjs +8 -898
  112. package/dist/config-Bnl7L-0H.d.ts +608 -0
  113. package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
  114. package/dist/core-leaf-6JVHEVFB.mjs +8 -0
  115. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  116. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  117. package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
  118. package/dist/distribution/coordinator/etcd.d.ts +63 -7
  119. package/dist/distribution/coordinator/etcd.mjs +725 -81
  120. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  121. package/dist/distribution/coordinator/in-memory.mjs +65 -10
  122. package/dist/distribution/transport/grpc.d.ts +2 -2
  123. package/dist/distribution/transport/grpc.mjs +39 -10
  124. package/dist/distribution/transport/in-memory.d.ts +2 -2
  125. package/dist/distribution/transport/in-memory.mjs +9 -6
  126. package/dist/distribution/transport/tcp.d.ts +2 -2
  127. package/dist/distribution/transport/tcp.mjs +20 -15
  128. package/dist/distribution.d.ts +267 -56
  129. package/dist/distribution.mjs +23156 -11362
  130. package/dist/durability-config-VWS2K24E.mjs +96 -0
  131. package/dist/embed-4PHHKGBK.mjs +263 -0
  132. package/dist/embeddings/openai.d.ts +1 -1
  133. package/dist/embeddings/openai.mjs +2 -2
  134. package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
  135. package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
  136. package/dist/gzip-2UCZW2MM.mjs +31 -0
  137. package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
  138. package/dist/index.browser.mjs +4 -4
  139. package/dist/index.d.ts +2133 -57
  140. package/dist/index.mjs +16 -32
  141. package/dist/index.react-native.mjs +12 -0
  142. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  143. package/dist/invalidation/filesystem.d.ts +1 -1
  144. package/dist/invalidation/filesystem.mjs +1 -2
  145. package/dist/invalidation/noop.d.ts +1 -1
  146. package/dist/languages/albanian.mjs +1 -1
  147. package/dist/languages/amharic.mjs +1 -1
  148. package/dist/languages/arabic.mjs +1 -1
  149. package/dist/languages/armenian.mjs +1 -1
  150. package/dist/languages/azerbaijani.mjs +1 -1
  151. package/dist/languages/bambara.mjs +1 -1
  152. package/dist/languages/basque.mjs +1 -1
  153. package/dist/languages/belarusian.mjs +1 -1
  154. package/dist/languages/bengali.mjs +1 -1
  155. package/dist/languages/bosnian.mjs +1 -1
  156. package/dist/languages/breton.mjs +1 -1
  157. package/dist/languages/bulgarian.mjs +1 -1
  158. package/dist/languages/burmese.mjs +1 -1
  159. package/dist/languages/catalan.mjs +1 -1
  160. package/dist/languages/chinese.mjs +1 -1
  161. package/dist/languages/croatian.mjs +1 -1
  162. package/dist/languages/czech.mjs +1 -1
  163. package/dist/languages/dagbani.mjs +1 -1
  164. package/dist/languages/danish.mjs +1 -1
  165. package/dist/languages/dutch.mjs +1 -1
  166. package/dist/languages/english.mjs +1 -1
  167. package/dist/languages/esperanto.mjs +1 -1
  168. package/dist/languages/estonian.mjs +1 -1
  169. package/dist/languages/ewe.mjs +1 -1
  170. package/dist/languages/faroese.mjs +1 -1
  171. package/dist/languages/fijian.mjs +1 -1
  172. package/dist/languages/finnish.mjs +1 -1
  173. package/dist/languages/french.mjs +1 -1
  174. package/dist/languages/ga.mjs +1 -1
  175. package/dist/languages/galician.mjs +1 -1
  176. package/dist/languages/georgian.mjs +1 -1
  177. package/dist/languages/german.mjs +1 -1
  178. package/dist/languages/greek.mjs +1 -1
  179. package/dist/languages/guarani.mjs +1 -1
  180. package/dist/languages/gujarati.mjs +1 -1
  181. package/dist/languages/haitian-creole.mjs +1 -1
  182. package/dist/languages/hausa.mjs +1 -1
  183. package/dist/languages/hawaiian.mjs +1 -1
  184. package/dist/languages/hebrew.mjs +1 -1
  185. package/dist/languages/hindi.mjs +1 -1
  186. package/dist/languages/hungarian.mjs +1 -1
  187. package/dist/languages/icelandic.mjs +1 -1
  188. package/dist/languages/igbo.mjs +1 -1
  189. package/dist/languages/indonesian.mjs +1 -1
  190. package/dist/languages/irish.mjs +1 -1
  191. package/dist/languages/italian.mjs +1 -1
  192. package/dist/languages/japanese.mjs +1 -1
  193. package/dist/languages/kannada.mjs +1 -1
  194. package/dist/languages/kazakh.mjs +1 -1
  195. package/dist/languages/khmer.mjs +1 -1
  196. package/dist/languages/kinyarwanda.mjs +1 -1
  197. package/dist/languages/kirundi.mjs +1 -1
  198. package/dist/languages/korean.mjs +1 -1
  199. package/dist/languages/kurmanji.mjs +1 -1
  200. package/dist/languages/kyrgyz.mjs +1 -1
  201. package/dist/languages/lao.mjs +1 -1
  202. package/dist/languages/latin.mjs +1 -1
  203. package/dist/languages/latvian.mjs +1 -1
  204. package/dist/languages/lingala.mjs +1 -1
  205. package/dist/languages/lithuanian.mjs +1 -1
  206. package/dist/languages/luxembourgish.mjs +1 -1
  207. package/dist/languages/macedonian.mjs +1 -1
  208. package/dist/languages/malagasy.mjs +1 -1
  209. package/dist/languages/malay.mjs +1 -1
  210. package/dist/languages/malayalam.mjs +1 -1
  211. package/dist/languages/maltese.mjs +1 -1
  212. package/dist/languages/maori.mjs +1 -1
  213. package/dist/languages/marathi.mjs +1 -1
  214. package/dist/languages/nepali.mjs +1 -1
  215. package/dist/languages/norwegian.mjs +1 -1
  216. package/dist/languages/oromo.mjs +1 -1
  217. package/dist/languages/persian.mjs +1 -1
  218. package/dist/languages/polish.mjs +1 -1
  219. package/dist/languages/portuguese.mjs +1 -1
  220. package/dist/languages/punjabi.mjs +1 -1
  221. package/dist/languages/romanian.mjs +1 -1
  222. package/dist/languages/russian.mjs +1 -1
  223. package/dist/languages/samoan.mjs +1 -1
  224. package/dist/languages/sanskrit.mjs +1 -1
  225. package/dist/languages/scottish-gaelic.mjs +1 -1
  226. package/dist/languages/serbian.mjs +1 -1
  227. package/dist/languages/shona.mjs +1 -1
  228. package/dist/languages/sinhala.mjs +1 -1
  229. package/dist/languages/slovak.mjs +1 -1
  230. package/dist/languages/slovenian.mjs +1 -1
  231. package/dist/languages/sorani.mjs +1 -1
  232. package/dist/languages/spanish.mjs +1 -1
  233. package/dist/languages/swahili.mjs +1 -1
  234. package/dist/languages/swedish.mjs +1 -1
  235. package/dist/languages/tagalog.mjs +1 -1
  236. package/dist/languages/tamil.mjs +1 -1
  237. package/dist/languages/tatar.mjs +1 -1
  238. package/dist/languages/telugu.mjs +1 -1
  239. package/dist/languages/thai.mjs +1 -1
  240. package/dist/languages/tibetan.mjs +1 -1
  241. package/dist/languages/tigrinya.mjs +1 -1
  242. package/dist/languages/tongan.mjs +1 -1
  243. package/dist/languages/turkish.mjs +1 -1
  244. package/dist/languages/twi.mjs +1 -1
  245. package/dist/languages/ukrainian.mjs +1 -1
  246. package/dist/languages/urdu.mjs +1 -1
  247. package/dist/languages/vietnamese.mjs +1 -1
  248. package/dist/languages/welsh.mjs +1 -1
  249. package/dist/languages/wolof.mjs +1 -1
  250. package/dist/languages/xhosa.mjs +1 -1
  251. package/dist/languages/yoruba.mjs +1 -1
  252. package/dist/languages/zulu.mjs +1 -1
  253. package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
  254. package/dist/module-5ZSLUQII.mjs +3729 -0
  255. package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
  256. package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
  257. package/dist/optional-writes-4UOQKETU.mjs +247 -0
  258. package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
  259. package/dist/partial-words.browser.mjs +1650 -0
  260. package/dist/partial-words.mjs +17 -11
  261. package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
  262. package/dist/pattern.mjs +6 -7
  263. package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
  264. package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
  265. package/dist/phrase-module-EFESWOCR.mjs +635 -0
  266. package/dist/query-actions-HMJUVWGT.mjs +119 -0
  267. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  268. package/dist/query-suggestions/merge-worker.mjs +53 -0
  269. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  270. package/dist/query-suggestions.browser.mjs +47 -0
  271. package/dist/query-suggestions.d.ts +20 -0
  272. package/dist/query-suggestions.mjs +19 -0
  273. package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
  274. package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
  275. package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
  276. package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
  277. package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
  278. package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
  279. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  280. package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
  281. package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
  282. package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
  283. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  284. package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
  285. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  286. package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
  287. package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
  288. package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
  289. package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
  290. package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
  291. package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
  292. package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
  293. package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
  294. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  295. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  296. package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
  297. package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
  298. package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
  299. package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
  300. package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
  301. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  302. package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
  303. package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
  304. package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
  305. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  306. package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
  307. package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
  308. package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
  309. package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
  310. package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
  311. package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
  312. package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
  313. package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
  314. package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
  315. package/dist/react-native/partial-words.mjs +1658 -0
  316. package/dist/react-native/pattern.mjs +2570 -0
  317. package/dist/react-native/query-suggestions.mjs +55 -0
  318. package/dist/react-native/sync.mjs +2759 -0
  319. package/dist/react-native/synonyms.mjs +676 -0
  320. package/dist/react.d.ts +12 -8
  321. package/dist/react.mjs +45 -1
  322. package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
  323. package/dist/rerankers/http.d.ts +1 -1
  324. package/dist/rerankers/http.mjs +2 -2
  325. package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
  326. package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
  327. package/dist/runtime-USQYKN3I.mjs +615 -0
  328. package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
  329. package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
  330. package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
  331. package/dist/server.d.ts +348 -22
  332. package/dist/server.mjs +2211 -327
  333. package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
  334. package/dist/store-leaf-6VMMSVBE.mjs +20 -0
  335. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  336. package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
  337. package/dist/sync/filesystem.d.ts +30 -0
  338. package/dist/sync/filesystem.mjs +288 -0
  339. package/dist/sync/indexeddb.browser.mjs +287 -0
  340. package/dist/sync/indexeddb.d.ts +30 -0
  341. package/dist/sync/indexeddb.mjs +242 -0
  342. package/dist/sync.browser.mjs +1271 -0
  343. package/dist/sync.d.ts +23 -0
  344. package/dist/sync.mjs +2712 -0
  345. package/dist/synonyms.browser.mjs +658 -0
  346. package/dist/synonyms.mjs +17 -7
  347. package/dist/tasks-65b6t6lk.d.ts +782 -0
  348. package/dist/thread-UJ3W5LVI.mjs +31 -0
  349. package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
  350. package/dist/types-CpCg-Gq-.d.ts +227 -0
  351. package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
  352. package/dist/vector/search-worker.d.ts +10 -2
  353. package/dist/vector/search-worker.mjs +11 -11
  354. package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
  355. package/dist/workers/entry.mjs +75 -15
  356. package/openapi.json +6840 -0
  357. package/package.json +406 -19
  358. package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
  359. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  360. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  361. package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
  362. package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
  363. package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
  364. package/dist/browser/chunk-QWES44AW.mjs +0 -473
  365. package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
  366. package/dist/browser/module-PDXYHT75.mjs +0 -2822
  367. package/dist/browser/release-76MXRPGV.mjs +0 -33
  368. package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
  369. package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
  370. package/dist/browser/write-L7FAQA47.mjs +0 -208
  371. package/dist/chunk-37CD3OKS.mjs +0 -87
  372. package/dist/chunk-5XNIRLFC.mjs +0 -96
  373. package/dist/chunk-7X5JST5L.mjs +0 -114
  374. package/dist/chunk-AV4IJKD2.mjs +0 -35
  375. package/dist/chunk-AY3AQ3UH.mjs +0 -73
  376. package/dist/chunk-BKB3XTMI.mjs +0 -707
  377. package/dist/chunk-C7SLA6CM.mjs +0 -19
  378. package/dist/chunk-COVCQ73F.mjs +0 -587
  379. package/dist/chunk-DJHHF2L5.mjs +0 -17
  380. package/dist/chunk-E74YLAIG.mjs +0 -24
  381. package/dist/chunk-FMWRCP2C.mjs +0 -57
  382. package/dist/chunk-GES7U2PM.mjs +0 -68
  383. package/dist/chunk-HJSN7UYG.mjs +0 -10
  384. package/dist/chunk-JFRNNBOM.mjs +0 -35
  385. package/dist/chunk-K25LZA54.mjs +0 -8484
  386. package/dist/chunk-KEHK2O6P.mjs +0 -13
  387. package/dist/chunk-KOKM4PQN.mjs +0 -52
  388. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  389. package/dist/chunk-L3L5YFXC.mjs +0 -614
  390. package/dist/chunk-L5ORED7V.mjs +0 -25
  391. package/dist/chunk-LH4NZNJJ.mjs +0 -1731
  392. package/dist/chunk-LU2CE5AE.mjs +0 -507
  393. package/dist/chunk-LXWDQLTW.mjs +0 -30
  394. package/dist/chunk-MHOSCQCQ.mjs +0 -948
  395. package/dist/chunk-NKRBEGJZ.mjs +0 -1705
  396. package/dist/chunk-NTRMFARG.mjs +0 -462
  397. package/dist/chunk-NW4XDSP2.mjs +0 -212
  398. package/dist/chunk-OOEXLUGR.mjs +0 -117
  399. package/dist/chunk-ORBLD3EL.mjs +0 -6
  400. package/dist/chunk-OWNZYMRE.mjs +0 -3591
  401. package/dist/chunk-PPY2GSIB.mjs +0 -44
  402. package/dist/chunk-QY7GC2MW.mjs +0 -33
  403. package/dist/chunk-R6A773WK.mjs +0 -226
  404. package/dist/chunk-R6PR4WYZ.mjs +0 -68
  405. package/dist/chunk-R7EEKAJE.mjs +0 -593
  406. package/dist/chunk-RONEOEON.mjs +0 -426
  407. package/dist/chunk-VWULPWRJ.mjs +0 -7469
  408. package/dist/chunk-WCJKUGHB.mjs +0 -262
  409. package/dist/chunk-WQBCBKY3.mjs +0 -8
  410. package/dist/chunk-X5CV2E2B.mjs +0 -1
  411. package/dist/chunk-YA7XLE2R.mjs +0 -1256
  412. package/dist/chunk-YTJCGKM4.mjs +0 -56
  413. package/dist/chunk-ZG477IPV.mjs +0 -526
  414. package/dist/chunk-ZYNAMIYX.mjs +0 -113
  415. package/dist/config-BGB_fDtq.d.ts +0 -237
  416. package/dist/direct-executor-CXZ66ACP.mjs +0 -22
  417. package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
  418. package/dist/engine-BGCEm9E_.d.ts +0 -577
  419. package/dist/module-3NONKBZX.mjs +0 -1860
  420. package/dist/release-BH3DKW3T.mjs +0 -33
  421. package/dist/restore-queue-JPRBUYII.mjs +0 -35
  422. package/dist/runtime-ZKYZFJJC.mjs +0 -177
  423. package/dist/snapshot-HOGXNO5W.mjs +0 -793
  424. package/dist/tasks-HG3rlxx6.d.ts +0 -302
  425. package/dist/thread-B657LQXK.mjs +0 -692
  426. package/dist/write-XLIGTXBU.mjs +0 -238
@@ -0,0 +1,4596 @@
1
+ import { loadVectorField, openColdVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-FT4KLZXW.mjs';
2
+ import { engineChangeSource, ChangeCursor, createSemaphore, changePositionList, changesNotEnabled, previousDocumentRequired, invalidPositions, changePositionsOf, requireConfigurationShape, RESTORE_ESTIMATE_MARGIN, persistEpochStates, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR, RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE } from './chunk-AVBPB3MI.mjs';
3
+ import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
4
+ import { ENTRIES_MESSAGE_LIMIT_BYTES, FEED_HEAD_RETRY_MS, decodeSegmentManifest, isRecord, isUint32, isSequenceNumber, ARCHIVE_CONFIG_ENTRY, ARCHIVE_MANIFEST_ENTRY, tarEnd, tarEntryHeaders, tarPadding, ARCHIVE_FILE_CHUNK_BYTES } from './chunk-ZPJFGGGI.mjs';
5
+ import { isPagedSegmentFile, openPagedSegmentScan, countsAsLatestHolder, writeCheckpointSegments, commitCheckpointManifest, readSegmentFile, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, removeUnusedSegmentFiles, readWalSegments, durableRecordMissing, checkpointLastSeqNo, pagedListingOf, residentSectionsOf, partitionContentsOf, checkPagedSegmentFully, openPagedSnapshot, createPagedFrozenSegment, SEGMENT_HEADER_SIZE, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-SLAC5PAB.mjs';
6
+ import { decodeSections, READS_AHEAD_AT_ONCE, readFileRangeLater, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, READ_AHEAD_CHUNK_BYTES, STABLE_HEAD_ATTEMPTS, directoryCheckpointStore } from './chunk-DXZ6JDWG.mjs';
7
+ import { manifestKey, segmentsPrefix, vectorFileKey, manifestFileRefs, graphPartKey } from './chunk-C4ZMXXTL.mjs';
8
+ import { DEFAULT_COMPACTION_THRESHOLD } from './chunk-MSIN4NM3.mjs';
9
+ import { encode } from './chunk-S5QMWECL.mjs';
10
+ import { setNestedValue, restoreVectorFields } from './chunk-R2PJKSNL.mjs';
11
+ import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, REBALANCE_CHUNK_SIZE, rebuildTextIndexes, textRebuildTargetOf } from './chunk-MFPSQZ6P.mjs';
12
+ import { DEFAULT_VECTOR_METRIC, defaultQuantizationFor, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-R724DJSG.mjs';
13
+ import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
14
+ import { openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, readHeader, crc32, engineVersionParts, writeHeader, storedStateToWire, storedStateFromWire } from './chunk-TKKXPKEO.mjs';
15
+ import { resolveIndexAnalysis, getLanguage, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, oversizedConfigurationPayloadOf, requireEncodedConfigurationWithinSizeLimit, loadSchemaCodeFor, VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, noteDocumentChurn, createPartitionIndex, writePartitionDocument, yieldToEventLoop, createPublicationGate, removedSurfaceSpellings, readHeapStatistics, readProcessMemoryUse, prepareConfiguredSynonyms, withOpenIndex, loadGzip, prepareRemovalText, prepareDocumentText, updateDocumentVectors, holdSyncEpochChange, decodeStoredDocument, emitSyncChange, ARRAY_POSITION_GAP } from './chunk-BY6NALRK.mjs';
16
+ import { decode } from './chunk-GXCDWLQI.mjs';
17
+ import { isSyncEpoch, newSyncEpoch } from './chunk-BUMREPLU.mjs';
18
+ import { extractVectorFieldsFromSchema, flattenSchema, VECTOR_PATTERN, validateDocument, validateDocumentStrict, schemaFieldsOf, parseVectorDimension, requireValidFieldTypes, isQuantizationMode, isStorageMode, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-7P5YWDRQ.mjs';
19
+ import { collectAndReturnPages, turnsAreBudgeted, turnIsSpent, yieldTurn, countWrite } from './chunk-5EI5CXO6.mjs';
20
+ import { DEFAULT_MAX_PARTITIONS, ANALYSIS_CHANGE_MEMBERS, EXPORT_CHUNK_BYTES, MAX_PARTITIONS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-3ECIRFS5.mjs';
21
+ import { VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
22
+ import { NarsilError, ErrorCodes } from './chunk-ZKOQR7EA.mjs';
23
+ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
24
+
25
+ // src/changes/constants.ts
26
+ var CHANGES_IDLE_RECHECK_MS = 1e3;
27
+ var MAX_FOLLOWER_CATCH_UPS = 4;
28
+
29
+ // src/changes/follow.ts
30
+ function requestOf(indexName, options) {
31
+ const { epoch, positions, filter } = options;
32
+ if (epoch === void 0 !== (positions === void 0)) {
33
+ throw invalidPositions(indexName, "Pass epoch and positions together, or leave both out");
34
+ }
35
+ const request = filter === void 0 ? {} : { filter };
36
+ if (epoch === void 0) return request;
37
+ if (!isSyncEpoch(epoch)) throw invalidPositions(indexName, "The epoch must be 32 lowercase hexadecimal digits");
38
+ const valid = changePositionsOf(positions);
39
+ if (valid === null) {
40
+ throw invalidPositions(indexName, "Each position needs a whole partitionId and a whole, non-negative seqNo");
41
+ }
42
+ return { ...request, epoch, positions: valid };
43
+ }
44
+ function documentOf(bytes, vectorFieldPaths) {
45
+ const document = decode(bytes);
46
+ restoreVectorFields(document, vectorFieldPaths);
47
+ return document;
48
+ }
49
+ function eventOf(batch, vectorFieldPaths) {
50
+ const changes = batch.changes.map((change) => {
51
+ const shaped = { seqNo: change.seqNo, type: change.type, documentId: change.documentId };
52
+ if (change.document !== null) shaped.document = documentOf(change.document, vectorFieldPaths);
53
+ if (change.previousDocument !== null) {
54
+ shaped.previousDocument = documentOf(change.previousDocument, vectorFieldPaths);
55
+ }
56
+ return shaped;
57
+ });
58
+ return { type: "changes", partitionId: batch.partitionId, changes, scannedSeqNo: batch.scannedSeqNo };
59
+ }
60
+ function wakeupOn(source, indexName, signal) {
61
+ let pending = false;
62
+ let wake = null;
63
+ const fire = () => {
64
+ pending = true;
65
+ wake?.();
66
+ };
67
+ const stopListening = source.listen((change) => {
68
+ if (change.indexName === indexName) fire();
69
+ });
70
+ const stopWatching = source.watchIndex?.(indexName);
71
+ signal?.addEventListener("abort", fire);
72
+ return {
73
+ async wait(ms) {
74
+ if (pending) {
75
+ pending = false;
76
+ return;
77
+ }
78
+ let timer;
79
+ await new Promise((resolve) => {
80
+ wake = resolve;
81
+ timer = setTimeout(resolve, ms);
82
+ });
83
+ clearTimeout(timer);
84
+ wake = null;
85
+ pending = false;
86
+ },
87
+ stop() {
88
+ stopListening();
89
+ stopWatching?.();
90
+ signal?.removeEventListener("abort", fire);
91
+ }
92
+ };
93
+ }
94
+ function followIndexChanges(core, indexName, options = {}) {
95
+ core.guardShutdown();
96
+ core.requireIndex(indexName);
97
+ return followSourceChanges(
98
+ { source: engineChangeSource(core), guard: () => core.guardShutdown() },
99
+ indexName,
100
+ options
101
+ );
102
+ }
103
+ async function* followSourceChanges(deps, indexName, options = {}) {
104
+ deps.guard();
105
+ const request = requestOf(indexName, options);
106
+ const { source } = deps;
107
+ const { signal } = options;
108
+ const aborted = () => signal?.aborted === true;
109
+ if (aborted()) return;
110
+ const wakeup = wakeupOn(source, indexName, signal);
111
+ try {
112
+ const opened = await ChangeCursor.open(
113
+ { source, indexName, reads: createSemaphore(MAX_FOLLOWER_CATCH_UPS) },
114
+ request
115
+ );
116
+ if (typeof opened === "string") {
117
+ yield { type: "resync", reason: opened };
118
+ return;
119
+ }
120
+ const cursor = opened;
121
+ const { state } = cursor;
122
+ const vectorFieldPaths = await source.vectorFieldPaths(indexName);
123
+ yield {
124
+ type: "subscribed",
125
+ epoch: state.epoch,
126
+ partitionCount: state.partitionCount,
127
+ positions: changePositionList(cursor.subscribedSeqNos),
128
+ heads: changePositionList(cursor.heads),
129
+ bytesBehind: cursor.bytesBehind,
130
+ previousDocument: state.previousDocument
131
+ };
132
+ while (!aborted()) {
133
+ deps.guard();
134
+ const current = source.changesState(indexName);
135
+ if (current === null) throw changesNotEnabled(indexName);
136
+ if (current.epoch !== state.epoch) {
137
+ yield { type: "resync", reason: "epoch" };
138
+ return;
139
+ }
140
+ if (cursor.filtered && !current.previousDocument) throw previousDocumentRequired(indexName);
141
+ await cursor.refreshHeads();
142
+ let progressed = false;
143
+ for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
144
+ if (aborted() || !cursor.behind(partitionId)) continue;
145
+ const batch = await cursor.readBatch(partitionId, ENTRIES_MESSAGE_LIMIT_BYTES);
146
+ if (batch === "history") {
147
+ yield { type: "resync", reason: "history" };
148
+ return;
149
+ }
150
+ if (batch === null) continue;
151
+ const afterRead = source.changesState(indexName);
152
+ if (afterRead === null || afterRead.epoch !== state.epoch) break;
153
+ if (cursor.filtered && !afterRead.previousDocument) throw previousDocumentRequired(indexName);
154
+ cursor.commit(batch);
155
+ progressed = true;
156
+ yield eventOf(batch, vectorFieldPaths);
157
+ }
158
+ if (!progressed) await wakeup.wait(cursor.awaitsDurability() ? FEED_HEAD_RETRY_MS : CHANGES_IDLE_RECHECK_MS);
159
+ }
160
+ } finally {
161
+ wakeup.stop();
162
+ }
163
+ }
164
+
165
+ // src/engine/analysis-change/pending-target.ts
166
+ function configUnderAnalysis(base, language, tokenizer, stopWords) {
167
+ const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
168
+ return {
169
+ ...rest,
170
+ language,
171
+ ...tokenizer !== void 0 ? { tokenizer } : {},
172
+ ...stopWords !== void 0 ? { stopWords } : {}
173
+ };
174
+ }
175
+ function configOfNamedSettings(base, settings) {
176
+ const { language, tokenizer, stopWords, stopWordList } = settings;
177
+ const listed = stopWordList === void 0 ? void 0 : new Set(stopWordList);
178
+ return configUnderAnalysis(base, language, tokenizer, stopWords ?? listed);
179
+ }
180
+ function targetOfPending(entry, pending) {
181
+ const config = configOfNamedSettings(entry.config, pending);
182
+ return { config, language: getLanguage(pending.language), analysis: resolveIndexAnalysis(config) };
183
+ }
184
+
185
+ // src/partitioning/checkpoint-hold.ts
186
+ var heldCheckpointErrors = /* @__PURE__ */ new WeakSet();
187
+ function rebalanceCheckpointHeld() {
188
+ const error = new NarsilError(
189
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
190
+ "Queued writes from the previous partition layout must be replayed before a checkpoint"
191
+ );
192
+ heldCheckpointErrors.add(error);
193
+ return error;
194
+ }
195
+ async function holdScheduledCheckpoint(checkpoint, requested) {
196
+ try {
197
+ await checkpoint;
198
+ } catch (error) {
199
+ if (requested || typeof error !== "object" || error === null || !heldCheckpointErrors.has(error)) throw error;
200
+ }
201
+ }
202
+
203
+ // src/partitioning/manager/checkpoint-prefix.ts
204
+ var queues = /* @__PURE__ */ new WeakMap();
205
+ var priorLayouts = /* @__PURE__ */ new WeakMap();
206
+ var heldLayouts = /* @__PURE__ */ new WeakSet();
207
+ function registerCheckpointQueue(manager, queue, holdsLayout = false) {
208
+ queues.set(manager, queue);
209
+ priorLayouts.set(manager, manager.partitionCount);
210
+ if (holdsLayout) heldLayouts.add(manager);
211
+ }
212
+ function releaseCheckpointQueue(manager, queue) {
213
+ if (queues.get(manager) !== queue) return;
214
+ queues.delete(manager);
215
+ priorLayouts.delete(manager);
216
+ heldLayouts.delete(manager);
217
+ }
218
+ function releaseLayoutCheckpoint(manager) {
219
+ heldLayouts.delete(manager);
220
+ }
221
+ function metadataPartitionCount(manager) {
222
+ return priorLayouts.get(manager) ?? manager.partitionCount;
223
+ }
224
+ function requiresLayoutCheckpoint(manager) {
225
+ const prior = priorLayouts.get(manager);
226
+ return prior !== void 0 && prior !== manager.partitionCount;
227
+ }
228
+ async function captureAppliedPrefix(manager, capture2) {
229
+ for (; ; ) {
230
+ const queue = queues.get(manager);
231
+ await queue?.settleAdmissions();
232
+ const captured = await manager.publication.read(async () => {
233
+ if (queue !== queues.get(manager) || (queue?.provisional ?? 0) > 0) return void 0;
234
+ queue?.requireSuccess();
235
+ if (heldLayouts.has(manager)) throw rebalanceCheckpointHeld();
236
+ return { value: await capture2(queue?.checkpointCutoffs(manager.partitionCount) ?? /* @__PURE__ */ new Map()) };
237
+ });
238
+ if (captured !== void 0) return captured.value;
239
+ }
240
+ }
241
+ async function requireCheckpointLayout(manager) {
242
+ if (manager !== void 0) await captureAppliedPrefix(manager, () => void 0);
243
+ }
244
+
245
+ // src/partitioning/wal-routing.ts
246
+ var routings = /* @__PURE__ */ new WeakMap();
247
+ function routingOf(manager) {
248
+ let routing = routings.get(manager);
249
+ if (routing === void 0) {
250
+ routing = { gate: createPublicationGate(), partitionCount: null, failure: null };
251
+ routings.set(manager, routing);
252
+ }
253
+ return routing;
254
+ }
255
+ function beginDurableRebalance(manager) {
256
+ routingOf(manager).partitionCount = manager.partitionCount;
257
+ }
258
+ function finishDurableRebalance(manager) {
259
+ routingOf(manager).partitionCount = null;
260
+ }
261
+ function admitDurableWrite(manager, write) {
262
+ if (manager === void 0) return write(1);
263
+ const routing = routingOf(manager);
264
+ return routing.gate.read(() => {
265
+ if (routing.failure !== null) throw routing.failure;
266
+ return write(routing.partitionCount ?? manager.partitionCount);
267
+ });
268
+ }
269
+ function switchDurableRouting(manager, commit) {
270
+ const routing = routingOf(manager);
271
+ return routing.gate.write(async () => {
272
+ try {
273
+ await commit(routing.partitionCount ?? manager.partitionCount);
274
+ routing.partitionCount = manager.partitionCount;
275
+ } catch (error) {
276
+ routing.failure = error instanceof Error ? error : new Error(String(error));
277
+ throw error;
278
+ }
279
+ });
280
+ }
281
+
282
+ // src/persistence/durability/checkpoint-turn.ts
283
+ function createIndexLoads(vectorIndexesOf) {
284
+ const loads = /* @__PURE__ */ new Map();
285
+ const startGraphRepairs = (indexName) => {
286
+ for (const vectorIndex of vectorIndexesOf(indexName).values()) {
287
+ vectorIndex.scheduleBuild();
288
+ vectorIndex.startShuffledRebuildWhenDue();
289
+ }
290
+ };
291
+ return {
292
+ track(indexName, load, startsGraphRepairs = true) {
293
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
294
+ loads.set(
295
+ indexName,
296
+ loaded.catch(() => void 0)
297
+ );
298
+ return loaded;
299
+ },
300
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
301
+ startGraphRepairs
302
+ };
303
+ }
304
+ async function switchInTurn(turns, turn) {
305
+ let swapped = false;
306
+ await checkpointInTurn(turns, {
307
+ joinsInFlight: false,
308
+ mayStart: turn.mayStart,
309
+ perform: async () => {
310
+ swapped = await turn.swap();
311
+ if (swapped) await turn.checkpoint();
312
+ },
313
+ afterSettling: turn.afterSettling
314
+ });
315
+ return swapped;
316
+ }
317
+ async function checkpointInTurn(turns, turn) {
318
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
319
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
320
+ if (!turn.mayStart()) return;
321
+ const run = turn.perform().finally(() => {
322
+ turns.checkpointInFlight = null;
323
+ turn.afterSettling();
324
+ });
325
+ turns.checkpointInFlight = run;
326
+ return run;
327
+ }
328
+
329
+ // src/persistence/durability/seq-owner.ts
330
+ var SINGLE_NODE_PRIMARY_TERM = 1;
331
+ function createSeqOwner(startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
332
+ let current = startSeqNo;
333
+ let term = primaryTerm;
334
+ return {
335
+ next() {
336
+ current += 1;
337
+ return current;
338
+ },
339
+ advanceTo(seqNo, nextPrimaryTerm) {
340
+ current = seqNo;
341
+ term = nextPrimaryTerm;
342
+ },
343
+ get current() {
344
+ return current;
345
+ },
346
+ get primaryTerm() {
347
+ return term;
348
+ }
349
+ };
350
+ }
351
+
352
+ // src/persistence/durability/layout-transition/codec.ts
353
+ var LAYOUT_TRANSITION_VERSION = 1;
354
+ var LAYOUT_TRANSITION_ENVELOPE_VERSION = 1;
355
+ function transitionKey(indexName) {
356
+ return `${indexName}/layout-transition`;
357
+ }
358
+ function invalidTransition(reason) {
359
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Partition layout transition ${reason}`);
360
+ }
361
+ function readPayload(raw) {
362
+ if (!isRecord(raw) || raw.version !== LAYOUT_TRANSITION_VERSION) invalidTransition("has an invalid version");
363
+ const { oldPartitionCount, newPartitionCount, oldHeads } = raw;
364
+ if (!isUint32(oldPartitionCount) || oldPartitionCount === 0 || !isUint32(newPartitionCount) || newPartitionCount === 0 || newPartitionCount === oldPartitionCount)
365
+ invalidTransition("has invalid partition counts");
366
+ if (!Array.isArray(oldHeads) || oldHeads.length !== oldPartitionCount)
367
+ invalidTransition("has an incomplete old head list");
368
+ const heads = oldHeads.map((head, partitionId) => {
369
+ if (!isRecord(head) || head.partitionId !== partitionId || !isSequenceNumber(head.lastSeqNo) || !isSequenceNumber(head.primaryTerm))
370
+ invalidTransition("has an invalid old head");
371
+ return { partitionId, lastSeqNo: head.lastSeqNo, primaryTerm: head.primaryTerm };
372
+ });
373
+ return { version: LAYOUT_TRANSITION_VERSION, oldPartitionCount, newPartitionCount, oldHeads: heads };
374
+ }
375
+ async function readTransition(store, indexName) {
376
+ const bytes = await store.read(transitionKey(indexName));
377
+ if (bytes === null) return null;
378
+ const { payload } = await openChecksummedEnvelope(bytes, {
379
+ envelopeFormatVersion: LAYOUT_TRANSITION_ENVELOPE_VERSION,
380
+ what: "partition layout transition",
381
+ allowsCompression: false
382
+ });
383
+ let raw;
384
+ try {
385
+ raw = decode(payload);
386
+ } catch {
387
+ invalidTransition("has an invalid MessagePack payload");
388
+ }
389
+ return readPayload(raw);
390
+ }
391
+ async function writeTransition(store, indexName, transition) {
392
+ const packed = await packChecksummedEnvelope(
393
+ [encode(transition)],
394
+ LAYOUT_TRANSITION_ENVELOPE_VERSION,
395
+ "partition layout transition"
396
+ );
397
+ try {
398
+ await store.write(transitionKey(indexName), packed.chunks);
399
+ } catch (error) {
400
+ if (error instanceof NarsilError) throw error;
401
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Failed to persist the partition layout transition", {
402
+ indexName,
403
+ cause: error instanceof Error ? error.message : String(error)
404
+ });
405
+ }
406
+ }
407
+
408
+ // src/persistence/durability/layout-transition/index.ts
409
+ async function transitionWalLayout(registry, indexName, manager, replay) {
410
+ const state = registry.indexState(indexName);
411
+ await switchDurableRouting(
412
+ manager,
413
+ (oldPartitionCount) => checkpointInTurn(state, {
414
+ joinsInFlight: false,
415
+ mayStart: () => {
416
+ if (state.unloading) {
417
+ throw new NarsilError(
418
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
419
+ "The index is unloading during rebalance"
420
+ );
421
+ }
422
+ return true;
423
+ },
424
+ afterSettling: () => void 0,
425
+ perform: async () => {
426
+ await replay();
427
+ const oldHeads = [];
428
+ for (let partitionId = 0; partitionId < oldPartitionCount; partitionId++) {
429
+ const partition = state.partitions.get(partitionId);
430
+ if (partition !== void 0) {
431
+ await partition.appendChain;
432
+ if (partition.failed !== null) throw partition.failed;
433
+ await partition.walWriter.commit();
434
+ }
435
+ oldHeads.push({
436
+ partitionId,
437
+ lastSeqNo: partition?.appliedSeqNo ?? 0,
438
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
439
+ });
440
+ }
441
+ await writeTransition(directoryCheckpointStore(registry.directory), indexName, {
442
+ version: 1,
443
+ oldPartitionCount,
444
+ newPartitionCount: manager.partitionCount,
445
+ oldHeads
446
+ });
447
+ releaseLayoutCheckpoint(manager);
448
+ }
449
+ })
450
+ );
451
+ }
452
+
453
+ // src/partitioning/rebalancer.ts
454
+ function createRebalancer() {
455
+ const activeManagers = /* @__PURE__ */ new WeakSet();
456
+ let activeCount = 0;
457
+ async function rebalance(manager, newPartitionCount, router, onProgress, spill = null) {
458
+ if (activeManagers.has(manager)) {
459
+ throw new NarsilError(
460
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
461
+ "A rebalance operation is already in progress"
462
+ );
463
+ }
464
+ if (newPartitionCount <= 0) {
465
+ throw new NarsilError(
466
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
467
+ `New partition count must be greater than 0, received ${newPartitionCount}`,
468
+ { newPartitionCount }
469
+ );
470
+ }
471
+ if (newPartitionCount === manager.partitionCount) {
472
+ throw new NarsilError(
473
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
474
+ `New partition count (${newPartitionCount}) is the same as the current count`,
475
+ { newPartitionCount, currentPartitionCount: manager.partitionCount }
476
+ );
477
+ }
478
+ activeManagers.add(manager);
479
+ activeCount++;
480
+ try {
481
+ const currentPartitions = manager.getAllPartitions();
482
+ let documentsTotal = 0;
483
+ for (const partition of currentPartitions) {
484
+ documentsTotal += partition.count();
485
+ }
486
+ onProgress?.({
487
+ phase: "scanning",
488
+ documentsProcessed: 0,
489
+ documentsTotal
490
+ });
491
+ const newPartitions = [];
492
+ for (let i = 0; i < newPartitionCount; i++) {
493
+ newPartitions.push(spill?.newPartition(i) ?? createPartitionIndex(i, manager.config.trackPositions ?? true));
494
+ }
495
+ let documentsProcessed = 0;
496
+ let chunkFill = 0;
497
+ const insertOptions = {
498
+ validate: false,
499
+ skipClone: true,
500
+ stopWordOverride: manager.analysis.stopWords,
501
+ customTokenizer: manager.analysis.customTokenizer,
502
+ collectSurfaces: manager.config.surfaceForms !== false
503
+ };
504
+ for (const partition of currentPartitions) {
505
+ const docIds = [...partition.docIds()];
506
+ for (const docId of docIds) {
507
+ const document = partition.getRef(docId);
508
+ if (!document) continue;
509
+ const targetPartitionId = router.route(docId, newPartitionCount);
510
+ const targetPartition = newPartitions[targetPartitionId];
511
+ if (!targetPartition.has(docId)) {
512
+ newPartitions[targetPartitionId] = writePartitionDocument(
513
+ targetPartition,
514
+ docId,
515
+ document,
516
+ manager.schema,
517
+ manager.language,
518
+ insertOptions
519
+ ).partition;
520
+ documentsProcessed++;
521
+ }
522
+ chunkFill++;
523
+ if (chunkFill >= REBALANCE_CHUNK_SIZE) {
524
+ chunkFill = 0;
525
+ onProgress?.({
526
+ phase: "moving",
527
+ documentsProcessed,
528
+ documentsTotal
529
+ });
530
+ for (let at = 0; at < newPartitions.length; at++) await spill?.spillIfFull(newPartitions[at], at);
531
+ await yieldToEventLoop();
532
+ }
533
+ }
534
+ }
535
+ if (chunkFill > 0) {
536
+ onProgress?.({
537
+ phase: "moving",
538
+ documentsProcessed,
539
+ documentsTotal
540
+ });
541
+ await yieldToEventLoop();
542
+ }
543
+ onProgress?.({
544
+ phase: "swapping",
545
+ documentsProcessed,
546
+ documentsTotal
547
+ });
548
+ await manager.publication.write(() => manager.setPartitions(newPartitions));
549
+ onProgress?.({
550
+ phase: "complete",
551
+ documentsProcessed,
552
+ documentsTotal
553
+ });
554
+ } finally {
555
+ activeManagers.delete(manager);
556
+ activeCount--;
557
+ }
558
+ }
559
+ return {
560
+ rebalance,
561
+ isRebalancing() {
562
+ return activeCount > 0;
563
+ }
564
+ };
565
+ }
566
+
567
+ // src/partitioning/write-ahead-queue.ts
568
+ function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
569
+ const entries = /* @__PURE__ */ new Map();
570
+ const byDocument = /* @__PURE__ */ new Map();
571
+ let nextSequence = 1;
572
+ let failed = false;
573
+ let failure;
574
+ let replayFailed = false;
575
+ let provisional = 0;
576
+ let rejectReplay = () => void 0;
577
+ const replayFailure = new Promise((_resolve, reject) => {
578
+ rejectReplay = reject;
579
+ });
580
+ void replayFailure.catch(() => void 0);
581
+ let resolveReplayed = () => void 0;
582
+ let rejectReplayed = () => void 0;
583
+ const replayed = new Promise((resolve, reject) => {
584
+ resolveReplayed = resolve;
585
+ rejectReplayed = reject;
586
+ });
587
+ void replayed.catch(() => void 0);
588
+ function assertWritable() {
589
+ if (failed) throw failure;
590
+ if (entries.size >= maxSize) {
591
+ throw new NarsilError(
592
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
593
+ `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
594
+ { maxSize, currentSize: entries.size }
595
+ );
596
+ }
597
+ }
598
+ function refuseWrites(error) {
599
+ if (failed) return;
600
+ failed = true;
601
+ failure = error;
602
+ rejectReplayed(error);
603
+ }
604
+ function requireReplayable() {
605
+ if (replayFailed) throw failure;
606
+ }
607
+ function discard(entry) {
608
+ entries.delete(entry.sequenceNumber);
609
+ if (byDocument.get(entry.docId) !== entry) return;
610
+ byDocument.delete(entry.docId);
611
+ for (const remaining of entries.values()) {
612
+ if (remaining.docId === entry.docId) byDocument.set(entry.docId, remaining);
613
+ }
614
+ }
615
+ return {
616
+ assertWritable,
617
+ push(entry, awaitsDurability) {
618
+ assertWritable();
619
+ const sequenceNumber = nextSequence++;
620
+ let settle = () => void 0;
621
+ let settled = !awaitsDurability;
622
+ if (awaitsDurability) provisional++;
623
+ const ready = awaitsDurability ? new Promise((resolve) => {
624
+ settle = resolve;
625
+ }) : Promise.resolve(true);
626
+ const prepared = { ...entry, sequenceNumber, ready };
627
+ entries.set(sequenceNumber, prepared);
628
+ byDocument.set(entry.docId, prepared);
629
+ return {
630
+ commit(partitionId, partitionCount, seqNo) {
631
+ if (settled) return;
632
+ settled = true;
633
+ provisional--;
634
+ prepared.durability = { partitionId, partitionCount, seqNo };
635
+ settle(true);
636
+ },
637
+ fail(error) {
638
+ if (settled) return;
639
+ settled = true;
640
+ provisional--;
641
+ discard(prepared);
642
+ refuseWrites(error);
643
+ settle(false);
644
+ }
645
+ };
646
+ },
647
+ first: () => entries.values().next().value,
648
+ complete(entry) {
649
+ requireReplayable();
650
+ if (entries.values().next().value !== entry) {
651
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "Queued writes must complete in sequence order");
652
+ }
653
+ entries.delete(entry.sequenceNumber);
654
+ if (byDocument.get(entry.docId) === entry) byDocument.delete(entry.docId);
655
+ },
656
+ latest: (docId) => byDocument.get(docId),
657
+ awaitReady: (entry) => Promise.race([entry.ready, replayFailure]),
658
+ requireReplayable,
659
+ requireSuccess() {
660
+ if (failed) throw failure;
661
+ },
662
+ async settleAdmissions() {
663
+ if (failed) throw failure;
664
+ await Promise.race([Promise.all([...entries.values()].map((entry) => entry.ready)), replayed]);
665
+ if (failed) throw failure;
666
+ },
667
+ checkpointCutoffs(partitionCount) {
668
+ if (failed) throw failure;
669
+ const cutoffs = /* @__PURE__ */ new Map();
670
+ for (const entry of entries.values()) {
671
+ const position = entry.durability;
672
+ if (position === void 0) continue;
673
+ if (position.partitionCount !== partitionCount) {
674
+ throw rebalanceCheckpointHeld();
675
+ }
676
+ const previous = cutoffs.get(position.partitionId);
677
+ cutoffs.set(position.partitionId, Math.min(previous ?? position.seqNo - 1, position.seqNo - 1));
678
+ }
679
+ return cutoffs;
680
+ },
681
+ get provisional() {
682
+ return provisional;
683
+ },
684
+ bufferedDocState(docId) {
685
+ const entry = byDocument.get(docId);
686
+ return entry === void 0 ? void 0 : entry.action === "remove" ? "absent" : "present";
687
+ },
688
+ whenReplayed() {
689
+ return replayed;
690
+ },
691
+ markReplayed() {
692
+ if (failed) throw failure;
693
+ if (entries.size > 0) {
694
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "Queued writes remain unapplied");
695
+ }
696
+ resolveReplayed();
697
+ },
698
+ fail(error) {
699
+ refuseWrites(error);
700
+ replayFailed = true;
701
+ rejectReplay(error);
702
+ },
703
+ get failed() {
704
+ return failed;
705
+ },
706
+ get size() {
707
+ return entries.size;
708
+ },
709
+ get isFull() {
710
+ return entries.size >= maxSize;
711
+ }
712
+ };
713
+ }
714
+
715
+ // src/engine/queued-mutation.ts
716
+ function bufferPreparedMutation(queue, manager, entry, vectorFieldPaths, awaitsDurability) {
717
+ queue.assertWritable();
718
+ const previous = queue.latest(entry.docId);
719
+ const previousDocument = previous === void 0 ? manager.get(entry.docId) : previous.document;
720
+ if (entry.action === "insert" && previousDocument !== void 0) {
721
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${entry.docId}" already exists`);
722
+ }
723
+ if (entry.action === "remove" && previousDocument === void 0) {
724
+ throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${entry.docId}" not found`);
725
+ }
726
+ const document = entry.document === void 0 ? void 0 : structuredClone(entry.document);
727
+ const { options, language } = manager.textAnalysis;
728
+ if (document !== void 0) {
729
+ validateDocument(document, manager.schema, options?.patternValueLimit);
730
+ if (options?.strict) validateDocumentStrict(document, manager.schema);
731
+ }
732
+ const fields = schemaFieldsOf(manager.schema);
733
+ const surfaceFields = fields.filter((field) => field.base === "string" || field.base === "string[]").map((field) => field.path);
734
+ const prepareText = (value) => prepareRemovalText(prepareDocumentText(value, fields, language, options), value, surfaceFields, language, options);
735
+ const before = previousDocument === void 0 ? void 0 : previous?.text.after ?? prepareText(previousDocument);
736
+ const after = document === void 0 ? void 0 : prepareText(document);
737
+ const partitionDoc = document === void 0 ? void 0 : prepareDocumentVectors(document, vectorFieldPaths);
738
+ const vectors = /* @__PURE__ */ new Map();
739
+ for (const field of vectorFieldPaths) vectors.set(field, partitionDoc?.extractedVectors.get(field) ?? null);
740
+ return queue.push(
741
+ structuredClone({
742
+ ...entry,
743
+ document,
744
+ partitionDoc: partitionDoc?.partitionDoc,
745
+ previousDocument,
746
+ previousPartitionDoc: previousDocument === void 0 ? void 0 : previous?.partitionDoc ?? prepareDocumentVectors(previousDocument, vectorFieldPaths).partitionDoc,
747
+ text: { before, after },
748
+ vectors,
749
+ removedSpellings: before === void 0 ? removedSurfaceSpellings(void 0) : previous?.afterSpellings ?? removedSurfaceSpellings(before),
750
+ afterSpellings: removedSurfaceSpellings(after)
751
+ }),
752
+ awaitsDurability
753
+ );
754
+ }
755
+
756
+ // src/engine/rebalance-replay.ts
757
+ function applyEntry(manager, entry, ctx) {
758
+ const { docId, indexName, partitionDoc, previousPartitionDoc, text, removedSpellings } = entry;
759
+ const options = { preparedText: text, validate: false };
760
+ if (entry.action === "remove") {
761
+ manager.remove(docId, removedSpellings, options);
762
+ } else if (partitionDoc === void 0) {
763
+ throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, "A queued write has no prepared document");
764
+ } else if (previousPartitionDoc === void 0) {
765
+ manager.insert(docId, partitionDoc, options);
766
+ } else {
767
+ manager.update(docId, partitionDoc, options, removedSpellings);
768
+ }
769
+ try {
770
+ updateDocumentVectors(docId, entry.vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
771
+ } catch (error) {
772
+ const rollback = {
773
+ preparedText: { before: text.after, after: text.before },
774
+ validate: false
775
+ };
776
+ if (previousPartitionDoc === void 0) manager.remove(docId, entry.afterSpellings, rollback);
777
+ else if (entry.action === "remove") manager.insert(docId, previousPartitionDoc, rollback);
778
+ else manager.update(docId, previousPartitionDoc, rollback, entry.afterSpellings);
779
+ throw error;
780
+ }
781
+ if (entry.action === "remove") {
782
+ return ctx.orchestrator.publishToWorkers({
783
+ type: "remove",
784
+ indexName,
785
+ docId,
786
+ removedSpellings,
787
+ requestId: `replicate-remove-${docId}`
788
+ });
789
+ } else if (partitionDoc !== void 0) {
790
+ return ctx.orchestrator.publishToWorkers({
791
+ type: previousPartitionDoc === void 0 ? "insert" : "update",
792
+ indexName,
793
+ docId,
794
+ document: partitionDoc,
795
+ removedSpellings,
796
+ requestId: `replicate-${entry.action}-${docId}`
797
+ });
798
+ }
799
+ }
800
+ async function withinRebalanceHook(ctx, indexName, hook) {
801
+ ctx.indexesInRebalanceHooks ??= /* @__PURE__ */ new Set();
802
+ const running = ctx.indexesInRebalanceHooks;
803
+ running.add(indexName);
804
+ try {
805
+ await hook();
806
+ } finally {
807
+ running.delete(indexName);
808
+ }
809
+ }
810
+ async function afterEntry(manager, entry, ctx) {
811
+ for (const field of entry.vectors.keys()) manager.getVectorIndexes().get(field)?.scheduleBuild();
812
+ await withinRebalanceHook(ctx, entry.indexName, () => runEntryHooks(entry, ctx));
813
+ }
814
+ async function runEntryHooks(entry, ctx) {
815
+ try {
816
+ if (entry.action === "remove") {
817
+ await ctx.pluginRegistry.runHook("afterRemove", { indexName: entry.indexName, docId: entry.docId });
818
+ } else if (entry.document !== void 0) {
819
+ if (entry.action === "insert") {
820
+ await ctx.pluginRegistry.runHook("afterInsert", {
821
+ indexName: entry.indexName,
822
+ docId: entry.docId,
823
+ document: structuredClone(entry.document)
824
+ });
825
+ } else {
826
+ await ctx.pluginRegistry.runHook("afterUpdate", {
827
+ indexName: entry.indexName,
828
+ docId: entry.docId,
829
+ oldDocument: structuredClone(entry.previousDocument ?? {}),
830
+ newDocument: structuredClone(entry.document)
831
+ });
832
+ }
833
+ }
834
+ } catch (error) {
835
+ console.warn("Queued write plugin hook error:", error instanceof Error ? error.message : String(error));
836
+ }
837
+ }
838
+ async function replayQueued(manager, queue, ctx, deferredHooks) {
839
+ for (; ; ) {
840
+ const entry = queue.first();
841
+ if (entry === void 0) {
842
+ queue.requireSuccess();
843
+ return;
844
+ }
845
+ try {
846
+ if (!await queue.awaitReady(entry)) continue;
847
+ let publication;
848
+ await countWrite(
849
+ () => manager.publication.write(() => {
850
+ try {
851
+ queue.requireReplayable();
852
+ publication = applyEntry(manager, entry, ctx);
853
+ queue.complete(entry);
854
+ } catch (error) {
855
+ queue.fail(error);
856
+ throw error;
857
+ }
858
+ })
859
+ );
860
+ await publication;
861
+ } catch (error) {
862
+ queue.fail(error);
863
+ throw error;
864
+ }
865
+ if (deferredHooks === void 0) await afterEntry(manager, entry, ctx);
866
+ else deferredHooks.push(() => afterEntry(manager, entry, ctx));
867
+ }
868
+ }
869
+
870
+ // src/engine/rebalance-executor.ts
871
+ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
872
+ if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
873
+ throw new NarsilError(
874
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
875
+ `Target partition count must be a positive integer, got ${targetPartitionCount}`,
876
+ { targetPartitionCount }
877
+ );
878
+ }
879
+ const entry = ctx.requireIndex(indexName);
880
+ const maxPartitions = entry.config.partitions?.maxPartitions ?? DEFAULT_MAX_PARTITIONS;
881
+ if (targetPartitionCount > maxPartitions) {
882
+ throw new NarsilError(
883
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
884
+ `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
885
+ { targetPartitionCount, maxPartitions }
886
+ );
887
+ }
888
+ if (ctx.rebalancingIndexes.has(indexName)) {
889
+ throw new NarsilError(
890
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
891
+ `Index "${indexName}" is already being rebalanced`
892
+ );
893
+ }
894
+ const registered = ctx.requireIndex(indexName);
895
+ if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
896
+ throw new NarsilError(
897
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
898
+ `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
899
+ { indexName }
900
+ );
901
+ }
902
+ if (targetPartitionCount === manager.partitionCount) {
903
+ return;
904
+ }
905
+ ctx.prepareQueuedMutation = bufferPreparedMutation;
906
+ const waq = createWriteAheadQueue();
907
+ const registry = ctx.durabilityManager?.partitionRegistry;
908
+ registerCheckpointQueue(manager, waq, registry !== void 0);
909
+ if (registry !== void 0) beginDurableRebalance(manager);
910
+ ctx.waqMap.set(indexName, waq);
911
+ ctx.rebalancingIndexes.add(indexName);
912
+ ctx.rebalanceTargets.set(indexName, targetPartitionCount);
913
+ const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
914
+ let spill = null;
915
+ const oldCount = manager.partitionCount;
916
+ try {
917
+ spill = await rewriteSpillOf(ctx.durabilityManager?.rewriteStorage?.(), manager);
918
+ await createRebalancer().rebalance(
919
+ manager,
920
+ targetPartitionCount,
921
+ ctx.router,
922
+ (progress) => {
923
+ if (progress.phase === "complete") {
924
+ emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
925
+ indexName,
926
+ oldCount,
927
+ newCount: targetPartitionCount
928
+ });
929
+ }
930
+ },
931
+ spill
932
+ );
933
+ await withinRebalanceHook(ctx, indexName, async () => {
934
+ try {
935
+ await ctx.pluginRegistry.runHook("onPartitionSplit", {
936
+ indexName,
937
+ oldPartitionCount: oldCount,
938
+ newPartitionCount: targetPartitionCount
939
+ });
940
+ } catch (hookErr) {
941
+ console.warn(
942
+ "onPartitionSplit plugin hook failed:",
943
+ hookErr instanceof Error ? hookErr.message : String(hookErr)
944
+ );
945
+ }
946
+ });
947
+ if (registry === void 0) await replayQueued(manager, waq, ctx);
948
+ else {
949
+ const hooks = [];
950
+ await transitionWalLayout(registry, indexName, manager, () => replayQueued(manager, waq, ctx, hooks));
951
+ for (const hook of hooks) await hook();
952
+ }
953
+ if (ctx.durabilityManager) {
954
+ await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
955
+ await spill?.removeUnread(manager);
956
+ await replayQueued(manager, waq, ctx);
957
+ }
958
+ await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
959
+ do {
960
+ await replayQueued(manager, waq, ctx);
961
+ } while (waq.size > 0);
962
+ } catch (err) {
963
+ waq.fail(err);
964
+ await spill?.removeUnread(manager).catch(() => void 0);
965
+ throw err;
966
+ }
967
+ ctx.rebalancingIndexes.delete(indexName);
968
+ ctx.rebalanceTargets.delete(indexName);
969
+ ctx.waqMap.delete(indexName);
970
+ waq.markReplayed();
971
+ releaseCheckpointQueue(manager, waq);
972
+ finishDurableRebalance(manager);
973
+ ctx.checkWatermark(indexName);
974
+ }
975
+
976
+ // src/engine/analysis-change/adopt-saved.ts
977
+ function adoptSavedAnalysis(manager, entry, metadata) {
978
+ const saved = reconstructSchemaFromMetadata(metadata);
979
+ const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = entry.config;
980
+ const config = {
981
+ ...rest,
982
+ language: saved.language,
983
+ ...saved.tokenizer !== void 0 ? { tokenizer: saved.tokenizer } : {},
984
+ ...saved.stopWords !== void 0 ? { stopWords: saved.stopWords } : {}
985
+ };
986
+ const language = getLanguage(config.language ?? "english");
987
+ const analysis = resolveIndexAnalysis(config);
988
+ const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
989
+ entry.config = config;
990
+ entry.language = language;
991
+ entry.analysisGeneration = metadata.analysisGeneration ?? 0;
992
+ entry.analysisRevision = metadata.analysisRevision ?? null;
993
+ entry.arrayPositionGap = metadata.arrayPositionGap ?? null;
994
+ manager?.adoptAnalysis({
995
+ config,
996
+ language,
997
+ analysis,
998
+ partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis),
999
+ synonyms: synonyms?.lookup ?? null
1000
+ });
1001
+ }
1002
+
1003
+ // src/engine/snapshot/memory-budget.ts
1004
+ var MEGABYTE_BYTES = 1048576;
1005
+ var ARCHIVE_RATES = {
1006
+ worst: {
1007
+ heap: {
1008
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
1009
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
1010
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
1011
+ },
1012
+ external: {
1013
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
1014
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
1015
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
1016
+ }
1017
+ },
1018
+ floor: {
1019
+ heap: {
1020
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
1021
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
1022
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
1023
+ },
1024
+ external: {
1025
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
1026
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
1027
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
1028
+ }
1029
+ }
1030
+ };
1031
+ var EARLIER_SNAPSHOT_RATES = {
1032
+ worst: {
1033
+ heap: {
1034
+ perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
1035
+ perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
1036
+ perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
1037
+ },
1038
+ external: {
1039
+ perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
1040
+ perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
1041
+ perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
1042
+ }
1043
+ },
1044
+ floor: {
1045
+ heap: {
1046
+ perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
1047
+ perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
1048
+ perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
1049
+ },
1050
+ external: {
1051
+ perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
1052
+ perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
1053
+ perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
1054
+ }
1055
+ }
1056
+ };
1057
+ function bytesAt(rate, storedBytes, vectors) {
1058
+ let bytes = storedBytes * rate.perStoredByte;
1059
+ for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
1060
+ return bytes;
1061
+ }
1062
+ function footprintAt(rates, replacesIndex, storedBytes, vectors) {
1063
+ const chosen = replacesIndex ? rates.worst : rates.floor;
1064
+ const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
1065
+ return {
1066
+ heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
1067
+ externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
1068
+ };
1069
+ }
1070
+ function deadCountOf(dead) {
1071
+ if (dead === null) return 0;
1072
+ let count = 0;
1073
+ for (const byte of dead) {
1074
+ for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
1075
+ }
1076
+ return count;
1077
+ }
1078
+ function storedBytesOfSegments(manifest) {
1079
+ let bytes = 0;
1080
+ for (const partition of manifest.partitions) {
1081
+ for (const segment of partition.segments) bytes += segment.bytes;
1082
+ }
1083
+ return bytes;
1084
+ }
1085
+ function archiveFootprint(prepared, manifest, replacesIndex, segmentBytes) {
1086
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
1087
+ const vectors = [];
1088
+ for (const field of manifest.vectors) {
1089
+ const dimension = dimensions.get(field.fieldPath) ?? 0;
1090
+ for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
1091
+ }
1092
+ return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
1093
+ }
1094
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
1095
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
1096
+ const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
1097
+ return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
1098
+ }
1099
+ function inMegabytes(bytes) {
1100
+ return Math.ceil(bytes / MEGABYTE_BYTES);
1101
+ }
1102
+ function shortfallOf(heapBytes, processBytes, countInUse) {
1103
+ const heap = readHeapStatistics();
1104
+ if (heap !== null) {
1105
+ const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
1106
+ const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
1107
+ if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
1108
+ }
1109
+ const memory = readProcessMemoryUse();
1110
+ if (memory !== null) {
1111
+ const availableBytes = countInUse ? memory.limitBytes - memory.processBytes : memory.limitBytes;
1112
+ if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
1113
+ }
1114
+ return null;
1115
+ }
1116
+ function refuse(indexName, replacesIndex, shortfall) {
1117
+ const { budget, neededBytes, availableBytes } = shortfall;
1118
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
1119
+ const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
1120
+ throw new NarsilError(
1121
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
1122
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
1123
+ { indexName, budget, neededBytes, availableBytes }
1124
+ );
1125
+ }
1126
+ function requireAnalysisRebuildMemory(indexName, termBytes) {
1127
+ if (shortfallOf(termBytes, termBytes, true) === null) return;
1128
+ const shortfall = shortfallOf(termBytes, termBytes, collectAndReturnPages());
1129
+ if (shortfall === null) return;
1130
+ const { budget, neededBytes, availableBytes } = shortfall;
1131
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
1132
+ throw new NarsilError(
1133
+ ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
1134
+ `The analysis change of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where} for a second copy of its terms, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its current settings. Give the process more memory, then change the settings again`,
1135
+ { indexName, budget, neededBytes, availableBytes }
1136
+ );
1137
+ }
1138
+ function requireRestoreMemory(indexName, footprint, replacesIndex) {
1139
+ const processBytes = footprint.heapBytes + footprint.externalBytes;
1140
+ if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
1141
+ const collected = collectAndReturnPages();
1142
+ const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
1143
+ if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
1144
+ }
1145
+
1146
+ // src/engine/analysis-change/run.ts
1147
+ function createHold(generation) {
1148
+ let markReady = () => void 0;
1149
+ let refuse5 = () => void 0;
1150
+ let wake = () => void 0;
1151
+ const ready = new Promise((resolve, reject) => {
1152
+ markReady = resolve;
1153
+ refuse5 = reject;
1154
+ });
1155
+ ready.catch(() => void 0);
1156
+ return {
1157
+ generation,
1158
+ built: false,
1159
+ released: false,
1160
+ ready,
1161
+ markReady,
1162
+ refuse: refuse5,
1163
+ wake: () => wake(),
1164
+ pause: () => new Promise((resolve) => {
1165
+ const timer = setTimeout(resolve, ANALYSIS_HOLD_APPLY_INTERVAL_MS);
1166
+ wake = () => {
1167
+ clearTimeout(timer);
1168
+ resolve();
1169
+ };
1170
+ })
1171
+ };
1172
+ }
1173
+ var runsByCore = /* @__PURE__ */ new WeakMap();
1174
+ function runsOf(core) {
1175
+ let runs = runsByCore.get(core);
1176
+ if (runs === void 0) {
1177
+ runs = /* @__PURE__ */ new Map();
1178
+ runsByCore.set(core, runs);
1179
+ }
1180
+ return runs;
1181
+ }
1182
+ function createRun(target, hold = null) {
1183
+ let finish = () => void 0;
1184
+ let rejectFinished = () => void 0;
1185
+ let settle = () => void 0;
1186
+ const finished = new Promise((resolve, reject) => {
1187
+ finish = resolve;
1188
+ rejectFinished = reject;
1189
+ });
1190
+ finished.catch(() => void 0);
1191
+ const settled = new Promise((resolve) => {
1192
+ settle = resolve;
1193
+ });
1194
+ const fail = (error) => {
1195
+ rejectFinished(error);
1196
+ hold?.refuse(error);
1197
+ hold?.wake();
1198
+ };
1199
+ return { target, stopReason: null, switched: false, settled, finish, fail, finished, settle, hold };
1200
+ }
1201
+ async function waitForRelease(run, hold, keepCurrent) {
1202
+ hold.built = true;
1203
+ hold.markReady();
1204
+ while (!hold.released) {
1205
+ if (run.stopReason !== null || !keepCurrent()) {
1206
+ hold.built = false;
1207
+ return false;
1208
+ }
1209
+ await hold.pause();
1210
+ }
1211
+ return true;
1212
+ }
1213
+ function stopRun(run, indexName) {
1214
+ run.stopReason = new NarsilError(
1215
+ ErrorCodes.ANALYSIS_CHANGE_REPLACED,
1216
+ `A later change or a restore replaced this analysis change of index "${indexName}"`,
1217
+ { indexName }
1218
+ );
1219
+ run.fail(run.stopReason);
1220
+ }
1221
+
1222
+ // src/engine/snapshot/index-configuration-limit.ts
1223
+ function requireConfigurationWithinSizeLimit(config, language) {
1224
+ const payload = oversizedConfigurationPayloadOf(config, language);
1225
+ if (payload !== null) requireEncodedConfigurationWithinSizeLimit(encode(payload).byteLength);
1226
+ }
1227
+
1228
+ // src/engine/analysis-change/settings.ts
1229
+ function refuseTokenizer(indexName) {
1230
+ throw new NarsilError(
1231
+ ErrorCodes.CONFIG_INVALID,
1232
+ `The tokenizer of index "${indexName}" must change to a name registered with registerTokenizer, because an instance survives neither a worker boundary nor a restart`,
1233
+ { indexName }
1234
+ );
1235
+ }
1236
+ function refuseStopWords(indexName) {
1237
+ throw new NarsilError(
1238
+ ErrorCodes.CONFIG_INVALID,
1239
+ `The stop words of index "${indexName}" must change to a name registered with registerStopWords or to a list of words, because a function survives neither a worker boundary nor a restart`,
1240
+ { indexName }
1241
+ );
1242
+ }
1243
+ function stopWordsOfChange(indexName, value) {
1244
+ if (value === void 0 || value === null || typeof value === "string") return value;
1245
+ const words = value instanceof Set ? [...value] : value;
1246
+ if (!Array.isArray(words) || words.some((word) => typeof word !== "string")) refuseStopWords(indexName);
1247
+ return new Set(words);
1248
+ }
1249
+ function tokenizerOfChange(indexName, value) {
1250
+ if (value === void 0 || value === null || typeof value === "string") return value;
1251
+ refuseTokenizer(indexName);
1252
+ }
1253
+ function requireRegisteredLanguage(name) {
1254
+ if (typeof name !== "string") {
1255
+ throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, "The language of an analysis change must be a name");
1256
+ }
1257
+ return getLanguage(name);
1258
+ }
1259
+ function resolveAnalysisTarget(indexName, base, change) {
1260
+ if (change === null || typeof change !== "object" || Array.isArray(change)) {
1261
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "An analysis change must be an object", { indexName });
1262
+ }
1263
+ const unknownMember = Object.keys(change).find((member) => !ANALYSIS_CHANGE_MEMBERS.includes(member));
1264
+ if (unknownMember !== void 0) {
1265
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `"${unknownMember}" is not an analysis setting`, { indexName });
1266
+ }
1267
+ const language = change.language === void 0 ? getLanguage(base.language ?? "english") : requireRegisteredLanguage(change.language);
1268
+ const tokenizer = tokenizerOfChange(indexName, change.tokenizer);
1269
+ const stopWords = stopWordsOfChange(indexName, change.stopWords);
1270
+ const config = configUnderAnalysis(
1271
+ base,
1272
+ language.name,
1273
+ tokenizer === void 0 ? base.tokenizer : tokenizer ?? void 0,
1274
+ stopWords === void 0 ? base.stopWords : stopWords ?? void 0
1275
+ );
1276
+ requireConfigurationWithinSizeLimit(config, language);
1277
+ return { config, language, analysis: resolveIndexAnalysis(config) };
1278
+ }
1279
+ function currentConfigUnderTarget(current, target) {
1280
+ return configUnderAnalysis(current, target.language.name, target.config.tokenizer, target.config.stopWords);
1281
+ }
1282
+ function sameStopWords(left, right) {
1283
+ if (left === right) return true;
1284
+ if (!(left instanceof Set) || !(right instanceof Set) || left.size !== right.size) return false;
1285
+ for (const word of left) if (!right.has(word)) return false;
1286
+ return true;
1287
+ }
1288
+ function holdsSettings(config, target) {
1289
+ return (config.language ?? "english") === target.language.name && config.tokenizer === target.config.tokenizer && sameStopWords(config.stopWords, target.config.stopWords);
1290
+ }
1291
+ function namedSettingsOf(config, language) {
1292
+ return {
1293
+ language,
1294
+ ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
1295
+ ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
1296
+ ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {}
1297
+ };
1298
+ }
1299
+ function durableSettingsOf(entry) {
1300
+ return {
1301
+ ...namedSettingsOf(entry.config, entry.language.name),
1302
+ generation: entry.analysisGeneration,
1303
+ analysisRevision: entry.analysisRevision,
1304
+ arrayPositionGap: entry.arrayPositionGap
1305
+ };
1306
+ }
1307
+ function requireSynonymsUnderTarget(entry, target) {
1308
+ if (entry.synonyms === null) return;
1309
+ prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
1310
+ }
1311
+
1312
+ // src/engine/analysis-change/index.ts
1313
+ function termBytesOf(core, indexName) {
1314
+ let bytes = 0;
1315
+ for (const partition of core.requireManager(indexName).getAllPartitions()) {
1316
+ bytes += partition.estimateMemoryBytes();
1317
+ for (const state of partition.readStates()) bytes -= state.docStore.contentBytes();
1318
+ }
1319
+ return Math.max(0, bytes);
1320
+ }
1321
+ function toError(value) {
1322
+ return value instanceof Error ? value : new Error(String(value));
1323
+ }
1324
+ async function requireNoUndurableSwitch(core, indexName, entry) {
1325
+ if (entry.durableAnalysis === null) return;
1326
+ const durable = core.durability?.manager;
1327
+ await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
1328
+ if (entry.durableAnalysis === null) return;
1329
+ throw new NarsilError(
1330
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
1331
+ `The previous analysis change of index "${indexName}" has no durable checkpoint yet, so try this change again once a checkpoint succeeds`,
1332
+ { indexName }
1333
+ );
1334
+ }
1335
+ function switchIndex({ core, indexName, entry, run, generation }) {
1336
+ const { language, analysis } = run.target;
1337
+ const config = currentConfigUnderTarget(entry.config, run.target);
1338
+ const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
1339
+ return () => {
1340
+ run.switched = true;
1341
+ entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
1342
+ entry.config = config;
1343
+ entry.language = language;
1344
+ entry.analysisGeneration = generation;
1345
+ entry.analysisRevision = language.revision;
1346
+ entry.arrayPositionGap = ARRAY_POSITION_GAP;
1347
+ entry.pendingAnalysis = null;
1348
+ entry.synonyms = synonyms?.encoded ?? null;
1349
+ core.requireManager(indexName).adoptAnalysis({
1350
+ ...textRebuildTargetOf(config, language, analysis),
1351
+ synonyms: synonyms?.lookup ?? null
1352
+ });
1353
+ core.analysisRebuild.clearStale(indexName);
1354
+ };
1355
+ }
1356
+ async function buildAndSwitch(scope) {
1357
+ const { core, indexName, entry, run } = scope;
1358
+ const { hold } = run;
1359
+ const manager = core.requireManager(indexName);
1360
+ const emit = (status, partitionsRebuilt2, error) => {
1361
+ emitEngineEvent(core.eventHandlers, "analysisRebuild", {
1362
+ indexName,
1363
+ status,
1364
+ partitionsRebuilt: partitionsRebuilt2,
1365
+ partitionCount: manager.partitionCount,
1366
+ ...error !== void 0 ? { error } : {}
1367
+ });
1368
+ };
1369
+ let partitionsRebuilt = 0;
1370
+ const refusal = { byRestore: null };
1371
+ const isCurrent = () => run.stopReason === null && refusal.byRestore === null && !core.shutdownState.isShutdown && core.indexRegistry.get(indexName) === entry && core.executor.getManager(indexName) === manager;
1372
+ emit("started", 0);
1373
+ const wasPromoted = core.orchestrator.desyncIndex(indexName);
1374
+ try {
1375
+ const durable = core.durability?.manager;
1376
+ let spill = null;
1377
+ const rebuilt = await rebuildTextIndexes(
1378
+ manager,
1379
+ {
1380
+ isCurrent,
1381
+ rewriteStorage: () => durable?.rewriteStorage?.(),
1382
+ onRewriteSpill(kept) {
1383
+ spill = kept;
1384
+ },
1385
+ onPartitionBuilt(built) {
1386
+ partitionsRebuilt = built;
1387
+ },
1388
+ onSwitch: () => switchIndex(scope)(),
1389
+ ...hold === null ? {} : { holdBeforeSwitch: (keepCurrent) => waitForRelease(run, hold, keepCurrent) },
1390
+ async runSwitch(swap) {
1391
+ const swapWithSynonyms = () => {
1392
+ if (!isCurrent()) return false;
1393
+ if (core.restoresInFlight.has(indexName)) {
1394
+ refusal.byRestore = restoreInFlightError(indexName);
1395
+ return false;
1396
+ }
1397
+ return swap();
1398
+ };
1399
+ const releaseSyncEpoch = await holdSyncEpochChange(core, indexName);
1400
+ try {
1401
+ return await (durable?.checkpointAfterSwitch?.(indexName, swapWithSynonyms) ?? swapWithSynonyms());
1402
+ } finally {
1403
+ releaseSyncEpoch();
1404
+ }
1405
+ }
1406
+ },
1407
+ textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
1408
+ );
1409
+ if (!rebuilt) throw run.stopReason ?? refusal.byRestore ?? stoppedError(core, indexName);
1410
+ await spill?.removeUnread(manager);
1411
+ if (entry.durableAnalysis !== null) {
1412
+ throw new NarsilError(
1413
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1414
+ `The new terms of index "${indexName}" answer queries, but they have no durable checkpoint yet`,
1415
+ { indexName }
1416
+ );
1417
+ }
1418
+ emit("completed", partitionsRebuilt);
1419
+ } catch (error) {
1420
+ emit("failed", partitionsRebuilt, toError(error));
1421
+ throw error;
1422
+ } finally {
1423
+ if (core.executor.getManager(indexName) === manager) {
1424
+ await core.orchestrator.resyncIndex(indexName, wasPromoted).catch(() => void 0);
1425
+ }
1426
+ }
1427
+ }
1428
+ function stoppedError(core, indexName) {
1429
+ if (core.shutdownState.isShutdown) {
1430
+ return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
1431
+ }
1432
+ return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" closed during its analysis change`, {
1433
+ indexName
1434
+ });
1435
+ }
1436
+ async function clearPending(core, indexName, entry) {
1437
+ entry.pendingAnalysis = null;
1438
+ await core.durability?.manager.persistMetadata(indexName);
1439
+ }
1440
+ function restoreInFlightError(indexName) {
1441
+ return new NarsilError(
1442
+ ErrorCodes.ANALYSIS_CHANGE_REPLACED,
1443
+ `A restore of index "${indexName}" is in flight and replaces this analysis change, so try the change again once the restore ends`,
1444
+ { indexName }
1445
+ );
1446
+ }
1447
+ function requireNoRestoreInFlight(core, indexName) {
1448
+ if (core.restoresInFlight.has(indexName)) throw restoreInFlightError(indexName);
1449
+ }
1450
+ function requireNotRebalancing(core, indexName) {
1451
+ if (!core.rebalancingIndexes.has(indexName)) return;
1452
+ throw new NarsilError(
1453
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
1454
+ `Index "${indexName}" is being rebalanced, so try its analysis change again once the rebalance ends`,
1455
+ { indexName }
1456
+ );
1457
+ }
1458
+ async function driveRun(core, indexName, entry, run, previous, resumedGeneration) {
1459
+ const runs = runsOf(core);
1460
+ try {
1461
+ if (previous !== void 0) await previous.settled;
1462
+ if (runs.get(indexName) !== run) return;
1463
+ await requireNoUndurableSwitch(core, indexName, entry);
1464
+ if (run.stopReason !== null) return;
1465
+ if (core.indexRegistry.get(indexName) !== entry) throw stoppedError(core, indexName);
1466
+ requireNotRebalancing(core, indexName);
1467
+ requireNoRestoreInFlight(core, indexName);
1468
+ if (resumedGeneration === null && holdsSettings(entry.config, run.target)) {
1469
+ if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
1470
+ run.finish();
1471
+ return;
1472
+ }
1473
+ const generation = resumedGeneration ?? Math.max(entry.analysisGeneration, entry.pendingAnalysis?.generation ?? 0) + 1;
1474
+ if (entry.pendingAnalysis?.generation !== generation || run.hold !== null) {
1475
+ entry.pendingAnalysis = { generation, ...namedSettingsOf(run.target.config, run.target.language.name) };
1476
+ await core.durability?.manager.persistMetadata(indexName);
1477
+ }
1478
+ if (run.stopReason !== null) return;
1479
+ await core.analysisRebuild.abandon(indexName);
1480
+ await core.analysisRebuild.runExclusive(
1481
+ indexName,
1482
+ () => buildAndSwitch({ core, indexName, entry, run, generation }),
1483
+ run.hold?.ready
1484
+ );
1485
+ run.finish();
1486
+ } catch (error) {
1487
+ run.fail(error);
1488
+ const refusedByRestore = run.stopReason === null && error instanceof NarsilError && error.code === ErrorCodes.ANALYSIS_CHANGE_REPLACED;
1489
+ const keepsPending = !refusedByRestore && (resumedGeneration !== null || core.shutdownState.isShutdown);
1490
+ const stillOwnsPending = runs.get(indexName) === run && run.stopReason === null;
1491
+ if (!keepsPending && stillOwnsPending && core.indexRegistry.get(indexName) === entry) {
1492
+ if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry).catch(() => void 0);
1493
+ }
1494
+ } finally {
1495
+ if (runs.get(indexName) === run) runs.delete(indexName);
1496
+ run.settle();
1497
+ }
1498
+ }
1499
+ async function changeIndexAnalysis(core, indexName, change) {
1500
+ const changedTokenizer = change !== null && typeof change === "object" ? change.tokenizer : void 0;
1501
+ const loadingTokenizerCode = loadSchemaCodeFor(null, changedTokenizer);
1502
+ if (loadingTokenizerCode !== null) await loadingTokenizerCode;
1503
+ const entry = core.requireIndex(indexName);
1504
+ requireNotRebalancing(core, indexName);
1505
+ requireNoRestoreInFlight(core, indexName);
1506
+ const target = resolveAnalysisTarget(indexName, entry.config, change);
1507
+ const runs = runsOf(core);
1508
+ const previous = runs.get(indexName);
1509
+ if (previous !== void 0 && holdsSettings(previous.target.config, target)) return previous.finished;
1510
+ const heldAlready = holdsSettings(entry.config, target);
1511
+ if (previous === void 0 && entry.durableAnalysis === null && heldAlready) return;
1512
+ if (!heldAlready) {
1513
+ requireSynonymsUnderTarget(entry, target);
1514
+ requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
1515
+ }
1516
+ const run = createRun(target);
1517
+ if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
1518
+ runs.set(indexName, run);
1519
+ void driveRun(core, indexName, entry, run, previous, null);
1520
+ return run.finished;
1521
+ }
1522
+ function resumeAnalysisChange(core, indexName) {
1523
+ const entry = core.requireIndex(indexName);
1524
+ const loadingTokenizerCode = loadSchemaCodeFor(null, entry.pendingAnalysis?.tokenizer);
1525
+ if (loadingTokenizerCode !== null) return loadingTokenizerCode.then(() => resumeAnalysisChange(core, indexName));
1526
+ const runs = runsOf(core);
1527
+ const running = runs.get(indexName);
1528
+ if (running !== void 0) return running.finished;
1529
+ const pending = entry.pendingAnalysis;
1530
+ if (pending === null) return Promise.resolve();
1531
+ let target;
1532
+ try {
1533
+ target = targetOfPending(entry, pending);
1534
+ requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
1535
+ } catch (error) {
1536
+ emitEngineEvent(core.eventHandlers, "analysisRebuild", {
1537
+ indexName,
1538
+ status: "failed",
1539
+ partitionsRebuilt: 0,
1540
+ partitionCount: entry.partitionCount,
1541
+ error: toError(error)
1542
+ });
1543
+ return Promise.reject(error);
1544
+ }
1545
+ const run = createRun(target);
1546
+ runs.set(indexName, run);
1547
+ void driveRun(core, indexName, entry, run, void 0, pending.generation);
1548
+ return run.finished;
1549
+ }
1550
+ async function abandonAnalysisChange(core, indexName) {
1551
+ const run = runsOf(core).get(indexName);
1552
+ if (run === void 0) return;
1553
+ stopRun(run, indexName);
1554
+ await run.settled;
1555
+ }
1556
+ function holdServes(run, entry, pending) {
1557
+ return run.hold?.generation === pending.generation && holdsSettings(run.target.config, targetOfPending(entry, pending));
1558
+ }
1559
+ function startHeldRun(core, indexName, pending) {
1560
+ const entry = core.requireIndex(indexName);
1561
+ if (entry.analysisGeneration >= pending.generation) return null;
1562
+ const runs = runsOf(core);
1563
+ const previous = runs.get(indexName);
1564
+ if (previous !== void 0 && holdServes(previous, entry, pending)) return previous;
1565
+ const target = targetOfPending(entry, pending);
1566
+ requireSynonymsUnderTarget(entry, target);
1567
+ requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
1568
+ const run = createRun(target, createHold(pending.generation));
1569
+ if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
1570
+ runs.set(indexName, run);
1571
+ void driveRun(core, indexName, entry, run, previous, pending.generation);
1572
+ return run;
1573
+ }
1574
+ async function buildHeldAnalysisChange(core, indexName, pending) {
1575
+ const loadingTokenizerCode = loadSchemaCodeFor(null, pending.tokenizer);
1576
+ if (loadingTokenizerCode !== null) await loadingTokenizerCode;
1577
+ const running = runsOf(core).get(indexName);
1578
+ const entry = core.indexRegistry.get(indexName);
1579
+ if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
1580
+ const release = await core.indexState.acquire(indexName, false);
1581
+ let run;
1582
+ try {
1583
+ run = startHeldRun(core, indexName, pending);
1584
+ } catch (error) {
1585
+ release();
1586
+ throw error;
1587
+ }
1588
+ if (run?.hold === void 0 || run.hold === null) {
1589
+ release();
1590
+ return;
1591
+ }
1592
+ void run.settled.finally(release);
1593
+ return run.hold.ready;
1594
+ }
1595
+ function heldAnalysisIsBuilt(core, indexName, generation) {
1596
+ const hold = runsOf(core).get(indexName)?.hold;
1597
+ return hold !== void 0 && hold !== null && hold.generation === generation && hold.built;
1598
+ }
1599
+ async function releaseHeldAnalysisChange(core, indexName, generation) {
1600
+ const run = runsOf(core).get(indexName);
1601
+ if (run?.hold?.generation !== generation) return;
1602
+ run.hold.released = true;
1603
+ run.hold.wake();
1604
+ await run.finished;
1605
+ }
1606
+ async function dropHeldAnalysisChange(core, indexName) {
1607
+ await abandonAnalysisChange(core, indexName);
1608
+ const entry = core.indexRegistry.get(indexName);
1609
+ if (entry !== void 0 && entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
1610
+ }
1611
+ function holdsNamedSettings(entry, settings) {
1612
+ return holdsSettings(entry.config, targetOfPending(entry, { ...settings }));
1613
+ }
1614
+ async function stampAnalysisGeneration(core, indexName, generation) {
1615
+ const entry = core.requireIndex(indexName);
1616
+ if (entry.analysisGeneration >= generation) return;
1617
+ await requireNoUndurableSwitch(core, indexName, entry);
1618
+ const durable = core.durability?.manager;
1619
+ const stamp = () => {
1620
+ entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
1621
+ entry.analysisGeneration = generation;
1622
+ entry.pendingAnalysis = null;
1623
+ return true;
1624
+ };
1625
+ await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
1626
+ }
1627
+
1628
+ // src/persistence/archive/byte-reader.ts
1629
+ async function* chunksOf(source) {
1630
+ if (source instanceof Uint8Array) {
1631
+ yield source;
1632
+ return;
1633
+ }
1634
+ if (typeof source.getReader === "function") {
1635
+ const reader = source.getReader();
1636
+ try {
1637
+ for (; ; ) {
1638
+ const step = await reader.read();
1639
+ if (step.done) return;
1640
+ yield step.value;
1641
+ }
1642
+ } finally {
1643
+ reader.releaseLock();
1644
+ }
1645
+ }
1646
+ yield* source;
1647
+ }
1648
+ function createByteReader(source) {
1649
+ const iterator = chunksOf(source);
1650
+ const pending = [];
1651
+ let pendingBytes = 0;
1652
+ let ended = false;
1653
+ async function fill(length) {
1654
+ while (pendingBytes < length && !ended) {
1655
+ const step = await iterator.next();
1656
+ if (step.done) {
1657
+ ended = true;
1658
+ break;
1659
+ }
1660
+ if (step.value.byteLength === 0) continue;
1661
+ pending.push(step.value);
1662
+ pendingBytes += step.value.byteLength;
1663
+ }
1664
+ }
1665
+ function take(length) {
1666
+ const out = new Uint8Array(length);
1667
+ let filled = 0;
1668
+ while (filled < length) {
1669
+ const head = pending[0];
1670
+ const needed = length - filled;
1671
+ if (head.byteLength <= needed) {
1672
+ out.set(head, filled);
1673
+ filled += head.byteLength;
1674
+ pending.shift();
1675
+ } else {
1676
+ out.set(head.subarray(0, needed), filled);
1677
+ pending[0] = head.subarray(needed);
1678
+ filled += needed;
1679
+ }
1680
+ }
1681
+ pendingBytes -= length;
1682
+ return out;
1683
+ }
1684
+ return {
1685
+ async readExactly(length) {
1686
+ await fill(length);
1687
+ if (pendingBytes === 0 && length > 0) return null;
1688
+ if (pendingBytes < length) return take(pendingBytes);
1689
+ return take(length);
1690
+ },
1691
+ async peek(length) {
1692
+ await fill(length);
1693
+ const available = Math.min(length, pendingBytes);
1694
+ const copy = take(available);
1695
+ pending.unshift(copy);
1696
+ pendingBytes += copy.byteLength;
1697
+ return copy;
1698
+ },
1699
+ async close() {
1700
+ await iterator.return(void 0);
1701
+ }
1702
+ };
1703
+ }
1704
+
1705
+ // src/engine/snapshot/stored-schema.ts
1706
+ function isRecord2(value) {
1707
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1708
+ }
1709
+ function conflictingStoredPath(path) {
1710
+ return new NarsilError(
1711
+ ErrorCodes.SCHEMA_INVALID_TYPE,
1712
+ `Field path "${path}" is declared both as a field and as an object of fields`,
1713
+ { path }
1714
+ );
1715
+ }
1716
+ function defineOwn(level, key, value) {
1717
+ Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
1718
+ }
1719
+ function placeStoredField(nested, path, type) {
1720
+ const segments = path.split(".");
1721
+ let level = nested;
1722
+ for (let at = 0; at < segments.length; at++) {
1723
+ const segment = segments[at];
1724
+ const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
1725
+ if (at === segments.length - 1) {
1726
+ if (held !== void 0) throw conflictingStoredPath(path);
1727
+ defineOwn(level, segment, type);
1728
+ return;
1729
+ }
1730
+ if (held === void 0) {
1731
+ const child = {};
1732
+ defineOwn(level, segment, child);
1733
+ level = child;
1734
+ } else if (isRecord2(held)) {
1735
+ level = held;
1736
+ } else {
1737
+ throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
1738
+ }
1739
+ }
1740
+ }
1741
+ function placeStoredFields(nested, stored, prefix) {
1742
+ for (const [key, type] of Object.entries(stored)) {
1743
+ const path = prefix ? `${prefix}.${key}` : key;
1744
+ if (isRecord2(type)) {
1745
+ placeStoredFields(nested, type, path);
1746
+ } else {
1747
+ placeStoredField(nested, path, type);
1748
+ }
1749
+ }
1750
+ }
1751
+ function nestedSchemaOf(stored) {
1752
+ if (!isRecord2(stored)) return stored;
1753
+ const nested = {};
1754
+ placeStoredFields(nested, stored, "");
1755
+ return nested;
1756
+ }
1757
+
1758
+ // src/engine/export/lines.ts
1759
+ var NEWLINE = 10;
1760
+ function joined(parts, total) {
1761
+ if (parts.length === 1) return parts[0];
1762
+ const bytes = new Uint8Array(total);
1763
+ let offset = 0;
1764
+ for (const part of parts) {
1765
+ bytes.set(part, offset);
1766
+ offset += part.byteLength;
1767
+ }
1768
+ return bytes;
1769
+ }
1770
+ async function* exportLines(source, maxLineBytes) {
1771
+ const decoder2 = new TextDecoder("utf-8", { fatal: true });
1772
+ let parts = [];
1773
+ let pending = 0;
1774
+ let overflowing = false;
1775
+ let line = 0;
1776
+ const finishLine = () => {
1777
+ line += 1;
1778
+ if (overflowing) {
1779
+ overflowing = false;
1780
+ parts = [];
1781
+ pending = 0;
1782
+ return { line, refused: "too-long" };
1783
+ }
1784
+ const bytes = joined(parts, pending);
1785
+ parts = [];
1786
+ pending = 0;
1787
+ try {
1788
+ return { line, text: decoder2.decode(bytes) };
1789
+ } catch {
1790
+ return { line, refused: "not-utf8" };
1791
+ }
1792
+ };
1793
+ for await (const chunk of chunksOf(source)) {
1794
+ let start = 0;
1795
+ for (; ; ) {
1796
+ const newline = chunk.indexOf(NEWLINE, start);
1797
+ const end = newline < 0 ? chunk.byteLength : newline;
1798
+ const piece = chunk.subarray(start, end);
1799
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
1800
+ if (piece.byteLength > 0) parts.push(piece.slice());
1801
+ pending += piece.byteLength;
1802
+ } else {
1803
+ overflowing = true;
1804
+ parts = [];
1805
+ }
1806
+ if (newline < 0) break;
1807
+ yield finishLine();
1808
+ start = newline + 1;
1809
+ }
1810
+ }
1811
+ if (pending > 0 || overflowing) yield finishLine();
1812
+ }
1813
+
1814
+ // src/serialization/base64.ts
1815
+ var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
1816
+ var PAD = 61;
1817
+ var BYTES_PER_GROUP = 3;
1818
+ var CHARACTERS_PER_GROUP = 4;
1819
+ var ASCII_LIMIT = 128;
1820
+ var ENCODE = new Uint8Array(ALPHABET.length);
1821
+ var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
1822
+ for (let at = 0; at < ALPHABET.length; at++) {
1823
+ ENCODE[at] = ALPHABET.charCodeAt(at);
1824
+ DECODE[ALPHABET.charCodeAt(at)] = at;
1825
+ }
1826
+ var asciiDecoder = new TextDecoder();
1827
+ function encodeBase64(bytes) {
1828
+ const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
1829
+ let write = 0;
1830
+ let read = 0;
1831
+ for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
1832
+ const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
1833
+ out[write++] = ENCODE[group >>> 18];
1834
+ out[write++] = ENCODE[group >>> 12 & 63];
1835
+ out[write++] = ENCODE[group >>> 6 & 63];
1836
+ out[write++] = ENCODE[group & 63];
1837
+ }
1838
+ const left = bytes.byteLength - read;
1839
+ if (left > 0) {
1840
+ const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
1841
+ out[write++] = ENCODE[group >>> 18];
1842
+ out[write++] = ENCODE[group >>> 12 & 63];
1843
+ out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
1844
+ out[write++] = PAD;
1845
+ }
1846
+ return asciiDecoder.decode(out);
1847
+ }
1848
+ function sextetAt(text, at) {
1849
+ const code = text.charCodeAt(at);
1850
+ return code < ASCII_LIMIT ? DECODE[code] : -1;
1851
+ }
1852
+ function decodeExactBase64(text) {
1853
+ if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
1854
+ const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
1855
+ const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
1856
+ const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
1857
+ let write = 0;
1858
+ for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
1859
+ const a2 = sextetAt(text, at);
1860
+ const b2 = sextetAt(text, at + 1);
1861
+ const c2 = sextetAt(text, at + 2);
1862
+ const d = sextetAt(text, at + 3);
1863
+ if ((a2 | b2 | c2 | d) < 0) return null;
1864
+ const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
1865
+ out[write++] = group >>> 16;
1866
+ out[write++] = group >>> 8 & 255;
1867
+ out[write++] = group & 255;
1868
+ }
1869
+ if (padding === 0) return out;
1870
+ const a = sextetAt(text, fullEnd);
1871
+ const b = sextetAt(text, fullEnd + 1);
1872
+ const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
1873
+ if ((a | b | c) < 0) return null;
1874
+ if (padding === 2 && (b & 15) !== 0) return null;
1875
+ if (padding === 1 && (c & 3) !== 0) return null;
1876
+ out[write++] = a << 2 | b >>> 4;
1877
+ if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
1878
+ return out;
1879
+ }
1880
+
1881
+ // src/engine/export/tags.ts
1882
+ var TAG_PREFIX = "$";
1883
+ var DATE_TAG = "$date";
1884
+ var BYTES_TAG = "$bytes";
1885
+ var NUMBER_TAG = "$number";
1886
+ var OBJECT_TAG = "$object";
1887
+ var MILLISECONDS_PER_SECOND = 1e3;
1888
+ var NANOSECONDS_PER_MILLISECOND = 1e6;
1889
+ var MAX_NANOSECONDS = 999999999;
1890
+ var MAX_DATE_MILLISECONDS = 864e13;
1891
+ var ESCAPED_TAG_PREFIX = "\\u0024";
1892
+ var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
1893
+ ["NaN", Number.NaN],
1894
+ ["Infinity", Number.POSITIVE_INFINITY],
1895
+ ["-Infinity", Number.NEGATIVE_INFINITY]
1896
+ ]);
1897
+ function refuse2(path, reason) {
1898
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
1899
+ field: path
1900
+ });
1901
+ }
1902
+ function isRecord3(value) {
1903
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1904
+ }
1905
+ function startsWithTagPrefix(key) {
1906
+ return key.startsWith(TAG_PREFIX);
1907
+ }
1908
+ function soleTagKey(record) {
1909
+ let sole = null;
1910
+ for (const key of Object.keys(record)) {
1911
+ if (sole !== null) return null;
1912
+ sole = key;
1913
+ }
1914
+ return sole !== null && startsWithTagPrefix(sole) ? sole : null;
1915
+ }
1916
+ function dateOf(value, path) {
1917
+ if (!Array.isArray(value) || value.length !== 2)
1918
+ refuse2(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
1919
+ const seconds = value[0];
1920
+ const nanoseconds = value[1];
1921
+ if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
1922
+ refuse2(
1923
+ path,
1924
+ `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
1925
+ );
1926
+ }
1927
+ if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
1928
+ refuse2(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
1929
+ }
1930
+ const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
1931
+ if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
1932
+ refuse2(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
1933
+ return new Date(milliseconds);
1934
+ }
1935
+ function bytesOf(value, path) {
1936
+ const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
1937
+ if (bytes === null) refuse2(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
1938
+ return bytes;
1939
+ }
1940
+ function numberOf(value, path) {
1941
+ const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
1942
+ if (number === void 0) refuse2(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
1943
+ return number;
1944
+ }
1945
+ function requireDepth(path, depth) {
1946
+ if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse2(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
1947
+ }
1948
+ function untagMap(record, path, depth) {
1949
+ requireDepth(path, depth);
1950
+ for (const key of Object.keys(record)) {
1951
+ record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
1952
+ }
1953
+ return record;
1954
+ }
1955
+ function untagList(list, path, depth) {
1956
+ requireDepth(path, depth);
1957
+ for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
1958
+ return list;
1959
+ }
1960
+ function untagTag(tag, value, path, depth) {
1961
+ switch (tag) {
1962
+ case DATE_TAG:
1963
+ return dateOf(value, path);
1964
+ case BYTES_TAG:
1965
+ return bytesOf(value, path);
1966
+ case NUMBER_TAG:
1967
+ return numberOf(value, path);
1968
+ case OBJECT_TAG:
1969
+ if (!isRecord3(value)) refuse2(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
1970
+ return untagMap(value, path, depth);
1971
+ default:
1972
+ return refuse2(path, `holds an object whose only key, "${tag}", names no tag`);
1973
+ }
1974
+ }
1975
+ function untag(value, path, depth) {
1976
+ if (typeof value !== "object" || value === null) return value;
1977
+ if (Array.isArray(value)) return untagList(value, path, depth);
1978
+ const record = value;
1979
+ const tag = soleTagKey(record);
1980
+ return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
1981
+ }
1982
+ function untagDocumentLine(text, parsed) {
1983
+ if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
1984
+ return untagMap(parsed, "", 0);
1985
+ }
1986
+
1987
+ // src/engine/export/import.ts
1988
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
1989
+ var DOCUMENT_EXPORT_VERSION = 1;
1990
+ function invalidHeader(message) {
1991
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
1992
+ }
1993
+ function configOfHeader(line) {
1994
+ if (line === void 0) invalidHeader("is missing");
1995
+ if (!("text" in line)) {
1996
+ invalidHeader(
1997
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
1998
+ );
1999
+ }
2000
+ let raw;
2001
+ try {
2002
+ raw = JSON.parse(line.text);
2003
+ } catch {
2004
+ invalidHeader("is not JSON");
2005
+ }
2006
+ if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
2007
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
2008
+ }
2009
+ const configuration = requireConfigurationShape(raw.index, "document export header");
2010
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
2011
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
2012
+ return {
2013
+ schema: nestedSchemaOf(configuration.schema),
2014
+ language: configuration.language,
2015
+ ...fields,
2016
+ ...embedding === void 0 ? {} : {
2017
+ embedding: {
2018
+ fields: embedding.fields,
2019
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
2020
+ }
2021
+ }
2022
+ };
2023
+ }
2024
+ async function prepareIndex(engine, indexName, config) {
2025
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
2026
+ await engine.createIndex(indexName, config);
2027
+ return;
2028
+ }
2029
+ const existing = flattenSchema(engine.getStats(indexName).schema);
2030
+ const exported = flattenSchema(config.schema);
2031
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
2032
+ for (const field of fields) {
2033
+ if (existing[field] === exported[field]) continue;
2034
+ throw new NarsilError(
2035
+ ErrorCodes.CONFIG_INVALID,
2036
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
2037
+ { indexName, field }
2038
+ );
2039
+ }
2040
+ }
2041
+ function highestReportedAt(refused) {
2042
+ let highest = 0;
2043
+ for (let at = 1; at < refused.length; at++) {
2044
+ if (refused[at].line > refused[highest].line) highest = at;
2045
+ }
2046
+ return highest;
2047
+ }
2048
+ function refuse3(state, line, code, message) {
2049
+ state.result.refusedCount += 1;
2050
+ const refused = state.result.refused;
2051
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
2052
+ refused.push({ line, code, message });
2053
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
2054
+ return;
2055
+ }
2056
+ if (line >= refused[state.highestReported].line) return;
2057
+ refused[state.highestReported] = { line, code, message };
2058
+ state.highestReported = highestReportedAt(refused);
2059
+ }
2060
+ async function flush(state) {
2061
+ if (state.batch.length === 0) return;
2062
+ const batch = state.batch;
2063
+ state.batch = [];
2064
+ state.batchIds = /* @__PURE__ */ new Set();
2065
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
2066
+ const inserts = [];
2067
+ const updates = [];
2068
+ for (const entry of batch) {
2069
+ if (await state.engine.has(state.indexName, entry.docId))
2070
+ updates.push({ docId: entry.docId, document: entry.document });
2071
+ else inserts.push(entry.document);
2072
+ }
2073
+ for (const outcome of [
2074
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
2075
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
2076
+ ]) {
2077
+ if (outcome === null) continue;
2078
+ state.result.imported += outcome.succeeded.length;
2079
+ for (const failure of outcome.failed) {
2080
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
2081
+ refuse3(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
2082
+ }
2083
+ }
2084
+ }
2085
+ async function takeLine(state, entry) {
2086
+ if (!("text" in entry)) {
2087
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
2088
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
2089
+ return;
2090
+ }
2091
+ if (entry.text.length === 0) return;
2092
+ let document;
2093
+ try {
2094
+ document = JSON.parse(entry.text);
2095
+ } catch {
2096
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
2097
+ return;
2098
+ }
2099
+ if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
2100
+ refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
2101
+ return;
2102
+ }
2103
+ try {
2104
+ untagDocumentLine(entry.text, document);
2105
+ } catch (error) {
2106
+ if (!(error instanceof NarsilError)) throw error;
2107
+ refuse3(state, entry.line, error.code, error.message);
2108
+ return;
2109
+ }
2110
+ if (state.batchIds.has(document.id)) await flush(state);
2111
+ state.batch.push({ line: entry.line, docId: document.id, document });
2112
+ state.batchIds.add(document.id);
2113
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
2114
+ }
2115
+ async function readDocumentExport(engine, indexName, source) {
2116
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
2117
+ const header = await lines.next();
2118
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
2119
+ const state = {
2120
+ engine,
2121
+ indexName,
2122
+ batch: [],
2123
+ batchIds: /* @__PURE__ */ new Set(),
2124
+ result: { imported: 0, refusedCount: 0, refused: [] },
2125
+ highestReported: 0
2126
+ };
2127
+ for await (const entry of lines) {
2128
+ await takeLine(state, entry);
2129
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2130
+ }
2131
+ await flush(state);
2132
+ state.result.refused.sort((a, b) => a.line - b.line);
2133
+ return state.result;
2134
+ }
2135
+
2136
+ // src/engine/export/json-line.ts
2137
+ function refuse4(path, kind, docId) {
2138
+ throw new NarsilError(
2139
+ ErrorCodes.DOC_VALIDATION_FAILED,
2140
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
2141
+ { docId, field: path }
2142
+ );
2143
+ }
2144
+ function encodeNumber(value, path, target) {
2145
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
2146
+ if (!target.tagged) refuse4(path, String(value), target.docId);
2147
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
2148
+ }
2149
+ function encodeDate(value, path, target) {
2150
+ const milliseconds = value.getTime();
2151
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse4(path, "a date", target.docId);
2152
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
2153
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
2154
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
2155
+ }
2156
+ function encodeBytes(value, path, target) {
2157
+ if (!target.tagged) refuse4(path, "bytes", target.docId);
2158
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
2159
+ }
2160
+ function encodeList(value, path, target) {
2161
+ target.parts.push("[");
2162
+ for (let at = 0; at < value.length; at++) {
2163
+ if (at > 0) target.parts.push(",");
2164
+ encodeValue(value[at], `${path}[${at}]`, target);
2165
+ }
2166
+ target.parts.push("]");
2167
+ }
2168
+ function encodeMap(value, path, target) {
2169
+ const proto = Object.getPrototypeOf(value);
2170
+ if (proto !== Object.prototype && proto !== null) refuse4(path, "a value other than a plain object", target.docId);
2171
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
2172
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
2173
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
2174
+ for (let at = 0; at < entries.length; at++) {
2175
+ const [key, entry] = entries[at];
2176
+ if (at > 0) target.parts.push(",");
2177
+ target.parts.push(JSON.stringify(key), ":");
2178
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
2179
+ }
2180
+ target.parts.push(wrapped ? "}}" : "}");
2181
+ }
2182
+ function encodeValue(value, path, target) {
2183
+ if (value === null) {
2184
+ target.parts.push("null");
2185
+ return;
2186
+ }
2187
+ switch (typeof value) {
2188
+ case "string":
2189
+ target.parts.push(JSON.stringify(value));
2190
+ return;
2191
+ case "boolean":
2192
+ target.parts.push(value ? "true" : "false");
2193
+ return;
2194
+ case "number":
2195
+ target.parts.push(encodeNumber(value, path, target));
2196
+ return;
2197
+ case "object":
2198
+ break;
2199
+ default:
2200
+ refuse4(path, `a ${typeof value}`, target.docId);
2201
+ }
2202
+ if (Array.isArray(value)) encodeList(value, path, target);
2203
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
2204
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
2205
+ else encodeMap(value, path, target);
2206
+ }
2207
+ function encodeExactJson(value, docId) {
2208
+ const target = { docId, tagged: false, parts: [] };
2209
+ encodeValue(value, "", target);
2210
+ return target.parts.join("");
2211
+ }
2212
+ function encodeDocumentJson(document, docId) {
2213
+ const target = { docId, tagged: true, parts: [] };
2214
+ encodeValue(document, "", target);
2215
+ return target.parts.join("");
2216
+ }
2217
+
2218
+ // src/engine/export/paged-lines.ts
2219
+ async function coldFieldOf(checkpoint, field) {
2220
+ const { directory } = checkpoint;
2221
+ if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
2222
+ if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
2223
+ const fieldType = checkpoint.manifest.schema[field.fieldPath];
2224
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
2225
+ if (dimension === null) return null;
2226
+ const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
2227
+ return openColdVectorField({
2228
+ fieldPath: field.fieldPath,
2229
+ dimension,
2230
+ metric: "cosine",
2231
+ codeBits: null,
2232
+ files: await Promise.all(field.files.map(withPath)),
2233
+ graphParts: await Promise.all(field.graphParts.map(withPath)),
2234
+ resolvePartition: () => void 0
2235
+ });
2236
+ }
2237
+ async function openFileVectorReads(checkpoint, wholeFileReads) {
2238
+ const fields = /* @__PURE__ */ new Map();
2239
+ for (const field of checkpoint.manifest.vectors) {
2240
+ const opened = await coldFieldOf(checkpoint, field);
2241
+ if (opened !== null) fields.set(field.fieldPath, opened);
2242
+ }
2243
+ const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
2244
+ return {
2245
+ async vectorOf(fieldPath, docId) {
2246
+ const field = fields.get(fieldPath);
2247
+ if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
2248
+ const number = field.numberOf(docId);
2249
+ return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
2250
+ },
2251
+ close() {
2252
+ for (const field of fields.values()) field.close();
2253
+ fallback?.close();
2254
+ }
2255
+ };
2256
+ }
2257
+ async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
2258
+ const { directory } = checkpoint;
2259
+ if (directory === void 0) return;
2260
+ const scans = [];
2261
+ try {
2262
+ for (const ref of partition.segments) {
2263
+ const paged = await isPagedSegmentFile(directory, ref.key);
2264
+ scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
2265
+ }
2266
+ for (let at = 0; at < scans.length; at++) {
2267
+ const scan = scans[at];
2268
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) {
2269
+ const docId = scan.idAt(ordinal);
2270
+ if (!countsAsLatestHolder(scans, at, docId)) continue;
2271
+ const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
2272
+ document.id = docId;
2273
+ for (const field of checkpoint.manifest.vectors) {
2274
+ const vector = await vectors.vectorOf(field.fieldPath, docId);
2275
+ if (vector !== null) setNestedValue(document, field.fieldPath, vector);
2276
+ }
2277
+ yield `${encodeDocumentJson(document, docId)}
2278
+ `;
2279
+ }
2280
+ }
2281
+ } finally {
2282
+ for (const scan of scans) scan.close();
2283
+ }
2284
+ }
2285
+
2286
+ // src/engine/export/write.ts
2287
+ var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
2288
+ var FLOAT32_BYTES = 4;
2289
+ var ENVELOPE_VERSION_OFFSET = 4;
2290
+ async function locateVectors(checkpoint) {
2291
+ const located = /* @__PURE__ */ new Map();
2292
+ for (const field of checkpoint.manifest.vectors) {
2293
+ const byDocId = /* @__PURE__ */ new Map();
2294
+ for (const ref of field.files) {
2295
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
2296
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
2297
+ what: `vector file "${ref.key}"`,
2298
+ allowsCompression: false
2299
+ });
2300
+ const decoded = decodeVectorFilePayload(decode(payload));
2301
+ for (let position = 0; position < decoded.docIds.length; position++) {
2302
+ if (isDead(ref.dead, position)) continue;
2303
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
2304
+ }
2305
+ }
2306
+ located.set(field.fieldPath, byDocId);
2307
+ }
2308
+ return located;
2309
+ }
2310
+ async function readVector(checkpoint, location) {
2311
+ const width = location.dimension * FLOAT32_BYTES;
2312
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
2313
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
2314
+ }
2315
+ async function readSegment(checkpoint, ref) {
2316
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
2317
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
2318
+ what: `segment "${ref.key}"`,
2319
+ allowsCompression: true
2320
+ });
2321
+ return decodeKeywordSegment(payload, false);
2322
+ }
2323
+ var decoder = new TextDecoder();
2324
+ function idAt(contents, ordinal) {
2325
+ const ids = contents.arrays.idTable;
2326
+ return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
2327
+ }
2328
+ async function winnersOf(checkpoint, partition) {
2329
+ const winners = /* @__PURE__ */ new Map();
2330
+ for (let at = 0; at < partition.segments.length; at++) {
2331
+ const contents = await readSegment(checkpoint, partition.segments[at]);
2332
+ for (const docId of contents.tombstones) winners.delete(docId);
2333
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
2334
+ }
2335
+ return winners;
2336
+ }
2337
+ async function wholeSegmentScan(checkpoint, ref) {
2338
+ const contents = await readSegment(checkpoint, ref);
2339
+ const table = contents.arrays.documentTable;
2340
+ const ids = /* @__PURE__ */ new Set();
2341
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
2342
+ return {
2343
+ count: contents.arrays.documentCount,
2344
+ schema: contents.schema,
2345
+ tombstones: new Set(contents.tombstones),
2346
+ idAt: (ordinal) => idAt(contents, ordinal),
2347
+ holds: (docId) => ids.has(docId),
2348
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
2349
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
2350
+ table.discardedFields
2351
+ ),
2352
+ close: () => void 0
2353
+ };
2354
+ }
2355
+ async function wholeFileVectorReads(checkpoint) {
2356
+ const located = await locateVectors(checkpoint);
2357
+ return {
2358
+ async vectorOf(fieldPath, docId) {
2359
+ const location = located.get(fieldPath)?.get(docId);
2360
+ return location === void 0 ? null : readVector(checkpoint, location);
2361
+ },
2362
+ close: () => located.clear()
2363
+ };
2364
+ }
2365
+ async function* pagedDocumentLines(checkpoint) {
2366
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
2367
+ try {
2368
+ for (const partition of checkpoint.manifest.partitions) {
2369
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
2370
+ }
2371
+ } finally {
2372
+ vectors.close();
2373
+ }
2374
+ }
2375
+ async function anyPartitionIsPaged(checkpoint) {
2376
+ for (const partition of checkpoint.manifest.partitions) {
2377
+ for (const ref of partition.segments) {
2378
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
2379
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
2380
+ }
2381
+ }
2382
+ return false;
2383
+ }
2384
+ async function* documentLines(checkpoint) {
2385
+ if (await anyPartitionIsPaged(checkpoint)) {
2386
+ yield* pagedDocumentLines(checkpoint);
2387
+ return;
2388
+ }
2389
+ const vectors = await locateVectors(checkpoint);
2390
+ for (const partition of checkpoint.manifest.partitions) {
2391
+ const winners = await winnersOf(checkpoint, partition);
2392
+ for (let at = 0; at < partition.segments.length; at++) {
2393
+ const contents = await readSegment(checkpoint, partition.segments[at]);
2394
+ const table = contents.arrays.documentTable;
2395
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
2396
+ const docId = idAt(contents, ordinal);
2397
+ if (winners.get(docId) !== at) continue;
2398
+ const stored = decodeStoredDocument(
2399
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
2400
+ table.discardedFields
2401
+ );
2402
+ const document = { id: docId, ...stored };
2403
+ document.id = docId;
2404
+ for (const [fieldPath, located] of vectors) {
2405
+ const location = located.get(docId);
2406
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
2407
+ }
2408
+ yield `${encodeDocumentJson(document, docId)}
2409
+ `;
2410
+ }
2411
+ }
2412
+ }
2413
+ }
2414
+ async function* exportLines2(header, checkpoint, positions) {
2415
+ const exportHeader = { format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header };
2416
+ yield `${encodeExactJson(positions === null ? exportHeader : { ...exportHeader, ...positions }, "")}
2417
+ `;
2418
+ yield* documentLines(checkpoint);
2419
+ }
2420
+ function documentExportStream(header, checkpoint, positions = null) {
2421
+ const lines = exportLines2(header, checkpoint, positions);
2422
+ const encoder = new TextEncoder();
2423
+ let released = false;
2424
+ const release = () => {
2425
+ if (released) return;
2426
+ released = true;
2427
+ checkpoint.release();
2428
+ };
2429
+ return new ReadableStream({
2430
+ async pull(controller) {
2431
+ try {
2432
+ let text = "";
2433
+ while (text.length < EXPORT_CHUNK_BYTES) {
2434
+ const step = await lines.next();
2435
+ if (step.done) {
2436
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
2437
+ release();
2438
+ controller.close();
2439
+ return;
2440
+ }
2441
+ text += step.value;
2442
+ }
2443
+ controller.enqueue(encoder.encode(text));
2444
+ } catch (error) {
2445
+ release();
2446
+ controller.error(error);
2447
+ }
2448
+ },
2449
+ async cancel() {
2450
+ release();
2451
+ await lines.return(void 0);
2452
+ }
2453
+ });
2454
+ }
2455
+
2456
+ // src/partitioning/saved-partitions.ts
2457
+ function requireSavedPartitionCount(partitionCount, source, indexName) {
2458
+ if (partitionCount <= MAX_PARTITIONS) return;
2459
+ throw new NarsilError(
2460
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2461
+ `The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
2462
+ { indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
2463
+ );
2464
+ }
2465
+ function highestPartitionIdOf(partitions) {
2466
+ let highest = partitions.length - 1;
2467
+ for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
2468
+ return highest;
2469
+ }
2470
+
2471
+ // src/persistence/paged/read-ahead.ts
2472
+ var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
2473
+ function rangeOf(path, section) {
2474
+ return { path, offset: HEADER_SIZE + section.offset, length: section.length };
2475
+ }
2476
+ async function readRangesAhead(ranges) {
2477
+ let rangeAt = 0;
2478
+ let doneInRange = 0;
2479
+ function nextChunk() {
2480
+ while (rangeAt < ranges.length && doneInRange >= ranges[rangeAt].length) {
2481
+ rangeAt += 1;
2482
+ doneInRange = 0;
2483
+ }
2484
+ if (rangeAt >= ranges.length) return null;
2485
+ const { path, offset, length } = ranges[rangeAt];
2486
+ const chunk = { path, offset: offset + doneInRange, length: Math.min(READ_AHEAD_CHUNK_BYTES, length - doneInRange) };
2487
+ doneInRange += chunk.length;
2488
+ return chunk;
2489
+ }
2490
+ const scratch = new Uint8Array(READ_AHEAD_CHUNK_BYTES);
2491
+ async function readChunks() {
2492
+ for (let chunk = nextChunk(); chunk !== null; chunk = nextChunk()) {
2493
+ const target = scratch.subarray(0, chunk.length);
2494
+ if (!await readFileRangeLater(chunk.path, chunk.offset, target)) return;
2495
+ }
2496
+ }
2497
+ await Promise.all(Array.from({ length: READS_AHEAD_AT_ONCE }, readChunks));
2498
+ }
2499
+ async function headOf(path, ref) {
2500
+ const { headOffset, headLength } = ref;
2501
+ if (headOffset === void 0 || headLength === void 0) return null;
2502
+ const head = new Uint8Array(headLength);
2503
+ if (!await readFileRangeLater(path, HEADER_SIZE + headOffset, head)) return null;
2504
+ return decode(head);
2505
+ }
2506
+ async function levelsOfPart(directory, ref) {
2507
+ const path = await directory.pathOf(ref.key);
2508
+ const head = await headOf(path, ref);
2509
+ const graphs = head?.graphs;
2510
+ if (!Array.isArray(graphs) || graphs.length !== 1) return [];
2511
+ const { levels } = graphs[0];
2512
+ return decodeSections([levels], ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, FILE_PAGE_SIZES).map(
2513
+ (section) => rangeOf(path, section)
2514
+ );
2515
+ }
2516
+ async function residentSectionsOfSegment(directory, ref) {
2517
+ if (!await isPagedSegmentFile(directory, ref.key)) return [];
2518
+ const listing = await pagedListingOf(directory, ref);
2519
+ return residentSectionsOf(listing).map((section) => rangeOf(listing.path, section));
2520
+ }
2521
+ async function startReadingOpeningSectionsAhead(directory, manifest) {
2522
+ try {
2523
+ const groups = await Promise.all([
2524
+ ...manifest.partitions.flatMap(
2525
+ (partition) => partition.segments.map((ref) => residentSectionsOfSegment(directory, ref))
2526
+ ),
2527
+ ...manifest.vectors.flatMap((field) => field.graphParts.map((ref) => levelsOfPart(directory, ref)))
2528
+ ]);
2529
+ void readRangesAhead(groups.flat());
2530
+ } catch {
2531
+ return;
2532
+ }
2533
+ }
2534
+
2535
+ // src/engine/changes/operations.ts
2536
+ function invalidLimit(name, value) {
2537
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The changes setting ${name} must be a positive whole number`, {
2538
+ [name]: String(value)
2539
+ });
2540
+ }
2541
+ function requireChangesSetting(setting) {
2542
+ if (setting === null || typeof setting !== "object" || Array.isArray(setting)) {
2543
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting must be an object");
2544
+ }
2545
+ const { enabled, previousDocument, historyMaxAgeMs, historyMaxBytes } = setting;
2546
+ if (typeof enabled !== "boolean") {
2547
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting needs enabled set to true or false");
2548
+ }
2549
+ if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
2550
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The changes setting previousDocument must be true or false");
2551
+ }
2552
+ const isLimit = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
2553
+ if (historyMaxAgeMs !== void 0 && !isLimit(historyMaxAgeMs)) invalidLimit("historyMaxAgeMs", historyMaxAgeMs);
2554
+ if (historyMaxBytes !== void 0 && !isLimit(historyMaxBytes)) invalidLimit("historyMaxBytes", historyMaxBytes);
2555
+ return {
2556
+ enabled,
2557
+ ...previousDocument !== void 0 ? { previousDocument } : {},
2558
+ ...historyMaxAgeMs !== void 0 ? { historyMaxAgeMs } : {},
2559
+ ...historyMaxBytes !== void 0 ? { historyMaxBytes } : {}
2560
+ };
2561
+ }
2562
+ function changesStorageOf(core, indexName) {
2563
+ const storage = core.durability?.manager.syncStorage?.();
2564
+ if (storage === void 0) {
2565
+ throw new NarsilError(
2566
+ ErrorCodes.CHANGES_DURABILITY_REQUIRED,
2567
+ `Index "${indexName}" needs write-ahead log durability for change capture, because the feed reads the log`,
2568
+ { indexName }
2569
+ );
2570
+ }
2571
+ return storage;
2572
+ }
2573
+ function changesStatusOf(changes) {
2574
+ if (changes === null) return { enabled: false, previousDocument: false, epoch: null };
2575
+ return {
2576
+ enabled: true,
2577
+ previousDocument: changes.previousDocument,
2578
+ ...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
2579
+ ...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {},
2580
+ epoch: changes.epoch
2581
+ };
2582
+ }
2583
+ function nextState(setting, epoch) {
2584
+ return {
2585
+ previousDocument: setting.previousDocument === true,
2586
+ ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
2587
+ ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
2588
+ epoch
2589
+ };
2590
+ }
2591
+ async function turnOff(core, indexName, storage) {
2592
+ const { sync, changes } = core.requireIndex(indexName);
2593
+ if (changes === null) return;
2594
+ await persistEpochStates(core, indexName, { sync, changes: null });
2595
+ if (sync?.active !== true) await storage.release(indexName);
2596
+ emitSyncChange(core.indexRegistry, { kind: "changes-off", indexName });
2597
+ }
2598
+ async function turnOn(core, indexName, storage, setting) {
2599
+ const { sync, changes } = core.requireIndex(indexName);
2600
+ if (changes !== null) {
2601
+ await persistEpochStates(core, indexName, { sync, changes: nextState(setting, changes.epoch) });
2602
+ if (changes.previousDocument && setting.previousDocument !== true) {
2603
+ emitSyncChange(core.indexRegistry, { kind: "previous-document-off", indexName });
2604
+ }
2605
+ return;
2606
+ }
2607
+ if (sync !== null) {
2608
+ await persistEpochStates(core, indexName, { sync, changes: nextState(setting, sync.epoch) });
2609
+ return;
2610
+ }
2611
+ await storage.release(indexName);
2612
+ await persistEpochStates(core, indexName, { sync: null, changes: nextState(setting, newSyncEpoch()) });
2613
+ }
2614
+ function updateIndexChanges(core, indexName, value) {
2615
+ const setting = requireChangesSetting(value);
2616
+ core.requireIndex(indexName);
2617
+ const storage = changesStorageOf(core, indexName);
2618
+ return withOpenIndex(
2619
+ core,
2620
+ indexName,
2621
+ () => oneSyncChangeAtATime(
2622
+ core.indexRegistry,
2623
+ indexName,
2624
+ () => setting.enabled ? turnOn(core, indexName, storage, setting) : turnOff(core, indexName, storage)
2625
+ )
2626
+ );
2627
+ }
2628
+ async function updateChangesOf(core, indexName, setting) {
2629
+ core.guardShutdown();
2630
+ core.requireIndex(indexName);
2631
+ await updateIndexChanges(core, indexName, setting);
2632
+ }
2633
+ function changesOf(core, indexName) {
2634
+ core.guardShutdown();
2635
+ return changesStatusOf(core.requireIndex(indexName).changes);
2636
+ }
2637
+
2638
+ // src/persistence/durability/complete-graphs.ts
2639
+ async function whileGraphsStayComplete(vectorIndexes, capture2) {
2640
+ const letBuildsIn = [];
2641
+ try {
2642
+ for (const fieldPath of [...vectorIndexes.keys()].sort()) {
2643
+ const vectorIndex = vectorIndexes.get(fieldPath);
2644
+ if (vectorIndex !== void 0) letBuildsIn.push(await vectorIndex.holdGraphComplete());
2645
+ }
2646
+ return await capture2();
2647
+ } finally {
2648
+ for (const release of letBuildsIn) release();
2649
+ }
2650
+ }
2651
+
2652
+ // src/serialization/crc32-combine.ts
2653
+ var IEEE_POLYNOMIAL = 3988292384;
2654
+ var CRC_BITS = 32;
2655
+ var zeroByteOperators = [];
2656
+ function timesVector(operator, vector) {
2657
+ let sum = 0;
2658
+ let rest = vector >>> 0;
2659
+ for (let row = 0; rest !== 0; row++, rest >>>= 1) {
2660
+ if (rest & 1) sum ^= operator[row];
2661
+ }
2662
+ return sum >>> 0;
2663
+ }
2664
+ function squared(operator) {
2665
+ const result = new Uint32Array(CRC_BITS);
2666
+ for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
2667
+ return result;
2668
+ }
2669
+ function firstOperator() {
2670
+ let oneZeroBit = new Uint32Array(CRC_BITS);
2671
+ oneZeroBit[0] = IEEE_POLYNOMIAL;
2672
+ for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
2673
+ for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
2674
+ return oneZeroBit;
2675
+ }
2676
+ function operatorForZeroBytesPower(power) {
2677
+ if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
2678
+ while (zeroByteOperators.length <= power) {
2679
+ zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
2680
+ }
2681
+ return zeroByteOperators[power];
2682
+ }
2683
+ function crc32Combine(first, second, secondByteLength) {
2684
+ let crc = first >>> 0;
2685
+ let rest = secondByteLength;
2686
+ for (let power = 0; rest > 0; power++) {
2687
+ if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
2688
+ rest = Math.floor(rest / 2);
2689
+ }
2690
+ return (crc ^ second) >>> 0;
2691
+ }
2692
+
2693
+ // src/persistence/paged/vector-bytes.ts
2694
+ var BIN8 = 196;
2695
+ var BIN16 = 197;
2696
+ var BIN32 = 198;
2697
+ var ARRAY16 = 220;
2698
+ var ARRAY32 = 221;
2699
+ var FIXMAP = 128;
2700
+ var FIXARRAY = 144;
2701
+ var FIXED_LIST_MAX = 15;
2702
+ var FIXSTR = 160;
2703
+ var FIXSTR_MAX_LENGTH = 31;
2704
+ var ASCII_MAX = 127;
2705
+ var UTF8_MAX_BYTES_PER_UNIT = 3;
2706
+ var STRING_HEADER_MAX_BYTES = 5;
2707
+ var BIN8_MAX_BYTES = 255;
2708
+ var UINT16_MAX = 65535;
2709
+ function binHeader(byteLength) {
2710
+ if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
2711
+ if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
2712
+ const header = new Uint8Array(5);
2713
+ header[0] = BIN32;
2714
+ new DataView(header.buffer).setUint32(1, byteLength, false);
2715
+ return header;
2716
+ }
2717
+ function arrayHeader(count) {
2718
+ if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
2719
+ if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
2720
+ const header = new Uint8Array(5);
2721
+ header[0] = ARRAY32;
2722
+ new DataView(header.buffer).setUint32(1, count, false);
2723
+ return header;
2724
+ }
2725
+ function createPayloadBuilder() {
2726
+ const chunks = [];
2727
+ let offset = 0;
2728
+ function push(chunk) {
2729
+ chunks.push(chunk);
2730
+ offset += chunk.byteLength;
2731
+ }
2732
+ return {
2733
+ chunks,
2734
+ get offset() {
2735
+ return offset;
2736
+ },
2737
+ push,
2738
+ entry(key, value) {
2739
+ push(encode(key));
2740
+ push(encode(value));
2741
+ },
2742
+ rawEntry(key, encoded) {
2743
+ push(encode(key));
2744
+ const at = offset;
2745
+ push(encoded);
2746
+ return at;
2747
+ },
2748
+ bytesEntry(key, bytes) {
2749
+ push(encode(key));
2750
+ push(binHeader(bytes.byteLength));
2751
+ const at = offset;
2752
+ push(bytes);
2753
+ return at;
2754
+ }
2755
+ };
2756
+ }
2757
+ function pageChecksums(bytes, pageBytes) {
2758
+ const pages = Math.ceil(bytes.byteLength / pageBytes);
2759
+ const checksums = new Uint8Array(pages * 4);
2760
+ const view = new DataView(checksums.buffer);
2761
+ for (let page = 0; page < pages; page++) {
2762
+ const from = page * pageBytes;
2763
+ view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
2764
+ }
2765
+ return checksums;
2766
+ }
2767
+ function sectionOf(name, offset, bytes, pageBytes) {
2768
+ return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
2769
+ }
2770
+ function uint32Bytes(values) {
2771
+ const bytes = new Uint8Array(values.length * 4);
2772
+ const view = new DataView(bytes.buffer);
2773
+ for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
2774
+ return bytes;
2775
+ }
2776
+ function writtenAsShortAscii(docId, list, at) {
2777
+ if (docId.length > FIXSTR_MAX_LENGTH) return false;
2778
+ for (let unit = 0; unit < docId.length; unit++) {
2779
+ const code = docId.charCodeAt(unit);
2780
+ if (code > ASCII_MAX) return false;
2781
+ list[at + 1 + unit] = code;
2782
+ }
2783
+ list[at] = FIXSTR | docId.length;
2784
+ return true;
2785
+ }
2786
+ function encodedIdList(docIds) {
2787
+ const header = arrayHeader(docIds.length);
2788
+ let capacity = header.byteLength;
2789
+ for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
2790
+ const list = new Uint8Array(capacity);
2791
+ list.set(header);
2792
+ const offsets = new Uint32Array(docIds.length + 1);
2793
+ let cursor = header.byteLength;
2794
+ for (let i = 0; i < docIds.length; i++) {
2795
+ offsets[i] = cursor;
2796
+ if (writtenAsShortAscii(docIds[i], list, cursor)) {
2797
+ cursor += docIds[i].length + 1;
2798
+ continue;
2799
+ }
2800
+ const encoded = encode(docIds[i]);
2801
+ list.set(encoded, cursor);
2802
+ cursor += encoded.byteLength;
2803
+ }
2804
+ offsets[docIds.length] = cursor;
2805
+ return { list: list.subarray(0, cursor), offsets };
2806
+ }
2807
+ function positionsInIdOrder(docIds) {
2808
+ const positions = Array.from(docIds.keys());
2809
+ positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
2810
+ return Uint32Array.from(positions);
2811
+ }
2812
+ function magnitudeBytes(magnitudes) {
2813
+ const bytes = new Uint8Array(magnitudes.length * 8);
2814
+ const view = new DataView(bytes.buffer);
2815
+ for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
2816
+ return bytes;
2817
+ }
2818
+ function checksumAcrossPages(checksums, byteLength, pageBytes) {
2819
+ const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
2820
+ let whole = 0;
2821
+ for (let page = 0; page * 4 < checksums.byteLength; page++) {
2822
+ const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
2823
+ whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
2824
+ }
2825
+ return whole;
2826
+ }
2827
+ function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
2828
+ let headLength = 0;
2829
+ let head = new Uint8Array(0);
2830
+ let vectorsSection = {
2831
+ name: "vectors",
2832
+ offset: 0,
2833
+ length: vectors.byteLength,
2834
+ pageBytes: SEGMENT_PAGE_BYTES,
2835
+ checksums
2836
+ };
2837
+ for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
2838
+ vectorsSection = {
2839
+ ...vectorsSection,
2840
+ offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
2841
+ };
2842
+ head = encode({ sections: [...sections, vectorsSection] });
2843
+ if (head.byteLength === headLength) break;
2844
+ headLength = head.byteLength;
2845
+ }
2846
+ return { head, vectorsSection };
2847
+ }
2848
+ function pagedVectorFileChunks(payload, heldMagnitudes) {
2849
+ const builder = createPayloadBuilder();
2850
+ const { codes } = payload;
2851
+ builder.push(Uint8Array.of(FIXMAP | 10));
2852
+ builder.entry("v", payload.v);
2853
+ builder.entry("fieldName", payload.fieldName);
2854
+ builder.entry("dimension", payload.dimension);
2855
+ const { list, offsets } = encodedIdList(payload.docIds);
2856
+ const docIdsAt = builder.rawEntry("docIds", list);
2857
+ const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
2858
+ builder.push(encode("codes"));
2859
+ if (codes === null) {
2860
+ builder.push(encode(null));
2861
+ } else {
2862
+ builder.push(Uint8Array.of(FIXMAP | 3));
2863
+ builder.entry("bits", codes.bits);
2864
+ builder.entry("centroid", codes.centroid);
2865
+ const recordsAt = builder.bytesEntry("records", codes.records);
2866
+ sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
2867
+ }
2868
+ const magnitudes = magnitudeBytes(heldMagnitudes);
2869
+ sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
2870
+ const idOffsets = uint32Bytes(offsets);
2871
+ sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
2872
+ const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
2873
+ sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
2874
+ const headKey = encode("head");
2875
+ const vectorsKey = encode("vectors");
2876
+ const vectorsHeader = binHeader(payload.vectors.byteLength);
2877
+ const beforeHead = builder.offset + headKey.byteLength;
2878
+ const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
2879
+ const { head } = encodedHeadAndVectorsPlace(
2880
+ sections,
2881
+ (headLength) => beforeHead + binHeader(headLength).byteLength,
2882
+ payload.vectors,
2883
+ vectorChecksums,
2884
+ vectorsKey.byteLength + vectorsHeader.byteLength
2885
+ );
2886
+ const headOffset = builder.bytesEntry("head", head);
2887
+ builder.push(vectorsKey);
2888
+ builder.push(vectorsHeader);
2889
+ const beforeVectors = [...builder.chunks];
2890
+ builder.push(payload.vectors);
2891
+ return {
2892
+ chunks: builder.chunks,
2893
+ headOffset,
2894
+ headLength: head.byteLength,
2895
+ headChecksum: crc32(head),
2896
+ beforeVectors,
2897
+ vectors: {
2898
+ byteLength: payload.vectors.byteLength,
2899
+ checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
2900
+ }
2901
+ };
2902
+ }
2903
+ function nodeOffsetsOf(graph) {
2904
+ const offsets = new Uint32Array(graph.levels.length + 1);
2905
+ const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
2906
+ let cursor = 0;
2907
+ for (let at = 0; at < graph.levels.length; at++) {
2908
+ offsets[at] = cursor;
2909
+ for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
2910
+ }
2911
+ offsets[graph.levels.length] = cursor;
2912
+ return uint32Bytes(offsets);
2913
+ }
2914
+ function pagedGraphPartChunks(part) {
2915
+ const builder = createPayloadBuilder();
2916
+ builder.push(Uint8Array.of(FIXMAP | 4));
2917
+ builder.entry("v", part.v);
2918
+ builder.entry("fieldName", part.fieldName);
2919
+ builder.push(encode("graphs"));
2920
+ builder.push(arrayHeader(part.graphs.length));
2921
+ const layouts = [];
2922
+ for (const graph of part.graphs) {
2923
+ const { levels, neighbours, ...header } = graph;
2924
+ const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
2925
+ builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
2926
+ for (const [key, value] of headerEntries) builder.entry(key, value);
2927
+ const levelsAt = builder.bytesEntry("levels", levels);
2928
+ const neighboursAt = builder.bytesEntry("neighbours", neighbours);
2929
+ layouts.push({
2930
+ ...header,
2931
+ levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
2932
+ nodeOffsets: nodeOffsetsOf(graph),
2933
+ neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
2934
+ });
2935
+ }
2936
+ const head = encode({ graphs: layouts });
2937
+ const headOffset = builder.bytesEntry("head", head);
2938
+ return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
2939
+ }
2940
+
2941
+ // src/persistence/durability/segment/vector-part-bytes.ts
2942
+ var EMPTY_BIN = new Uint8Array(0);
2943
+ var EMPTY_BIN_BYTES = 2;
2944
+ var BIN82 = 196;
2945
+ var BIN162 = 197;
2946
+ var BIN322 = 198;
2947
+ var BIN8_MAX_BYTES2 = 255;
2948
+ var BIN16_MAX_BYTES = 65535;
2949
+ function binHeader2(byteLength) {
2950
+ if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
2951
+ if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
2952
+ const header = new Uint8Array(5);
2953
+ header[0] = BIN322;
2954
+ new DataView(header.buffer).setUint32(1, byteLength, false);
2955
+ return header;
2956
+ }
2957
+ function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
2958
+ const head = encode(withEmptyBytesInTheirPlace);
2959
+ const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
2960
+ if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
2961
+ return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
2962
+ }
2963
+ function vectorPartChunks(part) {
2964
+ return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
2965
+ }
2966
+
2967
+ // src/persistence/durability/segment/vector-write.ts
2968
+ function keptFileRef(input, prior, kept) {
2969
+ const listed = prior?.files.find((file) => file.key === kept.key) ?? input.priorFileRefs?.get(kept.key);
2970
+ if (listed === void 0) {
2971
+ throw new NarsilError(
2972
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
2973
+ `The checkpoint keeps the vector file "${kept.key}", which the previous manifest does not list`,
2974
+ { key: kept.key }
2975
+ );
2976
+ }
2977
+ return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
2978
+ }
2979
+ function headFieldsOf(ref) {
2980
+ return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
2981
+ }
2982
+ function headRefOf(paged) {
2983
+ return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
2984
+ }
2985
+ async function writeGraphParts(input, fieldPath, plan, generation) {
2986
+ const parts = plan.graphParts ?? [];
2987
+ const refs = [];
2988
+ for (let part = 0; part < parts.length; part++) {
2989
+ const key = graphPartKey(input.indexName, fieldPath, generation, part);
2990
+ const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
2991
+ const packed = await packChecksummedEnvelope(
2992
+ paged === null ? [encode(parts[part])] : paged.chunks,
2993
+ GRAPH_PART_ENVELOPE_VERSION,
2994
+ `graph part "${key}"`
2995
+ );
2996
+ await input.store.write(key, packed.chunks);
2997
+ refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
2998
+ }
2999
+ return refs;
3000
+ }
3001
+ async function writeField(input, fieldPath, plan, outcome) {
3002
+ const prior = input.priorVectors.find((ref) => ref.fieldPath === fieldPath);
3003
+ if (plan.unchanged && prior !== void 0) {
3004
+ outcome.refs.push(prior);
3005
+ return;
3006
+ }
3007
+ let nextFileId = prior?.nextFileId ?? 0;
3008
+ const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
3009
+ const written = [];
3010
+ for (let index = 0; index < plan.newFiles; index++) {
3011
+ const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
3012
+ const id = nextFileId;
3013
+ nextFileId += 1;
3014
+ const key = vectorFileKey(input.indexName, fieldPath, id);
3015
+ const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
3016
+ const what = `vector file "${key}"`;
3017
+ const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
3018
+ paged.chunks,
3019
+ VECTOR_FILE_ENVELOPE_VERSION,
3020
+ what,
3021
+ crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
3022
+ );
3023
+ await input.store.write(key, packed.chunks);
3024
+ files.push({
3025
+ id,
3026
+ key,
3027
+ count: payload.docIds.length,
3028
+ dead,
3029
+ bytes: packed.byteLength,
3030
+ checksum: packed.checksum,
3031
+ ...headRefOf(paged)
3032
+ });
3033
+ written.push({ id, key, ordinals });
3034
+ outcome.layouts.push({
3035
+ fieldPath,
3036
+ key,
3037
+ docIds: payload.docIds,
3038
+ dead,
3039
+ vectorsOffset: packed.byteLength - payload.vectors.byteLength
3040
+ });
3041
+ }
3042
+ const priorGeneration = prior?.graphGeneration ?? 0;
3043
+ const holdsGraph = plan.graphParts !== null && plan.graphParts.length > 0;
3044
+ const graphGeneration = holdsGraph ? priorGeneration + 1 : priorGeneration;
3045
+ const graphParts = holdsGraph ? await writeGraphParts(input, fieldPath, plan, graphGeneration) : [];
3046
+ outcome.refs.push({ fieldPath, nextFileId, files, graphGeneration, graphParts });
3047
+ outcome.written.push({ fieldPath, plan, files: written });
3048
+ }
3049
+ async function writeLiveVectors(input) {
3050
+ const outcome = { refs: [], layouts: [], written: [] };
3051
+ for (const [fieldPath, plan] of input.plans) await writeField(input, fieldPath, plan, outcome);
3052
+ return outcome;
3053
+ }
3054
+
3055
+ // src/persistence/archive/memory-store.ts
3056
+ function joined2(chunks) {
3057
+ let total = 0;
3058
+ for (const chunk of chunks) total += chunk.byteLength;
3059
+ const bytes = new Uint8Array(total);
3060
+ let offset = 0;
3061
+ for (const chunk of chunks) {
3062
+ bytes.set(chunk, offset);
3063
+ offset += chunk.byteLength;
3064
+ }
3065
+ return bytes;
3066
+ }
3067
+ function memoryCheckpointStore() {
3068
+ const files = /* @__PURE__ */ new Map();
3069
+ return {
3070
+ files,
3071
+ read: async (key) => files.get(key) ?? null,
3072
+ write: async (key, chunks) => {
3073
+ files.set(key, joined2(chunks));
3074
+ },
3075
+ remove: async (key) => {
3076
+ files.delete(key);
3077
+ },
3078
+ list: async (prefix) => [...files.keys()].filter((key) => key.startsWith(prefix)).sort(),
3079
+ byteLength: async (key) => files.get(key)?.byteLength ?? null
3080
+ };
3081
+ }
3082
+
3083
+ // src/persistence/archive/from-memory.ts
3084
+ function frozenPlan(plan) {
3085
+ const files = [];
3086
+ for (let index = 0; index < plan.newFiles; index++) files.push(plan.readNewFile(index));
3087
+ return { ...plan, readNewFile: (index) => files[index] };
3088
+ }
3089
+ function partitionVectorPlan(vectorIndex, docIds) {
3090
+ const held = [];
3091
+ const vectors = [];
3092
+ for (const docId of docIds) {
3093
+ const vector = vectorIndex.getVector(docId);
3094
+ if (vector === null) continue;
3095
+ held.push(docId);
3096
+ vectors.push(Float32Array.from(vector));
3097
+ }
3098
+ const capacity = vectorsPerFile(vectorIndex.dimension, null);
3099
+ const files = [];
3100
+ for (let start = 0; start < held.length; start += capacity) {
3101
+ const slice = held.slice(start, start + capacity);
3102
+ const packed = new Float32Array(slice.length * vectorIndex.dimension);
3103
+ for (let i = 0; i < slice.length; i++) packed.set(vectors[start + i], i * vectorIndex.dimension);
3104
+ const payload = {
3105
+ v: VECTOR_FILE_PAYLOAD_VERSION,
3106
+ fieldName: vectorIndex.fieldName,
3107
+ dimension: vectorIndex.dimension,
3108
+ docIds: slice,
3109
+ codes: null,
3110
+ vectors: vectorsToBytes(packed)
3111
+ };
3112
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
3113
+ }
3114
+ return {
3115
+ unchanged: false,
3116
+ storesHeads: false,
3117
+ kept: [],
3118
+ keptFiles: [],
3119
+ newFiles: files.length,
3120
+ graphParts: null,
3121
+ signature: { revision: 0, graphSize: -1, calibration: null },
3122
+ readNewFile: (index) => files[index],
3123
+ release: () => void 0
3124
+ };
3125
+ }
3126
+ function capture(input) {
3127
+ const partitionIds = input.partitionId === null ? Array.from({ length: input.manager.partitionCount }, (_, partitionId) => partitionId) : [input.partitionId];
3128
+ const { indexName } = input;
3129
+ const writeRecords = input.writeRecords ?? null;
3130
+ const targets = partitionIds.map((partitionId) => {
3131
+ const target = {
3132
+ partitionId,
3133
+ lastSeqNo: input.appliedSeqNo(partitionId),
3134
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
3135
+ };
3136
+ if (writeRecords !== null) target.termHistory = writeRecords.termHistoryOf(indexName, partitionId, target.lastSeqNo);
3137
+ return target;
3138
+ });
3139
+ const contents = partitionIds.map(
3140
+ (partitionId) => partitionContentsOf(
3141
+ input.manager,
3142
+ partitionId,
3143
+ writeRecords === null ? void 0 : (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
3144
+ )
3145
+ );
3146
+ const plans = /* @__PURE__ */ new Map();
3147
+ const partitionDocIds = input.partitionId === null ? null : [...input.manager.getPartition(input.partitionId).docIds()];
3148
+ for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
3149
+ plans.set(
3150
+ fieldPath,
3151
+ partitionDocIds === null ? frozenPlan(vectorIndex.planCheckpoint(null)) : partitionVectorPlan(vectorIndex, partitionDocIds)
3152
+ );
3153
+ }
3154
+ return { targets, contents, plans };
3155
+ }
3156
+ async function buildArchiveFromMemory(input) {
3157
+ const captured = input.partitionId === null ? await whileGraphsStayComplete(input.vectorIndexes, () => capture(input)) : capture(input);
3158
+ const store = memoryCheckpointStore();
3159
+ const written = await writeCheckpointSegments({
3160
+ store,
3161
+ metadata: input.metadata,
3162
+ targets: captured.targets,
3163
+ compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3164
+ prior: null,
3165
+ wholePartitions: captured.contents,
3166
+ keepsWriteRecords: input.writeRecords != null
3167
+ });
3168
+ const vectors = await writeLiveVectors({ store, indexName: input.indexName, plans: captured.plans, priorVectors: [] });
3169
+ const refs = vectors.refs;
3170
+ await commitCheckpointManifest(store, input.metadata, written, refs);
3171
+ const manifestBytes = store.files.get(manifestKey(input.indexName));
3172
+ const decoded = manifestBytes === void 0 ? null : await decodeSegmentManifest(manifestBytes, input.indexName, input.partitionId);
3173
+ if (manifestBytes === void 0 || decoded === null || decoded.kind !== "current") {
3174
+ throw new NarsilError(
3175
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
3176
+ `The archive of index "${input.indexName}" holds no manifest`
3177
+ );
3178
+ }
3179
+ return {
3180
+ manifestBytes,
3181
+ manifest: decoded.manifest,
3182
+ readFile: async (ref) => {
3183
+ const bytes = store.files.get(ref.key);
3184
+ if (bytes === void 0) {
3185
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The archive file "${ref.key}" is missing`, {
3186
+ key: ref.key
3187
+ });
3188
+ }
3189
+ return bytes;
3190
+ },
3191
+ readRange: async (ref, offset, length) => {
3192
+ const bytes = store.files.get(ref.key);
3193
+ if (bytes === void 0 || offset + length > bytes.byteLength) {
3194
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive file "${ref.key}" is missing`, {
3195
+ key: ref.key
3196
+ });
3197
+ }
3198
+ return bytes.subarray(offset, offset + length);
3199
+ },
3200
+ release: () => store.files.clear()
3201
+ };
3202
+ }
3203
+
3204
+ // src/persistence/archive/write.ts
3205
+ function entryChunks(name, bytes) {
3206
+ return [...tarEntryHeaders(name, bytes.byteLength), bytes, tarPadding(bytes.byteLength)].filter(
3207
+ (chunk) => chunk.byteLength > 0
3208
+ );
3209
+ }
3210
+ async function* fileEntryChunks(source, name, ref) {
3211
+ yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
3212
+ for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
3213
+ yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
3214
+ }
3215
+ const padding = tarPadding(ref.bytes);
3216
+ if (padding.byteLength > 0) yield padding;
3217
+ }
3218
+ async function* archiveChunks(source) {
3219
+ yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
3220
+ yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
3221
+ const prefix = `${source.sourceIndexName}/`;
3222
+ for (const ref of manifestFileRefs(source.manifest)) {
3223
+ yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
3224
+ }
3225
+ if (source.trailingEntries !== void 0) {
3226
+ for await (const entry of source.trailingEntries()) yield* entryChunks(entry.name, entry.bytes);
3227
+ }
3228
+ yield tarEnd();
3229
+ }
3230
+ function archiveStream(source) {
3231
+ const chunks = archiveChunks(source);
3232
+ let released = false;
3233
+ const release = () => {
3234
+ if (released) return;
3235
+ released = true;
3236
+ source.release();
3237
+ };
3238
+ return new ReadableStream({
3239
+ async pull(controller) {
3240
+ try {
3241
+ const step = await chunks.next();
3242
+ if (step.done) {
3243
+ release();
3244
+ controller.close();
3245
+ return;
3246
+ }
3247
+ controller.enqueue(step.value);
3248
+ } catch (error) {
3249
+ release();
3250
+ controller.error(error);
3251
+ }
3252
+ },
3253
+ async cancel() {
3254
+ release();
3255
+ await chunks.return(void 0);
3256
+ }
3257
+ });
3258
+ }
3259
+
3260
+ // src/engine/durability-metadata.ts
3261
+ var VECTOR_TYPE = /^vector\[(\d+)\]$/;
3262
+ function vectorFieldsOf(schema, promotion) {
3263
+ const fields = {};
3264
+ for (const [fieldPath, type] of Object.entries(schema)) {
3265
+ const match = VECTOR_TYPE.exec(type);
3266
+ if (match === null) continue;
3267
+ const dimension = Number(match[1]);
3268
+ fields[fieldPath] = {
3269
+ dimension,
3270
+ metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
3271
+ quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
3272
+ };
3273
+ }
3274
+ return Object.keys(fields).length === 0 ? null : fields;
3275
+ }
3276
+ function metadataFromConfig(indexName, config, partitionCount, documentCount, savedBytes) {
3277
+ const metadata = {
3278
+ indexName,
3279
+ schema: config.schema,
3280
+ language: config.language,
3281
+ partitionCount: partitionCount ?? config.partitionCount,
3282
+ bm25Params: {
3283
+ k1: config.k1,
3284
+ b: config.b,
3285
+ ...config.bm25Fields !== void 0 ? { fields: config.bm25Fields } : {}
3286
+ },
3287
+ createdAt: Date.now(),
3288
+ engineVersion: VERSION
3289
+ };
3290
+ metadata.documentCount = documentCount ?? config.documentCount;
3291
+ const recordedSavedBytes = savedBytes ?? config.savedBytes;
3292
+ if (recordedSavedBytes !== void 0) metadata.savedBytes = recordedSavedBytes;
3293
+ const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
3294
+ if (vectorFields !== null) {
3295
+ metadata.vectorFields = vectorFields;
3296
+ }
3297
+ if (config.embedding) {
3298
+ metadata.embedding = config.embedding;
3299
+ }
3300
+ if (config.surfaceForms !== void 0) {
3301
+ metadata.surfaceForms = config.surfaceForms;
3302
+ }
3303
+ if (config.analysisRevision !== void 0) {
3304
+ metadata.analysisRevision = config.analysisRevision;
3305
+ }
3306
+ if (config.arrayPositionGap !== void 0) {
3307
+ metadata.arrayPositionGap = config.arrayPositionGap;
3308
+ }
3309
+ if (config.tokenizer !== void 0) {
3310
+ metadata.tokenizer = config.tokenizer;
3311
+ }
3312
+ if (config.stopWords !== void 0) {
3313
+ metadata.stopWords = config.stopWords;
3314
+ }
3315
+ if (config.stopWordList !== void 0) {
3316
+ metadata.stopWordList = config.stopWordList;
3317
+ }
3318
+ if (config.analysisGeneration !== void 0) {
3319
+ metadata.analysisGeneration = config.analysisGeneration;
3320
+ }
3321
+ if (config.pendingAnalysis !== void 0) {
3322
+ metadata.pendingAnalysis = config.pendingAnalysis;
3323
+ }
3324
+ if (config.partitionLimits !== void 0) {
3325
+ metadata.partitionLimits = config.partitionLimits;
3326
+ }
3327
+ if (config.defaultScoring !== void 0) {
3328
+ metadata.defaultScoring = config.defaultScoring;
3329
+ }
3330
+ if (config.trackPositions !== void 0) {
3331
+ metadata.trackPositions = config.trackPositions;
3332
+ }
3333
+ if (config.textStorage !== void 0) {
3334
+ metadata.textStorage = config.textStorage;
3335
+ }
3336
+ if (config.strict !== void 0) {
3337
+ metadata.strict = config.strict;
3338
+ }
3339
+ if (config.required !== void 0) {
3340
+ metadata.required = config.required;
3341
+ }
3342
+ if (config.vectorPromotion !== void 0) {
3343
+ metadata.vectorPromotion = config.vectorPromotion;
3344
+ }
3345
+ if (config.patternValueLimit !== void 0) {
3346
+ metadata.patternValueLimit = config.patternValueLimit;
3347
+ }
3348
+ if (config.indexUuid !== void 0) {
3349
+ metadata.indexUuid = config.indexUuid;
3350
+ }
3351
+ if (config.synonyms !== void 0) {
3352
+ metadata.synonyms = config.synonyms;
3353
+ }
3354
+ if (config.heldPartitions !== void 0) {
3355
+ metadata.heldPartitions = config.heldPartitions;
3356
+ }
3357
+ if (config.sync !== void 0) metadata.sync = config.sync;
3358
+ if (config.syncActive !== void 0) metadata.syncActive = config.syncActive;
3359
+ if (config.syncEpoch !== void 0) metadata.syncEpoch = config.syncEpoch;
3360
+ if (config.changes !== void 0) metadata.changes = config.changes;
3361
+ if (config.querySuggestions !== void 0) metadata.querySuggestions = config.querySuggestions;
3362
+ return metadata;
3363
+ }
3364
+ function metadataConfigOf(entry) {
3365
+ const embedding = entry.config.embedding ? {
3366
+ fields: entry.config.embedding.fields,
3367
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
3368
+ } : void 0;
3369
+ return {
3370
+ ...entry.indexUuid !== null ? { indexUuid: entry.indexUuid } : {},
3371
+ ...entry.heldPartitions !== null ? { heldPartitions: entry.heldPartitions } : {},
3372
+ documentCount: entry.documentCount,
3373
+ savedBytes: entry.savedBytes,
3374
+ partitionCount: entry.partitionCount,
3375
+ schema: flattenSchema(entry.config.schema),
3376
+ language: entry.language.name,
3377
+ k1: entry.config.bm25?.k1 ?? DEFAULT_BM25_K1,
3378
+ b: entry.config.bm25?.b ?? DEFAULT_BM25_B,
3379
+ ...entry.config.bm25?.fields !== void 0 ? { bm25Fields: entry.config.bm25.fields } : {},
3380
+ ...embedding !== void 0 ? { embedding } : {},
3381
+ surfaceForms: entry.config.surfaceForms !== false,
3382
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
3383
+ ...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
3384
+ ...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
3385
+ ...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
3386
+ ...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
3387
+ ...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
3388
+ ...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
3389
+ ...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
3390
+ ...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
3391
+ ...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
3392
+ ...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
3393
+ ...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
3394
+ ...entry.config.required !== void 0 ? { required: entry.config.required } : {},
3395
+ ...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
3396
+ ...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
3397
+ ...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {},
3398
+ ...syncMetadataOf(entry),
3399
+ ...entry.querySuggestions !== null ? { querySuggestions: entry.querySuggestions } : {}
3400
+ };
3401
+ }
3402
+ function syncMetadataOf(entry) {
3403
+ const { sync, changes } = entry;
3404
+ if (sync === null && changes === null) return {};
3405
+ return {
3406
+ ...sync === null ? {} : {
3407
+ sync: {
3408
+ enabled: true,
3409
+ ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
3410
+ ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {}
3411
+ },
3412
+ syncActive: sync.active
3413
+ },
3414
+ syncEpoch: sync?.epoch ?? changes?.epoch,
3415
+ ...changes === null ? {} : {
3416
+ changes: {
3417
+ enabled: true,
3418
+ ...changes.previousDocument ? { previousDocument: true } : {},
3419
+ ...changes.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: changes.historyMaxAgeMs } : {},
3420
+ ...changes.historyMaxBytes !== void 0 ? { historyMaxBytes: changes.historyMaxBytes } : {}
3421
+ }
3422
+ }
3423
+ };
3424
+ }
3425
+ function durableMetadataConfigOf(entry) {
3426
+ const terms = metadataConfigOf(entry);
3427
+ const durable = entry.durableAnalysis;
3428
+ if (durable === null) return terms;
3429
+ const {
3430
+ language,
3431
+ tokenizer,
3432
+ stopWords,
3433
+ stopWordList,
3434
+ analysisRevision: _analysisRevision,
3435
+ arrayPositionGap: _arrayPositionGap,
3436
+ analysisGeneration: _analysisGeneration,
3437
+ ...rest
3438
+ } = terms;
3439
+ return {
3440
+ ...rest,
3441
+ language: durable.language,
3442
+ ...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
3443
+ ...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
3444
+ ...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
3445
+ ...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
3446
+ ...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
3447
+ ...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
3448
+ pendingAnalysis: {
3449
+ generation: entry.analysisGeneration,
3450
+ language,
3451
+ ...tokenizer !== void 0 ? { tokenizer } : {},
3452
+ ...stopWords !== void 0 ? { stopWords } : {},
3453
+ ...stopWordList !== void 0 ? { stopWordList } : {}
3454
+ }
3455
+ };
3456
+ }
3457
+ function indexMetadataOf(indexName, entry, partitionCount) {
3458
+ return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
3459
+ }
3460
+
3461
+ // src/persistence/archive/file-entry.ts
3462
+ async function writeArchiveFile(store, key, chunks) {
3463
+ const { directory } = store;
3464
+ if (directory === void 0) {
3465
+ const held = [];
3466
+ for await (const chunk of chunks) held.push(chunk);
3467
+ await store.write(key, held);
3468
+ return;
3469
+ }
3470
+ const receiving = `${key}.receiving`;
3471
+ await directory.remove(receiving);
3472
+ const handle = await directory.appendHandle(receiving);
3473
+ try {
3474
+ for await (const chunk of chunks) await handle.append(chunk);
3475
+ await handle.sync();
3476
+ } finally {
3477
+ await handle.close();
3478
+ }
3479
+ await directory.move(receiving, key);
3480
+ }
3481
+
3482
+ // src/persistence/durability/segment/load-partition.ts
3483
+ function requireAgreedSchema(agreed, schema, key) {
3484
+ for (const [field, type] of Object.entries(schema)) {
3485
+ const existing = agreed[field];
3486
+ if (existing !== void 0 && existing !== type) {
3487
+ throw new NarsilError(
3488
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3489
+ `The segment "${key}" gives field "${field}" the type "${type}", while the index gives it "${existing}"`,
3490
+ { key, field, existing, found: type }
3491
+ );
3492
+ }
3493
+ }
3494
+ }
3495
+ function requireListedCounts(ref, documentCount, tombstoneCount) {
3496
+ if (ref.docCount === documentCount && ref.tombstoneCount === tombstoneCount) return;
3497
+ throw new NarsilError(
3498
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3499
+ `The segment "${ref.key}" holds ${documentCount} documents and ${tombstoneCount} tombstones where the manifest lists ${ref.docCount} and ${ref.tombstoneCount}`,
3500
+ {
3501
+ key: ref.key,
3502
+ documentCount,
3503
+ tombstoneCount,
3504
+ listedDocCount: ref.docCount,
3505
+ listedTombstoneCount: ref.tombstoneCount
3506
+ }
3507
+ );
3508
+ }
3509
+ function removeWhatTheSegmentReplaces(manager, segment, contents) {
3510
+ manager.beginBatchRemove();
3511
+ try {
3512
+ for (const docId of contents.tombstones) {
3513
+ if (manager.has(docId)) manager.remove(docId);
3514
+ }
3515
+ const ids = segment.arrays.idTable;
3516
+ for (let ordinal = 0; ordinal < ids.count; ordinal++) {
3517
+ const docId = ids.idAt(ordinal);
3518
+ if (manager.has(docId)) manager.remove(docId);
3519
+ }
3520
+ } finally {
3521
+ manager.endBatchRemove();
3522
+ }
3523
+ }
3524
+ function yieldToEventLoop2() {
3525
+ return new Promise((resolve) => setTimeout(resolve, 0));
3526
+ }
3527
+ function removeFromPartition(manager, partitionId, docIds) {
3528
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
3529
+ composite.beginBatch();
3530
+ try {
3531
+ for (const docId of docIds) {
3532
+ if (!composite.has(docId)) continue;
3533
+ composite.remove(docId, manager.config.schema, manager.language, insertOptions);
3534
+ docPartitionMap.delete(docId);
3535
+ }
3536
+ } finally {
3537
+ composite.endBatch();
3538
+ }
3539
+ }
3540
+ async function attachPagedSegment(directory, ref, partitionId, manager, options) {
3541
+ if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
3542
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
3543
+ requireListedCounts(ref, snapshot.documentCount, snapshot.tombstones.length);
3544
+ requireAgreedSchema(options.schema, snapshot.schema, ref.key);
3545
+ const segment = createPagedFrozenSegment(snapshot);
3546
+ removeFromPartition(manager, partitionId, snapshot.tombstones);
3547
+ options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
3548
+ if (snapshot.documentCount > 0) {
3549
+ manager.attachFrozenSegment(partitionId, segment);
3550
+ return segment;
3551
+ }
3552
+ segment.paged?.close();
3553
+ return null;
3554
+ }
3555
+ function documentsHeldBefore(manager, partitionId) {
3556
+ const held = manager.partitionAt(partitionId);
3557
+ return held === void 0 ? [] : [...held.docIds()];
3558
+ }
3559
+ function removeDocumentsMissingFromLoadedSegments(manager, partitionId, heldBefore, loaded) {
3560
+ const absent = heldBefore.filter(
3561
+ (docId) => manager.partitionIdOf(docId) === partitionId && !loaded.some((segment) => segment.hasDocument(docId))
3562
+ );
3563
+ if (absent.length === 0) return;
3564
+ manager.beginBatchRemove();
3565
+ try {
3566
+ for (const docId of absent) manager.remove(docId);
3567
+ } finally {
3568
+ manager.endBatchRemove();
3569
+ }
3570
+ }
3571
+ async function loadPartitionSegments(store, partition, manager, options) {
3572
+ const heldBefore = documentsHeldBefore(manager, partition.partitionId);
3573
+ const loaded = await attachPartitionSegments(store, partition, manager, options);
3574
+ removeDocumentsMissingFromLoadedSegments(manager, partition.partitionId, heldBefore, loaded);
3575
+ }
3576
+ async function attachPartitionSegments(store, partition, manager, options) {
3577
+ const loaded = [];
3578
+ const { directory } = store;
3579
+ for (const ref of partition.segments) {
3580
+ if (manager.config.textStorage === "disk" && directory !== void 0) {
3581
+ if (await isPagedSegmentFile(directory, ref.key)) {
3582
+ const paged = await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
3583
+ if (paged !== null) loaded.push(paged);
3584
+ await yieldToEventLoop2();
3585
+ continue;
3586
+ }
3587
+ }
3588
+ const contents = await readSegmentFile(store, ref);
3589
+ requireListedCounts(ref, contents.arrays.documentCount, contents.tombstones.length);
3590
+ requireAgreedSchema(options.schema, contents.schema, ref.key);
3591
+ if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
3592
+ const segment = createSharedFrozenSegment(shareSegmentArrays(contents.arrays, manager.partialWordAnalysis));
3593
+ removeWhatTheSegmentReplaces(manager, segment, contents);
3594
+ options.writeRecords?.loadSegment(manager.indexName, partition.partitionId, segment, contents);
3595
+ if (contents.arrays.documentCount > 0) {
3596
+ manager.attachFrozenSegment(partition.partitionId, segment);
3597
+ loaded.push(segment);
3598
+ }
3599
+ await yieldToEventLoop2();
3600
+ }
3601
+ return loaded;
3602
+ }
3603
+
3604
+ // src/persistence/durability/segment/load.ts
3605
+ async function readSegmentManifest(store, indexName) {
3606
+ const bytes = await store.read(manifestKey(indexName));
3607
+ if (bytes === null) return null;
3608
+ return decodeSegmentManifest(bytes, indexName);
3609
+ }
3610
+ async function readCurrentManifest(store, indexName) {
3611
+ const decoded = await readSegmentManifest(store, indexName);
3612
+ if (decoded === null) return null;
3613
+ if (decoded.kind === "current") return decoded.manifest;
3614
+ throw new NarsilError(
3615
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
3616
+ `Index "${indexName}" still holds the layout of an earlier release, which recovery upgrades before any checkpoint`,
3617
+ { indexName }
3618
+ );
3619
+ }
3620
+ function growPartitionsTo(manager, highestPartitionId) {
3621
+ manager.growToSavedPartitionCount(highestPartitionId + 1);
3622
+ }
3623
+ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
3624
+ const highestPartitionId = highestPartitionIdOf(manifest.partitions);
3625
+ requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
3626
+ growPartitionsTo(deps.manager, highestPartitionId);
3627
+ const { directory } = store;
3628
+ const opensVectorsOnDemand = manifest.vectors.some(
3629
+ (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
3630
+ );
3631
+ if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
3632
+ await startReadingOpeningSectionsAhead(directory, manifest);
3633
+ }
3634
+ for (const partition of manifest.partitions) {
3635
+ await loadPartitionSegments(store, partition, deps.manager, {
3636
+ schema: manifest.schema,
3637
+ decodesEveryDocument,
3638
+ writeRecords: deps.writeRecords ?? null
3639
+ });
3640
+ }
3641
+ for (const vector of manifest.vectors) {
3642
+ const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
3643
+ if (vectorIndex === void 0) continue;
3644
+ await loadVectorField(
3645
+ store,
3646
+ vector,
3647
+ vectorIndex,
3648
+ decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
3649
+ );
3650
+ }
3651
+ return manifest.checkpoint;
3652
+ }
3653
+ async function reclaimUnlistedKeys(store, indexName, referenced) {
3654
+ for (const key of await store.list(segmentsPrefix(indexName))) {
3655
+ if (!referenced.has(key)) await store.remove(key);
3656
+ }
3657
+ }
3658
+
3659
+ // src/serialization/stored-vector-values.ts
3660
+ function dropStoredVectorValues(documents, schema) {
3661
+ const vectorFieldPaths = [];
3662
+ for (const [fieldPath, fieldType] of Object.entries(schema)) {
3663
+ if (typeof fieldType === "string" && VECTOR_PATTERN.test(fieldType)) {
3664
+ vectorFieldPaths.push(fieldPath);
3665
+ }
3666
+ }
3667
+ if (vectorFieldPaths.length === 0) {
3668
+ return;
3669
+ }
3670
+ for (const document of Object.values(documents)) {
3671
+ for (const fieldPath of vectorFieldPaths) {
3672
+ removeFieldAtPath(document.fields, fieldPath);
3673
+ }
3674
+ }
3675
+ }
3676
+ function removeFieldAtPath(fields, fieldPath) {
3677
+ if (!fieldPath.includes(".")) {
3678
+ delete fields[fieldPath];
3679
+ return;
3680
+ }
3681
+ const segments = fieldPath.split(".");
3682
+ let current = fields;
3683
+ for (let i = 0; i < segments.length - 1; i += 1) {
3684
+ const next = current[segments[i]];
3685
+ if (next === null || typeof next !== "object") {
3686
+ return;
3687
+ }
3688
+ current = next;
3689
+ }
3690
+ delete current[segments[segments.length - 1]];
3691
+ }
3692
+
3693
+ // src/serialization/payload-v1/partition.ts
3694
+ function sanitizeSurfaceForms(raw) {
3695
+ if (raw === void 0 || raw === null || typeof raw !== "object" || Array.isArray(raw)) return void 0;
3696
+ const result = /* @__PURE__ */ Object.create(null);
3697
+ let count = 0;
3698
+ for (const [surface, value] of Object.entries(raw)) {
3699
+ if (typeof value === "number") {
3700
+ result[surface] = value;
3701
+ count++;
3702
+ } else if (Array.isArray(value) && typeof value[0] === "number" && typeof value[1] === "string") {
3703
+ result[surface] = [value[0], value[1]];
3704
+ count++;
3705
+ }
3706
+ }
3707
+ return count > 0 ? result : void 0;
3708
+ }
3709
+ function isWireRecord(value) {
3710
+ return value !== null && typeof value === "object" && !Array.isArray(value);
3711
+ }
3712
+ function corruptPatternEntry(field) {
3713
+ return new NarsilError(
3714
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3715
+ `The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
3716
+ { field }
3717
+ );
3718
+ }
3719
+ function patternIndexesFromWire(raw) {
3720
+ if (raw === void 0 || raw === null) return {};
3721
+ if (!isWireRecord(raw)) {
3722
+ throw new NarsilError(
3723
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3724
+ "The pattern entry of this partition must be a map of fields"
3725
+ );
3726
+ }
3727
+ const pattern = /* @__PURE__ */ Object.create(null);
3728
+ for (const [field, index] of Object.entries(raw)) {
3729
+ if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
3730
+ throw corruptPatternEntry(field);
3731
+ }
3732
+ const docIds = [];
3733
+ for (const docId of index.doc_ids) {
3734
+ if (typeof docId !== "string") throw corruptPatternEntry(field);
3735
+ docIds.push(docId);
3736
+ }
3737
+ pattern[field] = { docIds, runs: index.runs };
3738
+ }
3739
+ return { pattern };
3740
+ }
3741
+ function validateHnswMetric(value) {
3742
+ if (value === void 0 || value === null) return void 0;
3743
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
3744
+ return value;
3745
+ }
3746
+ return void 0;
3747
+ }
3748
+ function wireToPartition(raw) {
3749
+ const documents = {};
3750
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
3751
+ documents[docId] = {
3752
+ fields: doc.fields,
3753
+ fieldLengths: doc.field_lengths ?? {}
3754
+ };
3755
+ }
3756
+ requireValidFieldTypes(raw.schema ?? {});
3757
+ dropStoredVectorValues(documents, raw.schema ?? {});
3758
+ const invertedIndex = {};
3759
+ for (const [token, list] of Object.entries(raw.inverted_index ?? {})) {
3760
+ invertedIndex[token] = {
3761
+ docFrequency: list.doc_freq,
3762
+ postings: (list.postings ?? []).map((p) => ({
3763
+ docId: p.doc_id,
3764
+ termFrequency: p.term_freq,
3765
+ field: p.field,
3766
+ positions: p.positions ?? []
3767
+ }))
3768
+ };
3769
+ }
3770
+ const numeric = {};
3771
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
3772
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
3773
+ }
3774
+ const boolean = {};
3775
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
3776
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
3777
+ }
3778
+ const geopoint = {};
3779
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
3780
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
3781
+ }
3782
+ const vectorData = {};
3783
+ for (const [field, data] of Object.entries(raw.vector_data ?? {})) {
3784
+ vectorData[field] = {
3785
+ dimension: data.dimension,
3786
+ vectors: (data.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
3787
+ hnswGraph: data.hnsw_graph ? {
3788
+ entryPoint: data.hnsw_graph.entry_point,
3789
+ maxLayer: data.hnsw_graph.max_layer,
3790
+ m: data.hnsw_graph.m,
3791
+ efConstruction: data.hnsw_graph.ef_construction,
3792
+ metric: validateHnswMetric(data.hnsw_graph.metric),
3793
+ nodes: data.hnsw_graph.nodes
3794
+ } : null,
3795
+ codes: data.codes ? { bits: data.codes.bits, centroid: data.codes.centroid, records: data.codes.records } : null
3796
+ };
3797
+ }
3798
+ return {
3799
+ indexName: raw.index_name,
3800
+ partitionId: raw.partition_id,
3801
+ totalPartitions: raw.total_partitions,
3802
+ language: raw.language ?? "english",
3803
+ schema: raw.schema ?? {},
3804
+ docCount: raw.doc_count ?? 0,
3805
+ avgDocLength: raw.avg_doc_length ?? 0,
3806
+ documents,
3807
+ invertedIndex,
3808
+ fieldIndexes: {
3809
+ numeric,
3810
+ boolean,
3811
+ enum: raw.field_indexes?.enum ?? {},
3812
+ geopoint,
3813
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
3814
+ },
3815
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
3816
+ vectorData,
3817
+ statistics: {
3818
+ totalDocuments: raw.statistics?.total_documents ?? 0,
3819
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
3820
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
3821
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
3822
+ }
3823
+ };
3824
+ }
3825
+ function deserializePayloadV1(data) {
3826
+ const raw = decode(data);
3827
+ return wireToPartition(raw);
3828
+ }
3829
+
3830
+ // src/serialization/payload-v1/sync-metadata.ts
3831
+ function invalid(message, details) {
3832
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The device sync state in the index metadata ${message}`, details);
3833
+ }
3834
+ function invalidChanges(message, details) {
3835
+ throw new NarsilError(
3836
+ ErrorCodes.CONFIG_INVALID,
3837
+ `The change capture setting in the index metadata ${message}`,
3838
+ details
3839
+ );
3840
+ }
3841
+ function isPositiveSafeInteger(value) {
3842
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
3843
+ }
3844
+ function historyLimitsToWire(setting) {
3845
+ return {
3846
+ ...setting.historyMaxAgeMs !== void 0 ? { history_max_age_ms: setting.historyMaxAgeMs } : {},
3847
+ ...setting.historyMaxBytes !== void 0 ? { history_max_bytes: setting.historyMaxBytes } : {}
3848
+ };
3849
+ }
3850
+ function syncMetadataToWire(meta) {
3851
+ const wire = {};
3852
+ if (meta.sync !== void 0) {
3853
+ wire.sync = { enabled: meta.sync.enabled, ...historyLimitsToWire(meta.sync) };
3854
+ }
3855
+ if (meta.syncActive !== void 0) wire.sync_active = meta.syncActive;
3856
+ if (meta.syncEpoch !== void 0) wire.sync_epoch = meta.syncEpoch;
3857
+ if (meta.changes !== void 0) {
3858
+ wire.changes = {
3859
+ enabled: meta.changes.enabled,
3860
+ ...meta.changes.previousDocument !== void 0 ? { previous_document: meta.changes.previousDocument } : {},
3861
+ ...historyLimitsToWire(meta.changes)
3862
+ };
3863
+ }
3864
+ return wire;
3865
+ }
3866
+ function historyLimitsFromWire(raw, fail) {
3867
+ const { history_max_age_ms: maxAge, history_max_bytes: maxBytes } = raw;
3868
+ if (maxAge !== void 0 && !isPositiveSafeInteger(maxAge)) {
3869
+ fail("holds a history age limit that is not a positive whole number", { historyMaxAgeMs: String(maxAge) });
3870
+ }
3871
+ if (maxBytes !== void 0 && !isPositiveSafeInteger(maxBytes)) {
3872
+ fail("holds a history byte limit that is not a positive whole number", { historyMaxBytes: String(maxBytes) });
3873
+ }
3874
+ return {
3875
+ ...maxAge !== void 0 ? { historyMaxAgeMs: maxAge } : {},
3876
+ ...maxBytes !== void 0 ? { historyMaxBytes: maxBytes } : {}
3877
+ };
3878
+ }
3879
+ function storedSyncSetting(raw) {
3880
+ if (raw === null || typeof raw !== "object") invalid("holds a sync setting that is not a map", {});
3881
+ const fields = raw;
3882
+ if (typeof fields.enabled !== "boolean") invalid("holds a sync setting without a boolean enabled", {});
3883
+ return { enabled: fields.enabled, ...historyLimitsFromWire(fields, invalid) };
3884
+ }
3885
+ function storedChangesSetting(raw) {
3886
+ if (raw === null || typeof raw !== "object") invalidChanges("is not a map", {});
3887
+ const fields = raw;
3888
+ if (typeof fields.enabled !== "boolean") invalidChanges("lacks a boolean enabled", {});
3889
+ const previousDocument = fields.previous_document;
3890
+ if (previousDocument !== void 0 && typeof previousDocument !== "boolean") {
3891
+ invalidChanges("holds a previous_document that is not a boolean", {});
3892
+ }
3893
+ return {
3894
+ enabled: fields.enabled,
3895
+ ...previousDocument !== void 0 ? { previousDocument } : {},
3896
+ ...historyLimitsFromWire(fields, invalidChanges)
3897
+ };
3898
+ }
3899
+ function syncMetadataFromWire(raw, meta) {
3900
+ if (raw.sync !== void 0 && raw.sync !== null) meta.sync = storedSyncSetting(raw.sync);
3901
+ if (raw.sync_active !== void 0 && raw.sync_active !== null) {
3902
+ if (typeof raw.sync_active !== "boolean") invalid("holds a sync_active that is not a boolean", {});
3903
+ meta.syncActive = raw.sync_active;
3904
+ }
3905
+ if (raw.sync_epoch !== void 0 && raw.sync_epoch !== null) {
3906
+ if (!isSyncEpoch(raw.sync_epoch)) invalid("holds an epoch that is not 32 lowercase hexadecimal digits", {});
3907
+ meta.syncEpoch = raw.sync_epoch;
3908
+ }
3909
+ if (raw.changes !== void 0 && raw.changes !== null) meta.changes = storedChangesSetting(raw.changes);
3910
+ }
3911
+
3912
+ // src/serialization/payload-v1/metadata.ts
3913
+ function metadataToWire(meta) {
3914
+ const wire = {
3915
+ index_name: meta.indexName,
3916
+ schema: meta.schema,
3917
+ language: meta.language,
3918
+ partition_count: meta.partitionCount,
3919
+ bm25_params: meta.bm25Params,
3920
+ created_at: meta.createdAt,
3921
+ engine_version: meta.engineVersion
3922
+ };
3923
+ if (meta.vectorFields) {
3924
+ wire.vector_fields = meta.vectorFields;
3925
+ }
3926
+ if (isNonNegativeInteger(meta.documentCount) && Number.isSafeInteger(meta.documentCount)) {
3927
+ wire.document_count = meta.documentCount;
3928
+ }
3929
+ if (isNonNegativeInteger(meta.savedBytes) && Number.isSafeInteger(meta.savedBytes)) {
3930
+ wire.saved_bytes = meta.savedBytes;
3931
+ }
3932
+ if (meta.embedding) {
3933
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
3934
+ }
3935
+ if (meta.surfaceForms !== void 0) {
3936
+ wire.surface_forms_enabled = meta.surfaceForms;
3937
+ }
3938
+ if (meta.analysisRevision !== void 0) {
3939
+ wire.analysis_revision = meta.analysisRevision;
3940
+ }
3941
+ if (meta.arrayPositionGap !== void 0) {
3942
+ wire.array_position_gap = meta.arrayPositionGap;
3943
+ }
3944
+ if (meta.tokenizer !== void 0) {
3945
+ wire.tokenizer = meta.tokenizer;
3946
+ }
3947
+ if (meta.stopWords !== void 0) {
3948
+ wire.stop_words = meta.stopWords;
3949
+ }
3950
+ if (meta.stopWordList !== void 0) {
3951
+ wire.stop_word_list = meta.stopWordList;
3952
+ }
3953
+ if (meta.analysisGeneration !== void 0) {
3954
+ wire.analysis_generation = meta.analysisGeneration;
3955
+ }
3956
+ if (meta.pendingAnalysis !== void 0) {
3957
+ const pending = meta.pendingAnalysis;
3958
+ wire.pending_analysis = {
3959
+ generation: pending.generation,
3960
+ language: pending.language,
3961
+ ...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
3962
+ ...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
3963
+ ...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
3964
+ };
3965
+ }
3966
+ if (meta.partitionLimits !== void 0) {
3967
+ wire.partition_limits = {
3968
+ ...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
3969
+ ...meta.partitionLimits.maxPartitions !== void 0 ? { max_partitions: meta.partitionLimits.maxPartitions } : {},
3970
+ ...meta.partitionLimits.watermark !== void 0 ? { watermark: meta.partitionLimits.watermark } : {}
3971
+ };
3972
+ }
3973
+ if (meta.defaultScoring !== void 0) {
3974
+ wire.default_scoring = meta.defaultScoring;
3975
+ }
3976
+ if (meta.trackPositions !== void 0) {
3977
+ wire.track_positions = meta.trackPositions;
3978
+ }
3979
+ if (meta.textStorage !== void 0) {
3980
+ wire.text_storage = meta.textStorage;
3981
+ }
3982
+ if (meta.strict !== void 0) {
3983
+ wire.strict = meta.strict;
3984
+ }
3985
+ if (meta.required !== void 0) {
3986
+ wire.required = meta.required;
3987
+ }
3988
+ if (meta.vectorPromotion !== void 0) {
3989
+ const promotion = meta.vectorPromotion;
3990
+ wire.vector_promotion = {
3991
+ ...promotion.threshold !== void 0 ? { threshold: promotion.threshold } : {},
3992
+ ...promotion.filterThreshold !== void 0 ? { filter_threshold: promotion.filterThreshold } : {},
3993
+ ...promotion.hnswConfig !== void 0 ? {
3994
+ hnsw_config: {
3995
+ ...promotion.hnswConfig.m !== void 0 ? { m: promotion.hnswConfig.m } : {},
3996
+ ...promotion.hnswConfig.efConstruction !== void 0 ? { ef_construction: promotion.hnswConfig.efConstruction } : {},
3997
+ ...promotion.hnswConfig.metric !== void 0 ? { metric: promotion.hnswConfig.metric } : {}
3998
+ }
3999
+ } : {},
4000
+ ...promotion.quantization !== void 0 ? { quantization: promotion.quantization } : {},
4001
+ ...promotion.storage !== void 0 ? { storage: promotion.storage } : {}
4002
+ };
4003
+ }
4004
+ if (meta.patternValueLimit !== void 0) {
4005
+ wire.pattern_value_limit = meta.patternValueLimit;
4006
+ }
4007
+ if (meta.indexUuid !== void 0) {
4008
+ wire.index_uuid = meta.indexUuid;
4009
+ }
4010
+ if (meta.heldPartitions !== void 0) {
4011
+ wire.held_partitions = meta.heldPartitions;
4012
+ }
4013
+ if (Array.isArray(meta.synonyms) && meta.synonyms.length > 0) {
4014
+ wire.synonyms = meta.synonyms;
4015
+ }
4016
+ return {
4017
+ ...wire,
4018
+ ...syncMetadataToWire(meta),
4019
+ ...meta.querySuggestions !== void 0 ? storedStateToWire(meta.querySuggestions) : {}
4020
+ };
4021
+ }
4022
+ function wireToMetadata(raw) {
4023
+ const meta = {
4024
+ indexName: raw.index_name,
4025
+ schema: raw.schema ?? {},
4026
+ language: raw.language ?? "english",
4027
+ partitionCount: raw.partition_count ?? 1,
4028
+ bm25Params: raw.bm25_params ?? { k1: DEFAULT_BM25_K1, b: DEFAULT_BM25_B },
4029
+ createdAt: raw.created_at ?? 0,
4030
+ engineVersion: raw.engine_version ?? "0.0.0"
4031
+ };
4032
+ if (raw.vector_fields) {
4033
+ for (const [fieldPath, field] of Object.entries(raw.vector_fields)) {
4034
+ if (!isQuantizationMode(field.quantization)) {
4035
+ throw new NarsilError(
4036
+ ErrorCodes.CONFIG_INVALID,
4037
+ `Index metadata names a vector quantization "${String(field.quantization)}" for "${fieldPath}" this engine does not read`,
4038
+ { fieldPath, quantization: field.quantization }
4039
+ );
4040
+ }
4041
+ }
4042
+ meta.vectorFields = raw.vector_fields;
4043
+ }
4044
+ if (isNonNegativeInteger(raw.document_count) && Number.isSafeInteger(raw.document_count)) {
4045
+ meta.documentCount = raw.document_count;
4046
+ }
4047
+ if (isNonNegativeInteger(raw.saved_bytes) && Number.isSafeInteger(raw.saved_bytes)) {
4048
+ meta.savedBytes = raw.saved_bytes;
4049
+ }
4050
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
4051
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
4052
+ }
4053
+ if (typeof raw.surface_forms_enabled === "boolean") {
4054
+ meta.surfaceForms = raw.surface_forms_enabled;
4055
+ }
4056
+ if (typeof raw.analysis_revision === "string") {
4057
+ meta.analysisRevision = raw.analysis_revision;
4058
+ }
4059
+ if (typeof raw.array_position_gap === "number") {
4060
+ meta.arrayPositionGap = raw.array_position_gap;
4061
+ }
4062
+ if (typeof raw.tokenizer === "string") {
4063
+ meta.tokenizer = raw.tokenizer;
4064
+ }
4065
+ if (typeof raw.stop_words === "string") {
4066
+ meta.stopWords = raw.stop_words;
4067
+ }
4068
+ if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
4069
+ meta.stopWordList = raw.stop_word_list;
4070
+ }
4071
+ if (isPositiveInteger(raw.analysis_generation)) {
4072
+ meta.analysisGeneration = raw.analysis_generation;
4073
+ }
4074
+ const pending = raw.pending_analysis;
4075
+ if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
4076
+ meta.pendingAnalysis = {
4077
+ generation: pending.generation,
4078
+ language: pending.language,
4079
+ ...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
4080
+ ...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
4081
+ ...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
4082
+ };
4083
+ }
4084
+ if (raw.partition_limits && typeof raw.partition_limits === "object") {
4085
+ const limits = {
4086
+ ...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
4087
+ ...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: Math.min(raw.partition_limits.max_partitions, MAX_PARTITIONS) } : {},
4088
+ ...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
4089
+ };
4090
+ if (Object.keys(limits).length > 0) {
4091
+ meta.partitionLimits = limits;
4092
+ }
4093
+ }
4094
+ if (raw.default_scoring === "local" || raw.default_scoring === "dfs" || raw.default_scoring === "broadcast") {
4095
+ meta.defaultScoring = raw.default_scoring;
4096
+ }
4097
+ if (typeof raw.track_positions === "boolean") {
4098
+ meta.trackPositions = raw.track_positions;
4099
+ }
4100
+ if (raw.text_storage !== void 0) {
4101
+ if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
4102
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
4103
+ textStorage: raw.text_storage
4104
+ });
4105
+ }
4106
+ meta.textStorage = raw.text_storage;
4107
+ }
4108
+ if (typeof raw.strict === "boolean") {
4109
+ meta.strict = raw.strict;
4110
+ }
4111
+ if (Array.isArray(raw.required) && raw.required.every((field) => typeof field === "string")) {
4112
+ meta.required = raw.required;
4113
+ }
4114
+ if (raw.vector_promotion && typeof raw.vector_promotion === "object") {
4115
+ const promotion = raw.vector_promotion;
4116
+ const hnsw = promotion.hnsw_config && typeof promotion.hnsw_config === "object" ? promotion.hnsw_config : void 0;
4117
+ const metric = hnsw?.metric;
4118
+ const restored = {};
4119
+ const hnswConfig = {};
4120
+ if (isPositiveInteger(hnsw?.m)) {
4121
+ hnswConfig.m = hnsw?.m;
4122
+ }
4123
+ if (isPositiveInteger(hnsw?.ef_construction)) {
4124
+ hnswConfig.efConstruction = hnsw?.ef_construction;
4125
+ }
4126
+ if (metric === "cosine" || metric === "dotProduct" || metric === "euclidean") {
4127
+ hnswConfig.metric = metric;
4128
+ }
4129
+ if (isPositiveInteger(promotion.threshold)) {
4130
+ restored.threshold = promotion.threshold;
4131
+ }
4132
+ if (isSelectivityRatio(promotion.filter_threshold)) {
4133
+ restored.filterThreshold = promotion.filter_threshold;
4134
+ }
4135
+ if (Object.keys(hnswConfig).length > 0) {
4136
+ restored.hnswConfig = hnswConfig;
4137
+ }
4138
+ if (promotion.quantization !== void 0) {
4139
+ if (!isQuantizationMode(promotion.quantization)) {
4140
+ throw new NarsilError(
4141
+ ErrorCodes.CONFIG_INVALID,
4142
+ `Index metadata names a vector quantization "${String(promotion.quantization)}" this engine does not read`,
4143
+ { quantization: promotion.quantization }
4144
+ );
4145
+ }
4146
+ restored.quantization = promotion.quantization;
4147
+ }
4148
+ if (promotion.storage !== void 0) {
4149
+ if (!isStorageMode(promotion.storage)) {
4150
+ throw new NarsilError(
4151
+ ErrorCodes.CONFIG_INVALID,
4152
+ `Index metadata names a vector storage "${String(promotion.storage)}" this engine does not read`,
4153
+ { storage: promotion.storage }
4154
+ );
4155
+ }
4156
+ restored.storage = promotion.storage;
4157
+ }
4158
+ if (Object.keys(restored).length > 0) {
4159
+ meta.vectorPromotion = restored;
4160
+ }
4161
+ }
4162
+ if (raw.pattern_value_limit !== void 0) {
4163
+ if (!isPositiveInteger(raw.pattern_value_limit)) {
4164
+ throw new NarsilError(
4165
+ ErrorCodes.CONFIG_INVALID,
4166
+ `The pattern value limit in the index metadata is ${String(raw.pattern_value_limit)}, but the limit must be a whole number of code points`,
4167
+ { patternValueLimit: String(raw.pattern_value_limit) }
4168
+ );
4169
+ }
4170
+ meta.patternValueLimit = raw.pattern_value_limit;
4171
+ }
4172
+ if (typeof raw.index_uuid === "string" && raw.index_uuid.length > 0) {
4173
+ meta.indexUuid = raw.index_uuid;
4174
+ }
4175
+ if (Array.isArray(raw.held_partitions) && raw.held_partitions.every(isNonNegativeInteger)) {
4176
+ meta.heldPartitions = raw.held_partitions;
4177
+ }
4178
+ if (raw.synonyms !== void 0 && raw.synonyms !== null) meta.synonyms = raw.synonyms;
4179
+ syncMetadataFromWire(raw, meta);
4180
+ const querySuggestions = storedStateFromWire(raw);
4181
+ if (querySuggestions !== null) meta.querySuggestions = querySuggestions;
4182
+ return meta;
4183
+ }
4184
+ function isNonNegativeInteger(value) {
4185
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
4186
+ }
4187
+ function isStringList(value) {
4188
+ return Array.isArray(value) && value.every((word) => typeof word === "string");
4189
+ }
4190
+ function isPositiveInteger(value) {
4191
+ return typeof value === "number" && Number.isInteger(value) && value >= 1;
4192
+ }
4193
+ function isSelectivityRatio(value) {
4194
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1;
4195
+ }
4196
+ function isWatermarkFraction(value) {
4197
+ return typeof value === "number" && value > 0 && value <= 1;
4198
+ }
4199
+ function serializeMetadata(meta) {
4200
+ const wire = metadataToWire(meta);
4201
+ return encode(wire);
4202
+ }
4203
+ function deserializeMetadata(data) {
4204
+ const raw = decode(data);
4205
+ return wireToMetadata(raw);
4206
+ }
4207
+
4208
+ // src/serialization/envelope.ts
4209
+ var CURRENT_ENVELOPE_VERSION = 1;
4210
+ var SNAPSHOT_ENVELOPE_VERSION = 2;
4211
+ var INDEX_SNAPSHOT_ENVELOPE_VERSION = 3;
4212
+ function buildFlags(options) {
4213
+ const useCompression = options.compression === "gzip";
4214
+ return {
4215
+ compressionEnabled: useCompression,
4216
+ compressionAlgorithm: useCompression ? "gzip" : "none",
4217
+ checksumPresent: options.checksum ?? false,
4218
+ encryptionEnabled: false
4219
+ };
4220
+ }
4221
+ async function packEnvelope(payloadBytes, options) {
4222
+ const flags = buildFlags(options);
4223
+ let finalPayload = payloadBytes;
4224
+ if (flags.compressionEnabled) {
4225
+ const { gzip } = await loadGzip();
4226
+ finalPayload = await gzip(payloadBytes);
4227
+ }
4228
+ const checksum = flags.checksumPresent ? crc32(finalPayload) : 0;
4229
+ const [major, minor, patch] = engineVersionParts();
4230
+ const envelopeFormatVersion = options.envelopeFormatVersion ?? CURRENT_ENVELOPE_VERSION;
4231
+ const header = {
4232
+ magic: "NRSL",
4233
+ envelopeFormatVersion,
4234
+ engineVersionMajor: major,
4235
+ engineVersionMinor: minor,
4236
+ engineVersionPatch: patch,
4237
+ payloadLength: finalPayload.length,
4238
+ flags,
4239
+ checksum,
4240
+ reserved: new Uint8Array(14)
4241
+ };
4242
+ const headerBytes = writeHeader(header);
4243
+ const envelope = new Uint8Array(HEADER_SIZE + finalPayload.length);
4244
+ envelope.set(headerBytes, 0);
4245
+ envelope.set(finalPayload, HEADER_SIZE);
4246
+ return envelope;
4247
+ }
4248
+ async function unpackEnvelope(data, maxAcceptedVersion = CURRENT_ENVELOPE_VERSION) {
4249
+ const header = readHeader(data);
4250
+ if (header.envelopeFormatVersion > maxAcceptedVersion) {
4251
+ throw new NarsilError(
4252
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
4253
+ `This data holds Narsil envelope format v${header.envelopeFormatVersion}, while this build of @delali/narsil reads up to v${maxAcceptedVersion}, so upgrade the package to read it.`
4254
+ );
4255
+ }
4256
+ if (header.flags.encryptionEnabled) {
4257
+ throw new NarsilError(
4258
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
4259
+ "Encrypted envelopes are not supported in this version of Narsil"
4260
+ );
4261
+ }
4262
+ const compAlgo = header.flags.compressionAlgorithm;
4263
+ if (header.flags.compressionEnabled && compAlgo !== "gzip" && compAlgo !== "none") {
4264
+ throw new NarsilError(
4265
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
4266
+ `Compression algorithm "${compAlgo}" is not supported in this version of Narsil`
4267
+ );
4268
+ }
4269
+ const availablePayload = data.length - HEADER_SIZE;
4270
+ if (availablePayload < header.payloadLength) {
4271
+ throw new NarsilError(
4272
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
4273
+ `Envelope data is truncated: header declares ${header.payloadLength} bytes of payload but only ${availablePayload} bytes are available`
4274
+ );
4275
+ }
4276
+ let payloadBytes = new Uint8Array(data.slice(HEADER_SIZE, HEADER_SIZE + header.payloadLength));
4277
+ if (header.flags.checksumPresent) {
4278
+ const computed = crc32(payloadBytes);
4279
+ if (computed !== header.checksum) {
4280
+ throw new NarsilError(ErrorCodes.PERSISTENCE_CRC_MISMATCH, "CRC32 checksum mismatch: data may be corrupted", {
4281
+ expected: header.checksum,
4282
+ computed
4283
+ });
4284
+ }
4285
+ }
4286
+ if (header.flags.compressionEnabled) {
4287
+ const { gunzip } = await loadGzip();
4288
+ payloadBytes = await gunzip(payloadBytes);
4289
+ }
4290
+ return { header, payloadBytes };
4291
+ }
4292
+ async function unpackEnvelopeBytes(data) {
4293
+ return unpackEnvelope(data, SNAPSHOT_ENVELOPE_VERSION);
4294
+ }
4295
+ async function unpackIndexSnapshotEnvelope(data) {
4296
+ const { header, payloadBytes } = await unpackEnvelope(data, INDEX_SNAPSHOT_ENVELOPE_VERSION);
4297
+ if (header.envelopeFormatVersion !== INDEX_SNAPSHOT_ENVELOPE_VERSION) {
4298
+ throw new NarsilError(
4299
+ ErrorCodes.ENVELOPE_VERSION_MISMATCH,
4300
+ `Expected an index snapshot envelope of format v${INDEX_SNAPSHOT_ENVELOPE_VERSION}, received format v${header.envelopeFormatVersion}`
4301
+ );
4302
+ }
4303
+ return payloadBytes;
4304
+ }
4305
+ async function writeMetadataEnvelope(meta, options = {}) {
4306
+ const payloadBytes = serializeMetadata(meta);
4307
+ return packEnvelope(payloadBytes, options);
4308
+ }
4309
+ async function readMetadataEnvelope(data) {
4310
+ const { header, payloadBytes } = await unpackEnvelope(data);
4311
+ const metadata = deserializeMetadata(payloadBytes);
4312
+ return { header, metadata };
4313
+ }
4314
+
4315
+ // src/distribution/replication/replica.ts
4316
+ function validateReplicationEntry(entry, localPrimaryTerm, log) {
4317
+ if (!log.verifyChecksum(entry)) {
4318
+ return { valid: false, error: ErrorCodes.REPLICATION_ENTRY_CORRUPT };
4319
+ }
4320
+ if (entry.primaryTerm < localPrimaryTerm) {
4321
+ return { valid: false, error: ErrorCodes.REPLICATION_TERM_MISMATCH };
4322
+ }
4323
+ return { valid: true };
4324
+ }
4325
+ function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
4326
+ if (entry.document === null) {
4327
+ return;
4328
+ }
4329
+ const document = decode(entry.document);
4330
+ restoreVectorFields(document, vectorFieldPaths);
4331
+ const documentId = entry.documentId;
4332
+ if (manager.has(documentId)) {
4333
+ removeDocumentVectors(documentId, vecIndexes);
4334
+ manager.remove(documentId);
4335
+ }
4336
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths);
4337
+ manager.insert(documentId, partitionDoc);
4338
+ try {
4339
+ insertDocumentVectors(documentId, extractedVectors, vecIndexes, manager.partitionIdOf(documentId));
4340
+ } catch (err) {
4341
+ manager.remove(documentId);
4342
+ throw err;
4343
+ }
4344
+ if (!schedulesGraphBuild) return;
4345
+ for (const fieldPath of extractedVectors.keys()) {
4346
+ const vecIndex = vecIndexes.get(fieldPath);
4347
+ if (vecIndex) {
4348
+ vecIndex.scheduleBuild();
4349
+ }
4350
+ }
4351
+ }
4352
+ function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
4353
+ removeDocumentVectors(entry.documentId, vecIndexes);
4354
+ if (!manager.has(entry.documentId)) {
4355
+ return;
4356
+ }
4357
+ manager.remove(entry.documentId, removedSpellings);
4358
+ }
4359
+
4360
+ // src/workers/detached-index.ts
4361
+ function attachablePolicy(shared) {
4362
+ let attached = false;
4363
+ const policy = {
4364
+ get enabled() {
4365
+ return attached && shared.enabled;
4366
+ },
4367
+ get count() {
4368
+ return shared.count;
4369
+ },
4370
+ get host() {
4371
+ return shared.host;
4372
+ }
4373
+ };
4374
+ return {
4375
+ policy,
4376
+ attach() {
4377
+ attached = true;
4378
+ }
4379
+ };
4380
+ }
4381
+ function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
4382
+ const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
4383
+ const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
4384
+ entry.manager.synonyms = synonyms;
4385
+ return { indexName, manager: entry.manager, entry, attach };
4386
+ }
4387
+
4388
+ // src/persistence/durability/marker-writer.ts
4389
+ function markerKey(indexName, partitionId) {
4390
+ return `${indexName}/wal/${partitionId}/commit`;
4391
+ }
4392
+ async function createMarkerWriter(directory, indexName, partitionId) {
4393
+ const key = markerKey(indexName, partitionId);
4394
+ const handle = await directory.markerHandle(key);
4395
+ const existing = await handle.read();
4396
+ const current = readCommitMarker(existing);
4397
+ const createdFresh = existing.length === 0;
4398
+ let lastSlot = current?.slotIndex ?? null;
4399
+ let writeSeq = current?.state.writeSeq ?? 0;
4400
+ return {
4401
+ created: createdFresh,
4402
+ existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
4403
+ async commit(state) {
4404
+ const slotIndex = nextMarkerSlot(lastSlot);
4405
+ writeSeq += 1;
4406
+ const slot = encodeMarkerSlot({ ...state, writeSeq });
4407
+ await handle.writeSlot(markerSlotOffset(slotIndex), slot);
4408
+ lastSlot = slotIndex;
4409
+ },
4410
+ async close() {
4411
+ await handle.close();
4412
+ }
4413
+ };
4414
+ }
4415
+
4416
+ // src/persistence/durability/recovery.ts
4417
+ async function removeLeftoverRewriteFiles(directory, names) {
4418
+ for (const indexName of names) {
4419
+ for (const place of ["rebuilt", "received"]) {
4420
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
4421
+ }
4422
+ }
4423
+ }
4424
+ async function listPersistedIndexes(directory) {
4425
+ const metaKeys = await directory.list("");
4426
+ const names = [];
4427
+ for (const key of metaKeys) {
4428
+ if (key.endsWith("/meta")) {
4429
+ names.push(key.slice(0, -"/meta".length));
4430
+ }
4431
+ }
4432
+ return names;
4433
+ }
4434
+ async function loadMetadata(directory, indexName) {
4435
+ const bytes = await directory.read(`${indexName}/meta`);
4436
+ if (bytes === null) {
4437
+ return null;
4438
+ }
4439
+ const { metadata } = await readMetadataEnvelope(bytes);
4440
+ return metadata;
4441
+ }
4442
+ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
4443
+ await removeUnusedSegmentFiles(directory, indexName, partitionId);
4444
+ const read = await readWalSegments(directory, indexName, partitionId);
4445
+ if (read === null) {
4446
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
4447
+ }
4448
+ const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
4449
+ if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
4450
+ throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
4451
+ }
4452
+ let highestSeqNo = fromSeqNoExclusive;
4453
+ let highestPrimaryTerm = 0;
4454
+ const replay = async (entries) => {
4455
+ for (const entry of entries) {
4456
+ if (entry.seqNo <= highestSeqNo) {
4457
+ continue;
4458
+ }
4459
+ if (turnIsSpent()) await yieldTurn();
4460
+ applyEntry2(entry, deps);
4461
+ highestSeqNo = entry.seqNo;
4462
+ if (entry.primaryTerm > highestPrimaryTerm) {
4463
+ highestPrimaryTerm = entry.primaryTerm;
4464
+ }
4465
+ }
4466
+ };
4467
+ await replay(durableEntries);
4468
+ if (activeTail !== null) {
4469
+ await replay(activeTail.entries);
4470
+ }
4471
+ await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
4472
+ if (activeTail !== null) {
4473
+ await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
4474
+ }
4475
+ return { highestSeqNo, highestPrimaryTerm };
4476
+ }
4477
+ async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
4478
+ const lastReplayed = tail.entries.at(-1);
4479
+ if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
4480
+ const handle = await directory.appendHandle(tail.key);
4481
+ try {
4482
+ if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
4483
+ else await handle.sync();
4484
+ } finally {
4485
+ await handle.close();
4486
+ }
4487
+ if (lastReplayed === void 0) return;
4488
+ const writer = await createMarkerWriter(directory, indexName, partitionId);
4489
+ try {
4490
+ await writer.commit({
4491
+ activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
4492
+ durableByteLength: tail.cleanEnd,
4493
+ highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
4494
+ });
4495
+ } finally {
4496
+ await writer.close();
4497
+ }
4498
+ }
4499
+ async function replayWalUpTo(directory, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps) {
4500
+ const read = await readWalSegments(directory, indexName, partitionId);
4501
+ if (read === null) {
4502
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
4503
+ }
4504
+ return replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps);
4505
+ }
4506
+ async function replaySyncedWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
4507
+ await removeUnusedSegmentFiles(directory, indexName, partitionId);
4508
+ const read = await readWalSegments(directory, indexName, partitionId);
4509
+ if (read === null) {
4510
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
4511
+ }
4512
+ const replayed = replayDurableEntries(
4513
+ read,
4514
+ indexName,
4515
+ partitionId,
4516
+ fromSeqNoExclusive,
4517
+ Number.POSITIVE_INFINITY,
4518
+ deps
4519
+ );
4520
+ await deleteOrphanSegments(directory, read.segments, read.marker.state.activeSegmentSeqNo);
4521
+ if (read.activeTail !== null) {
4522
+ await truncateSegmentTail(directory, read.activeTail.key, read.marker.state.durableByteLength);
4523
+ }
4524
+ return replayed;
4525
+ }
4526
+ function replayDurableEntries(read, indexName, partitionId, fromSeqNoExclusive, upToSeqNoInclusive, deps) {
4527
+ if (Math.max(fromSeqNoExclusive, read.highestReadFromWal) < read.marker.state.highestDurableSeqNo) {
4528
+ throw durableRecordMissing(indexName, partitionId, read.highestReadFromWal, read.marker);
4529
+ }
4530
+ let highestSeqNo = fromSeqNoExclusive;
4531
+ let highestPrimaryTerm = 0;
4532
+ for (const entry of read.durableEntries) {
4533
+ if (entry.seqNo <= fromSeqNoExclusive || entry.seqNo > upToSeqNoInclusive) {
4534
+ continue;
4535
+ }
4536
+ applyEntry2(entry, deps);
4537
+ if (entry.seqNo > highestSeqNo) {
4538
+ highestSeqNo = entry.seqNo;
4539
+ }
4540
+ if (entry.primaryTerm > highestPrimaryTerm) {
4541
+ highestPrimaryTerm = entry.primaryTerm;
4542
+ }
4543
+ }
4544
+ return { highestSeqNo, highestPrimaryTerm };
4545
+ }
4546
+ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
4547
+ for (const { key, startSeqNo } of segments) {
4548
+ if (startSeqNo > activeSegmentSeqNo) {
4549
+ await directory.remove(key);
4550
+ }
4551
+ }
4552
+ }
4553
+ async function truncateSegmentTail(directory, key, durableByteLength) {
4554
+ const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
4555
+ const handle = await directory.appendHandle(key);
4556
+ try {
4557
+ const currentSize = await handle.size();
4558
+ if (cleanLength < currentSize) {
4559
+ await handle.truncate(cleanLength);
4560
+ }
4561
+ } finally {
4562
+ await handle.close();
4563
+ }
4564
+ }
4565
+ function applyEntry2(entry, deps) {
4566
+ deps.writeRecords?.replayed(entry);
4567
+ if (entry.operation === "DELETE" || deps.manager.has(entry.documentId)) noteDocumentChurn(entry.indexName, 1);
4568
+ if (entry.operation === "DELETE") {
4569
+ applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
4570
+ return;
4571
+ }
4572
+ applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
4573
+ }
4574
+ function snapshotCheckpointFor(checkpoint, partitionId) {
4575
+ return checkpointLastSeqNo(checkpoint, partitionId);
4576
+ }
4577
+
4578
+ // src/persistence/durability/manager-state.ts
4579
+ function closePartitionWriter(partition) {
4580
+ return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
4581
+ }
4582
+ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
4583
+ indexState.unloading = true;
4584
+ try {
4585
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
4586
+ await pendingMetadataWrite;
4587
+ for (const partition of indexState.partitions.values()) {
4588
+ await closeWalWriter(closePartitionWriter(partition));
4589
+ }
4590
+ } catch (error) {
4591
+ indexState.unloading = false;
4592
+ throw error;
4593
+ }
4594
+ }
4595
+
4596
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, SINGLE_NODE_PRIMARY_TERM, abandonAnalysisChange, admitDurableWrite, adoptSavedAnalysis, applyDeleteEntry, applyIndexEntry, archiveFootprint, archiveStream, buildArchiveFromMemory, buildHeldAnalysisChange, captureAppliedPrefix, changeIndexAnalysis, changesOf, changesStatusOf, checkpointInTurn, closePartitionWriter, configOfNamedSettings, createByteReader, createDetachedIndex, createIndexLoads, createMarkerWriter, createRebalancer, createSeqOwner, deserializePayloadV1, documentExportStream, drainIndexStateForUnload, dropHeldAnalysisChange, dropStoredVectorValues, durableMetadataConfigOf, earlierSnapshotFootprint, executeRebalance, followIndexChanges, followSourceChanges, growPartitionsTo, heldAnalysisIsBuilt, highestPartitionIdOf, holdScheduledCheckpoint, holdsNamedSettings, holdsSettings, indexMetadataOf, invalidTransition, listPersistedIndexes, loadMetadata, loadPartitionSegments, loadSegmentedCheckpoint, memoryCheckpointStore, metadataConfigOf, metadataFromConfig, metadataPartitionCount, namedSettingsOf, nestedSchemaOf, openFileVectorReads, pagedPartitionLines, patternIndexesFromWire, readCurrentManifest, readDocumentExport, readMetadataEnvelope, readSegmentManifest, readTransition, reclaimUnlistedKeys, releaseHeldAnalysisChange, removeLeftoverRewriteFiles, replaySyncedWal, replayWal, replayWalUpTo, requireChangesSetting, requireCheckpointLayout, requireConfigurationWithinSizeLimit, requireRestoreMemory, requireSavedPartitionCount, requiresLayoutCheckpoint, resolveAnalysisTarget, resumeAnalysisChange, sanitizeSurfaceForms, snapshotCheckpointFor, stampAnalysisGeneration, startReadingOpeningSectionsAhead, storedBytesOfSegments, switchInTurn, targetOfPending, transitionKey, transitionWalLayout, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, updateChangesOf, validateReplicationEntry, whileGraphsStayComplete, writeArchiveFile, writeLiveVectors, writeMetadataEnvelope };