@delali/narsil 0.3.1-next.153 → 0.3.1-next.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (330) hide show
  1. package/README.md +3 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
  8. package/dist/archive-history-E4R73RWD.mjs +29 -0
  9. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
  10. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
  11. package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
  12. package/dist/browser/chunk-23XRXIBW.mjs +85 -0
  13. package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
  14. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
  15. package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
  16. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
  17. package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
  18. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  19. package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
  20. package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
  21. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
  22. package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
  23. package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
  24. package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
  25. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  26. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  27. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
  28. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
  29. package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
  30. package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
  31. package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
  32. package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
  33. package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
  34. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
  35. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
  36. package/dist/browser/chunk-KKAMCABR.mjs +432 -0
  37. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
  38. package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
  39. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  40. package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
  41. package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
  42. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  43. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
  44. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
  45. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
  46. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
  47. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
  48. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
  49. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
  50. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  51. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
  52. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  53. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  54. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
  55. package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
  56. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
  57. package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
  58. package/dist/browser/import-OM6QAW45.mjs +18 -0
  59. package/dist/browser/methods-KTQCSIN4.mjs +30 -0
  60. package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
  61. package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
  62. package/dist/browser/operations-OJ3BXS55.mjs +5 -0
  63. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
  64. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
  65. package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
  66. package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
  67. package/dist/browser/retention-ASWUHAYI.mjs +209 -0
  68. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
  69. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
  70. package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
  71. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  72. package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
  73. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
  74. package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
  75. package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
  76. package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
  77. package/dist/chunk-3HB3PEC6.mjs +240 -0
  78. package/dist/chunk-3HHQLEEQ.mjs +1092 -0
  79. package/dist/chunk-3OH67G7R.mjs +214 -0
  80. package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
  81. package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
  82. package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
  83. package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
  84. package/dist/chunk-5FVU47TB.mjs +20 -0
  85. package/dist/chunk-5YNRKI2C.mjs +81 -0
  86. package/dist/chunk-6L3XJXGU.mjs +470 -0
  87. package/dist/chunk-6MSZ543Y.mjs +32 -0
  88. package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
  89. package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
  90. package/dist/chunk-7GXJUJ24.mjs +49 -0
  91. package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
  92. package/dist/chunk-BDRIVYOK.mjs +72 -0
  93. package/dist/chunk-BUMREPLU.mjs +15 -0
  94. package/dist/chunk-BY6ZDGYS.mjs +50 -0
  95. package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
  96. package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
  97. package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
  98. package/dist/chunk-GES7U2PM.mjs +68 -0
  99. package/dist/chunk-GH4WQ22V.mjs +34 -0
  100. package/dist/chunk-GQPTDQCL.mjs +54 -0
  101. package/dist/chunk-H4KHPLHJ.mjs +382 -0
  102. package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
  103. package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
  104. package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
  105. package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
  106. package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
  107. package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
  108. package/dist/chunk-IPN7S4KX.mjs +130 -0
  109. package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
  110. package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
  111. package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
  112. package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
  114. package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
  115. package/dist/chunk-KJ3RTZDJ.mjs +101 -0
  116. package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
  117. package/dist/chunk-KP7TYHWT.mjs +92 -0
  118. package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
  119. package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
  120. package/dist/chunk-KYYABCSW.mjs +72 -0
  121. package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
  122. package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
  123. package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
  124. package/dist/chunk-LNGE7SLY.mjs +151 -0
  125. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
  126. package/dist/chunk-MT2NRTA6.mjs +216 -0
  127. package/dist/chunk-NF4DT3IH.mjs +85 -0
  128. package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
  129. package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
  130. package/dist/chunk-PHQNACDT.mjs +63 -0
  131. package/dist/chunk-Q622CTMC.mjs +74 -0
  132. package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
  133. package/dist/chunk-QDZJM7NI.mjs +23 -0
  134. package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
  135. package/dist/chunk-QJVOLKU6.mjs +111 -0
  136. package/dist/chunk-QUJHMPTP.mjs +30 -0
  137. package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
  138. package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
  139. package/dist/chunk-RZWWB2XR.mjs +96 -0
  140. package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
  141. package/dist/chunk-S5LN746F.mjs +10 -0
  142. package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
  143. package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
  144. package/dist/chunk-SY3EWKQY.mjs +242 -0
  145. package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
  146. package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
  147. package/dist/chunk-UHICS3JI.mjs +83 -0
  148. package/dist/chunk-UZRZKNNZ.mjs +101 -0
  149. package/dist/chunk-V6EUBAIT.mjs +19 -0
  150. package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
  151. package/dist/chunk-VHBNF4MA.mjs +543 -0
  152. package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
  153. package/dist/chunk-WQRN64Q6.mjs +196 -0
  154. package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
  155. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  156. package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
  157. package/dist/chunk-YEKL4JBK.mjs +96 -0
  158. package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
  159. package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
  160. package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
  161. package/dist/chunk-ZTW5WTGG.mjs +167 -0
  162. package/dist/chunk-ZYAY26KG.mjs +29 -0
  163. package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
  164. package/dist/client.d.ts +9 -7
  165. package/dist/client.mjs +120 -7
  166. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
  167. package/dist/config-CoSig5Ox.d.ts +482 -0
  168. package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
  169. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
  170. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  171. package/dist/distribution/coordinator/etcd.mjs +1 -1
  172. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  173. package/dist/distribution/transport/grpc.mjs +4 -4
  174. package/dist/distribution/transport/in-memory.mjs +2 -2
  175. package/dist/distribution/transport/tcp.mjs +3 -3
  176. package/dist/distribution.d.ts +41 -9
  177. package/dist/distribution.mjs +9304 -7271
  178. package/dist/durability-config-GE4U7MCZ.mjs +89 -0
  179. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
  180. package/dist/embeddings/openai.mjs +2 -2
  181. package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
  182. package/dist/engine-hooks-DHCMOADY.mjs +47 -0
  183. package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
  184. package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
  185. package/dist/index.browser.mjs +9 -4
  186. package/dist/index.d.ts +7 -6
  187. package/dist/index.mjs +29 -21
  188. package/dist/index.react-native.mjs +19 -12
  189. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
  190. package/dist/methods-7IOINERF.mjs +30 -0
  191. package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
  192. package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
  193. package/dist/operations-G5EP6JPX.mjs +5 -0
  194. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
  195. package/dist/partial-words.mjs +7 -5
  196. package/dist/pattern.mjs +4 -3
  197. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  198. package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
  199. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
  200. package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
  201. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  202. package/dist/query-suggestions/merge-worker.mjs +81 -0
  203. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  204. package/dist/query-suggestions.browser.mjs +246 -0
  205. package/dist/query-suggestions.d.ts +20 -0
  206. package/dist/query-suggestions.mjs +62 -0
  207. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
  208. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
  209. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
  210. package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
  211. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  212. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
  213. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
  214. package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
  215. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
  216. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
  217. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
  218. package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
  219. package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
  220. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
  221. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
  222. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
  223. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
  224. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  225. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
  226. package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
  227. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
  228. package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
  229. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
  230. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
  231. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  232. package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
  233. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
  234. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  235. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
  236. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
  237. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  238. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
  239. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  240. package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
  241. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
  242. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  243. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
  244. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
  245. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
  246. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
  247. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  248. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
  249. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  250. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
  251. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
  252. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  253. package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
  254. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
  255. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
  256. package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
  257. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
  258. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  259. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
  260. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
  261. package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
  262. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
  263. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
  264. package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
  265. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
  266. package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
  267. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
  268. package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
  269. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
  270. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
  271. package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
  272. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
  273. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
  274. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
  275. package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
  276. package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
  277. package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
  278. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
  279. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
  280. package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
  281. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  282. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
  283. package/dist/react-native/partial-words.mjs +8 -5
  284. package/dist/react-native/pattern.mjs +4 -3
  285. package/dist/react-native/query-suggestions.mjs +258 -0
  286. package/dist/react-native/sync.mjs +2299 -0
  287. package/dist/react-native/synonyms.mjs +7 -4
  288. package/dist/react.d.ts +11 -10
  289. package/dist/react.mjs +8 -1
  290. package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
  291. package/dist/rerankers/http.mjs +2 -2
  292. package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
  293. package/dist/restore-queue-JVGENGGE.mjs +55 -0
  294. package/dist/retention-HROPV2QU.mjs +59 -0
  295. package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
  296. package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
  297. package/dist/server.d.ts +119 -20
  298. package/dist/server.mjs +1123 -58
  299. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
  300. package/dist/start-UTWHGH42.mjs +59 -0
  301. package/dist/sync/filesystem.d.ts +30 -0
  302. package/dist/sync/filesystem.mjs +271 -0
  303. package/dist/sync/indexeddb.d.ts +30 -0
  304. package/dist/sync/indexeddb.mjs +237 -0
  305. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  306. package/dist/sync.browser.mjs +2294 -0
  307. package/dist/sync.d.ts +23 -0
  308. package/dist/sync.mjs +2268 -0
  309. package/dist/synonyms.mjs +6 -4
  310. package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
  311. package/dist/thread-6MOXY2KD.mjs +45 -0
  312. package/dist/types-BZCGjfLn.d.ts +227 -0
  313. package/dist/vector/search-worker.d.ts +1 -1
  314. package/dist/vector/search-worker.mjs +7 -7
  315. package/dist/workers/entry.mjs +8 -8
  316. package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
  317. package/openapi.json +912 -5
  318. package/package.json +45 -8
  319. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  320. package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
  321. package/dist/chunk-7LDMWKP6.mjs +0 -209
  322. package/dist/engine-CJRUHXgf.d.ts +0 -664
  323. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  324. package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
  325. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  326. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  327. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  328. package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
  329. package/dist/restore-queue-AOF5OTQV.mjs +0 -41
  330. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,713 +1,316 @@
1
- import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-FNBNZCGP.mjs';
2
- import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, removeLeftoverRewriteFiles, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-4QCSHLNM.mjs';
3
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, isRecord, invalidManifest, unpackEnvelopeBytes, isId, isUint32, isSequenceNumber, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf, readCurrentManifest } from './chunk-ZS6CMXAE.mjs';
4
- import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
1
+ import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-AILVAOLX.mjs';
2
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-YEKL4JBK.mjs';
3
+ import { closePartitionWriter, listPersistedIndexes, removeLeftoverRewriteFiles, drainIndexStateForUnload, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-DUOLGBTE.mjs';
4
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, unpackEnvelopeBytes, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf } from './chunk-3R4UZWQG.mjs';
5
+ import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-KBDBPVLN.mjs';
5
6
  import './chunk-ZXNNR73R.mjs';
6
- import './chunk-DMNPYF54.mjs';
7
+ import './chunk-WZ7YV54N.mjs';
7
8
  import './chunk-MATMVH65.mjs';
8
9
  import './chunk-VD7J552I.mjs';
9
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-G5KHSM6E.mjs';
10
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-SY3EWKQY.mjs';
11
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-K4ZIGL2O.mjs';
12
+ import './chunk-BUMREPLU.mjs';
10
13
  import './chunk-37CD3OKS.mjs';
11
- import { createDurableDirectory } from './chunk-FDG4KXX5.mjs';
12
- import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
13
- import { getFs } from './chunk-HLPRHOSO.mjs';
14
- import { nextSegmentKey, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, readCommitMarker } from './chunk-KQJVNPRM.mjs';
14
+ import { createDurableDirectory } from './chunk-ZHCUZZE5.mjs';
15
+ import { SEGMENT_HEADER_SIZE, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, buildEntry, captureWholePartition, checkpointLastSeqNo } from './chunk-S2EQU2GE.mjs';
16
+ import './chunk-BY6ZDGYS.mjs';
17
+ import { readSegmentFile } from './chunk-J4IWBIVY.mjs';
18
+ import './chunk-KXCVQWOR.mjs';
19
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-UHICS3JI.mjs';
20
+ import './chunk-5FLRXF73.mjs';
15
21
  import './chunk-JDDXV6UF.mjs';
16
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, readSegmentFile, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-CSHHU7PA.mjs';
17
- import './chunk-NPXVQS33.mjs';
18
- import { directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-7LDMWKP6.mjs';
19
- import './chunk-5QMHMELB.mjs';
20
- import { toError, openChecksummedEnvelope, HEADER_SIZE } from './chunk-2PNYKG2D.mjs';
21
- import './chunk-6F6USYII.mjs';
22
- import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UGVUW27S.mjs';
23
- import './chunk-3HV7M5OJ.mjs';
24
- import { fnv1a } from './chunk-KAKQ74LS.mjs';
22
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-NF4DT3IH.mjs';
23
+ import { getFs } from './chunk-HLPRHOSO.mjs';
24
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
25
+ import { sha256 } from './chunk-LNGE7SLY.mjs';
26
+ import './chunk-UZRZKNNZ.mjs';
27
+ import './chunk-H4KHPLHJ.mjs';
28
+ import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
29
+ import './chunk-GQPTDQCL.mjs';
30
+ import { QUERY_HISTORY_FOLDER } from './chunk-5YNRKI2C.mjs';
31
+ import { toError, packChecksummedEnvelope } from './chunk-KMUNHGBQ.mjs';
32
+ import './chunk-L7ZD4L3K.mjs';
25
33
  import './chunk-HJSN7UYG.mjs';
26
- import { detectRuntime, resolveWorkerEntry } from './chunk-6BWLDLS3.mjs';
34
+ import { detectRuntime, resolveWorkerEntry } from './chunk-LHKOSI2J.mjs';
27
35
  import './chunk-XIHUI4KB.mjs';
28
- import './chunk-Q2UL2TPC.mjs';
29
36
  import './chunk-3MXV3VHR.mjs';
30
37
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
31
- import './chunk-2HJWQPR6.mjs';
38
+ import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-Z2BE2MAM.mjs';
39
+ import './chunk-ZAUJFLEN.mjs';
40
+ import './chunk-GH4WQ22V.mjs';
41
+ import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-3OH67G7R.mjs';
42
+ import './chunk-IPN7S4KX.mjs';
43
+ import { fnv1a } from './chunk-KAKQ74LS.mjs';
44
+ import { crc32 } from './chunk-7GXJUJ24.mjs';
45
+ import './chunk-Q2UL2TPC.mjs';
46
+ import './chunk-LC3FDWTX.mjs';
47
+ import './chunk-JUKYYXSP.mjs';
48
+ import './chunk-4KRC2APY.mjs';
49
+ import './chunk-FQBFZAWG.mjs';
50
+ import { encodeDocument, requireValidFieldTypes } from './chunk-WQRN64Q6.mjs';
51
+ import './chunk-GES7U2PM.mjs';
52
+ import './chunk-I54T6CQP.mjs';
53
+ import './chunk-YEDIDCR7.mjs';
32
54
  import './chunk-X5CV2E2B.mjs';
55
+ import './chunk-Y4WPNHMC.mjs';
56
+ import './chunk-RZWWB2XR.mjs';
33
57
  import './chunk-TGP5M5R7.mjs';
58
+ import './chunk-4BYWRYPM.mjs';
59
+ import './chunk-4UMZ7KQL.mjs';
34
60
  import './chunk-U2BKYEC7.mjs';
35
- import './chunk-DGRM4EZZ.mjs';
36
61
  import { createRequestId } from './chunk-QHPYS427.mjs';
37
62
  import './chunk-QYRNLLLI.mjs';
63
+ import './chunk-HNVVEQP7.mjs';
38
64
  import './chunk-ZXNQEW43.mjs';
65
+ import './chunk-L5ORED7V.mjs';
39
66
  import './chunk-KOKM4PQN.mjs';
40
- import './chunk-NVTFQFUL.mjs';
41
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
67
+ import './chunk-Q7G3BV3Y.mjs';
42
68
  import './chunk-BTPGQC6Z.mjs';
43
- import './chunk-RFBLQLLZ.mjs';
44
- import './chunk-U46IZNCW.mjs';
45
- import './chunk-JUKYYXSP.mjs';
46
- import { encodeDocument, requireValidFieldTypes } from './chunk-SOUZMPME.mjs';
47
- import './chunk-FQBFZAWG.mjs';
48
- import './chunk-65WCV5JO.mjs';
49
- import './chunk-4UMZ7KQL.mjs';
69
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
50
70
  import { MIN_M } from './chunk-ZXX5GZEX.mjs';
51
- import './chunk-KYVRZAYP.mjs';
52
- import './chunk-L5ORED7V.mjs';
71
+ import './chunk-RFBLQLLZ.mjs';
53
72
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
54
- import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
73
+ import { NarsilError, ErrorCodes } from './chunk-MP7ABPTD.mjs';
55
74
  import './chunk-R6A773WK.mjs';
56
75
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
57
- import { decode } from '@msgpack/msgpack';
76
+ import { decode, encode } from '@msgpack/msgpack';
58
77
 
59
- // src/persistence/durability/sha256.ts
60
- var ROUND_CONSTANTS = new Uint32Array([
61
- 1116352408,
62
- 1899447441,
63
- 3049323471,
64
- 3921009573,
65
- 961987163,
66
- 1508970993,
67
- 2453635748,
68
- 2870763221,
69
- 3624381080,
70
- 310598401,
71
- 607225278,
72
- 1426881987,
73
- 1925078388,
74
- 2162078206,
75
- 2614888103,
76
- 3248222580,
77
- 3835390401,
78
- 4022224774,
79
- 264347078,
80
- 604807628,
81
- 770255983,
82
- 1249150122,
83
- 1555081692,
84
- 1996064986,
85
- 2554220882,
86
- 2821834349,
87
- 2952996808,
88
- 3210313671,
89
- 3336571891,
90
- 3584528711,
91
- 113926993,
92
- 338241895,
93
- 666307205,
94
- 773529912,
95
- 1294757372,
96
- 1396182291,
97
- 1695183700,
98
- 1986661051,
99
- 2177026350,
100
- 2456956037,
101
- 2730485921,
102
- 2820302411,
103
- 3259730800,
104
- 3345764771,
105
- 3516065817,
106
- 3600352804,
107
- 4094571909,
108
- 275423344,
109
- 430227734,
110
- 506948616,
111
- 659060556,
112
- 883997877,
113
- 958139571,
114
- 1322822218,
115
- 1537002063,
116
- 1747873779,
117
- 1955562222,
118
- 2024104815,
119
- 2227730452,
120
- 2361852424,
121
- 2428436474,
122
- 2756734187,
123
- 3204031479,
124
- 3329325298
125
- ]);
126
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
127
- var BLOCK_BYTES = 64;
128
- var LENGTH_BYTES = 8;
129
- function rotateRight(value, bits) {
130
- return value >>> bits | value << 32 - bits;
131
- }
132
- function padded(bytes) {
133
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
134
- const message = new Uint8Array(total);
135
- message.set(bytes);
136
- message[bytes.byteLength] = 128;
137
- const view = new DataView(message.buffer);
138
- const bitLength = bytes.byteLength * 8;
139
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
140
- view.setUint32(total - 4, bitLength >>> 0, false);
141
- return view;
142
- }
143
- function sha256(bytes) {
144
- const view = padded(bytes);
145
- const state = INITIAL_STATE.slice();
146
- const words = new Uint32Array(64);
147
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
148
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
149
- for (let i = 16; i < 64; i++) {
150
- const w15 = words[i - 15];
151
- const w2 = words[i - 2];
152
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
153
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
154
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
155
- }
156
- let [a, b, c, d, e, f, g, h] = state;
157
- for (let i = 0; i < 64; i++) {
158
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
159
- const choice = e & f ^ ~e & g;
160
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
161
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
162
- const majority = a & b ^ a & c ^ b & c;
163
- const t2 = s0 + majority >>> 0;
164
- h = g;
165
- g = f;
166
- f = e;
167
- e = d + t1 >>> 0;
168
- d = c;
169
- c = b;
170
- b = a;
171
- a = t1 + t2 >>> 0;
172
- }
173
- state[0] = state[0] + a >>> 0;
174
- state[1] = state[1] + b >>> 0;
175
- state[2] = state[2] + c >>> 0;
176
- state[3] = state[3] + d >>> 0;
177
- state[4] = state[4] + e >>> 0;
178
- state[5] = state[5] + f >>> 0;
179
- state[6] = state[6] + g >>> 0;
180
- state[7] = state[7] + h >>> 0;
181
- }
182
- const digest = new Uint8Array(32);
183
- const out = new DataView(digest.buffer);
184
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
185
- return digest;
186
- }
187
-
188
- // src/persistence/durability/restore-journal.ts
189
- var RESTORE_JOURNAL_PREFIX = ".swap-";
190
- var RESTORE_STAGING_PREFIX = ".restore-";
191
- var RETIRED_INDEX_PREFIX = ".retired-";
192
- var LOCK_FILE_KEY = ".narsil.lock";
193
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
194
- function digestOf(indexName) {
195
- let hex = "";
196
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
197
- return hex;
78
+ // src/persistence/durability/sync/layout.ts
79
+ var SYNC_KEY_NUMBER_WIDTH = 16;
80
+ function syncDirectoryPrefix(indexName) {
81
+ return `${indexName}/sync/`;
198
82
  }
199
- function journalKey(indexName) {
200
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
83
+ function publishedSnapshotKey(indexName) {
84
+ return `${syncDirectoryPrefix(indexName)}published`;
201
85
  }
202
- function randomSuffix() {
203
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
86
+ function historyRoot(indexName) {
87
+ return `${indexName}/history/`;
204
88
  }
205
- function newStagingPrefix() {
206
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
89
+ function historyPrefix(indexName, partitionId) {
90
+ return `${historyRoot(indexName)}${partitionId}/`;
207
91
  }
208
- function newRetiredPrefix() {
209
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
92
+ function walSegmentPrefix(indexName, partitionId) {
93
+ return `${indexName}/wal/${partitionId}/`;
210
94
  }
211
- function encodeJournal(indexName, stagingPrefix) {
212
- return new TextEncoder().encode(`${indexName}
213
- ${stagingPrefix}`);
95
+ function formatSyncKeyNumber(value) {
96
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
97
+ }
98
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
99
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
100
+ }
101
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
102
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
103
+ }
104
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
105
+ function parseHistoryKey(indexName, key) {
106
+ const root = historyRoot(indexName);
107
+ if (!key.startsWith(root)) return null;
108
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
109
+ if (match === null) return null;
110
+ const partitionId = Number.parseInt(match[1], 10);
111
+ const startSeqNo = Number.parseInt(match[2], 10);
112
+ const movedAtMs = Number.parseInt(match[3], 10);
113
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
114
+ return null;
115
+ }
116
+ return { key, partitionId, startSeqNo, movedAtMs };
214
117
  }
215
- function isStagingPrefix(prefix) {
216
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
118
+ function parseWalSegmentStart(indexName, partitionId, key) {
119
+ const prefix = walSegmentPrefix(indexName, partitionId);
120
+ if (!key.startsWith(prefix)) return null;
121
+ const tail = key.slice(prefix.length);
122
+ if (!/^\d{16}$/.test(tail)) return null;
123
+ const value = Number.parseInt(tail, 10);
124
+ return Number.isSafeInteger(value) ? value : null;
217
125
  }
218
- function corruptJournal(key, reason) {
219
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
126
+
127
+ // src/persistence/durability/sync/history.ts
128
+ function compareHistorySegments(a, b) {
129
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
130
+ }
131
+ async function listHistorySegments(directory, indexName, partitionId) {
132
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
133
+ const segments = [];
134
+ for (const key of await directory.list(prefix)) {
135
+ const segment = parseHistoryKey(indexName, key);
136
+ if (segment !== null) segments.push(segment);
137
+ }
138
+ return segments.sort(compareHistorySegments);
139
+ }
140
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
141
+ await directory.move(
142
+ walSegmentKey(indexName, partitionId, startSeqNo),
143
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
144
+ );
220
145
  }
221
- function readJournal(key, bytes) {
222
- const text = new TextDecoder().decode(bytes);
223
- const lineEnd = text.indexOf("\n");
224
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
225
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
226
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
227
- corruptJournal(key, "names an index that no index name can match");
228
- }
229
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
230
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
231
- corruptJournal(key, "names no staging prefix that a restore writes");
146
+ async function removeHistory(directory, indexName, partitionId) {
147
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
148
+ await directory.remove(segment.key);
232
149
  }
233
- return { kind: "journal", indexName, stagingPrefix };
234
150
  }
235
- async function listedKeys(storage, indexName, stagingPrefix) {
236
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
237
- if (manifestBytes === null) {
238
- throw new NarsilError(
239
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
240
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
241
- { indexName, stagingPrefix }
242
- );
243
- }
244
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
245
- if (decoded.kind !== "current") {
246
- throw new NarsilError(
247
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
248
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
249
- { indexName }
250
- );
251
- }
252
- const root = `${indexName}/`;
253
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
254
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
255
- return keys;
151
+
152
+ // src/persistence/durability/checkpoint.ts
153
+ function walPrefix(indexName, partitionId) {
154
+ return `${indexName}/wal/${partitionId}/`;
256
155
  }
257
- function withoutTrailingSlash(prefix) {
258
- return prefix.slice(0, -1);
156
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
157
+ for (const { partitionId, lastSeqNo } of checkpoint) {
158
+ const prefix = walPrefix(indexName, partitionId);
159
+ const markerBytes = await directory.read(`${prefix}commit`);
160
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
161
+ if (marker === null) {
162
+ continue;
163
+ }
164
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
165
+ const segments = await directory.list(prefix);
166
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
167
+ for (let i = 0; i < startSeqNos.length; i += 1) {
168
+ const start = startSeqNos[i];
169
+ if (start >= activeSegmentSeqNo) {
170
+ break;
171
+ }
172
+ const nextStart = startSeqNos[i + 1];
173
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
174
+ if (segmentCoversBeyondCheckpoint) {
175
+ break;
176
+ }
177
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
178
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
179
+ }
180
+ }
259
181
  }
260
- async function finishJournal(storage, indexName, stagingPrefix) {
261
- const root = `${indexName}/`;
262
- const listed = await listedKeys(storage, indexName, stagingPrefix);
263
- const staged = await storage.list(stagingPrefix);
264
- let retiredPrefix = null;
265
- if (storage.moveTree !== void 0 && staged.length > 0) {
266
- retiredPrefix = newRetiredPrefix();
267
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
268
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
269
- } else {
270
- const stagedKeys = new Set(staged);
271
- for (const key of listed) {
272
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
182
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
183
+ for (const [partitionId, partition] of [...partitions]) {
184
+ if (partitionId < partitionCount) continue;
185
+ try {
186
+ await partition.walWriter.close();
187
+ } catch (err) {
188
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
273
189
  }
190
+ partitions.delete(partitionId);
274
191
  }
275
- for (const key of await storage.list(root)) {
276
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
192
+ const walRoot = `${indexName}/wal/`;
193
+ for (const key of await directory.list(walRoot)) {
194
+ const partitionId = walKeyPartitionId(key, walRoot);
195
+ if (partitionId === null || partitionId < partitionCount) continue;
196
+ await directory.remove(key);
277
197
  }
278
- await storage.remove(journalKey(indexName));
279
- return retiredPrefix;
280
198
  }
281
- async function removePrefix(storage, prefix) {
282
- if (storage.removeTree !== void 0) {
283
- await storage.removeTree(prefix);
284
- return;
199
+ function walKeyPartitionId(key, walRoot) {
200
+ const tail = key.slice(walRoot.length);
201
+ const separator = tail.indexOf("/");
202
+ if (separator <= 0) {
203
+ return null;
285
204
  }
286
- for (const key of await storage.list(prefix)) await storage.remove(key);
287
- }
288
- async function commitJournal(storage, indexName, stagingPrefix) {
289
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
290
- }
291
- function isTopLevelJournal(key) {
292
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
293
- }
294
- async function removeStagingPrefixes(storage) {
295
- const prefixes = /* @__PURE__ */ new Set();
296
- for (const key of await storage.list(".")) {
297
- const slash = key.indexOf("/");
298
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
205
+ const segment = tail.slice(0, separator);
206
+ if (!/^\d+$/.test(segment)) {
207
+ return null;
299
208
  }
300
- for (const prefix of prefixes) await removePrefix(storage, prefix);
209
+ const value = Number.parseInt(segment, 10);
210
+ return Number.isSafeInteger(value) ? value : null;
301
211
  }
302
- async function recoverRestoreJournals(storage) {
303
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
304
- if (!isTopLevelJournal(key)) continue;
305
- const bytes = await storage.read(key);
306
- if (bytes === null) continue;
307
- const reading = readJournal(key, bytes);
308
- if (reading.kind === "leftover") {
309
- await storage.remove(key);
310
- continue;
311
- }
312
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
212
+ function parseSegmentStartSeqNo(key, prefix) {
213
+ if (!key.startsWith(prefix)) {
214
+ return null;
313
215
  }
314
- await removeStagingPrefixes(storage);
315
- }
316
-
317
- // src/persistence/durability/restore-directory.ts
318
- function missing(error) {
319
- return error.code === "ENOENT";
216
+ const tail = key.slice(prefix.length);
217
+ if (!/^\d{16}$/.test(tail)) {
218
+ return null;
219
+ }
220
+ const value = Number.parseInt(tail, 10);
221
+ return Number.isSafeInteger(value) ? value : null;
320
222
  }
321
- function storageFailed(code, message, key, error) {
322
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
223
+ function formatSegmentStart(startSeqNo) {
224
+ return startSeqNo.toString().padStart(16, "0");
323
225
  }
324
- function directoryRestoreStorage(directory) {
325
- return {
326
- directory,
327
- read: (key) => directory.read(key),
328
- write: (key, chunks) => directory.atomicWrite(key, chunks),
329
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
330
- remove: (key) => directory.remove(key),
331
- list: (prefix) => directory.list(prefix),
332
- moveKey: (from, to) => directory.move(from, to),
333
- pathOf: (key) => directory.pathOf(key),
334
- async moveTree(from, to) {
335
- const source = await directory.pathOf(from);
336
- const target = await directory.pathOf(to);
337
- const fs = await getFs();
338
- try {
339
- await fs.rename(source, target);
340
- } catch (error) {
341
- if (missing(error)) return;
342
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
343
- }
344
- await directory.syncDirectoryOf(to);
345
- },
346
- async removeTree(prefix) {
347
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
348
- const fs = await getFs();
226
+
227
+ // src/persistence/durability/checkpoint-capture.ts
228
+ function whileNoMutationApplies(partitions, capture2) {
229
+ return new Promise((resolve, reject) => {
230
+ let release = () => void 0;
231
+ const released = new Promise((settle) => {
232
+ release = settle;
233
+ });
234
+ function captureNow() {
349
235
  try {
350
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
236
+ resolve(capture2());
351
237
  } catch (error) {
352
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
238
+ reject(error);
239
+ } finally {
240
+ release();
353
241
  }
354
- await directory.syncDirectoryOf(key);
355
242
  }
356
- };
357
- }
358
- function recoverDirectoryRestores(directory) {
359
- return recoverRestoreJournals(directoryRestoreStorage(directory));
360
- }
361
-
362
- // src/persistence/durability/restore-staging.ts
363
- function toError2(error) {
364
- return error instanceof Error ? error : new Error(String(error));
365
- }
366
- function pathUnder(path, root) {
367
- if (!path.startsWith(root)) return false;
368
- const next = path.charAt(root.length);
369
- return next === "" || next === "/" || next === "\\";
370
- }
371
- function stagedDirectory(directory, staged, unstaged) {
372
- const { readRange, byteLength } = directory;
373
- return {
374
- root: directory.root,
375
- appendHandle: (key) => directory.appendHandle(staged(key)),
376
- markerHandle: (key) => directory.markerHandle(staged(key)),
377
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
378
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
379
- read: (key) => directory.read(staged(key)),
380
- remove: (key) => directory.remove(staged(key)),
381
- move: (from, to) => directory.move(staged(from), staged(to)),
382
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
383
- pathOf: (key) => directory.pathOf(staged(key)),
384
- ...readRange === void 0 ? {} : {
385
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
386
- },
387
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
388
- };
389
- }
390
- function createRestoreStaging(storage, indexName, options = {}) {
391
- const stagingPrefix = newStagingPrefix();
392
- const indexRoot = `${indexName}/`;
393
- let committed = false;
394
- const staged = (key) => {
395
- if (!key.startsWith(indexRoot)) {
396
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
397
- key
398
- });
243
+ let waiting = partitions.length;
244
+ if (waiting === 0) {
245
+ captureNow();
246
+ return;
399
247
  }
400
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
401
- };
402
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
403
- const store = {
404
- read: (key) => storage.read(staged(key)),
405
- write: (key, chunks) => storage.write(staged(key), chunks),
406
- remove: (key) => storage.remove(staged(key)),
407
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
408
- };
409
- const pathOf = storage.pathOf;
410
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
411
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
412
- let retiredPrefix = null;
413
- async function finishOnStorage() {
414
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
415
- }
416
- return {
417
- store,
418
- async relocation() {
419
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
420
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
421
- const to = await pathOf.call(storage, indexName);
422
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
423
- },
424
- async commit() {
425
- try {
426
- await commitJournal(storage, indexName, stagingPrefix);
427
- } catch (error) {
428
- try {
429
- await storage.remove(journalKey(indexName));
430
- } catch (removal) {
431
- committed = true;
432
- options.onStorageLost?.(toError2(removal));
433
- }
434
- throw error;
435
- }
436
- committed = true;
437
- },
438
- async finish() {
439
- try {
440
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
441
- } catch (error) {
442
- options.onStorageLost?.(toError2(error));
443
- }
444
- },
445
- async removeRetired() {
446
- const prefix = retiredPrefix;
447
- retiredPrefix = null;
448
- if (prefix !== null) await removePrefix(storage, prefix);
449
- },
450
- async discard() {
451
- if (committed) return;
452
- await removePrefix(storage, stagingPrefix);
248
+ for (const partition of partitions) {
249
+ partition.appendChain = partition.appendChain.then(() => {
250
+ waiting -= 1;
251
+ if (waiting === 0) captureNow();
252
+ return released;
253
+ });
453
254
  }
454
- };
255
+ });
455
256
  }
456
- function adapterRestoreStorage(adapter) {
457
- const store = adapterCheckpointStore(adapter);
458
- return {
459
- read: (key) => store.read(key),
460
- write: (key, chunks) => store.write(key, chunks),
461
- writeJournal: (key, bytes) => store.write(key, [bytes]),
462
- remove: (key) => store.remove(key),
463
- list: (prefix) => store.list(prefix),
464
- async moveKey(from, to) {
465
- const bytes = await store.read(from);
466
- if (bytes === null) return;
467
- await store.write(to, [bytes]);
468
- await store.remove(from);
257
+ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
258
+ for (const [partitionId, partition] of [...indexState.partitions]) {
259
+ await partition.appendChain;
260
+ if (partition.failed !== null) continue;
261
+ try {
262
+ await partition.walWriter.commit();
263
+ } catch (error) {
264
+ failPartition2(partitionId, partition, error);
469
265
  }
470
- };
471
- }
472
-
473
- // src/persistence/durability/archive-checkpoint.ts
474
- function toError3(error) {
475
- return error instanceof Error ? error : new Error(String(error));
266
+ }
476
267
  }
477
- function stageDirectoryRestore(deps, indexName) {
478
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
479
- async finish(run) {
480
- try {
481
- await run();
482
- } catch (error) {
483
- deps.markFatal(toError3(error));
484
- throw error;
485
- }
486
- },
487
- onStorageLost: deps.markFatal
268
+ async function openNewLogSegments(indexState, failPartition2) {
269
+ const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
270
+ try {
271
+ if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
272
+ } catch (error) {
273
+ failPartition2(partitionId, partition, error);
274
+ return;
275
+ }
276
+ const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
277
+ failPartition2(partitionId, partition, error);
278
+ });
279
+ partition.appendChain = rolled;
280
+ await rolled;
488
281
  });
282
+ await Promise.all(rolls);
489
283
  }
490
- function releaser(held, indexName) {
491
- let released = false;
492
- return () => {
493
- if (released) return;
494
- released = true;
495
- const count = (held.get(indexName) ?? 1) - 1;
496
- if (count <= 0) held.delete(indexName);
497
- else held.set(indexName, count);
498
- };
284
+ function aTargetFailed(indexState, targets) {
285
+ return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
499
286
  }
500
- function createArchiveCheckpointAccess(deps) {
501
- const store = directoryCheckpointStore(deps.directory);
502
- return {
503
- async openArchiveCheckpoint(indexName) {
504
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
505
- const release = releaser(deps.held, indexName);
506
- try {
507
- await deps.checkpoint(indexName, false);
508
- await deps.checkpoint(indexName, true);
509
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
510
- if (manifestBytes === null) {
511
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
512
- indexName
513
- });
514
- }
515
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
516
- if (decoded.kind !== "current") {
517
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
518
- indexName
519
- });
520
- }
521
- return {
522
- manifestBytes,
523
- manifest: decoded.manifest,
524
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
525
- readRange: async (ref, offset, length) => {
526
- const readRange = deps.directory.readRange;
527
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
528
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
529
- return whole.slice(offset, offset + length);
530
- },
531
- release,
532
- directory: deps.directory
533
- };
534
- } catch (error) {
535
- release();
536
- throw error;
537
- }
538
- },
539
- stageRestore(indexName) {
540
- return stageDirectoryRestore(deps, indexName);
541
- },
542
- adoptRestoredCheckpoint(indexName, checkpoint) {
543
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
544
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
287
+ function noWholePartitions() {
288
+ return { contents: [], captured: [] };
289
+ }
290
+ function wholePartitionsWhereMostChanged(manager, rules) {
291
+ const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
292
+ return (targets) => {
293
+ const whole = { contents: [], captured: [] };
294
+ const changed = targets.filter(
295
+ (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
296
+ );
297
+ const mostOfEachChanged = changed.every((target) => {
298
+ const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
299
+ const documents = manager.getPartition(target.partitionId).count();
300
+ return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
301
+ });
302
+ const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
303
+ if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
304
+ for (const { partitionId } of changed) {
305
+ const capture2 = captureRecords?.(partitionId);
306
+ const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
307
+ if (captured !== null) {
308
+ whole.captured.push(captured);
309
+ } else if (everyPartition || !aWorkerCanSerialise) {
310
+ whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
545
311
  }
546
312
  }
547
- };
548
- }
549
-
550
- // src/persistence/durability/checkpoint.ts
551
- function walPrefix(indexName, partitionId) {
552
- return `${indexName}/wal/${partitionId}/`;
553
- }
554
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
555
- for (const { partitionId, lastSeqNo } of checkpoint) {
556
- const prefix = walPrefix(indexName, partitionId);
557
- const markerBytes = await directory.read(`${prefix}commit`);
558
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
559
- if (marker === null) {
560
- continue;
561
- }
562
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
563
- const segments = await directory.list(prefix);
564
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
565
- for (let i = 0; i < startSeqNos.length; i += 1) {
566
- const start = startSeqNos[i];
567
- if (start >= activeSegmentSeqNo) {
568
- break;
569
- }
570
- const nextStart = startSeqNos[i + 1];
571
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
572
- if (segmentCoversBeyondCheckpoint) {
573
- break;
574
- }
575
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
576
- }
577
- }
578
- }
579
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
580
- for (const [partitionId, partition] of [...partitions]) {
581
- if (partitionId < partitionCount) continue;
582
- try {
583
- await partition.walWriter.close();
584
- } catch (err) {
585
- onCloseError(err instanceof Error ? err : new Error(String(err)));
586
- }
587
- partitions.delete(partitionId);
588
- }
589
- const walRoot = `${indexName}/wal/`;
590
- for (const key of await directory.list(walRoot)) {
591
- const partitionId = walKeyPartitionId(key, walRoot);
592
- if (partitionId === null || partitionId < partitionCount) continue;
593
- await directory.remove(key);
594
- }
595
- }
596
- function walKeyPartitionId(key, walRoot) {
597
- const tail = key.slice(walRoot.length);
598
- const separator = tail.indexOf("/");
599
- if (separator <= 0) {
600
- return null;
601
- }
602
- const segment = tail.slice(0, separator);
603
- if (!/^\d+$/.test(segment)) {
604
- return null;
605
- }
606
- const value = Number.parseInt(segment, 10);
607
- return Number.isSafeInteger(value) ? value : null;
608
- }
609
- function parseSegmentStartSeqNo(key, prefix) {
610
- if (!key.startsWith(prefix)) {
611
- return null;
612
- }
613
- const tail = key.slice(prefix.length);
614
- if (!/^\d{16}$/.test(tail)) {
615
- return null;
616
- }
617
- const value = Number.parseInt(tail, 10);
618
- return Number.isSafeInteger(value) ? value : null;
619
- }
620
- function formatSegmentStart(startSeqNo) {
621
- return startSeqNo.toString().padStart(16, "0");
622
- }
623
-
624
- // src/persistence/durability/checkpoint-capture.ts
625
- function whileNoMutationApplies(partitions, capture2) {
626
- return new Promise((resolve, reject) => {
627
- let release = () => void 0;
628
- const released = new Promise((settle) => {
629
- release = settle;
630
- });
631
- function captureNow() {
632
- try {
633
- resolve(capture2());
634
- } catch (error) {
635
- reject(error);
636
- } finally {
637
- release();
638
- }
639
- }
640
- let waiting = partitions.length;
641
- if (waiting === 0) {
642
- captureNow();
643
- return;
644
- }
645
- for (const partition of partitions) {
646
- partition.appendChain = partition.appendChain.then(() => {
647
- waiting -= 1;
648
- if (waiting === 0) captureNow();
649
- return released;
650
- });
651
- }
652
- });
653
- }
654
- async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
655
- for (const [partitionId, partition] of [...indexState.partitions]) {
656
- await partition.appendChain;
657
- if (partition.failed !== null) continue;
658
- try {
659
- await partition.walWriter.commit();
660
- } catch (error) {
661
- failPartition2(partitionId, partition, error);
662
- }
663
- }
664
- }
665
- async function openNewLogSegments(indexState, failPartition2) {
666
- const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
667
- try {
668
- if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
669
- } catch (error) {
670
- failPartition2(partitionId, partition, error);
671
- return;
672
- }
673
- const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
674
- failPartition2(partitionId, partition, error);
675
- });
676
- partition.appendChain = rolled;
677
- await rolled;
678
- });
679
- await Promise.all(rolls);
680
- }
681
- function aTargetFailed(indexState, targets) {
682
- return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
683
- }
684
- function noWholePartitions() {
685
- return { contents: [], captured: [] };
686
- }
687
- function wholePartitionsWhereMostChanged(manager, rules) {
688
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
689
- return (targets) => {
690
- const whole = { contents: [], captured: [] };
691
- const changed = targets.filter(
692
- (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
693
- );
694
- const mostOfEachChanged = changed.every((target) => {
695
- const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
696
- const documents = manager.getPartition(target.partitionId).count();
697
- return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
698
- });
699
- const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
700
- if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
701
- for (const { partitionId } of changed) {
702
- const capture2 = captureRecords?.(partitionId);
703
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
704
- if (captured !== null) {
705
- whole.captured.push(captured);
706
- } else if (everyPartition || !aWorkerCanSerialise) {
707
- whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
708
- }
709
- }
710
- return whole;
313
+ return whole;
711
314
  };
712
315
  }
713
316
  async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
@@ -1117,7 +720,7 @@ async function writeOneBoundedCheckpoint(input) {
1117
720
  }
1118
721
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1119
722
  const { indexName, indexState, hooks } = input;
1120
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-6RLIKSZQ.mjs');
723
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-DWKPBSMN.mjs');
1121
724
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1122
725
  if (adoption === null) return [];
1123
726
  const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
@@ -1172,10 +775,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1172
775
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1173
776
  if (!input.holdsCheckpointFiles()) {
1174
777
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1175
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-6RLIKSZQ.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
778
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-DWKPBSMN.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
779
+ const keptForSync = await input.keepsCheckpointKey();
1176
780
  await removeCheckpointGarbage(
1177
781
  store,
1178
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
782
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
1179
783
  );
1180
784
  }
1181
785
  const checkpointDocumentCount = documentCount;
@@ -1184,222 +788,1036 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1184
788
  if (checkpointMetadata === void 0) {
1185
789
  return;
1186
790
  }
1187
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
1188
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1189
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1190
- });
1191
- await truncateCoveredSegments(input.directory, input.indexName, targets);
1192
- await reclaimWalBeyondCount(
1193
- input.directory,
1194
- input.indexName,
1195
- manager.partitionCount,
1196
- input.indexState.partitions,
1197
- input.markFatal
1198
- );
1199
- const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1200
- noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1201
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1202
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
791
+ const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
792
+ await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
793
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
794
+ });
795
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
796
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
797
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
798
+ await reclaimWalBeyondCount(
799
+ input.directory,
800
+ input.indexName,
801
+ manager.partitionCount,
802
+ input.indexState.partitions,
803
+ input.markFatal
804
+ );
805
+ const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
806
+ noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
807
+ const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
808
+ return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
809
+ }
810
+ function recordsBeyond(indexState, targets) {
811
+ let records = 0;
812
+ for (const [partitionId, partition] of indexState.partitions) {
813
+ if (partition.failed !== null) continue;
814
+ records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
815
+ }
816
+ return records;
817
+ }
818
+
819
+ // src/persistence/durability/checkpoint-turn.ts
820
+ function createIndexLoads(vectorIndexesOf2) {
821
+ const loads = /* @__PURE__ */ new Map();
822
+ const startGraphRepairs = (indexName) => {
823
+ for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
824
+ vectorIndex.scheduleBuild();
825
+ vectorIndex.startShuffledRebuildWhenDue();
826
+ }
827
+ };
828
+ return {
829
+ track(indexName, load, startsGraphRepairs = true) {
830
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
831
+ loads.set(
832
+ indexName,
833
+ loaded.catch(() => void 0)
834
+ );
835
+ return loaded;
836
+ },
837
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
838
+ startGraphRepairs
839
+ };
840
+ }
841
+ async function switchInTurn(turns, turn) {
842
+ let swapped = false;
843
+ await checkpointInTurn(turns, {
844
+ joinsInFlight: false,
845
+ mayStart: turn.mayStart,
846
+ perform: async () => {
847
+ swapped = turn.swap();
848
+ if (swapped) await turn.checkpoint();
849
+ },
850
+ afterSettling: turn.afterSettling
851
+ });
852
+ return swapped;
853
+ }
854
+ async function checkpointInTurn(turns, turn) {
855
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
856
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
857
+ if (!turn.mayStart()) return;
858
+ const run = turn.perform().finally(() => {
859
+ turns.checkpointInFlight = null;
860
+ turn.afterSettling();
861
+ });
862
+ turns.checkpointInFlight = run;
863
+ return run;
864
+ }
865
+
866
+ // src/persistence/durability/held-checkpoints.ts
867
+ function failedPartitionIds(indexState) {
868
+ const failed = [];
869
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
870
+ return failed;
871
+ }
872
+ function createHeldCheckpoints(hooks) {
873
+ const reported = /* @__PURE__ */ new WeakSet();
874
+ function heldReport(indexName, indexState) {
875
+ const partitionIds = failedPartitionIds(indexState);
876
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
877
+ return {
878
+ indexName,
879
+ partitionIds,
880
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
881
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
882
+ };
883
+ }
884
+ return {
885
+ refuse(indexName, indexState) {
886
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
887
+ if (partitionIds.length === 0) return;
888
+ throw new NarsilError(
889
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
890
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
891
+ { indexName, partitionIds }
892
+ );
893
+ },
894
+ holds(indexName, indexState) {
895
+ const held = heldReport(indexName, indexState);
896
+ if (held === null) return false;
897
+ const unreported = held.partitionIds.some((partitionId) => {
898
+ const partition = indexState.partitions.get(partitionId);
899
+ return partition !== void 0 && !reported.has(partition);
900
+ });
901
+ if (unreported) {
902
+ for (const partitionId of held.partitionIds) {
903
+ const partition = indexState.partitions.get(partitionId);
904
+ if (partition !== void 0) reported.add(partition);
905
+ }
906
+ hooks.onCheckpointHeld?.(held);
907
+ }
908
+ return true;
909
+ },
910
+ report(indexes) {
911
+ const held = [];
912
+ for (const [indexName, indexState] of indexes) {
913
+ const entry = heldReport(indexName, indexState);
914
+ if (entry !== null) held.push(entry);
915
+ }
916
+ return held;
917
+ }
918
+ };
919
+ }
920
+
921
+ // src/persistence/durability/restore-journal.ts
922
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
923
+ var RESTORE_STAGING_PREFIX = ".restore-";
924
+ var RETIRED_INDEX_PREFIX = ".retired-";
925
+ var LOCK_FILE_KEY = ".narsil.lock";
926
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
927
+ function digestOf(indexName) {
928
+ let hex = "";
929
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
930
+ return hex;
931
+ }
932
+ function journalKey(indexName) {
933
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
934
+ }
935
+ function randomSuffix() {
936
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
937
+ }
938
+ function newStagingPrefix() {
939
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
940
+ }
941
+ function newRetiredPrefix() {
942
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
943
+ }
944
+ function encodeJournal(indexName, stagingPrefix) {
945
+ return new TextEncoder().encode(`${indexName}
946
+ ${stagingPrefix}`);
947
+ }
948
+ function isStagingPrefix(prefix) {
949
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
950
+ }
951
+ function corruptJournal(key, reason) {
952
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
953
+ }
954
+ function readJournal(key, bytes) {
955
+ const text = new TextDecoder().decode(bytes);
956
+ const lineEnd = text.indexOf("\n");
957
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
958
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
959
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
960
+ corruptJournal(key, "names an index that no index name can match");
961
+ }
962
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
963
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
964
+ corruptJournal(key, "names no staging prefix that a restore writes");
965
+ }
966
+ return { kind: "journal", indexName, stagingPrefix };
967
+ }
968
+ async function listedKeys(storage, indexName, stagingPrefix) {
969
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
970
+ if (manifestBytes === null) {
971
+ throw new NarsilError(
972
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
973
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
974
+ { indexName, stagingPrefix }
975
+ );
976
+ }
977
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
978
+ if (decoded.kind !== "current") {
979
+ throw new NarsilError(
980
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
981
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
982
+ { indexName }
983
+ );
984
+ }
985
+ const root = `${indexName}/`;
986
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
987
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
988
+ return keys;
989
+ }
990
+ function withoutTrailingSlash(prefix) {
991
+ return prefix.slice(0, -1);
992
+ }
993
+ async function finishJournal(storage, indexName, stagingPrefix) {
994
+ const root = `${indexName}/`;
995
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
996
+ const staged = await storage.list(stagingPrefix);
997
+ let retiredPrefix = null;
998
+ if (storage.moveTree !== void 0 && staged.length > 0) {
999
+ retiredPrefix = newRetiredPrefix();
1000
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1001
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1002
+ } else {
1003
+ const stagedKeys = new Set(staged);
1004
+ for (const key of listed) {
1005
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1006
+ }
1007
+ }
1008
+ for (const key of await storage.list(root)) {
1009
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1010
+ }
1011
+ await storage.remove(journalKey(indexName));
1012
+ return retiredPrefix;
1013
+ }
1014
+ async function removePrefix(storage, prefix) {
1015
+ if (storage.removeTree !== void 0) {
1016
+ await storage.removeTree(prefix);
1017
+ return;
1018
+ }
1019
+ for (const key of await storage.list(prefix)) await storage.remove(key);
1020
+ }
1021
+ async function commitJournal(storage, indexName, stagingPrefix) {
1022
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1023
+ }
1024
+ function isTopLevelJournal(key) {
1025
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1026
+ }
1027
+ async function removeStagingPrefixes(storage) {
1028
+ const prefixes = /* @__PURE__ */ new Set();
1029
+ for (const key of await storage.list(".")) {
1030
+ const slash = key.indexOf("/");
1031
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1032
+ }
1033
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
1034
+ }
1035
+ async function recoverRestoreJournals(storage) {
1036
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1037
+ if (!isTopLevelJournal(key)) continue;
1038
+ const bytes = await storage.read(key);
1039
+ if (bytes === null) continue;
1040
+ const reading = readJournal(key, bytes);
1041
+ if (reading.kind === "leftover") {
1042
+ await storage.remove(key);
1043
+ continue;
1044
+ }
1045
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1046
+ }
1047
+ await removeStagingPrefixes(storage);
1048
+ }
1049
+
1050
+ // src/persistence/durability/restore-directory.ts
1051
+ function missing(error) {
1052
+ return error.code === "ENOENT";
1053
+ }
1054
+ function storageFailed(code, message, key, error) {
1055
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1056
+ }
1057
+ function directoryRestoreStorage(directory) {
1058
+ return {
1059
+ directory,
1060
+ read: (key) => directory.read(key),
1061
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
1062
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1063
+ remove: (key) => directory.remove(key),
1064
+ list: (prefix) => directory.list(prefix),
1065
+ moveKey: (from, to) => directory.move(from, to),
1066
+ pathOf: (key) => directory.pathOf(key),
1067
+ async moveTree(from, to) {
1068
+ const source = await directory.pathOf(from);
1069
+ const target = await directory.pathOf(to);
1070
+ const fs = await getFs();
1071
+ try {
1072
+ await fs.rename(source, target);
1073
+ } catch (error) {
1074
+ if (missing(error)) return;
1075
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1076
+ }
1077
+ await directory.syncDirectoryOf(to);
1078
+ },
1079
+ async removeTree(prefix) {
1080
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1081
+ const fs = await getFs();
1082
+ try {
1083
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1084
+ } catch (error) {
1085
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1086
+ }
1087
+ await directory.syncDirectoryOf(key);
1088
+ }
1089
+ };
1090
+ }
1091
+ function recoverDirectoryRestores(directory) {
1092
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
1093
+ }
1094
+
1095
+ // src/persistence/durability/restore-staging.ts
1096
+ function toError2(error) {
1097
+ return error instanceof Error ? error : new Error(String(error));
1098
+ }
1099
+ function pathUnder(path, root) {
1100
+ if (!path.startsWith(root)) return false;
1101
+ const next = path.charAt(root.length);
1102
+ return next === "" || next === "/" || next === "\\";
1103
+ }
1104
+ function stagedDirectory(directory, staged, unstaged) {
1105
+ const { readRange, byteLength } = directory;
1106
+ return {
1107
+ root: directory.root,
1108
+ appendHandle: (key) => directory.appendHandle(staged(key)),
1109
+ markerHandle: (key) => directory.markerHandle(staged(key)),
1110
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1111
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1112
+ read: (key) => directory.read(staged(key)),
1113
+ remove: (key) => directory.remove(staged(key)),
1114
+ move: (from, to) => directory.move(staged(from), staged(to)),
1115
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1116
+ pathOf: (key) => directory.pathOf(staged(key)),
1117
+ ...readRange === void 0 ? {} : {
1118
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1119
+ },
1120
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1121
+ };
1122
+ }
1123
+ function createRestoreStaging(storage, indexName, options = {}) {
1124
+ const stagingPrefix = newStagingPrefix();
1125
+ const indexRoot = `${indexName}/`;
1126
+ let committed = false;
1127
+ const staged = (key) => {
1128
+ if (!key.startsWith(indexRoot)) {
1129
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1130
+ key
1131
+ });
1132
+ }
1133
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1134
+ };
1135
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1136
+ const store = {
1137
+ read: (key) => storage.read(staged(key)),
1138
+ write: (key, chunks) => storage.write(staged(key), chunks),
1139
+ remove: (key) => storage.remove(staged(key)),
1140
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1141
+ };
1142
+ const pathOf = storage.pathOf;
1143
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1144
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1145
+ let retiredPrefix = null;
1146
+ async function finishOnStorage() {
1147
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1148
+ }
1149
+ return {
1150
+ store,
1151
+ async relocation() {
1152
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1153
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1154
+ const to = await pathOf.call(storage, indexName);
1155
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1156
+ },
1157
+ async commit() {
1158
+ try {
1159
+ await commitJournal(storage, indexName, stagingPrefix);
1160
+ } catch (error) {
1161
+ try {
1162
+ await storage.remove(journalKey(indexName));
1163
+ } catch (removal) {
1164
+ committed = true;
1165
+ options.onStorageLost?.(toError2(removal));
1166
+ }
1167
+ throw error;
1168
+ }
1169
+ committed = true;
1170
+ },
1171
+ async finish() {
1172
+ try {
1173
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1174
+ } catch (error) {
1175
+ options.onStorageLost?.(toError2(error));
1176
+ }
1177
+ },
1178
+ async removeRetired() {
1179
+ const prefix = retiredPrefix;
1180
+ retiredPrefix = null;
1181
+ if (prefix !== null) await removePrefix(storage, prefix);
1182
+ },
1183
+ async discard() {
1184
+ if (committed) return;
1185
+ await removePrefix(storage, stagingPrefix);
1186
+ }
1187
+ };
1188
+ }
1189
+ function adapterRestoreStorage(adapter) {
1190
+ const store = adapterCheckpointStore(adapter);
1191
+ return {
1192
+ read: (key) => store.read(key),
1193
+ write: (key, chunks) => store.write(key, chunks),
1194
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
1195
+ remove: (key) => store.remove(key),
1196
+ list: (prefix) => store.list(prefix),
1197
+ async moveKey(from, to) {
1198
+ const bytes = await store.read(from);
1199
+ if (bytes === null) return;
1200
+ await store.write(to, [bytes]);
1201
+ await store.remove(from);
1202
+ }
1203
+ };
1204
+ }
1205
+
1206
+ // src/persistence/durability/archive-checkpoint.ts
1207
+ function toError3(error) {
1208
+ return error instanceof Error ? error : new Error(String(error));
1209
+ }
1210
+ function stageDirectoryRestore(deps, indexName) {
1211
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1212
+ async finish(run) {
1213
+ try {
1214
+ await run();
1215
+ } catch (error) {
1216
+ deps.markFatal(toError3(error));
1217
+ throw error;
1218
+ }
1219
+ },
1220
+ onStorageLost: deps.markFatal
1221
+ });
1222
+ }
1223
+ function releaser(held, indexName) {
1224
+ let released = false;
1225
+ return () => {
1226
+ if (released) return;
1227
+ released = true;
1228
+ const count = (held.get(indexName) ?? 1) - 1;
1229
+ if (count <= 0) held.delete(indexName);
1230
+ else held.set(indexName, count);
1231
+ };
1232
+ }
1233
+ function createArchiveCheckpointAccess(deps) {
1234
+ const store = directoryCheckpointStore(deps.directory);
1235
+ return {
1236
+ async openArchiveCheckpoint(indexName) {
1237
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1238
+ const release = releaser(deps.held, indexName);
1239
+ try {
1240
+ await deps.checkpoint(indexName, false);
1241
+ await deps.checkpoint(indexName, true);
1242
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
1243
+ if (manifestBytes === null) {
1244
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1245
+ indexName
1246
+ });
1247
+ }
1248
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1249
+ if (decoded.kind !== "current") {
1250
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1251
+ indexName
1252
+ });
1253
+ }
1254
+ return {
1255
+ manifestBytes,
1256
+ manifest: decoded.manifest,
1257
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1258
+ readRange: async (ref, offset, length) => {
1259
+ const readRange = deps.directory.readRange;
1260
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1261
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1262
+ return whole.slice(offset, offset + length);
1263
+ },
1264
+ release,
1265
+ directory: deps.directory
1266
+ };
1267
+ } catch (error) {
1268
+ release();
1269
+ throw error;
1270
+ }
1271
+ },
1272
+ stageRestore(indexName) {
1273
+ return stageDirectoryRestore(deps, indexName);
1274
+ },
1275
+ adoptRestoredCheckpoint(indexName, checkpoint) {
1276
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1277
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1278
+ }
1279
+ }
1280
+ };
1203
1281
  }
1204
- function recordsBeyond(indexState, targets) {
1205
- let records = 0;
1206
- for (const [partitionId, partition] of indexState.partitions) {
1207
- if (partition.failed !== null) continue;
1208
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
1282
+
1283
+ // src/persistence/durability/query-history.ts
1284
+ async function removeQueryHistoryOf(storage, indexName) {
1285
+ const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
1286
+ if (storage.kind === "directory") {
1287
+ for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
1288
+ return;
1209
1289
  }
1210
- return records;
1290
+ for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
1211
1291
  }
1212
-
1213
- // src/persistence/durability/checkpoint-turn.ts
1214
- function createIndexLoads(vectorIndexesOf2) {
1215
- const loads = /* @__PURE__ */ new Map();
1216
- const startGraphRepairs = (indexName) => {
1217
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1218
- vectorIndex.scheduleBuild();
1219
- vectorIndex.startShuffledRebuildWhenDue();
1220
- }
1221
- };
1292
+ function queryHistoryAccess(storage) {
1222
1293
  return {
1223
- track(indexName, load, startsGraphRepairs = true) {
1224
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1225
- loads.set(
1226
- indexName,
1227
- loaded.catch(() => void 0)
1228
- );
1229
- return loaded;
1230
- },
1231
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1232
- startGraphRepairs
1294
+ historyStorage: () => storage,
1295
+ removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
1233
1296
  };
1234
1297
  }
1235
- async function switchInTurn(turns, turn) {
1236
- let swapped = false;
1237
- await checkpointInTurn(turns, {
1238
- joinsInFlight: false,
1239
- mayStart: turn.mayStart,
1240
- perform: async () => {
1241
- swapped = turn.swap();
1242
- if (swapped) await turn.checkpoint();
1298
+
1299
+ // src/persistence/durability/manager-unload.ts
1300
+ function createIndexUnloading(deps) {
1301
+ const { directory, hooks, indexes, metadataWrites } = deps;
1302
+ return {
1303
+ async removeIndex(indexName) {
1304
+ hooks.writeRecords?.dropIndex(indexName);
1305
+ const indexState = indexes.get(indexName);
1306
+ if (indexState !== void 0) {
1307
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
1308
+ indexes.delete(indexName);
1309
+ metadataWrites.forget(indexName);
1310
+ }
1311
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
1312
+ for (const key of await directory.list(`${indexName}/`)) {
1313
+ await directory.remove(key);
1314
+ }
1315
+ await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
1316
+ deps.forgetSyncState(indexName);
1243
1317
  },
1244
- afterSettling: turn.afterSettling
1245
- });
1246
- return swapped;
1247
- }
1248
- async function checkpointInTurn(turns, turn) {
1249
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1250
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1251
- if (!turn.mayStart()) return;
1252
- const run = turn.perform().finally(() => {
1253
- turns.checkpointInFlight = null;
1254
- turn.afterSettling();
1255
- });
1256
- turns.checkpointInFlight = run;
1257
- return run;
1318
+ async unloadIndex(indexName) {
1319
+ const indexState = indexes.get(indexName);
1320
+ if (indexState === void 0) {
1321
+ return;
1322
+ }
1323
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
1324
+ indexes.delete(indexName);
1325
+ metadataWrites.forget(indexName);
1326
+ deps.forgetSyncState(indexName);
1327
+ }
1328
+ };
1258
1329
  }
1259
1330
 
1260
- // src/persistence/durability/directory-watch.ts
1261
- function unrefInterval(tick, intervalMs) {
1262
- const timer = setInterval(tick, intervalMs);
1263
- if (typeof timer.unref === "function") timer.unref();
1264
- return timer;
1265
- }
1266
- function createDirectoryWatch(directory, isStopped, onLost) {
1267
- let expected = null;
1268
- let timer = null;
1269
- let lock = null;
1270
- function cannotCheck(err) {
1271
- return new NarsilError(
1272
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1273
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
1274
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
1275
- );
1331
+ // src/persistence/durability/sync/frame-cursor.ts
1332
+ var SegmentFileMissing = class extends Error {
1333
+ };
1334
+ function corrupt(key, message, details) {
1335
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
1336
+ }
1337
+ function isMissingFile(error) {
1338
+ return error?.code === "ENOENT";
1339
+ }
1340
+ var FrameCursor = class {
1341
+ constructor(directory, key, limit) {
1342
+ this.directory = directory;
1343
+ this.key = key;
1344
+ this.limit = limit;
1345
+ }
1346
+ buffer = new Uint8Array(0);
1347
+ bufferStart = 0;
1348
+ whole = null;
1349
+ async load(offset, length) {
1350
+ const readRange = this.directory.readRange;
1351
+ if (readRange === void 0) {
1352
+ if (this.whole === null) {
1353
+ const bytes = await this.directory.read(this.key);
1354
+ if (bytes === null) throw new SegmentFileMissing(this.key);
1355
+ this.whole = bytes;
1356
+ }
1357
+ this.buffer = this.whole;
1358
+ this.bufferStart = 0;
1359
+ return;
1360
+ }
1361
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
1362
+ try {
1363
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
1364
+ } catch (error) {
1365
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
1366
+ throw error;
1367
+ }
1368
+ this.bufferStart = offset;
1276
1369
  }
1277
- async function currentIdentity() {
1370
+ async bytesAt(offset, length) {
1371
+ const end = offset + length;
1372
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
1373
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
1374
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
1375
+ }
1376
+ async checkHeader() {
1377
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
1378
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
1379
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
1380
+ }
1381
+ async frameLengthAt(offset) {
1382
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
1383
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
1384
+ 0,
1385
+ false
1386
+ );
1387
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
1388
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
1389
+ }
1390
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
1391
+ }
1392
+ async frameAt(offset, frameBytes) {
1393
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
1394
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
1395
+ const payload = body.subarray(0, recordLength);
1396
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
1397
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
1398
+ let seqNo;
1278
1399
  try {
1279
- return await directory.identity?.() ?? null;
1280
- } catch (err) {
1281
- return cannotCheck(err);
1400
+ seqNo = decodeWalEntry(payload).seqNo;
1401
+ } catch (error) {
1402
+ corrupt(this.key, "A log record fails to decode", {
1403
+ offset,
1404
+ cause: error instanceof Error ? error.message : String(error)
1405
+ });
1282
1406
  }
1407
+ return { seqNo, payload: payload.slice(), frameBytes };
1408
+ }
1409
+ };
1410
+
1411
+ // src/persistence/durability/sync/log-reader.ts
1412
+ async function walSources(directory, indexName, partitionId) {
1413
+ const prefix = walSegmentPrefix(indexName, partitionId);
1414
+ const markerBytes = await directory.read(`${prefix}commit`);
1415
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1416
+ if (marker === null) return { sources: [], head: 0 };
1417
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
1418
+ const sources = [];
1419
+ for (const key of await directory.list(prefix)) {
1420
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
1421
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
1422
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
1423
+ if (readableBytes === null || readableBytes === void 0) continue;
1424
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
1425
+ }
1426
+ return { sources, head: highestDurableSeqNo };
1427
+ }
1428
+ async function openPartitionLog(directory, indexName, partitionId) {
1429
+ const { sources, head } = await walSources(directory, indexName, partitionId);
1430
+ const byStart = /* @__PURE__ */ new Map();
1431
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
1432
+ const readableBytes = await directory.byteLength?.(segment.key);
1433
+ if (readableBytes === null || readableBytes === void 0) continue;
1434
+ byStart.set(segment.startSeqNo, {
1435
+ key: segment.key,
1436
+ startSeqNo: segment.startSeqNo,
1437
+ readableBytes,
1438
+ inHistory: true
1439
+ });
1283
1440
  }
1284
- async function lockTakenOver() {
1285
- if (lock === null) return null;
1441
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
1442
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
1443
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
1444
+ }
1445
+ function sourceHolding(log, seqNo) {
1446
+ let found = -1;
1447
+ for (let i = 0; i < log.sources.length; i++) {
1448
+ if (log.sources[i].startSeqNo > seqNo) break;
1449
+ found = i;
1450
+ }
1451
+ return found;
1452
+ }
1453
+ async function offsetAfter(cursor, source, afterSeqNo) {
1454
+ let offset = SEGMENT_HEADER_SIZE;
1455
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
1456
+ offset += await cursor.frameLengthAt(offset);
1457
+ }
1458
+ return offset;
1459
+ }
1460
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
1461
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1462
+ await cursor.checkHeader();
1463
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
1464
+ let checksNumbering = true;
1465
+ while (offset < cursor.limit) {
1466
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
1467
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
1468
+ checksNumbering = false;
1469
+ if (frame.seqNo > afterSeqNo) {
1470
+ offset = SEGMENT_HEADER_SIZE;
1471
+ continue;
1472
+ }
1473
+ }
1474
+ checksNumbering = false;
1475
+ offset += frame.frameBytes;
1476
+ if (frame.seqNo <= afterSeqNo) continue;
1477
+ if (frame.seqNo > upToSeqNo) return true;
1478
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
1479
+ read.frames.push(frame);
1480
+ read.bytes += frame.payload.byteLength;
1481
+ if (frame.seqNo === upToSeqNo) return true;
1482
+ }
1483
+ return false;
1484
+ }
1485
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
1486
+ const read = { frames: [], bytes: 0, gone: false };
1487
+ if (afterSeqNo >= upToSeqNo) return read;
1488
+ for (let attempt = 0; attempt < 2; attempt++) {
1489
+ read.frames = [];
1490
+ read.bytes = 0;
1491
+ const log = await openPartitionLog(directory, indexName, partitionId);
1492
+ let index = sourceHolding(log, afterSeqNo + 1);
1493
+ if (index < 0) {
1494
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
1495
+ return read;
1496
+ }
1286
1497
  try {
1287
- if (await lock.holds()) return null;
1288
- } catch (err) {
1289
- return cannotCheck(err);
1498
+ for (; index < log.sources.length; index++) {
1499
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
1500
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
1501
+ }
1502
+ return read;
1503
+ } catch (error) {
1504
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
1290
1505
  }
1291
- return new NarsilError(
1292
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1293
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
1294
- { directory: directory.root }
1295
- );
1296
1506
  }
1297
- async function verify() {
1298
- if (directory.identity === void 0 || isStopped()) return true;
1299
- const current = await currentIdentity();
1300
- if (current instanceof Error) {
1301
- onLost(current);
1302
- return false;
1303
- }
1304
- if (expected === null) {
1305
- expected = current;
1306
- } else if (current !== expected) {
1307
- onLost(
1308
- new NarsilError(
1309
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1310
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
1311
- { directory: directory.root }
1312
- )
1313
- );
1314
- return false;
1507
+ return read;
1508
+ }
1509
+ async function logBytesAfter(directory, log, afterSeqNo) {
1510
+ const first = sourceHolding(log, afterSeqNo + 1);
1511
+ let bytes = 0;
1512
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
1513
+ const source = log.sources[i];
1514
+ if (i === first) {
1515
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1516
+ try {
1517
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
1518
+ } catch (error) {
1519
+ if (!(error instanceof SegmentFileMissing)) throw error;
1520
+ }
1521
+ continue;
1315
1522
  }
1316
- const takenOver = await lockTakenOver();
1317
- if (takenOver === null) return true;
1318
- onLost(takenOver);
1319
- return false;
1523
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
1320
1524
  }
1321
- function stop() {
1322
- if (timer !== null) {
1323
- clearInterval(timer);
1324
- timer = null;
1525
+ return bytes;
1526
+ }
1527
+
1528
+ // src/persistence/durability/sync/pins.ts
1529
+ function createFilePins(remove) {
1530
+ const counts = /* @__PURE__ */ new Map();
1531
+ const awaitingRemoval = /* @__PURE__ */ new Set();
1532
+ function unpin(key) {
1533
+ const count = (counts.get(key) ?? 1) - 1;
1534
+ if (count > 0) {
1535
+ counts.set(key, count);
1536
+ return;
1325
1537
  }
1538
+ counts.delete(key);
1539
+ if (!awaitingRemoval.delete(key)) return;
1540
+ void remove(key).catch(() => void 0);
1326
1541
  }
1327
1542
  return {
1328
- async claim() {
1329
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
1330
- },
1331
- async release() {
1332
- const held = lock;
1333
- lock = null;
1334
- await held?.release();
1335
- },
1336
- verify,
1337
- async verifyOnce() {
1338
- if (expected === null && directory.identity !== void 0) await verify();
1339
- },
1340
- start() {
1341
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
1342
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
1543
+ pin(keys) {
1544
+ const pinned = [...keys];
1545
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
1546
+ let released = false;
1547
+ return () => {
1548
+ if (released) return;
1549
+ released = true;
1550
+ for (const key of pinned) unpin(key);
1551
+ };
1343
1552
  },
1344
- stop
1553
+ isPinned: (key) => counts.has(key),
1554
+ removeWhenUnpinned(key) {
1555
+ if (!counts.has(key)) return false;
1556
+ awaitingRemoval.add(key);
1557
+ return true;
1558
+ }
1559
+ };
1560
+ }
1561
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
1562
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
1563
+ function hexOf(bytes) {
1564
+ let hex = "";
1565
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
1566
+ return hex;
1567
+ }
1568
+ function isBytes(value) {
1569
+ return value instanceof Uint8Array;
1570
+ }
1571
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
1572
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1573
+ if (decoded.kind !== "current") {
1574
+ throw new NarsilError(
1575
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1576
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
1577
+ { indexName }
1578
+ );
1579
+ }
1580
+ return {
1581
+ config,
1582
+ manifestBytes,
1583
+ manifest: decoded.manifest,
1584
+ entityTag: hexOf(sha256(payload)),
1585
+ keys: manifestReferencedKeys(decoded.manifest)
1345
1586
  };
1346
1587
  }
1588
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
1589
+ const payload = encode({ config, manifest: manifestBytes });
1590
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
1591
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
1592
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
1593
+ return published;
1594
+ }
1595
+ async function readPublishedSnapshot(directory, indexName) {
1596
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
1597
+ if (bytes === null) return null;
1598
+ const { payload } = await openChecksummedEnvelope(bytes, {
1599
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
1600
+ what: PUBLISHED_SNAPSHOT_WHAT,
1601
+ allowsCompression: false
1602
+ });
1603
+ const raw = decode(payload);
1604
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
1605
+ throw new NarsilError(
1606
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1607
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
1608
+ { indexName }
1609
+ );
1610
+ }
1611
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
1612
+ }
1347
1613
 
1348
- // src/persistence/durability/held-checkpoints.ts
1349
- function failedPartitionIds(indexState) {
1350
- const failed = [];
1351
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1352
- return failed;
1614
+ // src/persistence/durability/sync/retention.ts
1615
+ async function firstLogStart(directory, indexName, partitionId) {
1616
+ let first = Number.POSITIVE_INFINITY;
1617
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
1618
+ const start = parseWalSegmentStart(indexName, partitionId, key);
1619
+ if (start !== null && start < first) first = start;
1620
+ }
1621
+ return first;
1622
+ }
1623
+ async function sizedQueues(retention) {
1624
+ const { directory, indexName } = retention;
1625
+ const queues = /* @__PURE__ */ new Map();
1626
+ for (const segment of await listHistorySegments(directory, indexName)) {
1627
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
1628
+ if (bytes === void 0 || bytes === null) continue;
1629
+ const queue = queues.get(segment.partitionId) ?? [];
1630
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
1631
+ queues.set(segment.partitionId, queue);
1632
+ }
1633
+ for (const [partitionId, queue] of queues) {
1634
+ const logStart = await firstLogStart(directory, indexName, partitionId);
1635
+ for (let i = 0; i < queue.length; i++) {
1636
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
1637
+ queue[i].lastSeqNo = nextStart - 1;
1638
+ }
1639
+ }
1640
+ return queues;
1641
+ }
1642
+ function nextToDelete(queues, totalBytes, retention) {
1643
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
1644
+ let oldest = null;
1645
+ for (const queue of queues.values()) {
1646
+ const head = queue[0];
1647
+ if (head === void 0) continue;
1648
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
1649
+ }
1650
+ if (oldest === null) return null;
1651
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
1652
+ return null;
1653
+ }
1654
+ async function enforceHistoryLimits(retention) {
1655
+ const queues = await sizedQueues(retention);
1656
+ let totalBytes = 0;
1657
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
1658
+ let published = false;
1659
+ let deleted = 0;
1660
+ let segment = nextToDelete(queues, totalBytes, retention);
1661
+ while (segment !== null) {
1662
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
1663
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
1664
+ published = true;
1665
+ await retention.publish();
1666
+ }
1667
+ await retention.directory.remove(segment.key);
1668
+ queues.get(segment.partitionId)?.shift();
1669
+ totalBytes -= segment.bytes;
1670
+ deleted += 1;
1671
+ segment = nextToDelete(queues, totalBytes, retention);
1672
+ }
1673
+ return deleted;
1353
1674
  }
1354
- function createHeldCheckpoints(hooks) {
1355
- const reported = /* @__PURE__ */ new WeakSet();
1356
- function heldReport(indexName, indexState) {
1357
- const partitionIds = failedPartitionIds(indexState);
1358
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1359
- return {
1360
- indexName,
1361
- partitionIds,
1362
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1363
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1364
- };
1675
+
1676
+ // src/persistence/durability/sync/storage.ts
1677
+ function createSyncStorage(deps) {
1678
+ const { directory } = deps;
1679
+ const store = directoryCheckpointStore(directory);
1680
+ const cache = /* @__PURE__ */ new Map();
1681
+ const publishing = /* @__PURE__ */ new Map();
1682
+ const pins = createFilePins((key) => directory.remove(key));
1683
+ async function published(indexName) {
1684
+ const cached = cache.get(indexName);
1685
+ if (cached !== void 0) return cached;
1686
+ const read = await readPublishedSnapshot(directory, indexName);
1687
+ cache.set(indexName, read);
1688
+ return read;
1689
+ }
1690
+ async function removeUnlisted(indexName, previous, kept) {
1691
+ const manifest = await readCurrentManifest(store, indexName);
1692
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
1693
+ for (const key of previous) {
1694
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
1695
+ await directory.remove(key);
1696
+ }
1697
+ }
1698
+ async function publishNow(indexName, configurationOf) {
1699
+ const archive = await deps.openArchiveCheckpoint(indexName);
1700
+ try {
1701
+ const config = await configurationOf(archive.manifest);
1702
+ const previous = await published(indexName);
1703
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
1704
+ cache.set(indexName, next);
1705
+ if (previous !== null) {
1706
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
1707
+ }
1708
+ return next;
1709
+ } finally {
1710
+ archive.release();
1711
+ }
1365
1712
  }
1366
1713
  return {
1367
- refuse(indexName, indexState) {
1368
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1369
- if (partitionIds.length === 0) return;
1370
- throw new NarsilError(
1371
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1372
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1373
- { indexName, partitionIds }
1374
- );
1714
+ publish(indexName, configurationOf) {
1715
+ const inFlight = publishing.get(indexName);
1716
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
1717
+ const tracked = run.finally(() => {
1718
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
1719
+ });
1720
+ publishing.set(indexName, tracked);
1721
+ return tracked;
1722
+ },
1723
+ published,
1724
+ pin: (snapshot) => pins.pin(snapshot.keys),
1725
+ async readFileRange(key, offset, length) {
1726
+ const readRange = directory.readRange;
1727
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
1728
+ const whole = await directory.read(key);
1729
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
1730
+ },
1731
+ async release(indexName) {
1732
+ await deps.inCheckpointTurn(indexName, async () => {
1733
+ const previous = await published(indexName);
1734
+ await directory.remove(publishedSnapshotKey(indexName));
1735
+ cache.set(indexName, null);
1736
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
1737
+ await removeHistory(directory, indexName);
1738
+ });
1375
1739
  },
1376
- holds(indexName, indexState) {
1377
- const held = heldReport(indexName, indexState);
1378
- if (held === null) return false;
1379
- const unreported = held.partitionIds.some((partitionId) => {
1380
- const partition = indexState.partitions.get(partitionId);
1381
- return partition !== void 0 && !reported.has(partition);
1740
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
1741
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
1742
+ const current = await published(indexName);
1743
+ return enforceHistoryLimits({
1744
+ directory,
1745
+ indexName,
1746
+ limits,
1747
+ nowMs,
1748
+ publish,
1749
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
1382
1750
  });
1383
- if (unreported) {
1384
- for (const partitionId of held.partitionIds) {
1385
- const partition = indexState.partitions.get(partitionId);
1386
- if (partition !== void 0) reported.add(partition);
1387
- }
1388
- hooks.onCheckpointHeld?.(held);
1389
- }
1390
- return true;
1391
1751
  },
1392
- report(indexes) {
1393
- const held = [];
1394
- for (const [indexName, indexState] of indexes) {
1395
- const entry = heldReport(indexName, indexState);
1396
- if (entry !== null) held.push(entry);
1752
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
1753
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
1754
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
1755
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
1756
+ async keepsCheckpointKey(indexName) {
1757
+ const current = await published(indexName);
1758
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
1759
+ },
1760
+ async settleAtRecovery(indexName, active) {
1761
+ cache.delete(indexName);
1762
+ const publishedKey = publishedSnapshotKey(indexName);
1763
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
1764
+ if (key !== publishedKey) await directory.remove(key);
1397
1765
  }
1398
- return held;
1766
+ if (!active) {
1767
+ await directory.remove(publishedSnapshotKey(indexName));
1768
+ await removeHistory(directory, indexName);
1769
+ cache.set(indexName, null);
1770
+ return /* @__PURE__ */ new Set();
1771
+ }
1772
+ try {
1773
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
1774
+ } catch {
1775
+ await directory.remove(publishedSnapshotKey(indexName));
1776
+ cache.set(indexName, null);
1777
+ return /* @__PURE__ */ new Set();
1778
+ }
1779
+ },
1780
+ forget(indexName) {
1781
+ cache.delete(indexName);
1399
1782
  }
1400
1783
  };
1401
1784
  }
1402
1785
 
1786
+ // src/persistence/durability/manager-storage.ts
1787
+ function createIndexStorageAccess(deps) {
1788
+ const { directory, indexes } = deps;
1789
+ const archiveAccess = createArchiveCheckpointAccess({
1790
+ directory,
1791
+ held: deps.heldCheckpointFiles,
1792
+ checkpoint: deps.checkpoint,
1793
+ registerPartition: deps.registerPartition,
1794
+ markFatal: deps.markFatal
1795
+ });
1796
+ const syncStorage = createSyncStorage({
1797
+ directory,
1798
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
1799
+ inCheckpointTurn: (indexName, work) => {
1800
+ const indexState = indexes.get(indexName);
1801
+ if (indexState === void 0) return work();
1802
+ return checkpointInTurn(indexState, {
1803
+ joinsInFlight: false,
1804
+ mayStart: () => true,
1805
+ perform: work,
1806
+ afterSettling: () => releaseStalledWrites(indexState)
1807
+ });
1808
+ }
1809
+ });
1810
+ const unloading = createIndexUnloading({
1811
+ directory,
1812
+ hooks: deps.hooks,
1813
+ indexes,
1814
+ metadataWrites: deps.metadataWrites,
1815
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
1816
+ forgetSyncState: syncStorage.forget
1817
+ });
1818
+ return { archiveAccess, syncStorage, unloading };
1819
+ }
1820
+
1403
1821
  // src/persistence/durability/metadata-write-queue.ts
1404
1822
  function createMetadataWriteQueue() {
1405
1823
  const writes = /* @__PURE__ */ new Map();
@@ -1645,8 +2063,8 @@ function createGroupCommitCoordinator(sync) {
1645
2063
  // src/persistence/durability/wal-writer.ts
1646
2064
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
1647
2065
  function segmentKey(indexName, partitionId, startSeqNo) {
1648
- const padded2 = startSeqNo.toString().padStart(16, "0");
1649
- return `${indexName}/wal/${partitionId}/${padded2}`;
2066
+ const padded = startSeqNo.toString().padStart(16, "0");
2067
+ return `${indexName}/wal/${partitionId}/${padded}`;
1650
2068
  }
1651
2069
  function parseSegmentStartSeqNo2(key) {
1652
2070
  const tail = key.slice(key.lastIndexOf("/") + 1);
@@ -2370,7 +2788,7 @@ var decoder = new TextDecoder();
2370
2788
  async function countPagedPartition(store, partition) {
2371
2789
  const { directory } = store;
2372
2790
  if (directory === void 0 || partition.segments.length === 0) return null;
2373
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-6RLIKSZQ.mjs');
2791
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-DWKPBSMN.mjs');
2374
2792
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2375
2793
  const scans = [];
2376
2794
  try {
@@ -2653,12 +3071,14 @@ async function recoverCheckpoint(deps) {
2653
3071
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
2654
3072
  }
2655
3073
  if (decoded === null || decoded.kind !== "current") {
2656
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
3074
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
2657
3075
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
2658
3076
  }
2659
3077
  await deps.store.remove(snapshotBundleKey(deps.indexName));
2660
3078
  if (!deps.keepsUnlistedFiles) {
2661
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
3079
+ const referenced = manifestReferencedKeys(decoded.manifest);
3080
+ for (const key of deps.keptForSync) referenced.add(key);
3081
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
2662
3082
  }
2663
3083
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
2664
3084
  manager: deps.manager,
@@ -2779,6 +3199,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
2779
3199
  }
2780
3200
  return replayed;
2781
3201
  };
3202
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
3203
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
2782
3204
  const recovered = await recoverCheckpoint({
2783
3205
  store,
2784
3206
  indexName,
@@ -2788,6 +3210,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
2788
3210
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
2789
3211
  compactionThreshold: deps.compactionThreshold,
2790
3212
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3213
+ keptForSync,
2791
3214
  replay,
2792
3215
  onLayoutUpgrade: hooks.onLayoutUpgrade,
2793
3216
  writeRecords
@@ -2903,6 +3326,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2903
3326
  mutationThreshold: checkpointMutationThreshold,
2904
3327
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
2905
3328
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
3329
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
2906
3330
  queueMetadataWrite,
2907
3331
  markFatal,
2908
3332
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -2956,6 +3380,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2956
3380
  startAsyncFlushTimer();
2957
3381
  return outcomes;
2958
3382
  }
3383
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
3384
+ directory,
3385
+ hooks,
3386
+ indexes,
3387
+ metadataWrites,
3388
+ heldCheckpointFiles,
3389
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3390
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3391
+ markFatal,
3392
+ closeWriterReportingFailure
3393
+ });
2959
3394
  const recoveryDeps = {
2960
3395
  directory,
2961
3396
  hooks,
@@ -2964,7 +3399,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2964
3399
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
2965
3400
  },
2966
3401
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
2967
- queueMetadataWrite
3402
+ queueMetadataWrite,
3403
+ syncStorage
2968
3404
  };
2969
3405
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
2970
3406
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3052,35 +3488,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3052
3488
  await checkpointIndex(indexName);
3053
3489
  }
3054
3490
  },
3055
- async removeIndex(indexName) {
3056
- hooks.writeRecords?.dropIndex(indexName);
3057
- const indexState = indexes.get(indexName);
3058
- if (indexState !== void 0) {
3059
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3060
- indexes.delete(indexName);
3061
- metadataWrites.forget(indexName);
3062
- }
3063
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3064
- for (const key of await directory.list(`${indexName}/`)) {
3065
- await directory.remove(key);
3066
- }
3067
- },
3068
- async unloadIndex(indexName) {
3069
- const indexState = indexes.get(indexName);
3070
- if (indexState === void 0) {
3071
- return;
3072
- }
3073
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3074
- indexes.delete(indexName);
3075
- metadataWrites.forget(indexName);
3076
- },
3077
- ...createArchiveCheckpointAccess({
3078
- directory,
3079
- held: heldCheckpointFiles,
3080
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3081
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3082
- markFatal
3083
- }),
3491
+ ...unloading,
3492
+ syncStorage: () => syncStorage,
3493
+ ...queryHistoryAccess({ kind: "directory", directory }),
3494
+ ...archiveAccess,
3084
3495
  async shutdown() {
3085
3496
  shuttingDown = true;
3086
3497
  stopTimers();
@@ -3347,6 +3758,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3347
3758
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3348
3759
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3349
3760
  keepsUnlistedFiles: false,
3761
+ keptForSync: /* @__PURE__ */ new Set(),
3350
3762
  replay: async (checkpoint) => checkpoint,
3351
3763
  onLayoutUpgrade: hooks.onLayoutUpgrade
3352
3764
  });
@@ -3482,6 +3894,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3482
3894
  if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
3483
3895
  }
3484
3896
  },
3897
+ ...queryHistoryAccess({ kind: "adapter", adapter }),
3485
3898
  async removeIndex(indexName) {
3486
3899
  await settleRestore(indexName).catch(() => void 0);
3487
3900
  restores.forget(indexName);
@@ -3490,6 +3903,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3490
3903
  for (const key of await adapter.list(`${indexName}/`)) {
3491
3904
  await adapter.delete(key);
3492
3905
  }
3906
+ await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
3493
3907
  },
3494
3908
  async unloadIndex(indexName) {
3495
3909
  await settleRestore(indexName);
@@ -3544,7 +3958,9 @@ function createDurabilityIntegration(tier, hooks) {
3544
3958
  recordCheckpoint: hooks.recordCheckpoint,
3545
3959
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3546
3960
  onCheckpointHeld: hooks.onCheckpointHeld,
3547
- onFatalError: hooks.onFatalError
3961
+ onFatalError: hooks.onFatalError,
3962
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
3963
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
3548
3964
  };
3549
3965
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
3550
3966
  function partitionFor(indexName, docId) {
@@ -3629,7 +4045,12 @@ function createDurabilityFromTier(tier, wiring) {
3629
4045
  recordCheckpoint: wiring.recordCheckpoint,
3630
4046
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
3631
4047
  onCheckpointHeld: wiring.emitCheckpointHeld,
3632
- onFatalError: wiring.emitFatalError
4048
+ onFatalError: wiring.emitFatalError,
4049
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4050
+ onHistoryMoved(indexName) {
4051
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4052
+ import('./retention-HROPV2QU.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4053
+ }
3633
4054
  });
3634
4055
  }
3635
4056