@delali/narsil 0.3.1-next.153 → 0.3.1-next.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (330) hide show
  1. package/README.md +3 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
  8. package/dist/archive-history-E4R73RWD.mjs +29 -0
  9. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
  10. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
  11. package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
  12. package/dist/browser/chunk-23XRXIBW.mjs +85 -0
  13. package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
  14. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
  15. package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
  16. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
  17. package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
  18. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  19. package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
  20. package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
  21. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
  22. package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
  23. package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
  24. package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
  25. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  26. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  27. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
  28. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
  29. package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
  30. package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
  31. package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
  32. package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
  33. package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
  34. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
  35. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
  36. package/dist/browser/chunk-KKAMCABR.mjs +432 -0
  37. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
  38. package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
  39. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  40. package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
  41. package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
  42. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  43. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
  44. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
  45. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
  46. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
  47. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
  48. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
  49. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
  50. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  51. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
  52. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  53. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  54. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
  55. package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
  56. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
  57. package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
  58. package/dist/browser/import-OM6QAW45.mjs +18 -0
  59. package/dist/browser/methods-KTQCSIN4.mjs +30 -0
  60. package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
  61. package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
  62. package/dist/browser/operations-OJ3BXS55.mjs +5 -0
  63. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
  64. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
  65. package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
  66. package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
  67. package/dist/browser/retention-ASWUHAYI.mjs +209 -0
  68. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
  69. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
  70. package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
  71. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  72. package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
  73. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
  74. package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
  75. package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
  76. package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
  77. package/dist/chunk-3HB3PEC6.mjs +240 -0
  78. package/dist/chunk-3HHQLEEQ.mjs +1092 -0
  79. package/dist/chunk-3OH67G7R.mjs +214 -0
  80. package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
  81. package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
  82. package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
  83. package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
  84. package/dist/chunk-5FVU47TB.mjs +20 -0
  85. package/dist/chunk-5YNRKI2C.mjs +81 -0
  86. package/dist/chunk-6L3XJXGU.mjs +470 -0
  87. package/dist/chunk-6MSZ543Y.mjs +32 -0
  88. package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
  89. package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
  90. package/dist/chunk-7GXJUJ24.mjs +49 -0
  91. package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
  92. package/dist/chunk-BDRIVYOK.mjs +72 -0
  93. package/dist/chunk-BUMREPLU.mjs +15 -0
  94. package/dist/chunk-BY6ZDGYS.mjs +50 -0
  95. package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
  96. package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
  97. package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
  98. package/dist/chunk-GES7U2PM.mjs +68 -0
  99. package/dist/chunk-GH4WQ22V.mjs +34 -0
  100. package/dist/chunk-GQPTDQCL.mjs +54 -0
  101. package/dist/chunk-H4KHPLHJ.mjs +382 -0
  102. package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
  103. package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
  104. package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
  105. package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
  106. package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
  107. package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
  108. package/dist/chunk-IPN7S4KX.mjs +130 -0
  109. package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
  110. package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
  111. package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
  112. package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
  114. package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
  115. package/dist/chunk-KJ3RTZDJ.mjs +101 -0
  116. package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
  117. package/dist/chunk-KP7TYHWT.mjs +92 -0
  118. package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
  119. package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
  120. package/dist/chunk-KYYABCSW.mjs +72 -0
  121. package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
  122. package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
  123. package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
  124. package/dist/chunk-LNGE7SLY.mjs +151 -0
  125. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
  126. package/dist/chunk-MT2NRTA6.mjs +216 -0
  127. package/dist/chunk-NF4DT3IH.mjs +85 -0
  128. package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
  129. package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
  130. package/dist/chunk-PHQNACDT.mjs +63 -0
  131. package/dist/chunk-Q622CTMC.mjs +74 -0
  132. package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
  133. package/dist/chunk-QDZJM7NI.mjs +23 -0
  134. package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
  135. package/dist/chunk-QJVOLKU6.mjs +111 -0
  136. package/dist/chunk-QUJHMPTP.mjs +30 -0
  137. package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
  138. package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
  139. package/dist/chunk-RZWWB2XR.mjs +96 -0
  140. package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
  141. package/dist/chunk-S5LN746F.mjs +10 -0
  142. package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
  143. package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
  144. package/dist/chunk-SY3EWKQY.mjs +242 -0
  145. package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
  146. package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
  147. package/dist/chunk-UHICS3JI.mjs +83 -0
  148. package/dist/chunk-UZRZKNNZ.mjs +101 -0
  149. package/dist/chunk-V6EUBAIT.mjs +19 -0
  150. package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
  151. package/dist/chunk-VHBNF4MA.mjs +543 -0
  152. package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
  153. package/dist/chunk-WQRN64Q6.mjs +196 -0
  154. package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
  155. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  156. package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
  157. package/dist/chunk-YEKL4JBK.mjs +96 -0
  158. package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
  159. package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
  160. package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
  161. package/dist/chunk-ZTW5WTGG.mjs +167 -0
  162. package/dist/chunk-ZYAY26KG.mjs +29 -0
  163. package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
  164. package/dist/client.d.ts +9 -7
  165. package/dist/client.mjs +120 -7
  166. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
  167. package/dist/config-CoSig5Ox.d.ts +482 -0
  168. package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
  169. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
  170. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  171. package/dist/distribution/coordinator/etcd.mjs +1 -1
  172. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  173. package/dist/distribution/transport/grpc.mjs +4 -4
  174. package/dist/distribution/transport/in-memory.mjs +2 -2
  175. package/dist/distribution/transport/tcp.mjs +3 -3
  176. package/dist/distribution.d.ts +41 -9
  177. package/dist/distribution.mjs +9304 -7271
  178. package/dist/durability-config-GE4U7MCZ.mjs +89 -0
  179. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
  180. package/dist/embeddings/openai.mjs +2 -2
  181. package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
  182. package/dist/engine-hooks-DHCMOADY.mjs +47 -0
  183. package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
  184. package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
  185. package/dist/index.browser.mjs +9 -4
  186. package/dist/index.d.ts +7 -6
  187. package/dist/index.mjs +29 -21
  188. package/dist/index.react-native.mjs +19 -12
  189. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
  190. package/dist/methods-7IOINERF.mjs +30 -0
  191. package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
  192. package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
  193. package/dist/operations-G5EP6JPX.mjs +5 -0
  194. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
  195. package/dist/partial-words.mjs +7 -5
  196. package/dist/pattern.mjs +4 -3
  197. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  198. package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
  199. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
  200. package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
  201. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  202. package/dist/query-suggestions/merge-worker.mjs +81 -0
  203. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  204. package/dist/query-suggestions.browser.mjs +246 -0
  205. package/dist/query-suggestions.d.ts +20 -0
  206. package/dist/query-suggestions.mjs +62 -0
  207. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
  208. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
  209. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
  210. package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
  211. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  212. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
  213. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
  214. package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
  215. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
  216. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
  217. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
  218. package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
  219. package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
  220. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
  221. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
  222. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
  223. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
  224. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  225. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
  226. package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
  227. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
  228. package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
  229. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
  230. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
  231. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  232. package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
  233. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
  234. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  235. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
  236. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
  237. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  238. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
  239. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  240. package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
  241. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
  242. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  243. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
  244. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
  245. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
  246. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
  247. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  248. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
  249. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  250. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
  251. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
  252. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  253. package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
  254. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
  255. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
  256. package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
  257. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
  258. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  259. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
  260. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
  261. package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
  262. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
  263. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
  264. package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
  265. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
  266. package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
  267. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
  268. package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
  269. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
  270. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
  271. package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
  272. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
  273. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
  274. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
  275. package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
  276. package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
  277. package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
  278. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
  279. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
  280. package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
  281. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  282. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
  283. package/dist/react-native/partial-words.mjs +8 -5
  284. package/dist/react-native/pattern.mjs +4 -3
  285. package/dist/react-native/query-suggestions.mjs +258 -0
  286. package/dist/react-native/sync.mjs +2299 -0
  287. package/dist/react-native/synonyms.mjs +7 -4
  288. package/dist/react.d.ts +11 -10
  289. package/dist/react.mjs +8 -1
  290. package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
  291. package/dist/rerankers/http.mjs +2 -2
  292. package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
  293. package/dist/restore-queue-JVGENGGE.mjs +55 -0
  294. package/dist/retention-HROPV2QU.mjs +59 -0
  295. package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
  296. package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
  297. package/dist/server.d.ts +119 -20
  298. package/dist/server.mjs +1123 -58
  299. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
  300. package/dist/start-UTWHGH42.mjs +59 -0
  301. package/dist/sync/filesystem.d.ts +30 -0
  302. package/dist/sync/filesystem.mjs +271 -0
  303. package/dist/sync/indexeddb.d.ts +30 -0
  304. package/dist/sync/indexeddb.mjs +237 -0
  305. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  306. package/dist/sync.browser.mjs +2294 -0
  307. package/dist/sync.d.ts +23 -0
  308. package/dist/sync.mjs +2268 -0
  309. package/dist/synonyms.mjs +6 -4
  310. package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
  311. package/dist/thread-6MOXY2KD.mjs +45 -0
  312. package/dist/types-BZCGjfLn.d.ts +227 -0
  313. package/dist/vector/search-worker.d.ts +1 -1
  314. package/dist/vector/search-worker.mjs +7 -7
  315. package/dist/workers/entry.mjs +8 -8
  316. package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
  317. package/openapi.json +912 -5
  318. package/package.json +45 -8
  319. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  320. package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
  321. package/dist/chunk-7LDMWKP6.mjs +0 -209
  322. package/dist/engine-CJRUHXgf.d.ts +0 -664
  323. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  324. package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
  325. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  326. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  327. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  328. package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
  329. package/dist/restore-queue-AOF5OTQV.mjs +0 -41
  330. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,20 +1,34 @@
1
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, SEGMENT_HEADER_SIZE, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-GJKXW3SF.mjs';
1
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-G64Q6OJF.mjs';
2
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-RND5CLX3.mjs';
3
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-A5KUEGDY.mjs';
4
+ import './chunk-YJHGOJAC.mjs';
5
+ import { sha256 } from './chunk-M3MQE2X7.mjs';
6
+ import './chunk-CJSYBQ4J.mjs';
7
+ import './chunk-KKAMCABR.mjs';
8
+ import { QUERY_HISTORY_FOLDER } from './chunk-7IGKGWSQ.mjs';
2
9
  import './chunk-JNOAKDUM.mjs';
3
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QJCO3TJU.mjs';
4
- import { applyDeleteEntry, applyIndexEntry } from './chunk-B3JM6Z74.mjs';
5
- import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
6
- import './chunk-3I27KXQD.mjs';
7
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-55WYRLDZ.mjs';
8
- import { encodeDocument, toError, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-KWYQ6KX4.mjs';
9
- import { fnv1a, createRequestId } from './chunk-MPK76GVS.mjs';
10
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-56W3HQVJ.mjs';
11
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-XY4VVBBO.mjs';
12
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-ZI2V4VD7.mjs';
13
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-23XRXIBW.mjs';
14
+ import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-SMEQ36NT.mjs';
15
+ import './chunk-ML7THVLJ.mjs';
16
+ import { createRequestId } from './chunk-VI7BQR4U.mjs';
10
17
  import './chunk-JPTQZWJQ.mjs';
18
+ import { encodeDocument, openChecksummedEnvelope, crc32, requireValidFieldTypes, HEADER_SIZE } from './chunk-JIJXMI42.mjs';
19
+ import { fnv1a } from './chunk-RA6LSBCM.mjs';
11
20
  import './chunk-26A5JFT2.mjs';
12
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, compareCodePoints, waitOutsideCountedWrite, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-IAXZAVYL.mjs';
21
+ import './chunk-3I27KXQD.mjs';
22
+ import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
23
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-5TCXK5SB.mjs';
24
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime, resolveWorkerEntry } from './chunk-4WCVMOU3.mjs';
13
25
  import './chunk-TY7L6OXO.mjs';
14
26
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
27
+ import { compareCodePoints, MIN_M } from './chunk-LFIWRCEA.mjs';
28
+ import { MAX_FILE_BYTES } from './chunk-GPPOG6PT.mjs';
15
29
  import './chunk-WXLBZJMT.mjs';
16
- import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
17
- import { decode } from '@msgpack/msgpack';
30
+ import { NarsilError, ErrorCodes } from './chunk-SRMZXJSL.mjs';
31
+ import { decode, encode } from '@msgpack/msgpack';
18
32
 
19
33
  // src/persistence/durability/node-fs.browser.ts
20
34
  function filesystemUnavailable() {
@@ -516,7 +530,9 @@ function createDurableDirectory(root) {
516
530
  const fs = await getFs();
517
531
  const pathMod = await getPath();
518
532
  const resolvedBase = pathMod.resolve(root);
519
- const all = await listRecursive(resolvedBase, resolvedBase, pathMod, fs);
533
+ const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
534
+ const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
535
+ const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
520
536
  return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
521
537
  },
522
538
  async pathOf(key) {
@@ -526,714 +542,298 @@ function createDurableDirectory(root) {
526
542
  };
527
543
  }
528
544
 
529
- // src/persistence/durability/sha256.ts
530
- var ROUND_CONSTANTS = new Uint32Array([
531
- 1116352408,
532
- 1899447441,
533
- 3049323471,
534
- 3921009573,
535
- 961987163,
536
- 1508970993,
537
- 2453635748,
538
- 2870763221,
539
- 3624381080,
540
- 310598401,
541
- 607225278,
542
- 1426881987,
543
- 1925078388,
544
- 2162078206,
545
- 2614888103,
546
- 3248222580,
547
- 3835390401,
548
- 4022224774,
549
- 264347078,
550
- 604807628,
551
- 770255983,
552
- 1249150122,
553
- 1555081692,
554
- 1996064986,
555
- 2554220882,
556
- 2821834349,
557
- 2952996808,
558
- 3210313671,
559
- 3336571891,
560
- 3584528711,
561
- 113926993,
562
- 338241895,
563
- 666307205,
564
- 773529912,
565
- 1294757372,
566
- 1396182291,
567
- 1695183700,
568
- 1986661051,
569
- 2177026350,
570
- 2456956037,
571
- 2730485921,
572
- 2820302411,
573
- 3259730800,
574
- 3345764771,
575
- 3516065817,
576
- 3600352804,
577
- 4094571909,
578
- 275423344,
579
- 430227734,
580
- 506948616,
581
- 659060556,
582
- 883997877,
583
- 958139571,
584
- 1322822218,
585
- 1537002063,
586
- 1747873779,
587
- 1955562222,
588
- 2024104815,
589
- 2227730452,
590
- 2361852424,
591
- 2428436474,
592
- 2756734187,
593
- 3204031479,
594
- 3329325298
595
- ]);
596
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
597
- var BLOCK_BYTES = 64;
598
- var LENGTH_BYTES = 8;
599
- function rotateRight(value, bits) {
600
- return value >>> bits | value << 32 - bits;
601
- }
602
- function padded(bytes) {
603
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
604
- const message = new Uint8Array(total);
605
- message.set(bytes);
606
- message[bytes.byteLength] = 128;
607
- const view = new DataView(message.buffer);
608
- const bitLength = bytes.byteLength * 8;
609
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
610
- view.setUint32(total - 4, bitLength >>> 0, false);
611
- return view;
612
- }
613
- function sha256(bytes) {
614
- const view = padded(bytes);
615
- const state = INITIAL_STATE.slice();
616
- const words = new Uint32Array(64);
617
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
618
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
619
- for (let i = 16; i < 64; i++) {
620
- const w15 = words[i - 15];
621
- const w2 = words[i - 2];
622
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
623
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
624
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
625
- }
626
- let [a, b, c, d, e, f, g, h] = state;
627
- for (let i = 0; i < 64; i++) {
628
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
629
- const choice = e & f ^ ~e & g;
630
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
631
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
632
- const majority = a & b ^ a & c ^ b & c;
633
- const t2 = s0 + majority >>> 0;
634
- h = g;
635
- g = f;
636
- f = e;
637
- e = d + t1 >>> 0;
638
- d = c;
639
- c = b;
640
- b = a;
641
- a = t1 + t2 >>> 0;
642
- }
643
- state[0] = state[0] + a >>> 0;
644
- state[1] = state[1] + b >>> 0;
645
- state[2] = state[2] + c >>> 0;
646
- state[3] = state[3] + d >>> 0;
647
- state[4] = state[4] + e >>> 0;
648
- state[5] = state[5] + f >>> 0;
649
- state[6] = state[6] + g >>> 0;
650
- state[7] = state[7] + h >>> 0;
651
- }
652
- const digest = new Uint8Array(32);
653
- const out = new DataView(digest.buffer);
654
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
655
- return digest;
656
- }
657
-
658
- // src/persistence/durability/restore-journal.ts
659
- var RESTORE_JOURNAL_PREFIX = ".swap-";
660
- var RESTORE_STAGING_PREFIX = ".restore-";
661
- var RETIRED_INDEX_PREFIX = ".retired-";
662
- var LOCK_FILE_KEY = ".narsil.lock";
663
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
664
- function digestOf(indexName) {
665
- let hex = "";
666
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
667
- return hex;
545
+ // src/persistence/durability/sync/layout.ts
546
+ var SYNC_KEY_NUMBER_WIDTH = 16;
547
+ function syncDirectoryPrefix(indexName) {
548
+ return `${indexName}/sync/`;
668
549
  }
669
- function journalKey(indexName) {
670
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
550
+ function publishedSnapshotKey(indexName) {
551
+ return `${syncDirectoryPrefix(indexName)}published`;
671
552
  }
672
- function randomSuffix() {
673
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
553
+ function historyRoot(indexName) {
554
+ return `${indexName}/history/`;
674
555
  }
675
- function newStagingPrefix() {
676
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
556
+ function historyPrefix(indexName, partitionId) {
557
+ return `${historyRoot(indexName)}${partitionId}/`;
677
558
  }
678
- function newRetiredPrefix() {
679
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
559
+ function walSegmentPrefix(indexName, partitionId) {
560
+ return `${indexName}/wal/${partitionId}/`;
680
561
  }
681
- function encodeJournal(indexName, stagingPrefix) {
682
- return new TextEncoder().encode(`${indexName}
683
- ${stagingPrefix}`);
562
+ function formatSyncKeyNumber(value) {
563
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
564
+ }
565
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
566
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
567
+ }
568
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
569
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
570
+ }
571
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
572
+ function parseHistoryKey(indexName, key) {
573
+ const root = historyRoot(indexName);
574
+ if (!key.startsWith(root)) return null;
575
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
576
+ if (match === null) return null;
577
+ const partitionId = Number.parseInt(match[1], 10);
578
+ const startSeqNo = Number.parseInt(match[2], 10);
579
+ const movedAtMs = Number.parseInt(match[3], 10);
580
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
581
+ return null;
582
+ }
583
+ return { key, partitionId, startSeqNo, movedAtMs };
684
584
  }
685
- function isStagingPrefix(prefix) {
686
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
585
+ function parseWalSegmentStart(indexName, partitionId, key) {
586
+ const prefix = walSegmentPrefix(indexName, partitionId);
587
+ if (!key.startsWith(prefix)) return null;
588
+ const tail = key.slice(prefix.length);
589
+ if (!/^\d{16}$/.test(tail)) return null;
590
+ const value = Number.parseInt(tail, 10);
591
+ return Number.isSafeInteger(value) ? value : null;
687
592
  }
688
- function corruptJournal(key, reason) {
689
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
593
+
594
+ // src/persistence/durability/sync/history.ts
595
+ function compareHistorySegments(a, b) {
596
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
597
+ }
598
+ async function listHistorySegments(directory, indexName, partitionId) {
599
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
600
+ const segments = [];
601
+ for (const key of await directory.list(prefix)) {
602
+ const segment = parseHistoryKey(indexName, key);
603
+ if (segment !== null) segments.push(segment);
604
+ }
605
+ return segments.sort(compareHistorySegments);
606
+ }
607
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
608
+ await directory.move(
609
+ walSegmentKey(indexName, partitionId, startSeqNo),
610
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
611
+ );
690
612
  }
691
- function readJournal(key, bytes) {
692
- const text = new TextDecoder().decode(bytes);
693
- const lineEnd = text.indexOf("\n");
694
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
695
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
696
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
697
- corruptJournal(key, "names an index that no index name can match");
698
- }
699
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
700
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
701
- corruptJournal(key, "names no staging prefix that a restore writes");
613
+ async function removeHistory(directory, indexName, partitionId) {
614
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
615
+ await directory.remove(segment.key);
702
616
  }
703
- return { kind: "journal", indexName, stagingPrefix };
704
617
  }
705
- async function listedKeys(storage, indexName, stagingPrefix) {
706
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
707
- if (manifestBytes === null) {
708
- throw new NarsilError(
709
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
710
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
711
- { indexName, stagingPrefix }
712
- );
713
- }
714
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
715
- if (decoded.kind !== "current") {
716
- throw new NarsilError(
717
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
718
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
719
- { indexName }
720
- );
721
- }
722
- const root = `${indexName}/`;
723
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
724
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
725
- return keys;
618
+
619
+ // src/persistence/durability/checkpoint.ts
620
+ function walPrefix(indexName, partitionId) {
621
+ return `${indexName}/wal/${partitionId}/`;
726
622
  }
727
- function withoutTrailingSlash(prefix) {
728
- return prefix.slice(0, -1);
623
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
624
+ for (const { partitionId, lastSeqNo } of checkpoint) {
625
+ const prefix = walPrefix(indexName, partitionId);
626
+ const markerBytes = await directory.read(`${prefix}commit`);
627
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
628
+ if (marker === null) {
629
+ continue;
630
+ }
631
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
632
+ const segments = await directory.list(prefix);
633
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
634
+ for (let i = 0; i < startSeqNos.length; i += 1) {
635
+ const start = startSeqNos[i];
636
+ if (start >= activeSegmentSeqNo) {
637
+ break;
638
+ }
639
+ const nextStart = startSeqNos[i + 1];
640
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
641
+ if (segmentCoversBeyondCheckpoint) {
642
+ break;
643
+ }
644
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
645
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
646
+ }
647
+ }
729
648
  }
730
- async function finishJournal(storage, indexName, stagingPrefix) {
731
- const root = `${indexName}/`;
732
- const listed = await listedKeys(storage, indexName, stagingPrefix);
733
- const staged = await storage.list(stagingPrefix);
734
- let retiredPrefix = null;
735
- if (storage.moveTree !== void 0 && staged.length > 0) {
736
- retiredPrefix = newRetiredPrefix();
737
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
738
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
739
- } else {
740
- const stagedKeys = new Set(staged);
741
- for (const key of listed) {
742
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
649
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
650
+ for (const [partitionId, partition] of [...partitions]) {
651
+ if (partitionId < partitionCount) continue;
652
+ try {
653
+ await partition.walWriter.close();
654
+ } catch (err) {
655
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
743
656
  }
657
+ partitions.delete(partitionId);
744
658
  }
745
- for (const key of await storage.list(root)) {
746
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
659
+ const walRoot = `${indexName}/wal/`;
660
+ for (const key of await directory.list(walRoot)) {
661
+ const partitionId = walKeyPartitionId(key, walRoot);
662
+ if (partitionId === null || partitionId < partitionCount) continue;
663
+ await directory.remove(key);
747
664
  }
748
- await storage.remove(journalKey(indexName));
749
- return retiredPrefix;
750
665
  }
751
- async function removePrefix(storage, prefix) {
752
- if (storage.removeTree !== void 0) {
753
- await storage.removeTree(prefix);
754
- return;
666
+ function walKeyPartitionId(key, walRoot) {
667
+ const tail = key.slice(walRoot.length);
668
+ const separator = tail.indexOf("/");
669
+ if (separator <= 0) {
670
+ return null;
755
671
  }
756
- for (const key of await storage.list(prefix)) await storage.remove(key);
757
- }
758
- async function commitJournal(storage, indexName, stagingPrefix) {
759
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
760
- }
761
- function isTopLevelJournal(key) {
762
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
763
- }
764
- async function removeStagingPrefixes(storage) {
765
- const prefixes = /* @__PURE__ */ new Set();
766
- for (const key of await storage.list(".")) {
767
- const slash = key.indexOf("/");
768
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
672
+ const segment = tail.slice(0, separator);
673
+ if (!/^\d+$/.test(segment)) {
674
+ return null;
769
675
  }
770
- for (const prefix of prefixes) await removePrefix(storage, prefix);
676
+ const value = Number.parseInt(segment, 10);
677
+ return Number.isSafeInteger(value) ? value : null;
771
678
  }
772
- async function recoverRestoreJournals(storage) {
773
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
774
- if (!isTopLevelJournal(key)) continue;
775
- const bytes = await storage.read(key);
776
- if (bytes === null) continue;
777
- const reading = readJournal(key, bytes);
778
- if (reading.kind === "leftover") {
779
- await storage.remove(key);
780
- continue;
781
- }
782
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
679
+ function parseSegmentStartSeqNo(key, prefix) {
680
+ if (!key.startsWith(prefix)) {
681
+ return null;
783
682
  }
784
- await removeStagingPrefixes(storage);
683
+ const tail = key.slice(prefix.length);
684
+ if (!/^\d{16}$/.test(tail)) {
685
+ return null;
686
+ }
687
+ const value = Number.parseInt(tail, 10);
688
+ return Number.isSafeInteger(value) ? value : null;
785
689
  }
786
-
787
- // src/persistence/durability/restore-directory.ts
788
- function missing(error) {
789
- return error.code === "ENOENT";
690
+ function formatSegmentStart(startSeqNo) {
691
+ return startSeqNo.toString().padStart(16, "0");
790
692
  }
791
- function storageFailed(code, message, key, error) {
792
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
693
+
694
+ // src/persistence/durability/marker-writer.ts
695
+ function markerKey(indexName, partitionId) {
696
+ return `${indexName}/wal/${partitionId}/commit`;
793
697
  }
794
- function directoryRestoreStorage(directory) {
698
+ async function createMarkerWriter(directory, indexName, partitionId) {
699
+ const key = markerKey(indexName, partitionId);
700
+ const handle = await directory.markerHandle(key);
701
+ const existing = await handle.read();
702
+ const current = readCommitMarker(existing);
703
+ const createdFresh = existing.length === 0;
704
+ let lastSlot = current?.slotIndex ?? null;
705
+ let writeSeq = current?.state.writeSeq ?? 0;
795
706
  return {
796
- directory,
797
- read: (key) => directory.read(key),
798
- write: (key, chunks) => directory.atomicWrite(key, chunks),
799
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
800
- remove: (key) => directory.remove(key),
801
- list: (prefix) => directory.list(prefix),
802
- moveKey: (from, to) => directory.move(from, to),
803
- pathOf: (key) => directory.pathOf(key),
804
- async moveTree(from, to) {
805
- const source = await directory.pathOf(from);
806
- const target = await directory.pathOf(to);
807
- const fs = await getFs();
808
- try {
809
- await fs.rename(source, target);
810
- } catch (error) {
811
- if (missing(error)) return;
812
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
813
- }
814
- await directory.syncDirectoryOf(to);
707
+ created: createdFresh,
708
+ existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
709
+ async commit(state) {
710
+ const slotIndex = nextMarkerSlot(lastSlot);
711
+ writeSeq += 1;
712
+ const slot = encodeMarkerSlot({ ...state, writeSeq });
713
+ await handle.writeSlot(markerSlotOffset(slotIndex), slot);
714
+ lastSlot = slotIndex;
815
715
  },
816
- async removeTree(prefix) {
817
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
818
- const fs = await getFs();
819
- try {
820
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
821
- } catch (error) {
822
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
823
- }
824
- await directory.syncDirectoryOf(key);
716
+ async close() {
717
+ await handle.close();
825
718
  }
826
719
  };
827
720
  }
828
- function recoverDirectoryRestores(directory) {
829
- return recoverRestoreJournals(directoryRestoreStorage(directory));
830
- }
831
721
 
832
- // src/persistence/durability/restore-staging.ts
833
- function toError2(error) {
834
- return error instanceof Error ? error : new Error(String(error));
835
- }
836
- function pathUnder(path, root) {
837
- if (!path.startsWith(root)) return false;
838
- const next = path.charAt(root.length);
839
- return next === "" || next === "/" || next === "\\";
722
+ // src/persistence/durability/recovery.ts
723
+ async function removeLeftoverRewriteFiles(directory, names) {
724
+ for (const indexName of names) {
725
+ for (const place of ["rebuilt", "received"]) {
726
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
727
+ }
728
+ }
840
729
  }
841
- function stagedDirectory(directory, staged, unstaged) {
842
- const { readRange, byteLength } = directory;
843
- return {
844
- root: directory.root,
845
- appendHandle: (key) => directory.appendHandle(staged(key)),
846
- markerHandle: (key) => directory.markerHandle(staged(key)),
847
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
848
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
849
- read: (key) => directory.read(staged(key)),
850
- remove: (key) => directory.remove(staged(key)),
851
- move: (from, to) => directory.move(staged(from), staged(to)),
852
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
853
- pathOf: (key) => directory.pathOf(staged(key)),
854
- ...readRange === void 0 ? {} : {
855
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
856
- },
857
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
858
- };
859
- }
860
- function createRestoreStaging(storage, indexName, options = {}) {
861
- const stagingPrefix = newStagingPrefix();
862
- const indexRoot = `${indexName}/`;
863
- let committed = false;
864
- const staged = (key) => {
865
- if (!key.startsWith(indexRoot)) {
866
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
867
- key
868
- });
730
+ async function listPersistedIndexes(directory) {
731
+ const metaKeys = await directory.list("");
732
+ const names = [];
733
+ for (const key of metaKeys) {
734
+ if (key.endsWith("/meta")) {
735
+ names.push(key.slice(0, -"/meta".length));
869
736
  }
870
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
871
- };
872
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
873
- const store = {
874
- read: (key) => storage.read(staged(key)),
875
- write: (key, chunks) => storage.write(staged(key), chunks),
876
- remove: (key) => storage.remove(staged(key)),
877
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
878
- };
879
- const pathOf = storage.pathOf;
880
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
881
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
882
- let retiredPrefix = null;
883
- async function finishOnStorage() {
884
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
885
737
  }
886
- return {
887
- store,
888
- async relocation() {
889
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
890
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
891
- const to = await pathOf.call(storage, indexName);
892
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
893
- },
894
- async commit() {
895
- try {
896
- await commitJournal(storage, indexName, stagingPrefix);
897
- } catch (error) {
898
- try {
899
- await storage.remove(journalKey(indexName));
900
- } catch (removal) {
901
- committed = true;
902
- options.onStorageLost?.(toError2(removal));
903
- }
904
- throw error;
738
+ return names;
739
+ }
740
+ async function loadMetadata(directory, indexName) {
741
+ const bytes = await directory.read(`${indexName}/meta`);
742
+ if (bytes === null) {
743
+ return null;
744
+ }
745
+ const { metadata } = await readMetadataEnvelope(bytes);
746
+ return metadata;
747
+ }
748
+ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
749
+ await directory.remove(nextSegmentKey(indexName, partitionId));
750
+ const read = await readWalSegments(directory, indexName, partitionId);
751
+ if (read === null) {
752
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
753
+ }
754
+ const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
755
+ if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
756
+ throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
757
+ }
758
+ let highestSeqNo = fromSeqNoExclusive;
759
+ let highestPrimaryTerm = 0;
760
+ const replay = async (entries) => {
761
+ for (const entry of entries) {
762
+ if (entry.seqNo <= highestSeqNo) {
763
+ continue;
905
764
  }
906
- committed = true;
907
- },
908
- async finish() {
909
- try {
910
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
911
- } catch (error) {
912
- options.onStorageLost?.(toError2(error));
765
+ if (turnIsSpent()) await yieldTurn();
766
+ applyEntry(entry, deps);
767
+ highestSeqNo = entry.seqNo;
768
+ if (entry.primaryTerm > highestPrimaryTerm) {
769
+ highestPrimaryTerm = entry.primaryTerm;
913
770
  }
914
- },
915
- async removeRetired() {
916
- const prefix = retiredPrefix;
917
- retiredPrefix = null;
918
- if (prefix !== null) await removePrefix(storage, prefix);
919
- },
920
- async discard() {
921
- if (committed) return;
922
- await removePrefix(storage, stagingPrefix);
923
771
  }
924
772
  };
773
+ await replay(durableEntries);
774
+ if (activeTail !== null) {
775
+ await replay(activeTail.entries);
776
+ }
777
+ await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
778
+ if (activeTail !== null) {
779
+ await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
780
+ }
781
+ return { highestSeqNo, highestPrimaryTerm };
925
782
  }
926
- function adapterRestoreStorage(adapter) {
927
- const store = adapterCheckpointStore(adapter);
928
- return {
929
- read: (key) => store.read(key),
930
- write: (key, chunks) => store.write(key, chunks),
931
- writeJournal: (key, bytes) => store.write(key, [bytes]),
932
- remove: (key) => store.remove(key),
933
- list: (prefix) => store.list(prefix),
934
- async moveKey(from, to) {
935
- const bytes = await store.read(from);
936
- if (bytes === null) return;
937
- await store.write(to, [bytes]);
938
- await store.remove(from);
939
- }
940
- };
783
+ async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
784
+ const lastReplayed = tail.entries.at(-1);
785
+ if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
786
+ const handle = await directory.appendHandle(tail.key);
787
+ try {
788
+ if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
789
+ else await handle.sync();
790
+ } finally {
791
+ await handle.close();
792
+ }
793
+ if (lastReplayed === void 0) return;
794
+ const writer = await createMarkerWriter(directory, indexName, partitionId);
795
+ try {
796
+ await writer.commit({
797
+ activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
798
+ durableByteLength: tail.cleanEnd,
799
+ highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
800
+ });
801
+ } finally {
802
+ await writer.close();
803
+ }
941
804
  }
942
-
943
- // src/persistence/durability/archive-checkpoint.ts
944
- function toError3(error) {
945
- return error instanceof Error ? error : new Error(String(error));
805
+ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
806
+ for (const { key, startSeqNo } of segments) {
807
+ if (startSeqNo > activeSegmentSeqNo) {
808
+ await directory.remove(key);
809
+ }
810
+ }
946
811
  }
947
- function stageDirectoryRestore(deps, indexName) {
948
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
949
- async finish(run) {
950
- try {
951
- await run();
952
- } catch (error) {
953
- deps.markFatal(toError3(error));
954
- throw error;
955
- }
956
- },
957
- onStorageLost: deps.markFatal
958
- });
812
+ function applyEntry(entry, deps) {
813
+ deps.writeRecords?.replayed(entry);
814
+ if (entry.operation === "DELETE") {
815
+ applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
816
+ return;
817
+ }
818
+ applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
959
819
  }
960
- function releaser(held, indexName) {
961
- let released = false;
962
- return () => {
963
- if (released) return;
964
- released = true;
965
- const count = (held.get(indexName) ?? 1) - 1;
966
- if (count <= 0) held.delete(indexName);
967
- else held.set(indexName, count);
968
- };
820
+ function snapshotCheckpointFor(checkpoint, partitionId) {
821
+ return checkpointLastSeqNo(checkpoint, partitionId);
969
822
  }
970
- function createArchiveCheckpointAccess(deps) {
971
- const store = directoryCheckpointStore(deps.directory);
972
- return {
973
- async openArchiveCheckpoint(indexName) {
974
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
975
- const release = releaser(deps.held, indexName);
823
+
824
+ // src/persistence/durability/checkpoint-capture.ts
825
+ function whileNoMutationApplies(partitions, capture2) {
826
+ return new Promise((resolve, reject) => {
827
+ let release = () => void 0;
828
+ const released = new Promise((settle) => {
829
+ release = settle;
830
+ });
831
+ function captureNow() {
976
832
  try {
977
- await deps.checkpoint(indexName, false);
978
- await deps.checkpoint(indexName, true);
979
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
980
- if (manifestBytes === null) {
981
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
982
- indexName
983
- });
984
- }
985
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
986
- if (decoded.kind !== "current") {
987
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
988
- indexName
989
- });
990
- }
991
- return {
992
- manifestBytes,
993
- manifest: decoded.manifest,
994
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
995
- readRange: async (ref, offset, length) => {
996
- const readRange = deps.directory.readRange;
997
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
998
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
999
- return whole.slice(offset, offset + length);
1000
- },
1001
- release,
1002
- directory: deps.directory
1003
- };
833
+ resolve(capture2());
1004
834
  } catch (error) {
1005
- release();
1006
- throw error;
1007
- }
1008
- },
1009
- stageRestore(indexName) {
1010
- return stageDirectoryRestore(deps, indexName);
1011
- },
1012
- adoptRestoredCheckpoint(indexName, checkpoint) {
1013
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1014
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1015
- }
1016
- }
1017
- };
1018
- }
1019
-
1020
- // src/persistence/durability/checkpoint.ts
1021
- function walPrefix(indexName, partitionId) {
1022
- return `${indexName}/wal/${partitionId}/`;
1023
- }
1024
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
1025
- for (const { partitionId, lastSeqNo } of checkpoint) {
1026
- const prefix = walPrefix(indexName, partitionId);
1027
- const markerBytes = await directory.read(`${prefix}commit`);
1028
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1029
- if (marker === null) {
1030
- continue;
1031
- }
1032
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
1033
- const segments = await directory.list(prefix);
1034
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
1035
- for (let i = 0; i < startSeqNos.length; i += 1) {
1036
- const start = startSeqNos[i];
1037
- if (start >= activeSegmentSeqNo) {
1038
- break;
1039
- }
1040
- const nextStart = startSeqNos[i + 1];
1041
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
1042
- if (segmentCoversBeyondCheckpoint) {
1043
- break;
1044
- }
1045
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
1046
- }
1047
- }
1048
- }
1049
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
1050
- for (const [partitionId, partition] of [...partitions]) {
1051
- if (partitionId < partitionCount) continue;
1052
- try {
1053
- await partition.walWriter.close();
1054
- } catch (err) {
1055
- onCloseError(err instanceof Error ? err : new Error(String(err)));
1056
- }
1057
- partitions.delete(partitionId);
1058
- }
1059
- const walRoot = `${indexName}/wal/`;
1060
- for (const key of await directory.list(walRoot)) {
1061
- const partitionId = walKeyPartitionId(key, walRoot);
1062
- if (partitionId === null || partitionId < partitionCount) continue;
1063
- await directory.remove(key);
1064
- }
1065
- }
1066
- function walKeyPartitionId(key, walRoot) {
1067
- const tail = key.slice(walRoot.length);
1068
- const separator = tail.indexOf("/");
1069
- if (separator <= 0) {
1070
- return null;
1071
- }
1072
- const segment = tail.slice(0, separator);
1073
- if (!/^\d+$/.test(segment)) {
1074
- return null;
1075
- }
1076
- const value = Number.parseInt(segment, 10);
1077
- return Number.isSafeInteger(value) ? value : null;
1078
- }
1079
- function parseSegmentStartSeqNo(key, prefix) {
1080
- if (!key.startsWith(prefix)) {
1081
- return null;
1082
- }
1083
- const tail = key.slice(prefix.length);
1084
- if (!/^\d{16}$/.test(tail)) {
1085
- return null;
1086
- }
1087
- const value = Number.parseInt(tail, 10);
1088
- return Number.isSafeInteger(value) ? value : null;
1089
- }
1090
- function formatSegmentStart(startSeqNo) {
1091
- return startSeqNo.toString().padStart(16, "0");
1092
- }
1093
-
1094
- // src/persistence/durability/marker-writer.ts
1095
- function markerKey(indexName, partitionId) {
1096
- return `${indexName}/wal/${partitionId}/commit`;
1097
- }
1098
- async function createMarkerWriter(directory, indexName, partitionId) {
1099
- const key = markerKey(indexName, partitionId);
1100
- const handle = await directory.markerHandle(key);
1101
- const existing = await handle.read();
1102
- const current = readCommitMarker(existing);
1103
- const createdFresh = existing.length === 0;
1104
- let lastSlot = current?.slotIndex ?? null;
1105
- let writeSeq = current?.state.writeSeq ?? 0;
1106
- return {
1107
- created: createdFresh,
1108
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
1109
- async commit(state) {
1110
- const slotIndex = nextMarkerSlot(lastSlot);
1111
- writeSeq += 1;
1112
- const slot = encodeMarkerSlot({ ...state, writeSeq });
1113
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1114
- lastSlot = slotIndex;
1115
- },
1116
- async close() {
1117
- await handle.close();
1118
- }
1119
- };
1120
- }
1121
-
1122
- // src/persistence/durability/recovery.ts
1123
- async function removeLeftoverRewriteFiles(directory, names) {
1124
- for (const indexName of names) {
1125
- for (const place of ["rebuilt", "received"]) {
1126
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1127
- }
1128
- }
1129
- }
1130
- async function listPersistedIndexes(directory) {
1131
- const metaKeys = await directory.list("");
1132
- const names = [];
1133
- for (const key of metaKeys) {
1134
- if (key.endsWith("/meta")) {
1135
- names.push(key.slice(0, -"/meta".length));
1136
- }
1137
- }
1138
- return names;
1139
- }
1140
- async function loadMetadata(directory, indexName) {
1141
- const bytes = await directory.read(`${indexName}/meta`);
1142
- if (bytes === null) {
1143
- return null;
1144
- }
1145
- const { metadata } = await readMetadataEnvelope(bytes);
1146
- return metadata;
1147
- }
1148
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1149
- await directory.remove(nextSegmentKey(indexName, partitionId));
1150
- const read = await readWalSegments(directory, indexName, partitionId);
1151
- if (read === null) {
1152
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1153
- }
1154
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1155
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1156
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1157
- }
1158
- let highestSeqNo = fromSeqNoExclusive;
1159
- let highestPrimaryTerm = 0;
1160
- const replay = async (entries) => {
1161
- for (const entry of entries) {
1162
- if (entry.seqNo <= highestSeqNo) {
1163
- continue;
1164
- }
1165
- if (turnIsSpent()) await yieldTurn();
1166
- applyEntry(entry, deps);
1167
- highestSeqNo = entry.seqNo;
1168
- if (entry.primaryTerm > highestPrimaryTerm) {
1169
- highestPrimaryTerm = entry.primaryTerm;
1170
- }
1171
- }
1172
- };
1173
- await replay(durableEntries);
1174
- if (activeTail !== null) {
1175
- await replay(activeTail.entries);
1176
- }
1177
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1178
- if (activeTail !== null) {
1179
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1180
- }
1181
- return { highestSeqNo, highestPrimaryTerm };
1182
- }
1183
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1184
- const lastReplayed = tail.entries.at(-1);
1185
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1186
- const handle = await directory.appendHandle(tail.key);
1187
- try {
1188
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1189
- else await handle.sync();
1190
- } finally {
1191
- await handle.close();
1192
- }
1193
- if (lastReplayed === void 0) return;
1194
- const writer = await createMarkerWriter(directory, indexName, partitionId);
1195
- try {
1196
- await writer.commit({
1197
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1198
- durableByteLength: tail.cleanEnd,
1199
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1200
- });
1201
- } finally {
1202
- await writer.close();
1203
- }
1204
- }
1205
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1206
- for (const { key, startSeqNo } of segments) {
1207
- if (startSeqNo > activeSegmentSeqNo) {
1208
- await directory.remove(key);
1209
- }
1210
- }
1211
- }
1212
- function applyEntry(entry, deps) {
1213
- deps.writeRecords?.replayed(entry);
1214
- if (entry.operation === "DELETE") {
1215
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1216
- return;
1217
- }
1218
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1219
- }
1220
- function snapshotCheckpointFor(checkpoint, partitionId) {
1221
- return checkpointLastSeqNo(checkpoint, partitionId);
1222
- }
1223
-
1224
- // src/persistence/durability/checkpoint-capture.ts
1225
- function whileNoMutationApplies(partitions, capture2) {
1226
- return new Promise((resolve, reject) => {
1227
- let release = () => void 0;
1228
- const released = new Promise((settle) => {
1229
- release = settle;
1230
- });
1231
- function captureNow() {
1232
- try {
1233
- resolve(capture2());
1234
- } catch (error) {
1235
- reject(error);
1236
- } finally {
835
+ reject(error);
836
+ } finally {
1237
837
  release();
1238
838
  }
1239
839
  }
@@ -1717,7 +1317,7 @@ async function writeOneBoundedCheckpoint(input) {
1717
1317
  }
1718
1318
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1719
1319
  const { indexName, indexState, hooks } = input;
1720
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-KAMRAJLE.mjs');
1320
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-E33X4AOK.mjs');
1721
1321
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1722
1322
  if (adoption === null) return [];
1723
1323
  const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
@@ -1772,10 +1372,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1772
1372
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1773
1373
  if (!input.holdsCheckpointFiles()) {
1774
1374
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1775
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-KAMRAJLE.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1375
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-E33X4AOK.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1376
+ const keptForSync = await input.keepsCheckpointKey();
1776
1377
  await removeCheckpointGarbage(
1777
1378
  store,
1778
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
1379
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
1779
1380
  );
1780
1381
  }
1781
1382
  const checkpointDocumentCount = documentCount;
@@ -1788,7 +1389,9 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1788
1389
  await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1789
1390
  input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1790
1391
  });
1791
- await truncateCoveredSegments(input.directory, input.indexName, targets);
1392
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1393
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
1394
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
1792
1395
  await reclaimWalBeyondCount(
1793
1396
  input.directory,
1794
1397
  input.indexName,
@@ -1820,202 +1423,1014 @@ function createIndexLoads(vectorIndexesOf2) {
1820
1423
  }
1821
1424
  };
1822
1425
  return {
1823
- track(indexName, load, startsGraphRepairs = true) {
1824
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1825
- loads.set(
1826
- indexName,
1827
- loaded.catch(() => void 0)
1828
- );
1829
- return loaded;
1830
- },
1831
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1832
- startGraphRepairs
1426
+ track(indexName, load, startsGraphRepairs = true) {
1427
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1428
+ loads.set(
1429
+ indexName,
1430
+ loaded.catch(() => void 0)
1431
+ );
1432
+ return loaded;
1433
+ },
1434
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1435
+ startGraphRepairs
1436
+ };
1437
+ }
1438
+ async function switchInTurn(turns, turn) {
1439
+ let swapped = false;
1440
+ await checkpointInTurn(turns, {
1441
+ joinsInFlight: false,
1442
+ mayStart: turn.mayStart,
1443
+ perform: async () => {
1444
+ swapped = turn.swap();
1445
+ if (swapped) await turn.checkpoint();
1446
+ },
1447
+ afterSettling: turn.afterSettling
1448
+ });
1449
+ return swapped;
1450
+ }
1451
+ async function checkpointInTurn(turns, turn) {
1452
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1453
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1454
+ if (!turn.mayStart()) return;
1455
+ const run = turn.perform().finally(() => {
1456
+ turns.checkpointInFlight = null;
1457
+ turn.afterSettling();
1458
+ });
1459
+ turns.checkpointInFlight = run;
1460
+ return run;
1461
+ }
1462
+
1463
+ // src/persistence/durability/held-checkpoints.ts
1464
+ function failedPartitionIds(indexState) {
1465
+ const failed = [];
1466
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1467
+ return failed;
1468
+ }
1469
+ function createHeldCheckpoints(hooks) {
1470
+ const reported = /* @__PURE__ */ new WeakSet();
1471
+ function heldReport(indexName, indexState) {
1472
+ const partitionIds = failedPartitionIds(indexState);
1473
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1474
+ return {
1475
+ indexName,
1476
+ partitionIds,
1477
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1478
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1479
+ };
1480
+ }
1481
+ return {
1482
+ refuse(indexName, indexState) {
1483
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1484
+ if (partitionIds.length === 0) return;
1485
+ throw new NarsilError(
1486
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1487
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1488
+ { indexName, partitionIds }
1489
+ );
1490
+ },
1491
+ holds(indexName, indexState) {
1492
+ const held = heldReport(indexName, indexState);
1493
+ if (held === null) return false;
1494
+ const unreported = held.partitionIds.some((partitionId) => {
1495
+ const partition = indexState.partitions.get(partitionId);
1496
+ return partition !== void 0 && !reported.has(partition);
1497
+ });
1498
+ if (unreported) {
1499
+ for (const partitionId of held.partitionIds) {
1500
+ const partition = indexState.partitions.get(partitionId);
1501
+ if (partition !== void 0) reported.add(partition);
1502
+ }
1503
+ hooks.onCheckpointHeld?.(held);
1504
+ }
1505
+ return true;
1506
+ },
1507
+ report(indexes) {
1508
+ const held = [];
1509
+ for (const [indexName, indexState] of indexes) {
1510
+ const entry = heldReport(indexName, indexState);
1511
+ if (entry !== null) held.push(entry);
1512
+ }
1513
+ return held;
1514
+ }
1515
+ };
1516
+ }
1517
+
1518
+ // src/persistence/durability/manager-state.ts
1519
+ function closePartitionWriter(partition) {
1520
+ return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1521
+ }
1522
+ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1523
+ indexState.unloading = true;
1524
+ try {
1525
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1526
+ await pendingMetadataWrite;
1527
+ for (const partition of indexState.partitions.values()) {
1528
+ await closeWalWriter(closePartitionWriter(partition));
1529
+ }
1530
+ } catch (error) {
1531
+ indexState.unloading = false;
1532
+ throw error;
1533
+ }
1534
+ }
1535
+
1536
+ // src/persistence/durability/restore-journal.ts
1537
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
1538
+ var RESTORE_STAGING_PREFIX = ".restore-";
1539
+ var RETIRED_INDEX_PREFIX = ".retired-";
1540
+ var LOCK_FILE_KEY = ".narsil.lock";
1541
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
1542
+ function digestOf(indexName) {
1543
+ let hex = "";
1544
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
1545
+ return hex;
1546
+ }
1547
+ function journalKey(indexName) {
1548
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
1549
+ }
1550
+ function randomSuffix() {
1551
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
1552
+ }
1553
+ function newStagingPrefix() {
1554
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
1555
+ }
1556
+ function newRetiredPrefix() {
1557
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
1558
+ }
1559
+ function encodeJournal(indexName, stagingPrefix) {
1560
+ return new TextEncoder().encode(`${indexName}
1561
+ ${stagingPrefix}`);
1562
+ }
1563
+ function isStagingPrefix(prefix) {
1564
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
1565
+ }
1566
+ function corruptJournal(key, reason) {
1567
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
1568
+ }
1569
+ function readJournal(key, bytes) {
1570
+ const text = new TextDecoder().decode(bytes);
1571
+ const lineEnd = text.indexOf("\n");
1572
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
1573
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
1574
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
1575
+ corruptJournal(key, "names an index that no index name can match");
1576
+ }
1577
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
1578
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
1579
+ corruptJournal(key, "names no staging prefix that a restore writes");
1580
+ }
1581
+ return { kind: "journal", indexName, stagingPrefix };
1582
+ }
1583
+ async function listedKeys(storage, indexName, stagingPrefix) {
1584
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
1585
+ if (manifestBytes === null) {
1586
+ throw new NarsilError(
1587
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1588
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1589
+ { indexName, stagingPrefix }
1590
+ );
1591
+ }
1592
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1593
+ if (decoded.kind !== "current") {
1594
+ throw new NarsilError(
1595
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1596
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1597
+ { indexName }
1598
+ );
1599
+ }
1600
+ const root = `${indexName}/`;
1601
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1602
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1603
+ return keys;
1604
+ }
1605
+ function withoutTrailingSlash(prefix) {
1606
+ return prefix.slice(0, -1);
1607
+ }
1608
+ async function finishJournal(storage, indexName, stagingPrefix) {
1609
+ const root = `${indexName}/`;
1610
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
1611
+ const staged = await storage.list(stagingPrefix);
1612
+ let retiredPrefix = null;
1613
+ if (storage.moveTree !== void 0 && staged.length > 0) {
1614
+ retiredPrefix = newRetiredPrefix();
1615
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1616
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1617
+ } else {
1618
+ const stagedKeys = new Set(staged);
1619
+ for (const key of listed) {
1620
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1621
+ }
1622
+ }
1623
+ for (const key of await storage.list(root)) {
1624
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1625
+ }
1626
+ await storage.remove(journalKey(indexName));
1627
+ return retiredPrefix;
1628
+ }
1629
+ async function removePrefix(storage, prefix) {
1630
+ if (storage.removeTree !== void 0) {
1631
+ await storage.removeTree(prefix);
1632
+ return;
1633
+ }
1634
+ for (const key of await storage.list(prefix)) await storage.remove(key);
1635
+ }
1636
+ async function commitJournal(storage, indexName, stagingPrefix) {
1637
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1638
+ }
1639
+ function isTopLevelJournal(key) {
1640
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1641
+ }
1642
+ async function removeStagingPrefixes(storage) {
1643
+ const prefixes = /* @__PURE__ */ new Set();
1644
+ for (const key of await storage.list(".")) {
1645
+ const slash = key.indexOf("/");
1646
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1647
+ }
1648
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
1649
+ }
1650
+ async function recoverRestoreJournals(storage) {
1651
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1652
+ if (!isTopLevelJournal(key)) continue;
1653
+ const bytes = await storage.read(key);
1654
+ if (bytes === null) continue;
1655
+ const reading = readJournal(key, bytes);
1656
+ if (reading.kind === "leftover") {
1657
+ await storage.remove(key);
1658
+ continue;
1659
+ }
1660
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1661
+ }
1662
+ await removeStagingPrefixes(storage);
1663
+ }
1664
+
1665
+ // src/persistence/durability/restore-directory.ts
1666
+ function missing(error) {
1667
+ return error.code === "ENOENT";
1668
+ }
1669
+ function storageFailed(code, message, key, error) {
1670
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1671
+ }
1672
+ function directoryRestoreStorage(directory) {
1673
+ return {
1674
+ directory,
1675
+ read: (key) => directory.read(key),
1676
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
1677
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1678
+ remove: (key) => directory.remove(key),
1679
+ list: (prefix) => directory.list(prefix),
1680
+ moveKey: (from, to) => directory.move(from, to),
1681
+ pathOf: (key) => directory.pathOf(key),
1682
+ async moveTree(from, to) {
1683
+ const source = await directory.pathOf(from);
1684
+ const target = await directory.pathOf(to);
1685
+ const fs = await getFs();
1686
+ try {
1687
+ await fs.rename(source, target);
1688
+ } catch (error) {
1689
+ if (missing(error)) return;
1690
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1691
+ }
1692
+ await directory.syncDirectoryOf(to);
1693
+ },
1694
+ async removeTree(prefix) {
1695
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1696
+ const fs = await getFs();
1697
+ try {
1698
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1699
+ } catch (error) {
1700
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1701
+ }
1702
+ await directory.syncDirectoryOf(key);
1703
+ }
1704
+ };
1705
+ }
1706
+ function recoverDirectoryRestores(directory) {
1707
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
1708
+ }
1709
+
1710
+ // src/persistence/durability/restore-staging.ts
1711
+ function toError2(error) {
1712
+ return error instanceof Error ? error : new Error(String(error));
1713
+ }
1714
+ function pathUnder(path, root) {
1715
+ if (!path.startsWith(root)) return false;
1716
+ const next = path.charAt(root.length);
1717
+ return next === "" || next === "/" || next === "\\";
1718
+ }
1719
+ function stagedDirectory(directory, staged, unstaged) {
1720
+ const { readRange, byteLength } = directory;
1721
+ return {
1722
+ root: directory.root,
1723
+ appendHandle: (key) => directory.appendHandle(staged(key)),
1724
+ markerHandle: (key) => directory.markerHandle(staged(key)),
1725
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1726
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1727
+ read: (key) => directory.read(staged(key)),
1728
+ remove: (key) => directory.remove(staged(key)),
1729
+ move: (from, to) => directory.move(staged(from), staged(to)),
1730
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1731
+ pathOf: (key) => directory.pathOf(staged(key)),
1732
+ ...readRange === void 0 ? {} : {
1733
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1734
+ },
1735
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1736
+ };
1737
+ }
1738
+ function createRestoreStaging(storage, indexName, options = {}) {
1739
+ const stagingPrefix = newStagingPrefix();
1740
+ const indexRoot = `${indexName}/`;
1741
+ let committed = false;
1742
+ const staged = (key) => {
1743
+ if (!key.startsWith(indexRoot)) {
1744
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1745
+ key
1746
+ });
1747
+ }
1748
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1749
+ };
1750
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1751
+ const store = {
1752
+ read: (key) => storage.read(staged(key)),
1753
+ write: (key, chunks) => storage.write(staged(key), chunks),
1754
+ remove: (key) => storage.remove(staged(key)),
1755
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1756
+ };
1757
+ const pathOf = storage.pathOf;
1758
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1759
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1760
+ let retiredPrefix = null;
1761
+ async function finishOnStorage() {
1762
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1763
+ }
1764
+ return {
1765
+ store,
1766
+ async relocation() {
1767
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1768
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1769
+ const to = await pathOf.call(storage, indexName);
1770
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1771
+ },
1772
+ async commit() {
1773
+ try {
1774
+ await commitJournal(storage, indexName, stagingPrefix);
1775
+ } catch (error) {
1776
+ try {
1777
+ await storage.remove(journalKey(indexName));
1778
+ } catch (removal) {
1779
+ committed = true;
1780
+ options.onStorageLost?.(toError2(removal));
1781
+ }
1782
+ throw error;
1783
+ }
1784
+ committed = true;
1785
+ },
1786
+ async finish() {
1787
+ try {
1788
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1789
+ } catch (error) {
1790
+ options.onStorageLost?.(toError2(error));
1791
+ }
1792
+ },
1793
+ async removeRetired() {
1794
+ const prefix = retiredPrefix;
1795
+ retiredPrefix = null;
1796
+ if (prefix !== null) await removePrefix(storage, prefix);
1797
+ },
1798
+ async discard() {
1799
+ if (committed) return;
1800
+ await removePrefix(storage, stagingPrefix);
1801
+ }
1802
+ };
1803
+ }
1804
+ function adapterRestoreStorage(adapter) {
1805
+ const store = adapterCheckpointStore(adapter);
1806
+ return {
1807
+ read: (key) => store.read(key),
1808
+ write: (key, chunks) => store.write(key, chunks),
1809
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
1810
+ remove: (key) => store.remove(key),
1811
+ list: (prefix) => store.list(prefix),
1812
+ async moveKey(from, to) {
1813
+ const bytes = await store.read(from);
1814
+ if (bytes === null) return;
1815
+ await store.write(to, [bytes]);
1816
+ await store.remove(from);
1817
+ }
1818
+ };
1819
+ }
1820
+
1821
+ // src/persistence/durability/archive-checkpoint.ts
1822
+ function toError3(error) {
1823
+ return error instanceof Error ? error : new Error(String(error));
1824
+ }
1825
+ function stageDirectoryRestore(deps, indexName) {
1826
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1827
+ async finish(run) {
1828
+ try {
1829
+ await run();
1830
+ } catch (error) {
1831
+ deps.markFatal(toError3(error));
1832
+ throw error;
1833
+ }
1834
+ },
1835
+ onStorageLost: deps.markFatal
1836
+ });
1837
+ }
1838
+ function releaser(held, indexName) {
1839
+ let released = false;
1840
+ return () => {
1841
+ if (released) return;
1842
+ released = true;
1843
+ const count = (held.get(indexName) ?? 1) - 1;
1844
+ if (count <= 0) held.delete(indexName);
1845
+ else held.set(indexName, count);
1846
+ };
1847
+ }
1848
+ function createArchiveCheckpointAccess(deps) {
1849
+ const store = directoryCheckpointStore(deps.directory);
1850
+ return {
1851
+ async openArchiveCheckpoint(indexName) {
1852
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1853
+ const release = releaser(deps.held, indexName);
1854
+ try {
1855
+ await deps.checkpoint(indexName, false);
1856
+ await deps.checkpoint(indexName, true);
1857
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
1858
+ if (manifestBytes === null) {
1859
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1860
+ indexName
1861
+ });
1862
+ }
1863
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1864
+ if (decoded.kind !== "current") {
1865
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1866
+ indexName
1867
+ });
1868
+ }
1869
+ return {
1870
+ manifestBytes,
1871
+ manifest: decoded.manifest,
1872
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1873
+ readRange: async (ref, offset, length) => {
1874
+ const readRange = deps.directory.readRange;
1875
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1876
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1877
+ return whole.slice(offset, offset + length);
1878
+ },
1879
+ release,
1880
+ directory: deps.directory
1881
+ };
1882
+ } catch (error) {
1883
+ release();
1884
+ throw error;
1885
+ }
1886
+ },
1887
+ stageRestore(indexName) {
1888
+ return stageDirectoryRestore(deps, indexName);
1889
+ },
1890
+ adoptRestoredCheckpoint(indexName, checkpoint) {
1891
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1892
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1893
+ }
1894
+ }
1895
+ };
1896
+ }
1897
+
1898
+ // src/persistence/durability/query-history.ts
1899
+ async function removeQueryHistoryOf(storage, indexName) {
1900
+ const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
1901
+ if (storage.kind === "directory") {
1902
+ for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
1903
+ return;
1904
+ }
1905
+ for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
1906
+ }
1907
+ function queryHistoryAccess(storage) {
1908
+ return {
1909
+ historyStorage: () => storage,
1910
+ removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
1833
1911
  };
1834
1912
  }
1835
- async function switchInTurn(turns, turn) {
1836
- let swapped = false;
1837
- await checkpointInTurn(turns, {
1838
- joinsInFlight: false,
1839
- mayStart: turn.mayStart,
1840
- perform: async () => {
1841
- swapped = turn.swap();
1842
- if (swapped) await turn.checkpoint();
1913
+
1914
+ // src/persistence/durability/manager-unload.ts
1915
+ function createIndexUnloading(deps) {
1916
+ const { directory, hooks, indexes, metadataWrites } = deps;
1917
+ return {
1918
+ async removeIndex(indexName) {
1919
+ hooks.writeRecords?.dropIndex(indexName);
1920
+ const indexState = indexes.get(indexName);
1921
+ if (indexState !== void 0) {
1922
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
1923
+ indexes.delete(indexName);
1924
+ metadataWrites.forget(indexName);
1925
+ }
1926
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
1927
+ for (const key of await directory.list(`${indexName}/`)) {
1928
+ await directory.remove(key);
1929
+ }
1930
+ await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
1931
+ deps.forgetSyncState(indexName);
1843
1932
  },
1844
- afterSettling: turn.afterSettling
1845
- });
1846
- return swapped;
1847
- }
1848
- async function checkpointInTurn(turns, turn) {
1849
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1850
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1851
- if (!turn.mayStart()) return;
1852
- const run = turn.perform().finally(() => {
1853
- turns.checkpointInFlight = null;
1854
- turn.afterSettling();
1855
- });
1856
- turns.checkpointInFlight = run;
1857
- return run;
1933
+ async unloadIndex(indexName) {
1934
+ const indexState = indexes.get(indexName);
1935
+ if (indexState === void 0) {
1936
+ return;
1937
+ }
1938
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
1939
+ indexes.delete(indexName);
1940
+ metadataWrites.forget(indexName);
1941
+ deps.forgetSyncState(indexName);
1942
+ }
1943
+ };
1858
1944
  }
1859
1945
 
1860
- // src/persistence/durability/directory-watch.ts
1861
- function unrefInterval(tick, intervalMs) {
1862
- const timer = setInterval(tick, intervalMs);
1863
- if (typeof timer.unref === "function") timer.unref();
1864
- return timer;
1865
- }
1866
- function createDirectoryWatch(directory, isStopped, onLost) {
1867
- let expected = null;
1868
- let timer = null;
1869
- let lock = null;
1870
- function cannotCheck(err) {
1871
- return new NarsilError(
1872
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1873
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
1874
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
1875
- );
1946
+ // src/persistence/durability/sync/frame-cursor.ts
1947
+ var SegmentFileMissing = class extends Error {
1948
+ };
1949
+ function corrupt(key, message, details) {
1950
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
1951
+ }
1952
+ function isMissingFile(error) {
1953
+ return error?.code === "ENOENT";
1954
+ }
1955
+ var FrameCursor = class {
1956
+ constructor(directory, key, limit) {
1957
+ this.directory = directory;
1958
+ this.key = key;
1959
+ this.limit = limit;
1960
+ }
1961
+ buffer = new Uint8Array(0);
1962
+ bufferStart = 0;
1963
+ whole = null;
1964
+ async load(offset, length) {
1965
+ const readRange = this.directory.readRange;
1966
+ if (readRange === void 0) {
1967
+ if (this.whole === null) {
1968
+ const bytes = await this.directory.read(this.key);
1969
+ if (bytes === null) throw new SegmentFileMissing(this.key);
1970
+ this.whole = bytes;
1971
+ }
1972
+ this.buffer = this.whole;
1973
+ this.bufferStart = 0;
1974
+ return;
1975
+ }
1976
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
1977
+ try {
1978
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
1979
+ } catch (error) {
1980
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
1981
+ throw error;
1982
+ }
1983
+ this.bufferStart = offset;
1876
1984
  }
1877
- async function currentIdentity() {
1985
+ async bytesAt(offset, length) {
1986
+ const end = offset + length;
1987
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
1988
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
1989
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
1990
+ }
1991
+ async checkHeader() {
1992
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
1993
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
1994
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
1995
+ }
1996
+ async frameLengthAt(offset) {
1997
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
1998
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
1999
+ 0,
2000
+ false
2001
+ );
2002
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
2003
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
2004
+ }
2005
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
2006
+ }
2007
+ async frameAt(offset, frameBytes) {
2008
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
2009
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
2010
+ const payload = body.subarray(0, recordLength);
2011
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
2012
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
2013
+ let seqNo;
1878
2014
  try {
1879
- return await directory.identity?.() ?? null;
1880
- } catch (err) {
1881
- return cannotCheck(err);
2015
+ seqNo = decodeWalEntry(payload).seqNo;
2016
+ } catch (error) {
2017
+ corrupt(this.key, "A log record fails to decode", {
2018
+ offset,
2019
+ cause: error instanceof Error ? error.message : String(error)
2020
+ });
1882
2021
  }
2022
+ return { seqNo, payload: payload.slice(), frameBytes };
2023
+ }
2024
+ };
2025
+
2026
+ // src/persistence/durability/sync/log-reader.ts
2027
+ async function walSources(directory, indexName, partitionId) {
2028
+ const prefix = walSegmentPrefix(indexName, partitionId);
2029
+ const markerBytes = await directory.read(`${prefix}commit`);
2030
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
2031
+ if (marker === null) return { sources: [], head: 0 };
2032
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
2033
+ const sources = [];
2034
+ for (const key of await directory.list(prefix)) {
2035
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
2036
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
2037
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
2038
+ if (readableBytes === null || readableBytes === void 0) continue;
2039
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
2040
+ }
2041
+ return { sources, head: highestDurableSeqNo };
2042
+ }
2043
+ async function openPartitionLog(directory, indexName, partitionId) {
2044
+ const { sources, head } = await walSources(directory, indexName, partitionId);
2045
+ const byStart = /* @__PURE__ */ new Map();
2046
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
2047
+ const readableBytes = await directory.byteLength?.(segment.key);
2048
+ if (readableBytes === null || readableBytes === void 0) continue;
2049
+ byStart.set(segment.startSeqNo, {
2050
+ key: segment.key,
2051
+ startSeqNo: segment.startSeqNo,
2052
+ readableBytes,
2053
+ inHistory: true
2054
+ });
1883
2055
  }
1884
- async function lockTakenOver() {
1885
- if (lock === null) return null;
2056
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
2057
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
2058
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
2059
+ }
2060
+ function sourceHolding(log, seqNo) {
2061
+ let found = -1;
2062
+ for (let i = 0; i < log.sources.length; i++) {
2063
+ if (log.sources[i].startSeqNo > seqNo) break;
2064
+ found = i;
2065
+ }
2066
+ return found;
2067
+ }
2068
+ async function offsetAfter(cursor, source, afterSeqNo) {
2069
+ let offset = SEGMENT_HEADER_SIZE;
2070
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
2071
+ offset += await cursor.frameLengthAt(offset);
2072
+ }
2073
+ return offset;
2074
+ }
2075
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
2076
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2077
+ await cursor.checkHeader();
2078
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
2079
+ let checksNumbering = true;
2080
+ while (offset < cursor.limit) {
2081
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
2082
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
2083
+ checksNumbering = false;
2084
+ if (frame.seqNo > afterSeqNo) {
2085
+ offset = SEGMENT_HEADER_SIZE;
2086
+ continue;
2087
+ }
2088
+ }
2089
+ checksNumbering = false;
2090
+ offset += frame.frameBytes;
2091
+ if (frame.seqNo <= afterSeqNo) continue;
2092
+ if (frame.seqNo > upToSeqNo) return true;
2093
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
2094
+ read.frames.push(frame);
2095
+ read.bytes += frame.payload.byteLength;
2096
+ if (frame.seqNo === upToSeqNo) return true;
2097
+ }
2098
+ return false;
2099
+ }
2100
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
2101
+ const read = { frames: [], bytes: 0, gone: false };
2102
+ if (afterSeqNo >= upToSeqNo) return read;
2103
+ for (let attempt = 0; attempt < 2; attempt++) {
2104
+ read.frames = [];
2105
+ read.bytes = 0;
2106
+ const log = await openPartitionLog(directory, indexName, partitionId);
2107
+ let index = sourceHolding(log, afterSeqNo + 1);
2108
+ if (index < 0) {
2109
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
2110
+ return read;
2111
+ }
1886
2112
  try {
1887
- if (await lock.holds()) return null;
1888
- } catch (err) {
1889
- return cannotCheck(err);
2113
+ for (; index < log.sources.length; index++) {
2114
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
2115
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
2116
+ }
2117
+ return read;
2118
+ } catch (error) {
2119
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
1890
2120
  }
1891
- return new NarsilError(
1892
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1893
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
1894
- { directory: directory.root }
1895
- );
1896
2121
  }
1897
- async function verify() {
1898
- if (directory.identity === void 0 || isStopped()) return true;
1899
- const current = await currentIdentity();
1900
- if (current instanceof Error) {
1901
- onLost(current);
1902
- return false;
1903
- }
1904
- if (expected === null) {
1905
- expected = current;
1906
- } else if (current !== expected) {
1907
- onLost(
1908
- new NarsilError(
1909
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1910
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
1911
- { directory: directory.root }
1912
- )
1913
- );
1914
- return false;
2122
+ return read;
2123
+ }
2124
+ async function logBytesAfter(directory, log, afterSeqNo) {
2125
+ const first = sourceHolding(log, afterSeqNo + 1);
2126
+ let bytes = 0;
2127
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
2128
+ const source = log.sources[i];
2129
+ if (i === first) {
2130
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2131
+ try {
2132
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
2133
+ } catch (error) {
2134
+ if (!(error instanceof SegmentFileMissing)) throw error;
2135
+ }
2136
+ continue;
1915
2137
  }
1916
- const takenOver = await lockTakenOver();
1917
- if (takenOver === null) return true;
1918
- onLost(takenOver);
1919
- return false;
2138
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
1920
2139
  }
1921
- function stop() {
1922
- if (timer !== null) {
1923
- clearInterval(timer);
1924
- timer = null;
2140
+ return bytes;
2141
+ }
2142
+
2143
+ // src/persistence/durability/sync/pins.ts
2144
+ function createFilePins(remove) {
2145
+ const counts = /* @__PURE__ */ new Map();
2146
+ const awaitingRemoval = /* @__PURE__ */ new Set();
2147
+ function unpin(key) {
2148
+ const count = (counts.get(key) ?? 1) - 1;
2149
+ if (count > 0) {
2150
+ counts.set(key, count);
2151
+ return;
1925
2152
  }
2153
+ counts.delete(key);
2154
+ if (!awaitingRemoval.delete(key)) return;
2155
+ void remove(key).catch(() => void 0);
1926
2156
  }
1927
2157
  return {
1928
- async claim() {
1929
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
1930
- },
1931
- async release() {
1932
- const held = lock;
1933
- lock = null;
1934
- await held?.release();
1935
- },
1936
- verify,
1937
- async verifyOnce() {
1938
- if (expected === null && directory.identity !== void 0) await verify();
1939
- },
1940
- start() {
1941
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
1942
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
2158
+ pin(keys) {
2159
+ const pinned = [...keys];
2160
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
2161
+ let released = false;
2162
+ return () => {
2163
+ if (released) return;
2164
+ released = true;
2165
+ for (const key of pinned) unpin(key);
2166
+ };
1943
2167
  },
1944
- stop
2168
+ isPinned: (key) => counts.has(key),
2169
+ removeWhenUnpinned(key) {
2170
+ if (!counts.has(key)) return false;
2171
+ awaitingRemoval.add(key);
2172
+ return true;
2173
+ }
2174
+ };
2175
+ }
2176
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2177
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2178
+ function hexOf(bytes) {
2179
+ let hex = "";
2180
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
2181
+ return hex;
2182
+ }
2183
+ function isBytes(value) {
2184
+ return value instanceof Uint8Array;
2185
+ }
2186
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
2187
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2188
+ if (decoded.kind !== "current") {
2189
+ throw new NarsilError(
2190
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2191
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
2192
+ { indexName }
2193
+ );
2194
+ }
2195
+ return {
2196
+ config,
2197
+ manifestBytes,
2198
+ manifest: decoded.manifest,
2199
+ entityTag: hexOf(sha256(payload)),
2200
+ keys: manifestReferencedKeys(decoded.manifest)
1945
2201
  };
1946
2202
  }
2203
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
2204
+ const payload = encode({ config, manifest: manifestBytes });
2205
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
2206
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
2207
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
2208
+ return published;
2209
+ }
2210
+ async function readPublishedSnapshot(directory, indexName) {
2211
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
2212
+ if (bytes === null) return null;
2213
+ const { payload } = await openChecksummedEnvelope(bytes, {
2214
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
2215
+ what: PUBLISHED_SNAPSHOT_WHAT,
2216
+ allowsCompression: false
2217
+ });
2218
+ const raw = decode(payload);
2219
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
2220
+ throw new NarsilError(
2221
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2222
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
2223
+ { indexName }
2224
+ );
2225
+ }
2226
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
2227
+ }
1947
2228
 
1948
- // src/persistence/durability/held-checkpoints.ts
1949
- function failedPartitionIds(indexState) {
1950
- const failed = [];
1951
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1952
- return failed;
2229
+ // src/persistence/durability/sync/retention.ts
2230
+ async function firstLogStart(directory, indexName, partitionId) {
2231
+ let first = Number.POSITIVE_INFINITY;
2232
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
2233
+ const start = parseWalSegmentStart(indexName, partitionId, key);
2234
+ if (start !== null && start < first) first = start;
2235
+ }
2236
+ return first;
2237
+ }
2238
+ async function sizedQueues(retention) {
2239
+ const { directory, indexName } = retention;
2240
+ const queues = /* @__PURE__ */ new Map();
2241
+ for (const segment of await listHistorySegments(directory, indexName)) {
2242
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
2243
+ if (bytes === void 0 || bytes === null) continue;
2244
+ const queue = queues.get(segment.partitionId) ?? [];
2245
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
2246
+ queues.set(segment.partitionId, queue);
2247
+ }
2248
+ for (const [partitionId, queue] of queues) {
2249
+ const logStart = await firstLogStart(directory, indexName, partitionId);
2250
+ for (let i = 0; i < queue.length; i++) {
2251
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
2252
+ queue[i].lastSeqNo = nextStart - 1;
2253
+ }
2254
+ }
2255
+ return queues;
2256
+ }
2257
+ function nextToDelete(queues, totalBytes, retention) {
2258
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
2259
+ let oldest = null;
2260
+ for (const queue of queues.values()) {
2261
+ const head = queue[0];
2262
+ if (head === void 0) continue;
2263
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
2264
+ }
2265
+ if (oldest === null) return null;
2266
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
2267
+ return null;
2268
+ }
2269
+ async function enforceHistoryLimits(retention) {
2270
+ const queues = await sizedQueues(retention);
2271
+ let totalBytes = 0;
2272
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
2273
+ let published = false;
2274
+ let deleted = 0;
2275
+ let segment = nextToDelete(queues, totalBytes, retention);
2276
+ while (segment !== null) {
2277
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
2278
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
2279
+ published = true;
2280
+ await retention.publish();
2281
+ }
2282
+ await retention.directory.remove(segment.key);
2283
+ queues.get(segment.partitionId)?.shift();
2284
+ totalBytes -= segment.bytes;
2285
+ deleted += 1;
2286
+ segment = nextToDelete(queues, totalBytes, retention);
2287
+ }
2288
+ return deleted;
1953
2289
  }
1954
- function createHeldCheckpoints(hooks) {
1955
- const reported = /* @__PURE__ */ new WeakSet();
1956
- function heldReport(indexName, indexState) {
1957
- const partitionIds = failedPartitionIds(indexState);
1958
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1959
- return {
1960
- indexName,
1961
- partitionIds,
1962
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1963
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1964
- };
2290
+
2291
+ // src/persistence/durability/sync/storage.ts
2292
+ function createSyncStorage(deps) {
2293
+ const { directory } = deps;
2294
+ const store = directoryCheckpointStore(directory);
2295
+ const cache = /* @__PURE__ */ new Map();
2296
+ const publishing = /* @__PURE__ */ new Map();
2297
+ const pins = createFilePins((key) => directory.remove(key));
2298
+ async function published(indexName) {
2299
+ const cached = cache.get(indexName);
2300
+ if (cached !== void 0) return cached;
2301
+ const read = await readPublishedSnapshot(directory, indexName);
2302
+ cache.set(indexName, read);
2303
+ return read;
2304
+ }
2305
+ async function removeUnlisted(indexName, previous, kept) {
2306
+ const manifest = await readCurrentManifest(store, indexName);
2307
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
2308
+ for (const key of previous) {
2309
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
2310
+ await directory.remove(key);
2311
+ }
2312
+ }
2313
+ async function publishNow(indexName, configurationOf) {
2314
+ const archive = await deps.openArchiveCheckpoint(indexName);
2315
+ try {
2316
+ const config = await configurationOf(archive.manifest);
2317
+ const previous = await published(indexName);
2318
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
2319
+ cache.set(indexName, next);
2320
+ if (previous !== null) {
2321
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
2322
+ }
2323
+ return next;
2324
+ } finally {
2325
+ archive.release();
2326
+ }
1965
2327
  }
1966
2328
  return {
1967
- refuse(indexName, indexState) {
1968
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1969
- if (partitionIds.length === 0) return;
1970
- throw new NarsilError(
1971
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1972
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1973
- { indexName, partitionIds }
1974
- );
2329
+ publish(indexName, configurationOf) {
2330
+ const inFlight = publishing.get(indexName);
2331
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
2332
+ const tracked = run.finally(() => {
2333
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
2334
+ });
2335
+ publishing.set(indexName, tracked);
2336
+ return tracked;
1975
2337
  },
1976
- holds(indexName, indexState) {
1977
- const held = heldReport(indexName, indexState);
1978
- if (held === null) return false;
1979
- const unreported = held.partitionIds.some((partitionId) => {
1980
- const partition = indexState.partitions.get(partitionId);
1981
- return partition !== void 0 && !reported.has(partition);
2338
+ published,
2339
+ pin: (snapshot) => pins.pin(snapshot.keys),
2340
+ async readFileRange(key, offset, length) {
2341
+ const readRange = directory.readRange;
2342
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
2343
+ const whole = await directory.read(key);
2344
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
2345
+ },
2346
+ async release(indexName) {
2347
+ await deps.inCheckpointTurn(indexName, async () => {
2348
+ const previous = await published(indexName);
2349
+ await directory.remove(publishedSnapshotKey(indexName));
2350
+ cache.set(indexName, null);
2351
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
2352
+ await removeHistory(directory, indexName);
1982
2353
  });
1983
- if (unreported) {
1984
- for (const partitionId of held.partitionIds) {
1985
- const partition = indexState.partitions.get(partitionId);
1986
- if (partition !== void 0) reported.add(partition);
1987
- }
1988
- hooks.onCheckpointHeld?.(held);
1989
- }
1990
- return true;
1991
2354
  },
1992
- report(indexes) {
1993
- const held = [];
1994
- for (const [indexName, indexState] of indexes) {
1995
- const entry = heldReport(indexName, indexState);
1996
- if (entry !== null) held.push(entry);
2355
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
2356
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
2357
+ const current = await published(indexName);
2358
+ return enforceHistoryLimits({
2359
+ directory,
2360
+ indexName,
2361
+ limits,
2362
+ nowMs,
2363
+ publish,
2364
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
2365
+ });
2366
+ },
2367
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
2368
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
2369
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
2370
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
2371
+ async keepsCheckpointKey(indexName) {
2372
+ const current = await published(indexName);
2373
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
2374
+ },
2375
+ async settleAtRecovery(indexName, active) {
2376
+ cache.delete(indexName);
2377
+ const publishedKey = publishedSnapshotKey(indexName);
2378
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
2379
+ if (key !== publishedKey) await directory.remove(key);
1997
2380
  }
1998
- return held;
2381
+ if (!active) {
2382
+ await directory.remove(publishedSnapshotKey(indexName));
2383
+ await removeHistory(directory, indexName);
2384
+ cache.set(indexName, null);
2385
+ return /* @__PURE__ */ new Set();
2386
+ }
2387
+ try {
2388
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
2389
+ } catch {
2390
+ await directory.remove(publishedSnapshotKey(indexName));
2391
+ cache.set(indexName, null);
2392
+ return /* @__PURE__ */ new Set();
2393
+ }
2394
+ },
2395
+ forget(indexName) {
2396
+ cache.delete(indexName);
1999
2397
  }
2000
2398
  };
2001
2399
  }
2002
2400
 
2003
- // src/persistence/durability/manager-state.ts
2004
- function closePartitionWriter(partition) {
2005
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
2006
- }
2007
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
2008
- indexState.unloading = true;
2009
- try {
2010
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2011
- await pendingMetadataWrite;
2012
- for (const partition of indexState.partitions.values()) {
2013
- await closeWalWriter(closePartitionWriter(partition));
2401
+ // src/persistence/durability/manager-storage.ts
2402
+ function createIndexStorageAccess(deps) {
2403
+ const { directory, indexes } = deps;
2404
+ const archiveAccess = createArchiveCheckpointAccess({
2405
+ directory,
2406
+ held: deps.heldCheckpointFiles,
2407
+ checkpoint: deps.checkpoint,
2408
+ registerPartition: deps.registerPartition,
2409
+ markFatal: deps.markFatal
2410
+ });
2411
+ const syncStorage = createSyncStorage({
2412
+ directory,
2413
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
2414
+ inCheckpointTurn: (indexName, work) => {
2415
+ const indexState = indexes.get(indexName);
2416
+ if (indexState === void 0) return work();
2417
+ return checkpointInTurn(indexState, {
2418
+ joinsInFlight: false,
2419
+ mayStart: () => true,
2420
+ perform: work,
2421
+ afterSettling: () => releaseStalledWrites(indexState)
2422
+ });
2014
2423
  }
2015
- } catch (error) {
2016
- indexState.unloading = false;
2017
- throw error;
2018
- }
2424
+ });
2425
+ const unloading = createIndexUnloading({
2426
+ directory,
2427
+ hooks: deps.hooks,
2428
+ indexes,
2429
+ metadataWrites: deps.metadataWrites,
2430
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
2431
+ forgetSyncState: syncStorage.forget
2432
+ });
2433
+ return { archiveAccess, syncStorage, unloading };
2019
2434
  }
2020
2435
 
2021
2436
  // src/persistence/durability/metadata-write-queue.ts
@@ -2263,8 +2678,8 @@ function createGroupCommitCoordinator(sync) {
2263
2678
  // src/persistence/durability/wal-writer.ts
2264
2679
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
2265
2680
  function segmentKey(indexName, partitionId, startSeqNo) {
2266
- const padded2 = startSeqNo.toString().padStart(16, "0");
2267
- return `${indexName}/wal/${partitionId}/${padded2}`;
2681
+ const padded = startSeqNo.toString().padStart(16, "0");
2682
+ return `${indexName}/wal/${partitionId}/${padded}`;
2268
2683
  }
2269
2684
  function parseSegmentStartSeqNo2(key) {
2270
2685
  const tail = key.slice(key.lastIndexOf("/") + 1);
@@ -2988,7 +3403,7 @@ var decoder = new TextDecoder();
2988
3403
  async function countPagedPartition(store, partition) {
2989
3404
  const { directory } = store;
2990
3405
  if (directory === void 0 || partition.segments.length === 0) return null;
2991
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-KAMRAJLE.mjs');
3406
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-E33X4AOK.mjs');
2992
3407
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2993
3408
  const scans = [];
2994
3409
  try {
@@ -3271,12 +3686,14 @@ async function recoverCheckpoint(deps) {
3271
3686
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3272
3687
  }
3273
3688
  if (decoded === null || decoded.kind !== "current") {
3274
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
3689
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3275
3690
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3276
3691
  }
3277
3692
  await deps.store.remove(snapshotBundleKey(deps.indexName));
3278
3693
  if (!deps.keepsUnlistedFiles) {
3279
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
3694
+ const referenced = manifestReferencedKeys(decoded.manifest);
3695
+ for (const key of deps.keptForSync) referenced.add(key);
3696
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
3280
3697
  }
3281
3698
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
3282
3699
  manager: deps.manager,
@@ -3397,6 +3814,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3397
3814
  }
3398
3815
  return replayed;
3399
3816
  };
3817
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
3818
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
3400
3819
  const recovered = await recoverCheckpoint({
3401
3820
  store,
3402
3821
  indexName,
@@ -3406,6 +3825,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3406
3825
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3407
3826
  compactionThreshold: deps.compactionThreshold,
3408
3827
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3828
+ keptForSync,
3409
3829
  replay,
3410
3830
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3411
3831
  writeRecords
@@ -3521,6 +3941,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3521
3941
  mutationThreshold: checkpointMutationThreshold,
3522
3942
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3523
3943
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
3944
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3524
3945
  queueMetadataWrite,
3525
3946
  markFatal,
3526
3947
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -3574,6 +3995,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3574
3995
  startAsyncFlushTimer();
3575
3996
  return outcomes;
3576
3997
  }
3998
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
3999
+ directory,
4000
+ hooks,
4001
+ indexes,
4002
+ metadataWrites,
4003
+ heldCheckpointFiles,
4004
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
4005
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
4006
+ markFatal,
4007
+ closeWriterReportingFailure
4008
+ });
3577
4009
  const recoveryDeps = {
3578
4010
  directory,
3579
4011
  hooks,
@@ -3582,7 +4014,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3582
4014
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3583
4015
  },
3584
4016
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3585
- queueMetadataWrite
4017
+ queueMetadataWrite,
4018
+ syncStorage
3586
4019
  };
3587
4020
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3588
4021
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3670,35 +4103,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3670
4103
  await checkpointIndex(indexName);
3671
4104
  }
3672
4105
  },
3673
- async removeIndex(indexName) {
3674
- hooks.writeRecords?.dropIndex(indexName);
3675
- const indexState = indexes.get(indexName);
3676
- if (indexState !== void 0) {
3677
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3678
- indexes.delete(indexName);
3679
- metadataWrites.forget(indexName);
3680
- }
3681
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3682
- for (const key of await directory.list(`${indexName}/`)) {
3683
- await directory.remove(key);
3684
- }
3685
- },
3686
- async unloadIndex(indexName) {
3687
- const indexState = indexes.get(indexName);
3688
- if (indexState === void 0) {
3689
- return;
3690
- }
3691
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3692
- indexes.delete(indexName);
3693
- metadataWrites.forget(indexName);
3694
- },
3695
- ...createArchiveCheckpointAccess({
3696
- directory,
3697
- held: heldCheckpointFiles,
3698
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3699
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3700
- markFatal
3701
- }),
4106
+ ...unloading,
4107
+ syncStorage: () => syncStorage,
4108
+ ...queryHistoryAccess({ kind: "directory", directory }),
4109
+ ...archiveAccess,
3702
4110
  async shutdown() {
3703
4111
  shuttingDown = true;
3704
4112
  stopTimers();
@@ -3965,6 +4373,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3965
4373
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3966
4374
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3967
4375
  keepsUnlistedFiles: false,
4376
+ keptForSync: /* @__PURE__ */ new Set(),
3968
4377
  replay: async (checkpoint) => checkpoint,
3969
4378
  onLayoutUpgrade: hooks.onLayoutUpgrade
3970
4379
  });
@@ -4100,6 +4509,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4100
4509
  if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
4101
4510
  }
4102
4511
  },
4512
+ ...queryHistoryAccess({ kind: "adapter", adapter }),
4103
4513
  async removeIndex(indexName) {
4104
4514
  await settleRestore(indexName).catch(() => void 0);
4105
4515
  restores.forget(indexName);
@@ -4108,6 +4518,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4108
4518
  for (const key of await adapter.list(`${indexName}/`)) {
4109
4519
  await adapter.delete(key);
4110
4520
  }
4521
+ await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
4111
4522
  },
4112
4523
  async unloadIndex(indexName) {
4113
4524
  await settleRestore(indexName);
@@ -4162,7 +4573,9 @@ function createDurabilityIntegration(tier, hooks) {
4162
4573
  recordCheckpoint: hooks.recordCheckpoint,
4163
4574
  onLayoutUpgrade: hooks.onLayoutUpgrade,
4164
4575
  onCheckpointHeld: hooks.onCheckpointHeld,
4165
- onFatalError: hooks.onFatalError
4576
+ onFatalError: hooks.onFatalError,
4577
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4578
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4166
4579
  };
4167
4580
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4168
4581
  function partitionFor(indexName, docId) {
@@ -4247,7 +4660,12 @@ function createDurabilityFromTier(tier, wiring) {
4247
4660
  recordCheckpoint: wiring.recordCheckpoint,
4248
4661
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
4249
4662
  onCheckpointHeld: wiring.emitCheckpointHeld,
4250
- onFatalError: wiring.emitFatalError
4663
+ onFatalError: wiring.emitFatalError,
4664
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4665
+ onHistoryMoved(indexName) {
4666
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4667
+ import('./retention-ASWUHAYI.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4668
+ }
4251
4669
  });
4252
4670
  }
4253
4671