@delali/narsil 0.3.1-next.153 → 0.3.1-next.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (330) hide show
  1. package/README.md +3 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
  8. package/dist/archive-history-E4R73RWD.mjs +29 -0
  9. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
  10. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
  11. package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
  12. package/dist/browser/chunk-23XRXIBW.mjs +85 -0
  13. package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
  14. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
  15. package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
  16. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
  17. package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
  18. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  19. package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
  20. package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
  21. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
  22. package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
  23. package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
  24. package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
  25. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  26. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  27. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
  28. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
  29. package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
  30. package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
  31. package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
  32. package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
  33. package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
  34. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
  35. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
  36. package/dist/browser/chunk-KKAMCABR.mjs +432 -0
  37. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
  38. package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
  39. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  40. package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
  41. package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
  42. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  43. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
  44. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
  45. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
  46. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
  47. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
  48. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
  49. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
  50. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  51. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
  52. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  53. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  54. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
  55. package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
  56. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
  57. package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
  58. package/dist/browser/import-OM6QAW45.mjs +18 -0
  59. package/dist/browser/methods-KTQCSIN4.mjs +30 -0
  60. package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
  61. package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
  62. package/dist/browser/operations-OJ3BXS55.mjs +5 -0
  63. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
  64. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
  65. package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
  66. package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
  67. package/dist/browser/retention-ASWUHAYI.mjs +209 -0
  68. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
  69. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
  70. package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
  71. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  72. package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
  73. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
  74. package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
  75. package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
  76. package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
  77. package/dist/chunk-3HB3PEC6.mjs +240 -0
  78. package/dist/chunk-3HHQLEEQ.mjs +1092 -0
  79. package/dist/chunk-3OH67G7R.mjs +214 -0
  80. package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
  81. package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
  82. package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
  83. package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
  84. package/dist/chunk-5FVU47TB.mjs +20 -0
  85. package/dist/chunk-5YNRKI2C.mjs +81 -0
  86. package/dist/chunk-6L3XJXGU.mjs +470 -0
  87. package/dist/chunk-6MSZ543Y.mjs +32 -0
  88. package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
  89. package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
  90. package/dist/chunk-7GXJUJ24.mjs +49 -0
  91. package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
  92. package/dist/chunk-BDRIVYOK.mjs +72 -0
  93. package/dist/chunk-BUMREPLU.mjs +15 -0
  94. package/dist/chunk-BY6ZDGYS.mjs +50 -0
  95. package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
  96. package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
  97. package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
  98. package/dist/chunk-GES7U2PM.mjs +68 -0
  99. package/dist/chunk-GH4WQ22V.mjs +34 -0
  100. package/dist/chunk-GQPTDQCL.mjs +54 -0
  101. package/dist/chunk-H4KHPLHJ.mjs +382 -0
  102. package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
  103. package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
  104. package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
  105. package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
  106. package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
  107. package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
  108. package/dist/chunk-IPN7S4KX.mjs +130 -0
  109. package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
  110. package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
  111. package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
  112. package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
  114. package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
  115. package/dist/chunk-KJ3RTZDJ.mjs +101 -0
  116. package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
  117. package/dist/chunk-KP7TYHWT.mjs +92 -0
  118. package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
  119. package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
  120. package/dist/chunk-KYYABCSW.mjs +72 -0
  121. package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
  122. package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
  123. package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
  124. package/dist/chunk-LNGE7SLY.mjs +151 -0
  125. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
  126. package/dist/chunk-MT2NRTA6.mjs +216 -0
  127. package/dist/chunk-NF4DT3IH.mjs +85 -0
  128. package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
  129. package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
  130. package/dist/chunk-PHQNACDT.mjs +63 -0
  131. package/dist/chunk-Q622CTMC.mjs +74 -0
  132. package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
  133. package/dist/chunk-QDZJM7NI.mjs +23 -0
  134. package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
  135. package/dist/chunk-QJVOLKU6.mjs +111 -0
  136. package/dist/chunk-QUJHMPTP.mjs +30 -0
  137. package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
  138. package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
  139. package/dist/chunk-RZWWB2XR.mjs +96 -0
  140. package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
  141. package/dist/chunk-S5LN746F.mjs +10 -0
  142. package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
  143. package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
  144. package/dist/chunk-SY3EWKQY.mjs +242 -0
  145. package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
  146. package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
  147. package/dist/chunk-UHICS3JI.mjs +83 -0
  148. package/dist/chunk-UZRZKNNZ.mjs +101 -0
  149. package/dist/chunk-V6EUBAIT.mjs +19 -0
  150. package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
  151. package/dist/chunk-VHBNF4MA.mjs +543 -0
  152. package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
  153. package/dist/chunk-WQRN64Q6.mjs +196 -0
  154. package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
  155. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  156. package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
  157. package/dist/chunk-YEKL4JBK.mjs +96 -0
  158. package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
  159. package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
  160. package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
  161. package/dist/chunk-ZTW5WTGG.mjs +167 -0
  162. package/dist/chunk-ZYAY26KG.mjs +29 -0
  163. package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
  164. package/dist/client.d.ts +9 -7
  165. package/dist/client.mjs +120 -7
  166. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
  167. package/dist/config-CoSig5Ox.d.ts +482 -0
  168. package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
  169. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
  170. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  171. package/dist/distribution/coordinator/etcd.mjs +1 -1
  172. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  173. package/dist/distribution/transport/grpc.mjs +4 -4
  174. package/dist/distribution/transport/in-memory.mjs +2 -2
  175. package/dist/distribution/transport/tcp.mjs +3 -3
  176. package/dist/distribution.d.ts +41 -9
  177. package/dist/distribution.mjs +9304 -7271
  178. package/dist/durability-config-GE4U7MCZ.mjs +89 -0
  179. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
  180. package/dist/embeddings/openai.mjs +2 -2
  181. package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
  182. package/dist/engine-hooks-DHCMOADY.mjs +47 -0
  183. package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
  184. package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
  185. package/dist/index.browser.mjs +9 -4
  186. package/dist/index.d.ts +7 -6
  187. package/dist/index.mjs +29 -21
  188. package/dist/index.react-native.mjs +19 -12
  189. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
  190. package/dist/methods-7IOINERF.mjs +30 -0
  191. package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
  192. package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
  193. package/dist/operations-G5EP6JPX.mjs +5 -0
  194. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
  195. package/dist/partial-words.mjs +7 -5
  196. package/dist/pattern.mjs +4 -3
  197. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  198. package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
  199. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
  200. package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
  201. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  202. package/dist/query-suggestions/merge-worker.mjs +81 -0
  203. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  204. package/dist/query-suggestions.browser.mjs +246 -0
  205. package/dist/query-suggestions.d.ts +20 -0
  206. package/dist/query-suggestions.mjs +62 -0
  207. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
  208. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
  209. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
  210. package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
  211. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  212. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
  213. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
  214. package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
  215. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
  216. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
  217. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
  218. package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
  219. package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
  220. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
  221. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
  222. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
  223. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
  224. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  225. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
  226. package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
  227. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
  228. package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
  229. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
  230. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
  231. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  232. package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
  233. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
  234. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  235. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
  236. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
  237. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  238. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
  239. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  240. package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
  241. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
  242. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  243. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
  244. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
  245. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
  246. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
  247. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  248. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
  249. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  250. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
  251. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
  252. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  253. package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
  254. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
  255. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
  256. package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
  257. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
  258. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  259. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
  260. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
  261. package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
  262. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
  263. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
  264. package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
  265. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
  266. package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
  267. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
  268. package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
  269. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
  270. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
  271. package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
  272. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
  273. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
  274. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
  275. package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
  276. package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
  277. package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
  278. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
  279. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
  280. package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
  281. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  282. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
  283. package/dist/react-native/partial-words.mjs +8 -5
  284. package/dist/react-native/pattern.mjs +4 -3
  285. package/dist/react-native/query-suggestions.mjs +258 -0
  286. package/dist/react-native/sync.mjs +2299 -0
  287. package/dist/react-native/synonyms.mjs +7 -4
  288. package/dist/react.d.ts +11 -10
  289. package/dist/react.mjs +8 -1
  290. package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
  291. package/dist/rerankers/http.mjs +2 -2
  292. package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
  293. package/dist/restore-queue-JVGENGGE.mjs +55 -0
  294. package/dist/retention-HROPV2QU.mjs +59 -0
  295. package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
  296. package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
  297. package/dist/server.d.ts +119 -20
  298. package/dist/server.mjs +1123 -58
  299. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
  300. package/dist/start-UTWHGH42.mjs +59 -0
  301. package/dist/sync/filesystem.d.ts +30 -0
  302. package/dist/sync/filesystem.mjs +271 -0
  303. package/dist/sync/indexeddb.d.ts +30 -0
  304. package/dist/sync/indexeddb.mjs +237 -0
  305. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  306. package/dist/sync.browser.mjs +2294 -0
  307. package/dist/sync.d.ts +23 -0
  308. package/dist/sync.mjs +2268 -0
  309. package/dist/synonyms.mjs +6 -4
  310. package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
  311. package/dist/thread-6MOXY2KD.mjs +45 -0
  312. package/dist/types-BZCGjfLn.d.ts +227 -0
  313. package/dist/vector/search-worker.d.ts +1 -1
  314. package/dist/vector/search-worker.mjs +7 -7
  315. package/dist/workers/entry.mjs +8 -8
  316. package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
  317. package/openapi.json +912 -5
  318. package/package.json +45 -8
  319. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  320. package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
  321. package/dist/chunk-7LDMWKP6.mjs +0 -209
  322. package/dist/engine-CJRUHXgf.d.ts +0 -664
  323. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  324. package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
  325. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  326. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  327. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  328. package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
  329. package/dist/restore-queue-AOF5OTQV.mjs +0 -41
  330. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,30 +1,46 @@
1
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, SEGMENT_HEADER_SIZE, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-7IQR3SZO.mjs';
2
- import { nodeThreadId } from './chunk-PSWKQVVT.mjs';
1
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-VXGE4ZXL.mjs';
2
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-SUPXRTMD.mjs';
3
+ import './chunk-CKNRFDTZ.mjs';
3
4
  import './chunk-HJBJBH26.mjs';
4
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-VGUWIZFQ.mjs';
5
- import './chunk-LRGWK367.mjs';
6
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, PHONE_FILE_CHUNK_BYTES, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-FS7BYE23.mjs';
7
- import { applyDeleteEntry, applyIndexEntry } from './chunk-CLE3LZ77.mjs';
8
- import { toError, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-IDFMNRIC.mjs';
9
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime } from './chunk-LQAKO5QV.mjs';
10
- import { createRequestId } from './chunk-NNE4VDMM.mjs';
11
- import { fnv1a } from './chunk-L4M7QJNP.mjs';
5
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-CUH5F57W.mjs';
6
+ import './chunk-JWUCMVRK.mjs';
7
+ import './chunk-QJXTNPLQ.mjs';
8
+ import { sha256 } from './chunk-H3MDVPBO.mjs';
9
+ import './chunk-TYIRMPD3.mjs';
10
+ import { nodeThreadId } from './chunk-PSWKQVVT.mjs';
11
+ import { QUERY_HISTORY_FOLDER } from './chunk-PJHAB5ZY.mjs';
12
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-C3PTWYKT.mjs';
13
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-GJFXPHXI.mjs';
14
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, PHONE_FILE_CHUNK_BYTES, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-F753DA54.mjs';
15
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-6WA3C3DV.mjs';
16
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-DVDX4SDR.mjs';
17
+ import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-HUOYZJ3Q.mjs';
12
18
  import './chunk-IO2SMUQD.mjs';
19
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime } from './chunk-JLIIWJUV.mjs';
13
20
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-AM7RSBPF.mjs';
14
- import './chunk-ZCKNBPQ4.mjs';
15
- import { encodeDocument, requireValidFieldTypes } from './chunk-PU55IOG4.mjs';
16
- import { MIN_M, MAX_FILE_BYTES } from './chunk-TJMSKBEM.mjs';
17
- import './chunk-DKJTZ625.mjs';
18
- import './chunk-3CTWSQRW.mjs';
19
- import './chunk-2J2727V4.mjs';
21
+ import { reactNativeHost } from './chunk-MVXELMJX.mjs';
22
+ import { createRequestId } from './chunk-NNE4VDMM.mjs';
23
+ import './chunk-LU3Y3P63.mjs';
24
+ import './chunk-RDWG33MM.mjs';
25
+ import './chunk-4Y53T4YE.mjs';
26
+ import './chunk-UNUFZ2QY.mjs';
27
+ import './chunk-3NQOVO3K.mjs';
28
+ import './chunk-JDOEEMEF.mjs';
29
+ import './chunk-7PXRWYUU.mjs';
30
+ import { openChecksummedEnvelope, crc32, HEADER_SIZE } from './chunk-YH7QY657.mjs';
31
+ import { fnv1a } from './chunk-CHPTFFJM.mjs';
32
+ import { encodeDocument, requireValidFieldTypes } from './chunk-3JIBF3XT.mjs';
33
+ import { MIN_M } from './chunk-5CQX3XXV.mjs';
34
+ import { MAX_FILE_BYTES } from './chunk-6VBXNFLE.mjs';
35
+ import './chunk-C5G6TJ7A.mjs';
36
+ import './chunk-MJDQU5MW.mjs';
20
37
  import './chunk-KWRW5U65.mjs';
21
38
  import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
22
39
  import './chunk-LVIKKAA6.mjs';
23
40
  import './chunk-AAAETEJK.mjs';
24
41
  import './chunk-JV6VQY2S.mjs';
25
- import { reactNativeHost } from './chunk-NFGBOI46.mjs';
26
- import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
27
- import { decode } from '@msgpack/msgpack';
42
+ import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
43
+ import { decode, encode } from '@msgpack/msgpack';
28
44
 
29
45
  // src/runtime/posix-path.ts
30
46
  function normalizeParts(path, absolute) {
@@ -808,7 +824,9 @@ function createDurableDirectory(root) {
808
824
  const fs = await getFs();
809
825
  const pathMod = await getPath();
810
826
  const resolvedBase = pathMod.resolve(root);
811
- const all = await listRecursive(resolvedBase, resolvedBase, pathMod, fs);
827
+ const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
828
+ const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
829
+ const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
812
830
  return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
813
831
  },
814
832
  async pathOf(key) {
@@ -818,714 +836,298 @@ function createDurableDirectory(root) {
818
836
  };
819
837
  }
820
838
 
821
- // src/persistence/durability/sha256.ts
822
- var ROUND_CONSTANTS = new Uint32Array([
823
- 1116352408,
824
- 1899447441,
825
- 3049323471,
826
- 3921009573,
827
- 961987163,
828
- 1508970993,
829
- 2453635748,
830
- 2870763221,
831
- 3624381080,
832
- 310598401,
833
- 607225278,
834
- 1426881987,
835
- 1925078388,
836
- 2162078206,
837
- 2614888103,
838
- 3248222580,
839
- 3835390401,
840
- 4022224774,
841
- 264347078,
842
- 604807628,
843
- 770255983,
844
- 1249150122,
845
- 1555081692,
846
- 1996064986,
847
- 2554220882,
848
- 2821834349,
849
- 2952996808,
850
- 3210313671,
851
- 3336571891,
852
- 3584528711,
853
- 113926993,
854
- 338241895,
855
- 666307205,
856
- 773529912,
857
- 1294757372,
858
- 1396182291,
859
- 1695183700,
860
- 1986661051,
861
- 2177026350,
862
- 2456956037,
863
- 2730485921,
864
- 2820302411,
865
- 3259730800,
866
- 3345764771,
867
- 3516065817,
868
- 3600352804,
869
- 4094571909,
870
- 275423344,
871
- 430227734,
872
- 506948616,
873
- 659060556,
874
- 883997877,
875
- 958139571,
876
- 1322822218,
877
- 1537002063,
878
- 1747873779,
879
- 1955562222,
880
- 2024104815,
881
- 2227730452,
882
- 2361852424,
883
- 2428436474,
884
- 2756734187,
885
- 3204031479,
886
- 3329325298
887
- ]);
888
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
889
- var BLOCK_BYTES = 64;
890
- var LENGTH_BYTES = 8;
891
- function rotateRight(value, bits) {
892
- return value >>> bits | value << 32 - bits;
893
- }
894
- function padded(bytes) {
895
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
896
- const message = new Uint8Array(total);
897
- message.set(bytes);
898
- message[bytes.byteLength] = 128;
899
- const view = new DataView(message.buffer);
900
- const bitLength = bytes.byteLength * 8;
901
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
902
- view.setUint32(total - 4, bitLength >>> 0, false);
903
- return view;
904
- }
905
- function sha256(bytes) {
906
- const view = padded(bytes);
907
- const state = INITIAL_STATE.slice();
908
- const words = new Uint32Array(64);
909
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
910
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
911
- for (let i = 16; i < 64; i++) {
912
- const w15 = words[i - 15];
913
- const w2 = words[i - 2];
914
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
915
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
916
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
917
- }
918
- let [a, b, c, d, e, f, g, h] = state;
919
- for (let i = 0; i < 64; i++) {
920
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
921
- const choice = e & f ^ ~e & g;
922
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
923
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
924
- const majority = a & b ^ a & c ^ b & c;
925
- const t2 = s0 + majority >>> 0;
926
- h = g;
927
- g = f;
928
- f = e;
929
- e = d + t1 >>> 0;
930
- d = c;
931
- c = b;
932
- b = a;
933
- a = t1 + t2 >>> 0;
934
- }
935
- state[0] = state[0] + a >>> 0;
936
- state[1] = state[1] + b >>> 0;
937
- state[2] = state[2] + c >>> 0;
938
- state[3] = state[3] + d >>> 0;
939
- state[4] = state[4] + e >>> 0;
940
- state[5] = state[5] + f >>> 0;
941
- state[6] = state[6] + g >>> 0;
942
- state[7] = state[7] + h >>> 0;
943
- }
944
- const digest = new Uint8Array(32);
945
- const out = new DataView(digest.buffer);
946
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
947
- return digest;
948
- }
949
-
950
- // src/persistence/durability/restore-journal.ts
951
- var RESTORE_JOURNAL_PREFIX = ".swap-";
952
- var RESTORE_STAGING_PREFIX = ".restore-";
953
- var RETIRED_INDEX_PREFIX = ".retired-";
954
- var LOCK_FILE_KEY = ".narsil.lock";
955
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
956
- function digestOf(indexName) {
957
- let hex = "";
958
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
959
- return hex;
839
+ // src/persistence/durability/sync/layout.ts
840
+ var SYNC_KEY_NUMBER_WIDTH = 16;
841
+ function syncDirectoryPrefix(indexName) {
842
+ return `${indexName}/sync/`;
960
843
  }
961
- function journalKey(indexName) {
962
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
844
+ function publishedSnapshotKey(indexName) {
845
+ return `${syncDirectoryPrefix(indexName)}published`;
963
846
  }
964
- function randomSuffix() {
965
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
847
+ function historyRoot(indexName) {
848
+ return `${indexName}/history/`;
966
849
  }
967
- function newStagingPrefix() {
968
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
850
+ function historyPrefix(indexName, partitionId) {
851
+ return `${historyRoot(indexName)}${partitionId}/`;
969
852
  }
970
- function newRetiredPrefix() {
971
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
853
+ function walSegmentPrefix(indexName, partitionId) {
854
+ return `${indexName}/wal/${partitionId}/`;
972
855
  }
973
- function encodeJournal(indexName, stagingPrefix) {
974
- return new TextEncoder().encode(`${indexName}
975
- ${stagingPrefix}`);
856
+ function formatSyncKeyNumber(value) {
857
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
858
+ }
859
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
860
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
861
+ }
862
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
863
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
864
+ }
865
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
866
+ function parseHistoryKey(indexName, key) {
867
+ const root = historyRoot(indexName);
868
+ if (!key.startsWith(root)) return null;
869
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
870
+ if (match === null) return null;
871
+ const partitionId = Number.parseInt(match[1], 10);
872
+ const startSeqNo = Number.parseInt(match[2], 10);
873
+ const movedAtMs = Number.parseInt(match[3], 10);
874
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
875
+ return null;
876
+ }
877
+ return { key, partitionId, startSeqNo, movedAtMs };
976
878
  }
977
- function isStagingPrefix(prefix) {
978
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
879
+ function parseWalSegmentStart(indexName, partitionId, key) {
880
+ const prefix = walSegmentPrefix(indexName, partitionId);
881
+ if (!key.startsWith(prefix)) return null;
882
+ const tail = key.slice(prefix.length);
883
+ if (!/^\d{16}$/.test(tail)) return null;
884
+ const value = Number.parseInt(tail, 10);
885
+ return Number.isSafeInteger(value) ? value : null;
979
886
  }
980
- function corruptJournal(key, reason) {
981
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
887
+
888
+ // src/persistence/durability/sync/history.ts
889
+ function compareHistorySegments(a, b) {
890
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
891
+ }
892
+ async function listHistorySegments(directory, indexName, partitionId) {
893
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
894
+ const segments = [];
895
+ for (const key of await directory.list(prefix)) {
896
+ const segment = parseHistoryKey(indexName, key);
897
+ if (segment !== null) segments.push(segment);
898
+ }
899
+ return segments.sort(compareHistorySegments);
900
+ }
901
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
902
+ await directory.move(
903
+ walSegmentKey(indexName, partitionId, startSeqNo),
904
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
905
+ );
982
906
  }
983
- function readJournal(key, bytes) {
984
- const text = new TextDecoder().decode(bytes);
985
- const lineEnd = text.indexOf("\n");
986
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
987
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
988
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
989
- corruptJournal(key, "names an index that no index name can match");
990
- }
991
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
992
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
993
- corruptJournal(key, "names no staging prefix that a restore writes");
907
+ async function removeHistory(directory, indexName, partitionId) {
908
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
909
+ await directory.remove(segment.key);
994
910
  }
995
- return { kind: "journal", indexName, stagingPrefix };
996
911
  }
997
- async function listedKeys(storage, indexName, stagingPrefix) {
998
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
999
- if (manifestBytes === null) {
1000
- throw new NarsilError(
1001
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1002
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1003
- { indexName, stagingPrefix }
1004
- );
1005
- }
1006
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1007
- if (decoded.kind !== "current") {
1008
- throw new NarsilError(
1009
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1010
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1011
- { indexName }
1012
- );
1013
- }
1014
- const root = `${indexName}/`;
1015
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1016
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1017
- return keys;
912
+
913
+ // src/persistence/durability/checkpoint.ts
914
+ function walPrefix(indexName, partitionId) {
915
+ return `${indexName}/wal/${partitionId}/`;
1018
916
  }
1019
- function withoutTrailingSlash(prefix) {
1020
- return prefix.slice(0, -1);
917
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
918
+ for (const { partitionId, lastSeqNo } of checkpoint) {
919
+ const prefix = walPrefix(indexName, partitionId);
920
+ const markerBytes = await directory.read(`${prefix}commit`);
921
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
922
+ if (marker === null) {
923
+ continue;
924
+ }
925
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
926
+ const segments = await directory.list(prefix);
927
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
928
+ for (let i = 0; i < startSeqNos.length; i += 1) {
929
+ const start = startSeqNos[i];
930
+ if (start >= activeSegmentSeqNo) {
931
+ break;
932
+ }
933
+ const nextStart = startSeqNos[i + 1];
934
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
935
+ if (segmentCoversBeyondCheckpoint) {
936
+ break;
937
+ }
938
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
939
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
940
+ }
941
+ }
1021
942
  }
1022
- async function finishJournal(storage, indexName, stagingPrefix) {
1023
- const root = `${indexName}/`;
1024
- const listed = await listedKeys(storage, indexName, stagingPrefix);
1025
- const staged = await storage.list(stagingPrefix);
1026
- let retiredPrefix = null;
1027
- if (storage.moveTree !== void 0 && staged.length > 0) {
1028
- retiredPrefix = newRetiredPrefix();
1029
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1030
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1031
- } else {
1032
- const stagedKeys = new Set(staged);
1033
- for (const key of listed) {
1034
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
943
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
944
+ for (const [partitionId, partition] of [...partitions]) {
945
+ if (partitionId < partitionCount) continue;
946
+ try {
947
+ await partition.walWriter.close();
948
+ } catch (err) {
949
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
1035
950
  }
951
+ partitions.delete(partitionId);
1036
952
  }
1037
- for (const key of await storage.list(root)) {
1038
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
953
+ const walRoot = `${indexName}/wal/`;
954
+ for (const key of await directory.list(walRoot)) {
955
+ const partitionId = walKeyPartitionId(key, walRoot);
956
+ if (partitionId === null || partitionId < partitionCount) continue;
957
+ await directory.remove(key);
1039
958
  }
1040
- await storage.remove(journalKey(indexName));
1041
- return retiredPrefix;
1042
959
  }
1043
- async function removePrefix(storage, prefix) {
1044
- if (storage.removeTree !== void 0) {
1045
- await storage.removeTree(prefix);
1046
- return;
960
+ function walKeyPartitionId(key, walRoot) {
961
+ const tail = key.slice(walRoot.length);
962
+ const separator = tail.indexOf("/");
963
+ if (separator <= 0) {
964
+ return null;
1047
965
  }
1048
- for (const key of await storage.list(prefix)) await storage.remove(key);
1049
- }
1050
- async function commitJournal(storage, indexName, stagingPrefix) {
1051
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1052
- }
1053
- function isTopLevelJournal(key) {
1054
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1055
- }
1056
- async function removeStagingPrefixes(storage) {
1057
- const prefixes = /* @__PURE__ */ new Set();
1058
- for (const key of await storage.list(".")) {
1059
- const slash = key.indexOf("/");
1060
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
966
+ const segment = tail.slice(0, separator);
967
+ if (!/^\d+$/.test(segment)) {
968
+ return null;
1061
969
  }
1062
- for (const prefix of prefixes) await removePrefix(storage, prefix);
970
+ const value = Number.parseInt(segment, 10);
971
+ return Number.isSafeInteger(value) ? value : null;
1063
972
  }
1064
- async function recoverRestoreJournals(storage) {
1065
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1066
- if (!isTopLevelJournal(key)) continue;
1067
- const bytes = await storage.read(key);
1068
- if (bytes === null) continue;
1069
- const reading = readJournal(key, bytes);
1070
- if (reading.kind === "leftover") {
1071
- await storage.remove(key);
1072
- continue;
1073
- }
1074
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
973
+ function parseSegmentStartSeqNo(key, prefix) {
974
+ if (!key.startsWith(prefix)) {
975
+ return null;
1075
976
  }
1076
- await removeStagingPrefixes(storage);
977
+ const tail = key.slice(prefix.length);
978
+ if (!/^\d{16}$/.test(tail)) {
979
+ return null;
980
+ }
981
+ const value = Number.parseInt(tail, 10);
982
+ return Number.isSafeInteger(value) ? value : null;
1077
983
  }
1078
-
1079
- // src/persistence/durability/restore-directory.ts
1080
- function missing(error) {
1081
- return error.code === "ENOENT";
984
+ function formatSegmentStart(startSeqNo) {
985
+ return startSeqNo.toString().padStart(16, "0");
1082
986
  }
1083
- function storageFailed(code, message, key, error) {
1084
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
987
+
988
+ // src/persistence/durability/marker-writer.ts
989
+ function markerKey(indexName, partitionId) {
990
+ return `${indexName}/wal/${partitionId}/commit`;
1085
991
  }
1086
- function directoryRestoreStorage(directory) {
992
+ async function createMarkerWriter(directory, indexName, partitionId) {
993
+ const key = markerKey(indexName, partitionId);
994
+ const handle = await directory.markerHandle(key);
995
+ const existing = await handle.read();
996
+ const current = readCommitMarker(existing);
997
+ const createdFresh = existing.length === 0;
998
+ let lastSlot = current?.slotIndex ?? null;
999
+ let writeSeq = current?.state.writeSeq ?? 0;
1087
1000
  return {
1088
- directory,
1089
- read: (key) => directory.read(key),
1090
- write: (key, chunks) => directory.atomicWrite(key, chunks),
1091
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1092
- remove: (key) => directory.remove(key),
1093
- list: (prefix) => directory.list(prefix),
1094
- moveKey: (from, to) => directory.move(from, to),
1095
- pathOf: (key) => directory.pathOf(key),
1096
- async moveTree(from, to) {
1097
- const source = await directory.pathOf(from);
1098
- const target = await directory.pathOf(to);
1099
- const fs = await getFs();
1100
- try {
1101
- await fs.rename(source, target);
1102
- } catch (error) {
1103
- if (missing(error)) return;
1104
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1105
- }
1106
- await directory.syncDirectoryOf(to);
1001
+ created: createdFresh,
1002
+ existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
1003
+ async commit(state) {
1004
+ const slotIndex = nextMarkerSlot(lastSlot);
1005
+ writeSeq += 1;
1006
+ const slot = encodeMarkerSlot({ ...state, writeSeq });
1007
+ await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1008
+ lastSlot = slotIndex;
1107
1009
  },
1108
- async removeTree(prefix) {
1109
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1110
- const fs = await getFs();
1111
- try {
1112
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1113
- } catch (error) {
1114
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1115
- }
1116
- await directory.syncDirectoryOf(key);
1010
+ async close() {
1011
+ await handle.close();
1117
1012
  }
1118
1013
  };
1119
1014
  }
1120
- function recoverDirectoryRestores(directory) {
1121
- return recoverRestoreJournals(directoryRestoreStorage(directory));
1122
- }
1123
1015
 
1124
- // src/persistence/durability/restore-staging.ts
1125
- function toError2(error) {
1126
- return error instanceof Error ? error : new Error(String(error));
1127
- }
1128
- function pathUnder(path, root) {
1129
- if (!path.startsWith(root)) return false;
1130
- const next = path.charAt(root.length);
1131
- return next === "" || next === "/" || next === "\\";
1016
+ // src/persistence/durability/recovery.ts
1017
+ async function removeLeftoverRewriteFiles(directory, names) {
1018
+ for (const indexName of names) {
1019
+ for (const place of ["rebuilt", "received"]) {
1020
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1021
+ }
1022
+ }
1132
1023
  }
1133
- function stagedDirectory(directory, staged, unstaged) {
1134
- const { readRange, byteLength } = directory;
1135
- return {
1136
- root: directory.root,
1137
- appendHandle: (key) => directory.appendHandle(staged(key)),
1138
- markerHandle: (key) => directory.markerHandle(staged(key)),
1139
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1140
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1141
- read: (key) => directory.read(staged(key)),
1142
- remove: (key) => directory.remove(staged(key)),
1143
- move: (from, to) => directory.move(staged(from), staged(to)),
1144
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1145
- pathOf: (key) => directory.pathOf(staged(key)),
1146
- ...readRange === void 0 ? {} : {
1147
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1148
- },
1149
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1150
- };
1151
- }
1152
- function createRestoreStaging(storage, indexName, options = {}) {
1153
- const stagingPrefix = newStagingPrefix();
1154
- const indexRoot = `${indexName}/`;
1155
- let committed = false;
1156
- const staged = (key) => {
1157
- if (!key.startsWith(indexRoot)) {
1158
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1159
- key
1160
- });
1024
+ async function listPersistedIndexes(directory) {
1025
+ const metaKeys = await directory.list("");
1026
+ const names = [];
1027
+ for (const key of metaKeys) {
1028
+ if (key.endsWith("/meta")) {
1029
+ names.push(key.slice(0, -"/meta".length));
1161
1030
  }
1162
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1163
- };
1164
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1165
- const store = {
1166
- read: (key) => storage.read(staged(key)),
1167
- write: (key, chunks) => storage.write(staged(key), chunks),
1168
- remove: (key) => storage.remove(staged(key)),
1169
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1170
- };
1171
- const pathOf = storage.pathOf;
1172
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1173
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1174
- let retiredPrefix = null;
1175
- async function finishOnStorage() {
1176
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1177
1031
  }
1178
- return {
1179
- store,
1180
- async relocation() {
1181
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1182
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1183
- const to = await pathOf.call(storage, indexName);
1184
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1185
- },
1186
- async commit() {
1187
- try {
1188
- await commitJournal(storage, indexName, stagingPrefix);
1189
- } catch (error) {
1190
- try {
1191
- await storage.remove(journalKey(indexName));
1192
- } catch (removal) {
1193
- committed = true;
1194
- options.onStorageLost?.(toError2(removal));
1195
- }
1196
- throw error;
1032
+ return names;
1033
+ }
1034
+ async function loadMetadata(directory, indexName) {
1035
+ const bytes = await directory.read(`${indexName}/meta`);
1036
+ if (bytes === null) {
1037
+ return null;
1038
+ }
1039
+ const { metadata } = await readMetadataEnvelope(bytes);
1040
+ return metadata;
1041
+ }
1042
+ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1043
+ await directory.remove(nextSegmentKey(indexName, partitionId));
1044
+ const read = await readWalSegments(directory, indexName, partitionId);
1045
+ if (read === null) {
1046
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1047
+ }
1048
+ const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1049
+ if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1050
+ throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1051
+ }
1052
+ let highestSeqNo = fromSeqNoExclusive;
1053
+ let highestPrimaryTerm = 0;
1054
+ const replay = async (entries) => {
1055
+ for (const entry of entries) {
1056
+ if (entry.seqNo <= highestSeqNo) {
1057
+ continue;
1197
1058
  }
1198
- committed = true;
1199
- },
1200
- async finish() {
1201
- try {
1202
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1203
- } catch (error) {
1204
- options.onStorageLost?.(toError2(error));
1059
+ if (turnIsSpent()) await yieldTurn();
1060
+ applyEntry(entry, deps);
1061
+ highestSeqNo = entry.seqNo;
1062
+ if (entry.primaryTerm > highestPrimaryTerm) {
1063
+ highestPrimaryTerm = entry.primaryTerm;
1205
1064
  }
1206
- },
1207
- async removeRetired() {
1208
- const prefix = retiredPrefix;
1209
- retiredPrefix = null;
1210
- if (prefix !== null) await removePrefix(storage, prefix);
1211
- },
1212
- async discard() {
1213
- if (committed) return;
1214
- await removePrefix(storage, stagingPrefix);
1215
1065
  }
1216
1066
  };
1067
+ await replay(durableEntries);
1068
+ if (activeTail !== null) {
1069
+ await replay(activeTail.entries);
1070
+ }
1071
+ await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1072
+ if (activeTail !== null) {
1073
+ await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1074
+ }
1075
+ return { highestSeqNo, highestPrimaryTerm };
1217
1076
  }
1218
- function adapterRestoreStorage(adapter) {
1219
- const store = adapterCheckpointStore(adapter);
1220
- return {
1221
- read: (key) => store.read(key),
1222
- write: (key, chunks) => store.write(key, chunks),
1223
- writeJournal: (key, bytes) => store.write(key, [bytes]),
1224
- remove: (key) => store.remove(key),
1225
- list: (prefix) => store.list(prefix),
1226
- async moveKey(from, to) {
1227
- const bytes = await store.read(from);
1228
- if (bytes === null) return;
1229
- await store.write(to, [bytes]);
1230
- await store.remove(from);
1231
- }
1232
- };
1077
+ async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1078
+ const lastReplayed = tail.entries.at(-1);
1079
+ if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1080
+ const handle = await directory.appendHandle(tail.key);
1081
+ try {
1082
+ if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1083
+ else await handle.sync();
1084
+ } finally {
1085
+ await handle.close();
1086
+ }
1087
+ if (lastReplayed === void 0) return;
1088
+ const writer = await createMarkerWriter(directory, indexName, partitionId);
1089
+ try {
1090
+ await writer.commit({
1091
+ activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1092
+ durableByteLength: tail.cleanEnd,
1093
+ highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1094
+ });
1095
+ } finally {
1096
+ await writer.close();
1097
+ }
1233
1098
  }
1234
-
1235
- // src/persistence/durability/archive-checkpoint.ts
1236
- function toError3(error) {
1237
- return error instanceof Error ? error : new Error(String(error));
1099
+ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1100
+ for (const { key, startSeqNo } of segments) {
1101
+ if (startSeqNo > activeSegmentSeqNo) {
1102
+ await directory.remove(key);
1103
+ }
1104
+ }
1238
1105
  }
1239
- function stageDirectoryRestore(deps, indexName) {
1240
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1241
- async finish(run) {
1242
- try {
1243
- await run();
1244
- } catch (error) {
1245
- deps.markFatal(toError3(error));
1246
- throw error;
1247
- }
1248
- },
1249
- onStorageLost: deps.markFatal
1250
- });
1106
+ function applyEntry(entry, deps) {
1107
+ deps.writeRecords?.replayed(entry);
1108
+ if (entry.operation === "DELETE") {
1109
+ applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1110
+ return;
1111
+ }
1112
+ applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1251
1113
  }
1252
- function releaser(held, indexName) {
1253
- let released = false;
1254
- return () => {
1255
- if (released) return;
1256
- released = true;
1257
- const count = (held.get(indexName) ?? 1) - 1;
1258
- if (count <= 0) held.delete(indexName);
1259
- else held.set(indexName, count);
1260
- };
1114
+ function snapshotCheckpointFor(checkpoint, partitionId) {
1115
+ return checkpointLastSeqNo(checkpoint, partitionId);
1261
1116
  }
1262
- function createArchiveCheckpointAccess(deps) {
1263
- const store = directoryCheckpointStore(deps.directory);
1264
- return {
1265
- async openArchiveCheckpoint(indexName) {
1266
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1267
- const release = releaser(deps.held, indexName);
1117
+
1118
+ // src/persistence/durability/checkpoint-capture.ts
1119
+ function whileNoMutationApplies(partitions, capture2) {
1120
+ return new Promise((resolve, reject) => {
1121
+ let release = () => void 0;
1122
+ const released = new Promise((settle) => {
1123
+ release = settle;
1124
+ });
1125
+ function captureNow() {
1268
1126
  try {
1269
- await deps.checkpoint(indexName, false);
1270
- await deps.checkpoint(indexName, true);
1271
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
1272
- if (manifestBytes === null) {
1273
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1274
- indexName
1275
- });
1276
- }
1277
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1278
- if (decoded.kind !== "current") {
1279
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1280
- indexName
1281
- });
1282
- }
1283
- return {
1284
- manifestBytes,
1285
- manifest: decoded.manifest,
1286
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1287
- readRange: async (ref, offset, length) => {
1288
- const readRange = deps.directory.readRange;
1289
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1290
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1291
- return whole.slice(offset, offset + length);
1292
- },
1293
- release,
1294
- directory: deps.directory
1295
- };
1127
+ resolve(capture2());
1296
1128
  } catch (error) {
1297
- release();
1298
- throw error;
1299
- }
1300
- },
1301
- stageRestore(indexName) {
1302
- return stageDirectoryRestore(deps, indexName);
1303
- },
1304
- adoptRestoredCheckpoint(indexName, checkpoint) {
1305
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1306
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1307
- }
1308
- }
1309
- };
1310
- }
1311
-
1312
- // src/persistence/durability/checkpoint.ts
1313
- function walPrefix(indexName, partitionId) {
1314
- return `${indexName}/wal/${partitionId}/`;
1315
- }
1316
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
1317
- for (const { partitionId, lastSeqNo } of checkpoint) {
1318
- const prefix = walPrefix(indexName, partitionId);
1319
- const markerBytes = await directory.read(`${prefix}commit`);
1320
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1321
- if (marker === null) {
1322
- continue;
1323
- }
1324
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
1325
- const segments = await directory.list(prefix);
1326
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
1327
- for (let i = 0; i < startSeqNos.length; i += 1) {
1328
- const start = startSeqNos[i];
1329
- if (start >= activeSegmentSeqNo) {
1330
- break;
1331
- }
1332
- const nextStart = startSeqNos[i + 1];
1333
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
1334
- if (segmentCoversBeyondCheckpoint) {
1335
- break;
1336
- }
1337
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
1338
- }
1339
- }
1340
- }
1341
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
1342
- for (const [partitionId, partition] of [...partitions]) {
1343
- if (partitionId < partitionCount) continue;
1344
- try {
1345
- await partition.walWriter.close();
1346
- } catch (err) {
1347
- onCloseError(err instanceof Error ? err : new Error(String(err)));
1348
- }
1349
- partitions.delete(partitionId);
1350
- }
1351
- const walRoot = `${indexName}/wal/`;
1352
- for (const key of await directory.list(walRoot)) {
1353
- const partitionId = walKeyPartitionId(key, walRoot);
1354
- if (partitionId === null || partitionId < partitionCount) continue;
1355
- await directory.remove(key);
1356
- }
1357
- }
1358
- function walKeyPartitionId(key, walRoot) {
1359
- const tail = key.slice(walRoot.length);
1360
- const separator = tail.indexOf("/");
1361
- if (separator <= 0) {
1362
- return null;
1363
- }
1364
- const segment = tail.slice(0, separator);
1365
- if (!/^\d+$/.test(segment)) {
1366
- return null;
1367
- }
1368
- const value = Number.parseInt(segment, 10);
1369
- return Number.isSafeInteger(value) ? value : null;
1370
- }
1371
- function parseSegmentStartSeqNo(key, prefix) {
1372
- if (!key.startsWith(prefix)) {
1373
- return null;
1374
- }
1375
- const tail = key.slice(prefix.length);
1376
- if (!/^\d{16}$/.test(tail)) {
1377
- return null;
1378
- }
1379
- const value = Number.parseInt(tail, 10);
1380
- return Number.isSafeInteger(value) ? value : null;
1381
- }
1382
- function formatSegmentStart(startSeqNo) {
1383
- return startSeqNo.toString().padStart(16, "0");
1384
- }
1385
-
1386
- // src/persistence/durability/marker-writer.ts
1387
- function markerKey(indexName, partitionId) {
1388
- return `${indexName}/wal/${partitionId}/commit`;
1389
- }
1390
- async function createMarkerWriter(directory, indexName, partitionId) {
1391
- const key = markerKey(indexName, partitionId);
1392
- const handle = await directory.markerHandle(key);
1393
- const existing = await handle.read();
1394
- const current = readCommitMarker(existing);
1395
- const createdFresh = existing.length === 0;
1396
- let lastSlot = current?.slotIndex ?? null;
1397
- let writeSeq = current?.state.writeSeq ?? 0;
1398
- return {
1399
- created: createdFresh,
1400
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
1401
- async commit(state) {
1402
- const slotIndex = nextMarkerSlot(lastSlot);
1403
- writeSeq += 1;
1404
- const slot = encodeMarkerSlot({ ...state, writeSeq });
1405
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1406
- lastSlot = slotIndex;
1407
- },
1408
- async close() {
1409
- await handle.close();
1410
- }
1411
- };
1412
- }
1413
-
1414
- // src/persistence/durability/recovery.ts
1415
- async function removeLeftoverRewriteFiles(directory, names) {
1416
- for (const indexName of names) {
1417
- for (const place of ["rebuilt", "received"]) {
1418
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1419
- }
1420
- }
1421
- }
1422
- async function listPersistedIndexes(directory) {
1423
- const metaKeys = await directory.list("");
1424
- const names = [];
1425
- for (const key of metaKeys) {
1426
- if (key.endsWith("/meta")) {
1427
- names.push(key.slice(0, -"/meta".length));
1428
- }
1429
- }
1430
- return names;
1431
- }
1432
- async function loadMetadata(directory, indexName) {
1433
- const bytes = await directory.read(`${indexName}/meta`);
1434
- if (bytes === null) {
1435
- return null;
1436
- }
1437
- const { metadata } = await readMetadataEnvelope(bytes);
1438
- return metadata;
1439
- }
1440
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1441
- await directory.remove(nextSegmentKey(indexName, partitionId));
1442
- const read = await readWalSegments(directory, indexName, partitionId);
1443
- if (read === null) {
1444
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1445
- }
1446
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1447
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1448
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1449
- }
1450
- let highestSeqNo = fromSeqNoExclusive;
1451
- let highestPrimaryTerm = 0;
1452
- const replay = async (entries) => {
1453
- for (const entry of entries) {
1454
- if (entry.seqNo <= highestSeqNo) {
1455
- continue;
1456
- }
1457
- if (turnIsSpent()) await yieldTurn();
1458
- applyEntry(entry, deps);
1459
- highestSeqNo = entry.seqNo;
1460
- if (entry.primaryTerm > highestPrimaryTerm) {
1461
- highestPrimaryTerm = entry.primaryTerm;
1462
- }
1463
- }
1464
- };
1465
- await replay(durableEntries);
1466
- if (activeTail !== null) {
1467
- await replay(activeTail.entries);
1468
- }
1469
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1470
- if (activeTail !== null) {
1471
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1472
- }
1473
- return { highestSeqNo, highestPrimaryTerm };
1474
- }
1475
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1476
- const lastReplayed = tail.entries.at(-1);
1477
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1478
- const handle = await directory.appendHandle(tail.key);
1479
- try {
1480
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1481
- else await handle.sync();
1482
- } finally {
1483
- await handle.close();
1484
- }
1485
- if (lastReplayed === void 0) return;
1486
- const writer = await createMarkerWriter(directory, indexName, partitionId);
1487
- try {
1488
- await writer.commit({
1489
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1490
- durableByteLength: tail.cleanEnd,
1491
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1492
- });
1493
- } finally {
1494
- await writer.close();
1495
- }
1496
- }
1497
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1498
- for (const { key, startSeqNo } of segments) {
1499
- if (startSeqNo > activeSegmentSeqNo) {
1500
- await directory.remove(key);
1501
- }
1502
- }
1503
- }
1504
- function applyEntry(entry, deps) {
1505
- deps.writeRecords?.replayed(entry);
1506
- if (entry.operation === "DELETE") {
1507
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1508
- return;
1509
- }
1510
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1511
- }
1512
- function snapshotCheckpointFor(checkpoint, partitionId) {
1513
- return checkpointLastSeqNo(checkpoint, partitionId);
1514
- }
1515
-
1516
- // src/persistence/durability/checkpoint-capture.ts
1517
- function whileNoMutationApplies(partitions, capture2) {
1518
- return new Promise((resolve, reject) => {
1519
- let release = () => void 0;
1520
- const released = new Promise((settle) => {
1521
- release = settle;
1522
- });
1523
- function captureNow() {
1524
- try {
1525
- resolve(capture2());
1526
- } catch (error) {
1527
- reject(error);
1528
- } finally {
1129
+ reject(error);
1130
+ } finally {
1529
1131
  release();
1530
1132
  }
1531
1133
  }
@@ -1983,7 +1585,7 @@ async function writeOneBoundedCheckpoint(input) {
1983
1585
  }
1984
1586
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1985
1587
  const { indexName, indexState, hooks } = input;
1986
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-FHJ2GOBI.mjs');
1588
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-ZL5ZG6YB.mjs');
1987
1589
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1988
1590
  if (adoption === null) return [];
1989
1591
  const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
@@ -2038,10 +1640,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
2038
1640
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
2039
1641
  if (!input.holdsCheckpointFiles()) {
2040
1642
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
2041
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-FHJ2GOBI.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1643
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-ZL5ZG6YB.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1644
+ const keptForSync = await input.keepsCheckpointKey();
2042
1645
  await removeCheckpointGarbage(
2043
1646
  store,
2044
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
1647
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
2045
1648
  );
2046
1649
  }
2047
1650
  const checkpointDocumentCount = documentCount;
@@ -2054,7 +1657,9 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
2054
1657
  await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
2055
1658
  input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
2056
1659
  });
2057
- await truncateCoveredSegments(input.directory, input.indexName, targets);
1660
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1661
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
1662
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
2058
1663
  await reclaimWalBeyondCount(
2059
1664
  input.directory,
2060
1665
  input.indexName,
@@ -2086,202 +1691,1014 @@ function createIndexLoads(vectorIndexesOf2) {
2086
1691
  }
2087
1692
  };
2088
1693
  return {
2089
- track(indexName, load, startsGraphRepairs = true) {
2090
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
2091
- loads.set(
2092
- indexName,
2093
- loaded.catch(() => void 0)
2094
- );
2095
- return loaded;
2096
- },
2097
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
2098
- startGraphRepairs
1694
+ track(indexName, load, startsGraphRepairs = true) {
1695
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1696
+ loads.set(
1697
+ indexName,
1698
+ loaded.catch(() => void 0)
1699
+ );
1700
+ return loaded;
1701
+ },
1702
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1703
+ startGraphRepairs
1704
+ };
1705
+ }
1706
+ async function switchInTurn(turns, turn) {
1707
+ let swapped = false;
1708
+ await checkpointInTurn(turns, {
1709
+ joinsInFlight: false,
1710
+ mayStart: turn.mayStart,
1711
+ perform: async () => {
1712
+ swapped = turn.swap();
1713
+ if (swapped) await turn.checkpoint();
1714
+ },
1715
+ afterSettling: turn.afterSettling
1716
+ });
1717
+ return swapped;
1718
+ }
1719
+ async function checkpointInTurn(turns, turn) {
1720
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1721
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1722
+ if (!turn.mayStart()) return;
1723
+ const run = turn.perform().finally(() => {
1724
+ turns.checkpointInFlight = null;
1725
+ turn.afterSettling();
1726
+ });
1727
+ turns.checkpointInFlight = run;
1728
+ return run;
1729
+ }
1730
+
1731
+ // src/persistence/durability/held-checkpoints.ts
1732
+ function failedPartitionIds(indexState) {
1733
+ const failed = [];
1734
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1735
+ return failed;
1736
+ }
1737
+ function createHeldCheckpoints(hooks) {
1738
+ const reported = /* @__PURE__ */ new WeakSet();
1739
+ function heldReport(indexName, indexState) {
1740
+ const partitionIds = failedPartitionIds(indexState);
1741
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1742
+ return {
1743
+ indexName,
1744
+ partitionIds,
1745
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1746
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1747
+ };
1748
+ }
1749
+ return {
1750
+ refuse(indexName, indexState) {
1751
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1752
+ if (partitionIds.length === 0) return;
1753
+ throw new NarsilError(
1754
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1755
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1756
+ { indexName, partitionIds }
1757
+ );
1758
+ },
1759
+ holds(indexName, indexState) {
1760
+ const held = heldReport(indexName, indexState);
1761
+ if (held === null) return false;
1762
+ const unreported = held.partitionIds.some((partitionId) => {
1763
+ const partition = indexState.partitions.get(partitionId);
1764
+ return partition !== void 0 && !reported.has(partition);
1765
+ });
1766
+ if (unreported) {
1767
+ for (const partitionId of held.partitionIds) {
1768
+ const partition = indexState.partitions.get(partitionId);
1769
+ if (partition !== void 0) reported.add(partition);
1770
+ }
1771
+ hooks.onCheckpointHeld?.(held);
1772
+ }
1773
+ return true;
1774
+ },
1775
+ report(indexes) {
1776
+ const held = [];
1777
+ for (const [indexName, indexState] of indexes) {
1778
+ const entry = heldReport(indexName, indexState);
1779
+ if (entry !== null) held.push(entry);
1780
+ }
1781
+ return held;
1782
+ }
1783
+ };
1784
+ }
1785
+
1786
+ // src/persistence/durability/manager-state.ts
1787
+ function closePartitionWriter(partition) {
1788
+ return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1789
+ }
1790
+ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1791
+ indexState.unloading = true;
1792
+ try {
1793
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1794
+ await pendingMetadataWrite;
1795
+ for (const partition of indexState.partitions.values()) {
1796
+ await closeWalWriter(closePartitionWriter(partition));
1797
+ }
1798
+ } catch (error) {
1799
+ indexState.unloading = false;
1800
+ throw error;
1801
+ }
1802
+ }
1803
+
1804
+ // src/persistence/durability/restore-journal.ts
1805
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
1806
+ var RESTORE_STAGING_PREFIX = ".restore-";
1807
+ var RETIRED_INDEX_PREFIX = ".retired-";
1808
+ var LOCK_FILE_KEY = ".narsil.lock";
1809
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
1810
+ function digestOf(indexName) {
1811
+ let hex = "";
1812
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
1813
+ return hex;
1814
+ }
1815
+ function journalKey(indexName) {
1816
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
1817
+ }
1818
+ function randomSuffix() {
1819
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
1820
+ }
1821
+ function newStagingPrefix() {
1822
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
1823
+ }
1824
+ function newRetiredPrefix() {
1825
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
1826
+ }
1827
+ function encodeJournal(indexName, stagingPrefix) {
1828
+ return new TextEncoder().encode(`${indexName}
1829
+ ${stagingPrefix}`);
1830
+ }
1831
+ function isStagingPrefix(prefix) {
1832
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
1833
+ }
1834
+ function corruptJournal(key, reason) {
1835
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
1836
+ }
1837
+ function readJournal(key, bytes) {
1838
+ const text = new TextDecoder().decode(bytes);
1839
+ const lineEnd = text.indexOf("\n");
1840
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
1841
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
1842
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
1843
+ corruptJournal(key, "names an index that no index name can match");
1844
+ }
1845
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
1846
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
1847
+ corruptJournal(key, "names no staging prefix that a restore writes");
1848
+ }
1849
+ return { kind: "journal", indexName, stagingPrefix };
1850
+ }
1851
+ async function listedKeys(storage, indexName, stagingPrefix) {
1852
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
1853
+ if (manifestBytes === null) {
1854
+ throw new NarsilError(
1855
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1856
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1857
+ { indexName, stagingPrefix }
1858
+ );
1859
+ }
1860
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1861
+ if (decoded.kind !== "current") {
1862
+ throw new NarsilError(
1863
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1864
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1865
+ { indexName }
1866
+ );
1867
+ }
1868
+ const root = `${indexName}/`;
1869
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1870
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1871
+ return keys;
1872
+ }
1873
+ function withoutTrailingSlash(prefix) {
1874
+ return prefix.slice(0, -1);
1875
+ }
1876
+ async function finishJournal(storage, indexName, stagingPrefix) {
1877
+ const root = `${indexName}/`;
1878
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
1879
+ const staged = await storage.list(stagingPrefix);
1880
+ let retiredPrefix = null;
1881
+ if (storage.moveTree !== void 0 && staged.length > 0) {
1882
+ retiredPrefix = newRetiredPrefix();
1883
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1884
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1885
+ } else {
1886
+ const stagedKeys = new Set(staged);
1887
+ for (const key of listed) {
1888
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1889
+ }
1890
+ }
1891
+ for (const key of await storage.list(root)) {
1892
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1893
+ }
1894
+ await storage.remove(journalKey(indexName));
1895
+ return retiredPrefix;
1896
+ }
1897
+ async function removePrefix(storage, prefix) {
1898
+ if (storage.removeTree !== void 0) {
1899
+ await storage.removeTree(prefix);
1900
+ return;
1901
+ }
1902
+ for (const key of await storage.list(prefix)) await storage.remove(key);
1903
+ }
1904
+ async function commitJournal(storage, indexName, stagingPrefix) {
1905
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1906
+ }
1907
+ function isTopLevelJournal(key) {
1908
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1909
+ }
1910
+ async function removeStagingPrefixes(storage) {
1911
+ const prefixes = /* @__PURE__ */ new Set();
1912
+ for (const key of await storage.list(".")) {
1913
+ const slash = key.indexOf("/");
1914
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1915
+ }
1916
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
1917
+ }
1918
+ async function recoverRestoreJournals(storage) {
1919
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1920
+ if (!isTopLevelJournal(key)) continue;
1921
+ const bytes = await storage.read(key);
1922
+ if (bytes === null) continue;
1923
+ const reading = readJournal(key, bytes);
1924
+ if (reading.kind === "leftover") {
1925
+ await storage.remove(key);
1926
+ continue;
1927
+ }
1928
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1929
+ }
1930
+ await removeStagingPrefixes(storage);
1931
+ }
1932
+
1933
+ // src/persistence/durability/restore-directory.ts
1934
+ function missing(error) {
1935
+ return error.code === "ENOENT";
1936
+ }
1937
+ function storageFailed(code, message, key, error) {
1938
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1939
+ }
1940
+ function directoryRestoreStorage(directory) {
1941
+ return {
1942
+ directory,
1943
+ read: (key) => directory.read(key),
1944
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
1945
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1946
+ remove: (key) => directory.remove(key),
1947
+ list: (prefix) => directory.list(prefix),
1948
+ moveKey: (from, to) => directory.move(from, to),
1949
+ pathOf: (key) => directory.pathOf(key),
1950
+ async moveTree(from, to) {
1951
+ const source = await directory.pathOf(from);
1952
+ const target = await directory.pathOf(to);
1953
+ const fs = await getFs();
1954
+ try {
1955
+ await fs.rename(source, target);
1956
+ } catch (error) {
1957
+ if (missing(error)) return;
1958
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1959
+ }
1960
+ await directory.syncDirectoryOf(to);
1961
+ },
1962
+ async removeTree(prefix) {
1963
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1964
+ const fs = await getFs();
1965
+ try {
1966
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1967
+ } catch (error) {
1968
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1969
+ }
1970
+ await directory.syncDirectoryOf(key);
1971
+ }
1972
+ };
1973
+ }
1974
+ function recoverDirectoryRestores(directory) {
1975
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
1976
+ }
1977
+
1978
+ // src/persistence/durability/restore-staging.ts
1979
+ function toError2(error) {
1980
+ return error instanceof Error ? error : new Error(String(error));
1981
+ }
1982
+ function pathUnder(path, root) {
1983
+ if (!path.startsWith(root)) return false;
1984
+ const next = path.charAt(root.length);
1985
+ return next === "" || next === "/" || next === "\\";
1986
+ }
1987
+ function stagedDirectory(directory, staged, unstaged) {
1988
+ const { readRange, byteLength } = directory;
1989
+ return {
1990
+ root: directory.root,
1991
+ appendHandle: (key) => directory.appendHandle(staged(key)),
1992
+ markerHandle: (key) => directory.markerHandle(staged(key)),
1993
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1994
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1995
+ read: (key) => directory.read(staged(key)),
1996
+ remove: (key) => directory.remove(staged(key)),
1997
+ move: (from, to) => directory.move(staged(from), staged(to)),
1998
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1999
+ pathOf: (key) => directory.pathOf(staged(key)),
2000
+ ...readRange === void 0 ? {} : {
2001
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
2002
+ },
2003
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
2004
+ };
2005
+ }
2006
+ function createRestoreStaging(storage, indexName, options = {}) {
2007
+ const stagingPrefix = newStagingPrefix();
2008
+ const indexRoot = `${indexName}/`;
2009
+ let committed = false;
2010
+ const staged = (key) => {
2011
+ if (!key.startsWith(indexRoot)) {
2012
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
2013
+ key
2014
+ });
2015
+ }
2016
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
2017
+ };
2018
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
2019
+ const store = {
2020
+ read: (key) => storage.read(staged(key)),
2021
+ write: (key, chunks) => storage.write(staged(key), chunks),
2022
+ remove: (key) => storage.remove(staged(key)),
2023
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
2024
+ };
2025
+ const pathOf = storage.pathOf;
2026
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
2027
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
2028
+ let retiredPrefix = null;
2029
+ async function finishOnStorage() {
2030
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
2031
+ }
2032
+ return {
2033
+ store,
2034
+ async relocation() {
2035
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
2036
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
2037
+ const to = await pathOf.call(storage, indexName);
2038
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
2039
+ },
2040
+ async commit() {
2041
+ try {
2042
+ await commitJournal(storage, indexName, stagingPrefix);
2043
+ } catch (error) {
2044
+ try {
2045
+ await storage.remove(journalKey(indexName));
2046
+ } catch (removal) {
2047
+ committed = true;
2048
+ options.onStorageLost?.(toError2(removal));
2049
+ }
2050
+ throw error;
2051
+ }
2052
+ committed = true;
2053
+ },
2054
+ async finish() {
2055
+ try {
2056
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
2057
+ } catch (error) {
2058
+ options.onStorageLost?.(toError2(error));
2059
+ }
2060
+ },
2061
+ async removeRetired() {
2062
+ const prefix = retiredPrefix;
2063
+ retiredPrefix = null;
2064
+ if (prefix !== null) await removePrefix(storage, prefix);
2065
+ },
2066
+ async discard() {
2067
+ if (committed) return;
2068
+ await removePrefix(storage, stagingPrefix);
2069
+ }
2070
+ };
2071
+ }
2072
+ function adapterRestoreStorage(adapter) {
2073
+ const store = adapterCheckpointStore(adapter);
2074
+ return {
2075
+ read: (key) => store.read(key),
2076
+ write: (key, chunks) => store.write(key, chunks),
2077
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
2078
+ remove: (key) => store.remove(key),
2079
+ list: (prefix) => store.list(prefix),
2080
+ async moveKey(from, to) {
2081
+ const bytes = await store.read(from);
2082
+ if (bytes === null) return;
2083
+ await store.write(to, [bytes]);
2084
+ await store.remove(from);
2085
+ }
2086
+ };
2087
+ }
2088
+
2089
+ // src/persistence/durability/archive-checkpoint.ts
2090
+ function toError3(error) {
2091
+ return error instanceof Error ? error : new Error(String(error));
2092
+ }
2093
+ function stageDirectoryRestore(deps, indexName) {
2094
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
2095
+ async finish(run) {
2096
+ try {
2097
+ await run();
2098
+ } catch (error) {
2099
+ deps.markFatal(toError3(error));
2100
+ throw error;
2101
+ }
2102
+ },
2103
+ onStorageLost: deps.markFatal
2104
+ });
2105
+ }
2106
+ function releaser(held, indexName) {
2107
+ let released = false;
2108
+ return () => {
2109
+ if (released) return;
2110
+ released = true;
2111
+ const count = (held.get(indexName) ?? 1) - 1;
2112
+ if (count <= 0) held.delete(indexName);
2113
+ else held.set(indexName, count);
2114
+ };
2115
+ }
2116
+ function createArchiveCheckpointAccess(deps) {
2117
+ const store = directoryCheckpointStore(deps.directory);
2118
+ return {
2119
+ async openArchiveCheckpoint(indexName) {
2120
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
2121
+ const release = releaser(deps.held, indexName);
2122
+ try {
2123
+ await deps.checkpoint(indexName, false);
2124
+ await deps.checkpoint(indexName, true);
2125
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
2126
+ if (manifestBytes === null) {
2127
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
2128
+ indexName
2129
+ });
2130
+ }
2131
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2132
+ if (decoded.kind !== "current") {
2133
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
2134
+ indexName
2135
+ });
2136
+ }
2137
+ return {
2138
+ manifestBytes,
2139
+ manifest: decoded.manifest,
2140
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
2141
+ readRange: async (ref, offset, length) => {
2142
+ const readRange = deps.directory.readRange;
2143
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
2144
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
2145
+ return whole.slice(offset, offset + length);
2146
+ },
2147
+ release,
2148
+ directory: deps.directory
2149
+ };
2150
+ } catch (error) {
2151
+ release();
2152
+ throw error;
2153
+ }
2154
+ },
2155
+ stageRestore(indexName) {
2156
+ return stageDirectoryRestore(deps, indexName);
2157
+ },
2158
+ adoptRestoredCheckpoint(indexName, checkpoint) {
2159
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
2160
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
2161
+ }
2162
+ }
2163
+ };
2164
+ }
2165
+
2166
+ // src/persistence/durability/query-history.ts
2167
+ async function removeQueryHistoryOf(storage, indexName) {
2168
+ const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
2169
+ if (storage.kind === "directory") {
2170
+ for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
2171
+ return;
2172
+ }
2173
+ for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
2174
+ }
2175
+ function queryHistoryAccess(storage) {
2176
+ return {
2177
+ historyStorage: () => storage,
2178
+ removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
2099
2179
  };
2100
2180
  }
2101
- async function switchInTurn(turns, turn) {
2102
- let swapped = false;
2103
- await checkpointInTurn(turns, {
2104
- joinsInFlight: false,
2105
- mayStart: turn.mayStart,
2106
- perform: async () => {
2107
- swapped = turn.swap();
2108
- if (swapped) await turn.checkpoint();
2181
+
2182
+ // src/persistence/durability/manager-unload.ts
2183
+ function createIndexUnloading(deps) {
2184
+ const { directory, hooks, indexes, metadataWrites } = deps;
2185
+ return {
2186
+ async removeIndex(indexName) {
2187
+ hooks.writeRecords?.dropIndex(indexName);
2188
+ const indexState = indexes.get(indexName);
2189
+ if (indexState !== void 0) {
2190
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
2191
+ indexes.delete(indexName);
2192
+ metadataWrites.forget(indexName);
2193
+ }
2194
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
2195
+ for (const key of await directory.list(`${indexName}/`)) {
2196
+ await directory.remove(key);
2197
+ }
2198
+ await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
2199
+ deps.forgetSyncState(indexName);
2109
2200
  },
2110
- afterSettling: turn.afterSettling
2111
- });
2112
- return swapped;
2113
- }
2114
- async function checkpointInTurn(turns, turn) {
2115
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
2116
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
2117
- if (!turn.mayStart()) return;
2118
- const run = turn.perform().finally(() => {
2119
- turns.checkpointInFlight = null;
2120
- turn.afterSettling();
2121
- });
2122
- turns.checkpointInFlight = run;
2123
- return run;
2201
+ async unloadIndex(indexName) {
2202
+ const indexState = indexes.get(indexName);
2203
+ if (indexState === void 0) {
2204
+ return;
2205
+ }
2206
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
2207
+ indexes.delete(indexName);
2208
+ metadataWrites.forget(indexName);
2209
+ deps.forgetSyncState(indexName);
2210
+ }
2211
+ };
2124
2212
  }
2125
2213
 
2126
- // src/persistence/durability/directory-watch.ts
2127
- function unrefInterval(tick, intervalMs) {
2128
- const timer = setInterval(tick, intervalMs);
2129
- if (typeof timer.unref === "function") timer.unref();
2130
- return timer;
2131
- }
2132
- function createDirectoryWatch(directory, isStopped, onLost) {
2133
- let expected = null;
2134
- let timer = null;
2135
- let lock = null;
2136
- function cannotCheck(err) {
2137
- return new NarsilError(
2138
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2139
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
2140
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
2141
- );
2214
+ // src/persistence/durability/sync/frame-cursor.ts
2215
+ var SegmentFileMissing = class extends Error {
2216
+ };
2217
+ function corrupt(key, message, details) {
2218
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
2219
+ }
2220
+ function isMissingFile(error) {
2221
+ return error?.code === "ENOENT";
2222
+ }
2223
+ var FrameCursor = class {
2224
+ constructor(directory, key, limit) {
2225
+ this.directory = directory;
2226
+ this.key = key;
2227
+ this.limit = limit;
2228
+ }
2229
+ buffer = new Uint8Array(0);
2230
+ bufferStart = 0;
2231
+ whole = null;
2232
+ async load(offset, length) {
2233
+ const readRange = this.directory.readRange;
2234
+ if (readRange === void 0) {
2235
+ if (this.whole === null) {
2236
+ const bytes = await this.directory.read(this.key);
2237
+ if (bytes === null) throw new SegmentFileMissing(this.key);
2238
+ this.whole = bytes;
2239
+ }
2240
+ this.buffer = this.whole;
2241
+ this.bufferStart = 0;
2242
+ return;
2243
+ }
2244
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
2245
+ try {
2246
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
2247
+ } catch (error) {
2248
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
2249
+ throw error;
2250
+ }
2251
+ this.bufferStart = offset;
2252
+ }
2253
+ async bytesAt(offset, length) {
2254
+ const end = offset + length;
2255
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
2256
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
2257
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
2258
+ }
2259
+ async checkHeader() {
2260
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
2261
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
2262
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
2142
2263
  }
2143
- async function currentIdentity() {
2264
+ async frameLengthAt(offset) {
2265
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
2266
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
2267
+ 0,
2268
+ false
2269
+ );
2270
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
2271
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
2272
+ }
2273
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
2274
+ }
2275
+ async frameAt(offset, frameBytes) {
2276
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
2277
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
2278
+ const payload = body.subarray(0, recordLength);
2279
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
2280
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
2281
+ let seqNo;
2144
2282
  try {
2145
- return await directory.identity?.() ?? null;
2146
- } catch (err) {
2147
- return cannotCheck(err);
2283
+ seqNo = decodeWalEntry(payload).seqNo;
2284
+ } catch (error) {
2285
+ corrupt(this.key, "A log record fails to decode", {
2286
+ offset,
2287
+ cause: error instanceof Error ? error.message : String(error)
2288
+ });
2148
2289
  }
2290
+ return { seqNo, payload: payload.slice(), frameBytes };
2149
2291
  }
2150
- async function lockTakenOver() {
2151
- if (lock === null) return null;
2292
+ };
2293
+
2294
+ // src/persistence/durability/sync/log-reader.ts
2295
+ async function walSources(directory, indexName, partitionId) {
2296
+ const prefix = walSegmentPrefix(indexName, partitionId);
2297
+ const markerBytes = await directory.read(`${prefix}commit`);
2298
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
2299
+ if (marker === null) return { sources: [], head: 0 };
2300
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
2301
+ const sources = [];
2302
+ for (const key of await directory.list(prefix)) {
2303
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
2304
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
2305
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
2306
+ if (readableBytes === null || readableBytes === void 0) continue;
2307
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
2308
+ }
2309
+ return { sources, head: highestDurableSeqNo };
2310
+ }
2311
+ async function openPartitionLog(directory, indexName, partitionId) {
2312
+ const { sources, head } = await walSources(directory, indexName, partitionId);
2313
+ const byStart = /* @__PURE__ */ new Map();
2314
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
2315
+ const readableBytes = await directory.byteLength?.(segment.key);
2316
+ if (readableBytes === null || readableBytes === void 0) continue;
2317
+ byStart.set(segment.startSeqNo, {
2318
+ key: segment.key,
2319
+ startSeqNo: segment.startSeqNo,
2320
+ readableBytes,
2321
+ inHistory: true
2322
+ });
2323
+ }
2324
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
2325
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
2326
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
2327
+ }
2328
+ function sourceHolding(log, seqNo) {
2329
+ let found = -1;
2330
+ for (let i = 0; i < log.sources.length; i++) {
2331
+ if (log.sources[i].startSeqNo > seqNo) break;
2332
+ found = i;
2333
+ }
2334
+ return found;
2335
+ }
2336
+ async function offsetAfter(cursor, source, afterSeqNo) {
2337
+ let offset = SEGMENT_HEADER_SIZE;
2338
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
2339
+ offset += await cursor.frameLengthAt(offset);
2340
+ }
2341
+ return offset;
2342
+ }
2343
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
2344
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2345
+ await cursor.checkHeader();
2346
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
2347
+ let checksNumbering = true;
2348
+ while (offset < cursor.limit) {
2349
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
2350
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
2351
+ checksNumbering = false;
2352
+ if (frame.seqNo > afterSeqNo) {
2353
+ offset = SEGMENT_HEADER_SIZE;
2354
+ continue;
2355
+ }
2356
+ }
2357
+ checksNumbering = false;
2358
+ offset += frame.frameBytes;
2359
+ if (frame.seqNo <= afterSeqNo) continue;
2360
+ if (frame.seqNo > upToSeqNo) return true;
2361
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
2362
+ read.frames.push(frame);
2363
+ read.bytes += frame.payload.byteLength;
2364
+ if (frame.seqNo === upToSeqNo) return true;
2365
+ }
2366
+ return false;
2367
+ }
2368
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
2369
+ const read = { frames: [], bytes: 0, gone: false };
2370
+ if (afterSeqNo >= upToSeqNo) return read;
2371
+ for (let attempt = 0; attempt < 2; attempt++) {
2372
+ read.frames = [];
2373
+ read.bytes = 0;
2374
+ const log = await openPartitionLog(directory, indexName, partitionId);
2375
+ let index = sourceHolding(log, afterSeqNo + 1);
2376
+ if (index < 0) {
2377
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
2378
+ return read;
2379
+ }
2152
2380
  try {
2153
- if (await lock.holds()) return null;
2154
- } catch (err) {
2155
- return cannotCheck(err);
2381
+ for (; index < log.sources.length; index++) {
2382
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
2383
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
2384
+ }
2385
+ return read;
2386
+ } catch (error) {
2387
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
2156
2388
  }
2157
- return new NarsilError(
2158
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2159
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
2160
- { directory: directory.root }
2161
- );
2162
2389
  }
2163
- async function verify() {
2164
- if (directory.identity === void 0 || isStopped()) return true;
2165
- const current = await currentIdentity();
2166
- if (current instanceof Error) {
2167
- onLost(current);
2168
- return false;
2169
- }
2170
- if (expected === null) {
2171
- expected = current;
2172
- } else if (current !== expected) {
2173
- onLost(
2174
- new NarsilError(
2175
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2176
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
2177
- { directory: directory.root }
2178
- )
2179
- );
2180
- return false;
2390
+ return read;
2391
+ }
2392
+ async function logBytesAfter(directory, log, afterSeqNo) {
2393
+ const first = sourceHolding(log, afterSeqNo + 1);
2394
+ let bytes = 0;
2395
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
2396
+ const source = log.sources[i];
2397
+ if (i === first) {
2398
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2399
+ try {
2400
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
2401
+ } catch (error) {
2402
+ if (!(error instanceof SegmentFileMissing)) throw error;
2403
+ }
2404
+ continue;
2181
2405
  }
2182
- const takenOver = await lockTakenOver();
2183
- if (takenOver === null) return true;
2184
- onLost(takenOver);
2185
- return false;
2406
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
2186
2407
  }
2187
- function stop() {
2188
- if (timer !== null) {
2189
- clearInterval(timer);
2190
- timer = null;
2408
+ return bytes;
2409
+ }
2410
+
2411
+ // src/persistence/durability/sync/pins.ts
2412
+ function createFilePins(remove) {
2413
+ const counts = /* @__PURE__ */ new Map();
2414
+ const awaitingRemoval = /* @__PURE__ */ new Set();
2415
+ function unpin(key) {
2416
+ const count = (counts.get(key) ?? 1) - 1;
2417
+ if (count > 0) {
2418
+ counts.set(key, count);
2419
+ return;
2191
2420
  }
2421
+ counts.delete(key);
2422
+ if (!awaitingRemoval.delete(key)) return;
2423
+ void remove(key).catch(() => void 0);
2192
2424
  }
2193
2425
  return {
2194
- async claim() {
2195
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
2196
- },
2197
- async release() {
2198
- const held = lock;
2199
- lock = null;
2200
- await held?.release();
2201
- },
2202
- verify,
2203
- async verifyOnce() {
2204
- if (expected === null && directory.identity !== void 0) await verify();
2205
- },
2206
- start() {
2207
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
2208
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
2426
+ pin(keys) {
2427
+ const pinned = [...keys];
2428
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
2429
+ let released = false;
2430
+ return () => {
2431
+ if (released) return;
2432
+ released = true;
2433
+ for (const key of pinned) unpin(key);
2434
+ };
2209
2435
  },
2210
- stop
2436
+ isPinned: (key) => counts.has(key),
2437
+ removeWhenUnpinned(key) {
2438
+ if (!counts.has(key)) return false;
2439
+ awaitingRemoval.add(key);
2440
+ return true;
2441
+ }
2211
2442
  };
2212
2443
  }
2444
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2445
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2446
+ function hexOf(bytes) {
2447
+ let hex = "";
2448
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
2449
+ return hex;
2450
+ }
2451
+ function isBytes(value) {
2452
+ return value instanceof Uint8Array;
2453
+ }
2454
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
2455
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2456
+ if (decoded.kind !== "current") {
2457
+ throw new NarsilError(
2458
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2459
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
2460
+ { indexName }
2461
+ );
2462
+ }
2463
+ return {
2464
+ config,
2465
+ manifestBytes,
2466
+ manifest: decoded.manifest,
2467
+ entityTag: hexOf(sha256(payload)),
2468
+ keys: manifestReferencedKeys(decoded.manifest)
2469
+ };
2470
+ }
2471
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
2472
+ const payload = encode({ config, manifest: manifestBytes });
2473
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
2474
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
2475
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
2476
+ return published;
2477
+ }
2478
+ async function readPublishedSnapshot(directory, indexName) {
2479
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
2480
+ if (bytes === null) return null;
2481
+ const { payload } = await openChecksummedEnvelope(bytes, {
2482
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
2483
+ what: PUBLISHED_SNAPSHOT_WHAT,
2484
+ allowsCompression: false
2485
+ });
2486
+ const raw = decode(payload);
2487
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
2488
+ throw new NarsilError(
2489
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2490
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
2491
+ { indexName }
2492
+ );
2493
+ }
2494
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
2495
+ }
2213
2496
 
2214
- // src/persistence/durability/held-checkpoints.ts
2215
- function failedPartitionIds(indexState) {
2216
- const failed = [];
2217
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
2218
- return failed;
2497
+ // src/persistence/durability/sync/retention.ts
2498
+ async function firstLogStart(directory, indexName, partitionId) {
2499
+ let first = Number.POSITIVE_INFINITY;
2500
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
2501
+ const start = parseWalSegmentStart(indexName, partitionId, key);
2502
+ if (start !== null && start < first) first = start;
2503
+ }
2504
+ return first;
2505
+ }
2506
+ async function sizedQueues(retention) {
2507
+ const { directory, indexName } = retention;
2508
+ const queues = /* @__PURE__ */ new Map();
2509
+ for (const segment of await listHistorySegments(directory, indexName)) {
2510
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
2511
+ if (bytes === void 0 || bytes === null) continue;
2512
+ const queue = queues.get(segment.partitionId) ?? [];
2513
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
2514
+ queues.set(segment.partitionId, queue);
2515
+ }
2516
+ for (const [partitionId, queue] of queues) {
2517
+ const logStart = await firstLogStart(directory, indexName, partitionId);
2518
+ for (let i = 0; i < queue.length; i++) {
2519
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
2520
+ queue[i].lastSeqNo = nextStart - 1;
2521
+ }
2522
+ }
2523
+ return queues;
2524
+ }
2525
+ function nextToDelete(queues, totalBytes, retention) {
2526
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
2527
+ let oldest = null;
2528
+ for (const queue of queues.values()) {
2529
+ const head = queue[0];
2530
+ if (head === void 0) continue;
2531
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
2532
+ }
2533
+ if (oldest === null) return null;
2534
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
2535
+ return null;
2219
2536
  }
2220
- function createHeldCheckpoints(hooks) {
2221
- const reported = /* @__PURE__ */ new WeakSet();
2222
- function heldReport(indexName, indexState) {
2223
- const partitionIds = failedPartitionIds(indexState);
2224
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
2225
- return {
2226
- indexName,
2227
- partitionIds,
2228
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
2229
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
2230
- };
2537
+ async function enforceHistoryLimits(retention) {
2538
+ const queues = await sizedQueues(retention);
2539
+ let totalBytes = 0;
2540
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
2541
+ let published = false;
2542
+ let deleted = 0;
2543
+ let segment = nextToDelete(queues, totalBytes, retention);
2544
+ while (segment !== null) {
2545
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
2546
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
2547
+ published = true;
2548
+ await retention.publish();
2549
+ }
2550
+ await retention.directory.remove(segment.key);
2551
+ queues.get(segment.partitionId)?.shift();
2552
+ totalBytes -= segment.bytes;
2553
+ deleted += 1;
2554
+ segment = nextToDelete(queues, totalBytes, retention);
2555
+ }
2556
+ return deleted;
2557
+ }
2558
+
2559
+ // src/persistence/durability/sync/storage.ts
2560
+ function createSyncStorage(deps) {
2561
+ const { directory } = deps;
2562
+ const store = directoryCheckpointStore(directory);
2563
+ const cache = /* @__PURE__ */ new Map();
2564
+ const publishing = /* @__PURE__ */ new Map();
2565
+ const pins = createFilePins((key) => directory.remove(key));
2566
+ async function published(indexName) {
2567
+ const cached = cache.get(indexName);
2568
+ if (cached !== void 0) return cached;
2569
+ const read = await readPublishedSnapshot(directory, indexName);
2570
+ cache.set(indexName, read);
2571
+ return read;
2572
+ }
2573
+ async function removeUnlisted(indexName, previous, kept) {
2574
+ const manifest = await readCurrentManifest(store, indexName);
2575
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
2576
+ for (const key of previous) {
2577
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
2578
+ await directory.remove(key);
2579
+ }
2580
+ }
2581
+ async function publishNow(indexName, configurationOf) {
2582
+ const archive = await deps.openArchiveCheckpoint(indexName);
2583
+ try {
2584
+ const config = await configurationOf(archive.manifest);
2585
+ const previous = await published(indexName);
2586
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
2587
+ cache.set(indexName, next);
2588
+ if (previous !== null) {
2589
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
2590
+ }
2591
+ return next;
2592
+ } finally {
2593
+ archive.release();
2594
+ }
2231
2595
  }
2232
2596
  return {
2233
- refuse(indexName, indexState) {
2234
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
2235
- if (partitionIds.length === 0) return;
2236
- throw new NarsilError(
2237
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2238
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
2239
- { indexName, partitionIds }
2240
- );
2597
+ publish(indexName, configurationOf) {
2598
+ const inFlight = publishing.get(indexName);
2599
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
2600
+ const tracked = run.finally(() => {
2601
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
2602
+ });
2603
+ publishing.set(indexName, tracked);
2604
+ return tracked;
2241
2605
  },
2242
- holds(indexName, indexState) {
2243
- const held = heldReport(indexName, indexState);
2244
- if (held === null) return false;
2245
- const unreported = held.partitionIds.some((partitionId) => {
2246
- const partition = indexState.partitions.get(partitionId);
2247
- return partition !== void 0 && !reported.has(partition);
2606
+ published,
2607
+ pin: (snapshot) => pins.pin(snapshot.keys),
2608
+ async readFileRange(key, offset, length) {
2609
+ const readRange = directory.readRange;
2610
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
2611
+ const whole = await directory.read(key);
2612
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
2613
+ },
2614
+ async release(indexName) {
2615
+ await deps.inCheckpointTurn(indexName, async () => {
2616
+ const previous = await published(indexName);
2617
+ await directory.remove(publishedSnapshotKey(indexName));
2618
+ cache.set(indexName, null);
2619
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
2620
+ await removeHistory(directory, indexName);
2248
2621
  });
2249
- if (unreported) {
2250
- for (const partitionId of held.partitionIds) {
2251
- const partition = indexState.partitions.get(partitionId);
2252
- if (partition !== void 0) reported.add(partition);
2253
- }
2254
- hooks.onCheckpointHeld?.(held);
2255
- }
2256
- return true;
2257
2622
  },
2258
- report(indexes) {
2259
- const held = [];
2260
- for (const [indexName, indexState] of indexes) {
2261
- const entry = heldReport(indexName, indexState);
2262
- if (entry !== null) held.push(entry);
2623
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
2624
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
2625
+ const current = await published(indexName);
2626
+ return enforceHistoryLimits({
2627
+ directory,
2628
+ indexName,
2629
+ limits,
2630
+ nowMs,
2631
+ publish,
2632
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
2633
+ });
2634
+ },
2635
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
2636
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
2637
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
2638
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
2639
+ async keepsCheckpointKey(indexName) {
2640
+ const current = await published(indexName);
2641
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
2642
+ },
2643
+ async settleAtRecovery(indexName, active) {
2644
+ cache.delete(indexName);
2645
+ const publishedKey = publishedSnapshotKey(indexName);
2646
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
2647
+ if (key !== publishedKey) await directory.remove(key);
2263
2648
  }
2264
- return held;
2649
+ if (!active) {
2650
+ await directory.remove(publishedSnapshotKey(indexName));
2651
+ await removeHistory(directory, indexName);
2652
+ cache.set(indexName, null);
2653
+ return /* @__PURE__ */ new Set();
2654
+ }
2655
+ try {
2656
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
2657
+ } catch {
2658
+ await directory.remove(publishedSnapshotKey(indexName));
2659
+ cache.set(indexName, null);
2660
+ return /* @__PURE__ */ new Set();
2661
+ }
2662
+ },
2663
+ forget(indexName) {
2664
+ cache.delete(indexName);
2265
2665
  }
2266
2666
  };
2267
2667
  }
2268
2668
 
2269
- // src/persistence/durability/manager-state.ts
2270
- function closePartitionWriter(partition) {
2271
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
2272
- }
2273
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
2274
- indexState.unloading = true;
2275
- try {
2276
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2277
- await pendingMetadataWrite;
2278
- for (const partition of indexState.partitions.values()) {
2279
- await closeWalWriter(closePartitionWriter(partition));
2669
+ // src/persistence/durability/manager-storage.ts
2670
+ function createIndexStorageAccess(deps) {
2671
+ const { directory, indexes } = deps;
2672
+ const archiveAccess = createArchiveCheckpointAccess({
2673
+ directory,
2674
+ held: deps.heldCheckpointFiles,
2675
+ checkpoint: deps.checkpoint,
2676
+ registerPartition: deps.registerPartition,
2677
+ markFatal: deps.markFatal
2678
+ });
2679
+ const syncStorage = createSyncStorage({
2680
+ directory,
2681
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
2682
+ inCheckpointTurn: (indexName, work) => {
2683
+ const indexState = indexes.get(indexName);
2684
+ if (indexState === void 0) return work();
2685
+ return checkpointInTurn(indexState, {
2686
+ joinsInFlight: false,
2687
+ mayStart: () => true,
2688
+ perform: work,
2689
+ afterSettling: () => releaseStalledWrites(indexState)
2690
+ });
2280
2691
  }
2281
- } catch (error) {
2282
- indexState.unloading = false;
2283
- throw error;
2284
- }
2692
+ });
2693
+ const unloading = createIndexUnloading({
2694
+ directory,
2695
+ hooks: deps.hooks,
2696
+ indexes,
2697
+ metadataWrites: deps.metadataWrites,
2698
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
2699
+ forgetSyncState: syncStorage.forget
2700
+ });
2701
+ return { archiveAccess, syncStorage, unloading };
2285
2702
  }
2286
2703
 
2287
2704
  // src/persistence/durability/metadata-write-queue.ts
@@ -2529,8 +2946,8 @@ function createGroupCommitCoordinator(sync) {
2529
2946
  // src/persistence/durability/wal-writer.ts
2530
2947
  var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
2531
2948
  function segmentKey(indexName, partitionId, startSeqNo) {
2532
- const padded2 = startSeqNo.toString().padStart(16, "0");
2533
- return `${indexName}/wal/${partitionId}/${padded2}`;
2949
+ const padded = startSeqNo.toString().padStart(16, "0");
2950
+ return `${indexName}/wal/${partitionId}/${padded}`;
2534
2951
  }
2535
2952
  function parseSegmentStartSeqNo2(key) {
2536
2953
  const tail = key.slice(key.lastIndexOf("/") + 1);
@@ -3254,7 +3671,7 @@ var decoder = new TextDecoder();
3254
3671
  async function countPagedPartition(store, partition) {
3255
3672
  const { directory } = store;
3256
3673
  if (directory === void 0 || partition.segments.length === 0) return null;
3257
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-FHJ2GOBI.mjs');
3674
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-ZL5ZG6YB.mjs');
3258
3675
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
3259
3676
  const scans = [];
3260
3677
  try {
@@ -3537,12 +3954,14 @@ async function recoverCheckpoint(deps) {
3537
3954
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3538
3955
  }
3539
3956
  if (decoded === null || decoded.kind !== "current") {
3540
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
3957
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3541
3958
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3542
3959
  }
3543
3960
  await deps.store.remove(snapshotBundleKey(deps.indexName));
3544
3961
  if (!deps.keepsUnlistedFiles) {
3545
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
3962
+ const referenced = manifestReferencedKeys(decoded.manifest);
3963
+ for (const key of deps.keptForSync) referenced.add(key);
3964
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
3546
3965
  }
3547
3966
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
3548
3967
  manager: deps.manager,
@@ -3663,6 +4082,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3663
4082
  }
3664
4083
  return replayed;
3665
4084
  };
4085
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
4086
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
3666
4087
  const recovered = await recoverCheckpoint({
3667
4088
  store,
3668
4089
  indexName,
@@ -3672,6 +4093,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3672
4093
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3673
4094
  compactionThreshold: deps.compactionThreshold,
3674
4095
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
4096
+ keptForSync,
3675
4097
  replay,
3676
4098
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3677
4099
  writeRecords
@@ -3787,6 +4209,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3787
4209
  mutationThreshold: checkpointMutationThreshold,
3788
4210
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3789
4211
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
4212
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3790
4213
  queueMetadataWrite,
3791
4214
  markFatal,
3792
4215
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -3840,6 +4263,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3840
4263
  startAsyncFlushTimer();
3841
4264
  return outcomes;
3842
4265
  }
4266
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
4267
+ directory,
4268
+ hooks,
4269
+ indexes,
4270
+ metadataWrites,
4271
+ heldCheckpointFiles,
4272
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
4273
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
4274
+ markFatal,
4275
+ closeWriterReportingFailure
4276
+ });
3843
4277
  const recoveryDeps = {
3844
4278
  directory,
3845
4279
  hooks,
@@ -3848,7 +4282,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3848
4282
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3849
4283
  },
3850
4284
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3851
- queueMetadataWrite
4285
+ queueMetadataWrite,
4286
+ syncStorage
3852
4287
  };
3853
4288
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3854
4289
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3936,35 +4371,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3936
4371
  await checkpointIndex(indexName);
3937
4372
  }
3938
4373
  },
3939
- async removeIndex(indexName) {
3940
- hooks.writeRecords?.dropIndex(indexName);
3941
- const indexState = indexes.get(indexName);
3942
- if (indexState !== void 0) {
3943
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3944
- indexes.delete(indexName);
3945
- metadataWrites.forget(indexName);
3946
- }
3947
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3948
- for (const key of await directory.list(`${indexName}/`)) {
3949
- await directory.remove(key);
3950
- }
3951
- },
3952
- async unloadIndex(indexName) {
3953
- const indexState = indexes.get(indexName);
3954
- if (indexState === void 0) {
3955
- return;
3956
- }
3957
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3958
- indexes.delete(indexName);
3959
- metadataWrites.forget(indexName);
3960
- },
3961
- ...createArchiveCheckpointAccess({
3962
- directory,
3963
- held: heldCheckpointFiles,
3964
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3965
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3966
- markFatal
3967
- }),
4374
+ ...unloading,
4375
+ syncStorage: () => syncStorage,
4376
+ ...queryHistoryAccess({ kind: "directory", directory }),
4377
+ ...archiveAccess,
3968
4378
  async shutdown() {
3969
4379
  shuttingDown = true;
3970
4380
  stopTimers();
@@ -4231,6 +4641,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4231
4641
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
4232
4642
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
4233
4643
  keepsUnlistedFiles: false,
4644
+ keptForSync: /* @__PURE__ */ new Set(),
4234
4645
  replay: async (checkpoint) => checkpoint,
4235
4646
  onLayoutUpgrade: hooks.onLayoutUpgrade
4236
4647
  });
@@ -4366,6 +4777,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4366
4777
  if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
4367
4778
  }
4368
4779
  },
4780
+ ...queryHistoryAccess({ kind: "adapter", adapter }),
4369
4781
  async removeIndex(indexName) {
4370
4782
  await settleRestore(indexName).catch(() => void 0);
4371
4783
  restores.forget(indexName);
@@ -4374,6 +4786,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4374
4786
  for (const key of await adapter.list(`${indexName}/`)) {
4375
4787
  await adapter.delete(key);
4376
4788
  }
4789
+ await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
4377
4790
  },
4378
4791
  async unloadIndex(indexName) {
4379
4792
  await settleRestore(indexName);
@@ -4428,7 +4841,9 @@ function createDurabilityIntegration(tier, hooks) {
4428
4841
  recordCheckpoint: hooks.recordCheckpoint,
4429
4842
  onLayoutUpgrade: hooks.onLayoutUpgrade,
4430
4843
  onCheckpointHeld: hooks.onCheckpointHeld,
4431
- onFatalError: hooks.onFatalError
4844
+ onFatalError: hooks.onFatalError,
4845
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4846
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4432
4847
  };
4433
4848
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4434
4849
  function partitionFor(indexName, docId) {
@@ -4513,7 +4928,12 @@ function createDurabilityFromTier(tier, wiring) {
4513
4928
  recordCheckpoint: wiring.recordCheckpoint,
4514
4929
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
4515
4930
  onCheckpointHeld: wiring.emitCheckpointHeld,
4516
- onFatalError: wiring.emitFatalError
4931
+ onFatalError: wiring.emitFatalError,
4932
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4933
+ onHistoryMoved(indexName) {
4934
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4935
+ import('./retention-7I7UAWYK.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4936
+ }
4517
4937
  });
4518
4938
  }
4519
4939