@delali/narsil 0.3.1-next.153 → 0.3.1-next.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-4GCRVRBW.mjs} +27 -18
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KYGSXW4K.mjs} +1 -1
  8. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-IXWPXXJJ.mjs} +17 -7
  9. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-FCSHFQSS.mjs} +1 -1
  10. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-2FX6ROWR.mjs} +15 -6
  11. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-2SHAKW2Z.mjs} +119 -1922
  12. package/dist/browser/{chunk-55WYRLDZ.mjs → chunk-3U3QR5YP.mjs} +2 -2
  13. package/dist/browser/chunk-4PD752F6.mjs +85 -0
  14. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  15. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-7MEYN7V6.mjs} +1 -151
  16. package/dist/browser/chunk-7TTWXUH6.mjs +1828 -0
  17. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-AJ4HHOVT.mjs} +4 -4
  18. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-AYY6PIKL.mjs} +2 -2
  19. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-C4KS2QNF.mjs} +9 -8
  20. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  21. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  22. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-EUNBNLKM.mjs} +6 -5
  23. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-G7LUMBSG.mjs} +137 -253
  24. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-H6QFBMVE.mjs} +3 -2
  25. package/dist/browser/chunk-J433A7P3.mjs +241 -0
  26. package/dist/browser/chunk-ML6HY7T7.mjs +96 -0
  27. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  28. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-MYITLCHT.mjs} +7 -1
  29. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-QXTPSX2H.mjs} +9 -150
  30. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  31. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SIR3LMTV.mjs} +2 -2
  32. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-SJB6VIZG.mjs} +1 -1
  33. package/dist/browser/chunk-TTD27VXX.mjs +298 -0
  34. package/dist/browser/chunk-U2E3HMYW.mjs +151 -0
  35. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-VCOYGVP7.mjs} +5 -31
  36. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  37. package/dist/browser/chunk-W5X2UFWQ.mjs +102 -0
  38. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-XRV72TJD.mjs} +2 -2
  39. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  40. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  41. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-ZPTA75ZC.mjs} +2 -79
  42. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-VTUGDYNY.mjs} +20 -14
  43. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-GL4A7KDV.mjs} +1886 -1364
  44. package/dist/browser/import-YI5GNB4H.mjs +17 -0
  45. package/dist/browser/methods-LMUTS7OI.mjs +30 -0
  46. package/dist/browser/{module-XQLEOYNM.mjs → module-EC3FYW7N.mjs} +9 -8
  47. package/dist/browser/{module-KAMRAJLE.mjs → module-M4JNBV6V.mjs} +16 -10
  48. package/dist/browser/operations-WVYLF72Q.mjs +5 -0
  49. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-NPYUVX75.mjs} +1 -1
  50. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-WCGDZHJP.mjs} +3 -2
  51. package/dist/{restore-queue-AOF5OTQV.mjs → browser/restore-queue-TYIGDL5L.mjs} +2 -2
  52. package/dist/browser/retention-6QMRS5VK.mjs +207 -0
  53. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-3EW2AGB6.mjs} +1 -1
  54. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-DN3CRVUQ.mjs} +38 -307
  55. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  56. package/dist/browser/{write-I5P5MW4O.mjs → write-WAWAFGKQ.mjs} +16 -11
  57. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-B3Brs1Rc.d.ts} +62 -7
  58. package/dist/{chunk-DGRM4EZZ.mjs → chunk-37IVJLP6.mjs} +1 -1
  59. package/dist/{chunk-KYVRZAYP.mjs → chunk-3A5KI24C.mjs} +2 -69
  60. package/dist/{chunk-POUYICOU.mjs → chunk-3OFU3NJT.mjs} +1 -1
  61. package/dist/{chunk-3TVDTACC.mjs → chunk-3Y3J4V2R.mjs} +1 -1
  62. package/dist/{chunk-FNBNZCGP.mjs → chunk-466PYR4P.mjs} +2 -2
  63. package/dist/{chunk-SEAJS664.mjs → chunk-4HWZV5MK.mjs} +5 -5
  64. package/dist/{chunk-UGVUW27S.mjs → chunk-5FUOVJKG.mjs} +13 -9
  65. package/dist/{chunk-ZHAXP7SD.mjs → chunk-5KGYQT6W.mjs} +6 -235
  66. package/dist/{chunk-NPXVQS33.mjs → chunk-62G63NMH.mjs} +4 -3
  67. package/dist/chunk-6MSZ543Y.mjs +32 -0
  68. package/dist/chunk-7GXJUJ24.mjs +49 -0
  69. package/dist/{chunk-3E4RRF5J.mjs → chunk-7HCZWC7B.mjs} +58 -8
  70. package/dist/{chunk-6RKSY3BX.mjs → chunk-7NTMUDC5.mjs} +5 -5
  71. package/dist/chunk-7QUGDVFC.mjs +101 -0
  72. package/dist/{chunk-G5KHSM6E.mjs → chunk-AO2FAYIZ.mjs} +1 -1
  73. package/dist/chunk-B7S6YEBP.mjs +102 -0
  74. package/dist/{chunk-AWNHDBKV.mjs → chunk-BPSKLCS7.mjs} +15 -7
  75. package/dist/chunk-BUMREPLU.mjs +15 -0
  76. package/dist/{chunk-T4CJDBBE.mjs → chunk-C6ASEFZI.mjs} +1 -1
  77. package/dist/{chunk-NITCPDFL.mjs → chunk-C73JLU4E.mjs} +2 -2
  78. package/dist/{chunk-NYSONUB7.mjs → chunk-CP5RRPJR.mjs} +1 -1
  79. package/dist/{chunk-TTCKT2RT.mjs → chunk-DMQPFLGI.mjs} +134 -11
  80. package/dist/{chunk-3SBJVQLL.mjs → chunk-DVWYODEB.mjs} +2 -2
  81. package/dist/chunk-ECX4SGX6.mjs +167 -0
  82. package/dist/chunk-GES7U2PM.mjs +68 -0
  83. package/dist/{chunk-NVTFQFUL.mjs → chunk-HVKXNBFL.mjs} +1 -1
  84. package/dist/chunk-HVYICDY3.mjs +214 -0
  85. package/dist/{chunk-3HV7M5OJ.mjs → chunk-HWEDFLB7.mjs} +9 -133
  86. package/dist/{chunk-52QQWEBS.mjs → chunk-IBNLW3AV.mjs} +21 -6
  87. package/dist/chunk-IJU7LL7D.mjs +242 -0
  88. package/dist/chunk-IPN7S4KX.mjs +130 -0
  89. package/dist/{chunk-5QMHMELB.mjs → chunk-IQ4R2I6N.mjs} +4 -1
  90. package/dist/chunk-JCLTESFN.mjs +96 -0
  91. package/dist/{chunk-J4BDAH2W.mjs → chunk-JFNKIBDT.mjs} +2 -2
  92. package/dist/{chunk-LDGGR7YT.mjs → chunk-K4B7OH65.mjs} +2 -2
  93. package/dist/{chunk-7TE2AJIH.mjs → chunk-K7KLL263.mjs} +13 -10
  94. package/dist/{chunk-CSHHU7PA.mjs → chunk-KCUIFVVW.mjs} +11 -30
  95. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-KUZKUONM.mjs} +11 -2
  96. package/dist/chunk-KYYABCSW.mjs +72 -0
  97. package/dist/{chunk-B2FTZNNF.mjs → chunk-LKUE5GXS.mjs} +34 -14
  98. package/dist/{chunk-2PNYKG2D.mjs → chunk-LSM2YTKM.mjs} +100 -335
  99. package/dist/{chunk-DMNPYF54.mjs → chunk-LTUGDMV4.mjs} +2 -2
  100. package/dist/{chunk-FDG4KXX5.mjs → chunk-MLHNSAPK.mjs} +5 -3
  101. package/dist/{chunk-VP6XRMWR.mjs → chunk-MMBEANB2.mjs} +6 -32
  102. package/dist/{chunk-LJ4ENXOD.mjs → chunk-MMEMEUOO.mjs} +2 -2
  103. package/dist/{chunk-2HJWQPR6.mjs → chunk-NTZOXVUZ.mjs} +6 -122
  104. package/dist/{chunk-5WB2A3XU.mjs → chunk-NZMAXR2S.mjs} +397 -24
  105. package/dist/{chunk-IZT3G24Q.mjs → chunk-O43Z4MS6.mjs} +2 -2
  106. package/dist/{chunk-U46IZNCW.mjs → chunk-OKX2BBNW.mjs} +3 -2
  107. package/dist/{chunk-RBEHLIKX.mjs → chunk-PFKM7VNZ.mjs} +14 -8
  108. package/dist/chunk-PGVPE7GJ.mjs +96 -0
  109. package/dist/chunk-Q3TCTQCC.mjs +85 -0
  110. package/dist/chunk-Q7MS6P2J.mjs +74 -0
  111. package/dist/chunk-QDZJM7NI.mjs +23 -0
  112. package/dist/{chunk-ZS6CMXAE.mjs → chunk-QKZHLADX.mjs} +89 -250
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-REL7LFLR.mjs} +3 -3
  114. package/dist/chunk-RVKFI3A2.mjs +237 -0
  115. package/dist/chunk-S2N75OTN.mjs +83 -0
  116. package/dist/chunk-TRISKD4Q.mjs +50 -0
  117. package/dist/{chunk-6F6USYII.mjs → chunk-TV2L3BN7.mjs} +13 -11
  118. package/dist/{chunk-6BWLDLS3.mjs → chunk-URPCYQVJ.mjs} +1 -1
  119. package/dist/chunk-USR7FQRZ.mjs +34 -0
  120. package/dist/chunk-V6EUBAIT.mjs +19 -0
  121. package/dist/{chunk-SOUZMPME.mjs → chunk-VMXSVVXG.mjs} +7 -194
  122. package/dist/{chunk-4QCSHLNM.mjs → chunk-VREK3MYG.mjs} +3 -3
  123. package/dist/{chunk-YGQOPB2Z.mjs → chunk-WCBA2LVZ.mjs} +1 -1
  124. package/dist/{chunk-KQJVNPRM.mjs → chunk-X6NSJYQ3.mjs} +19 -16
  125. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  126. package/dist/{chunk-IRWP524D.mjs → chunk-YBJDOA2T.mjs} +1 -1
  127. package/dist/{chunk-IJPK5PGM.mjs → chunk-YPUEV6ZC.mjs} +1 -1
  128. package/dist/{chunk-J7M73VY7.mjs → chunk-YROWNOMH.mjs} +1 -1
  129. package/dist/chunk-Z3TNWCI5.mjs +196 -0
  130. package/dist/{chunk-65WCV5JO.mjs → chunk-ZIFJXCKA.mjs} +1 -1
  131. package/dist/chunk-ZYAY26KG.mjs +29 -0
  132. package/dist/{client-Cn3gNxUU.d.ts → client-Df76GNKm.d.ts} +76 -6
  133. package/dist/client.d.ts +8 -7
  134. package/dist/client.mjs +56 -5
  135. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-YLWMTTZG.mjs} +38 -28
  136. package/dist/{config-CfhH1IWQ.d.ts → config-05G_hy-1.d.ts} +3 -3
  137. package/dist/config-B8DP2VwR.d.ts +463 -0
  138. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-FI5GJHWX.mjs} +14 -9
  139. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  140. package/dist/distribution/coordinator/etcd.mjs +1 -1
  141. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  142. package/dist/distribution/transport/grpc.mjs +4 -4
  143. package/dist/distribution/transport/in-memory.mjs +2 -2
  144. package/dist/distribution/transport/tcp.mjs +3 -3
  145. package/dist/distribution.d.ts +41 -9
  146. package/dist/distribution.mjs +1296 -117
  147. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-RHLXDIVF.mjs} +1751 -1226
  148. package/dist/embeddings/openai.mjs +2 -2
  149. package/dist/{engine-CJRUHXgf.d.ts → engine-DYu-e0oR.d.ts} +50 -40
  150. package/dist/{import-GJOMB2HC.mjs → import-SRBNC6TE.mjs} +27 -19
  151. package/dist/{index-MsF7e5hL.d.ts → index-nRqDM6uS.d.ts} +1 -1
  152. package/dist/index.browser.mjs +8 -4
  153. package/dist/index.d.ts +7 -6
  154. package/dist/index.mjs +24 -16
  155. package/dist/index.react-native.mjs +17 -11
  156. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-EMYTGSJP.mjs} +28 -15
  157. package/dist/methods-ABM5ELBR.mjs +30 -0
  158. package/dist/{module-AVQEIB7O.mjs → module-3OMJZUFT.mjs} +12 -12
  159. package/dist/{module-6RLIKSZQ.mjs → module-MG7JMV2U.mjs} +33 -23
  160. package/dist/operations-PGOAY2KS.mjs +5 -0
  161. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-UOLHOEZJ.mjs} +37 -26
  162. package/dist/partial-words.mjs +6 -4
  163. package/dist/pattern.mjs +4 -3
  164. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -1
  165. package/dist/persistence/durability/checkpoint-worker.mjs +34 -23
  166. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-O3MDRQXP.mjs} +1 -1
  167. package/dist/{pool-UPV5PVPY.mjs → pool-CQXYPFLG.mjs} +2 -2
  168. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-EAYO3Y6Q.mjs} +27 -15
  169. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-TNLUC6IZ.mjs} +1 -1
  170. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-2CGSQM4O.mjs} +12 -10
  171. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  172. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-3O57C2ZO.mjs} +2 -2
  173. package/dist/react-native/chunks/chunk-3XKMRNRE.mjs +298 -0
  174. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-63YSZOXB.mjs} +97 -333
  175. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-75VWIY35.mjs} +1 -1
  176. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7GWPJDOA.mjs} +1 -1
  177. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-AUT7PMA4.mjs} +6 -190
  178. package/dist/react-native/chunks/chunk-C2H2QQFW.mjs +288 -0
  179. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  180. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-CUNGPR5R.mjs} +11 -2
  181. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-DCKMICOB.mjs} +2 -79
  182. package/dist/react-native/chunks/chunk-DH3TVWWK.mjs +102 -0
  183. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-EH5CFV6Z.mjs} +2 -2
  184. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  185. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-GFYFQLJD.mjs} +2 -2
  186. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-I5PGWQJF.mjs} +142 -257
  187. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-IOY2SM2P.mjs} +1 -1
  188. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  189. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-JPAJWO6D.mjs} +1 -1
  190. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  191. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-JX2RIAXH.mjs} +4 -4
  192. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  193. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-NCJS3WIY.mjs} +2 -2
  194. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-NKIZX26F.mjs} +2 -2
  195. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-NP7DD4G5.mjs} +5 -4
  196. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  197. package/dist/react-native/chunks/chunk-O32RB3BL.mjs +191 -0
  198. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-OQQVRWZ2.mjs} +2 -69
  199. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  200. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  201. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-RVAEBN74.mjs} +1 -1
  202. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-RYMDHGL4.mjs} +16 -8
  203. package/dist/react-native/chunks/chunk-SJWDBRNP.mjs +85 -0
  204. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  205. package/dist/react-native/chunks/chunk-UWDNZDWK.mjs +242 -0
  206. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-VQSKZ3EZ.mjs} +9 -152
  207. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-VVFYOADG.mjs} +7 -6
  208. package/dist/react-native/chunks/{chunk-VGUWIZFQ.mjs → chunk-YBEW6YTR.mjs} +1 -1
  209. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-YHKZSLOC.mjs} +7 -33
  210. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-ZBSWRSBG.mjs} +4 -30
  211. package/dist/react-native/chunks/chunk-ZYDKJXQJ.mjs +96 -0
  212. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-TJL6AFAD.mjs} +29 -21
  213. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-RRDYJLLM.mjs} +1882 -1358
  214. package/dist/react-native/chunks/gzip.react-native-DNB5YNFA.mjs +3 -0
  215. package/dist/react-native/chunks/import-JYBMEJPQ.mjs +29 -0
  216. package/dist/react-native/chunks/methods-X6F36SJE.mjs +30 -0
  217. package/dist/react-native/chunks/module-QT5HZAY4.mjs +33 -0
  218. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-S4DKSWMF.mjs} +10 -9
  219. package/dist/react-native/chunks/operations-RYKTSZQA.mjs +5 -0
  220. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-JZ37TB7N.mjs} +1 -1
  221. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-RTGXJXPU.mjs} +3 -2
  222. package/dist/{browser/restore-queue-UQ6AAUVX.mjs → react-native/chunks/restore-queue-ZBVIKJWI.mjs} +2 -2
  223. package/dist/react-native/chunks/retention-PPCTP2VY.mjs +219 -0
  224. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-KUATOLYB.mjs} +1 -1
  225. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-ZBIEOZTZ.mjs} +47 -314
  226. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  227. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-2W4FTX7J.mjs} +25 -18
  228. package/dist/react-native/partial-words.mjs +6 -4
  229. package/dist/react-native/pattern.mjs +4 -3
  230. package/dist/react-native/sync.mjs +2236 -0
  231. package/dist/react-native/synonyms.mjs +7 -4
  232. package/dist/react.d.ts +10 -9
  233. package/dist/react.mjs +7 -1
  234. package/dist/{rerank-3W54N6P2.mjs → rerank-IZOB6NII.mjs} +4 -4
  235. package/dist/rerankers/http.mjs +2 -2
  236. package/dist/{react-native/chunks/restore-queue-H5VCN453.mjs → restore-queue-COI2GSSK.mjs} +2 -2
  237. package/dist/retention-55WA7GPU.mjs +58 -0
  238. package/dist/{runtime-L2DZYY6N.mjs → runtime-UZOC3NPY.mjs} +5 -5
  239. package/dist/{schema-D33CyXCy.d.ts → schema-DDauvDOv.d.ts} +1 -1
  240. package/dist/server.d.ts +22 -19
  241. package/dist/server.mjs +868 -50
  242. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-NFX2EYN2.mjs} +62 -36
  243. package/dist/sync/filesystem.d.ts +30 -0
  244. package/dist/sync/filesystem.mjs +271 -0
  245. package/dist/sync/indexeddb.d.ts +30 -0
  246. package/dist/sync/indexeddb.mjs +237 -0
  247. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  248. package/dist/{config-aIsy7mcL.d.ts → sync-uXR7MmM7.d.ts} +53 -444
  249. package/dist/sync.browser.mjs +2231 -0
  250. package/dist/sync.d.ts +23 -0
  251. package/dist/sync.mjs +2206 -0
  252. package/dist/synonyms.mjs +5 -3
  253. package/dist/{tasks-C4ElXr4P.d.ts → tasks-BShx8pdK.d.ts} +4 -4
  254. package/dist/thread-P2G2FWM2.mjs +44 -0
  255. package/dist/types-SnO7_tAI.d.ts +228 -0
  256. package/dist/vector/search-worker.d.ts +1 -1
  257. package/dist/vector/search-worker.mjs +6 -6
  258. package/dist/workers/entry.mjs +7 -7
  259. package/dist/{write-SKJ6HW7O.mjs → write-FGB62BHQ.mjs} +31 -23
  260. package/openapi.json +119 -3
  261. package/package.json +28 -8
  262. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  263. package/dist/chunk-7LDMWKP6.mjs +0 -209
  264. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  265. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  266. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  267. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  268. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,30 +1,41 @@
1
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, SEGMENT_HEADER_SIZE, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-7IQR3SZO.mjs';
1
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-ZYDKJXQJ.mjs';
2
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-I5PGWQJF.mjs';
2
3
  import { nodeThreadId } from './chunk-PSWKQVVT.mjs';
3
4
  import './chunk-HJBJBH26.mjs';
4
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-VGUWIZFQ.mjs';
5
- import './chunk-LRGWK367.mjs';
6
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, PHONE_FILE_CHUNK_BYTES, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-FS7BYE23.mjs';
7
- import { applyDeleteEntry, applyIndexEntry } from './chunk-CLE3LZ77.mjs';
8
- import { toError, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-IDFMNRIC.mjs';
9
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime } from './chunk-LQAKO5QV.mjs';
10
- import { createRequestId } from './chunk-NNE4VDMM.mjs';
11
- import { fnv1a } from './chunk-L4M7QJNP.mjs';
5
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-UWDNZDWK.mjs';
6
+ import './chunk-JWUCMVRK.mjs';
7
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-VQSKZ3EZ.mjs';
8
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, PHONE_FILE_CHUNK_BYTES, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-F753DA54.mjs';
9
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-NP7DD4G5.mjs';
10
+ import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-63YSZOXB.mjs';
12
11
  import './chunk-IO2SMUQD.mjs';
12
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime } from './chunk-NCJS3WIY.mjs';
13
13
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-AM7RSBPF.mjs';
14
- import './chunk-ZCKNBPQ4.mjs';
15
- import { encodeDocument, requireValidFieldTypes } from './chunk-PU55IOG4.mjs';
14
+ import { createRequestId } from './chunk-NNE4VDMM.mjs';
15
+ import './chunk-LU3Y3P63.mjs';
16
+ import './chunk-EH5CFV6Z.mjs';
17
+ import './chunk-AUT7PMA4.mjs';
18
+ import './chunk-UNUFZ2QY.mjs';
19
+ import './chunk-3NQOVO3K.mjs';
20
+ import './chunk-JDOEEMEF.mjs';
21
+ import './chunk-7GWPJDOA.mjs';
22
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-SJWDBRNP.mjs';
23
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-YBEW6YTR.mjs';
24
+ import { openChecksummedEnvelope, crc32, HEADER_SIZE } from './chunk-C2H2QQFW.mjs';
25
+ import { fnv1a } from './chunk-CHPTFFJM.mjs';
26
+ import { encodeDocument, requireValidFieldTypes } from './chunk-O32RB3BL.mjs';
16
27
  import { MIN_M, MAX_FILE_BYTES } from './chunk-TJMSKBEM.mjs';
17
- import './chunk-DKJTZ625.mjs';
18
- import './chunk-3CTWSQRW.mjs';
19
- import './chunk-2J2727V4.mjs';
28
+ import './chunk-OQQVRWZ2.mjs';
29
+ import './chunk-ZBSWRSBG.mjs';
20
30
  import './chunk-KWRW5U65.mjs';
21
31
  import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
22
32
  import './chunk-LVIKKAA6.mjs';
23
33
  import './chunk-AAAETEJK.mjs';
24
34
  import './chunk-JV6VQY2S.mjs';
25
- import { reactNativeHost } from './chunk-NFGBOI46.mjs';
26
- import { NarsilError, ErrorCodes } from './chunk-YH7W556M.mjs';
27
- import { decode } from '@msgpack/msgpack';
35
+ import './chunk-RVAEBN74.mjs';
36
+ import { reactNativeHost } from './chunk-IOY2SM2P.mjs';
37
+ import { NarsilError, ErrorCodes } from './chunk-CUNGPR5R.mjs';
38
+ import { decode, encode } from '@msgpack/msgpack';
28
39
 
29
40
  // src/runtime/posix-path.ts
30
41
  function normalizeParts(path, absolute) {
@@ -808,7 +819,9 @@ function createDurableDirectory(root) {
808
819
  const fs = await getFs();
809
820
  const pathMod = await getPath();
810
821
  const resolvedBase = pathMod.resolve(root);
811
- const all = await listRecursive(resolvedBase, resolvedBase, pathMod, fs);
822
+ const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
823
+ const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
824
+ const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
812
825
  return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
813
826
  },
814
827
  async pathOf(key) {
@@ -818,1470 +831,1981 @@ function createDurableDirectory(root) {
818
831
  };
819
832
  }
820
833
 
821
- // src/persistence/durability/sha256.ts
822
- var ROUND_CONSTANTS = new Uint32Array([
823
- 1116352408,
824
- 1899447441,
825
- 3049323471,
826
- 3921009573,
827
- 961987163,
828
- 1508970993,
829
- 2453635748,
830
- 2870763221,
831
- 3624381080,
832
- 310598401,
833
- 607225278,
834
- 1426881987,
835
- 1925078388,
836
- 2162078206,
837
- 2614888103,
838
- 3248222580,
839
- 3835390401,
840
- 4022224774,
841
- 264347078,
842
- 604807628,
843
- 770255983,
844
- 1249150122,
845
- 1555081692,
846
- 1996064986,
847
- 2554220882,
848
- 2821834349,
849
- 2952996808,
850
- 3210313671,
851
- 3336571891,
852
- 3584528711,
853
- 113926993,
854
- 338241895,
855
- 666307205,
856
- 773529912,
857
- 1294757372,
858
- 1396182291,
859
- 1695183700,
860
- 1986661051,
861
- 2177026350,
862
- 2456956037,
863
- 2730485921,
864
- 2820302411,
865
- 3259730800,
866
- 3345764771,
867
- 3516065817,
868
- 3600352804,
869
- 4094571909,
870
- 275423344,
871
- 430227734,
872
- 506948616,
873
- 659060556,
874
- 883997877,
875
- 958139571,
876
- 1322822218,
877
- 1537002063,
878
- 1747873779,
879
- 1955562222,
880
- 2024104815,
881
- 2227730452,
882
- 2361852424,
883
- 2428436474,
884
- 2756734187,
885
- 3204031479,
886
- 3329325298
887
- ]);
888
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
889
- var BLOCK_BYTES = 64;
890
- var LENGTH_BYTES = 8;
891
- function rotateRight(value, bits) {
892
- return value >>> bits | value << 32 - bits;
834
+ // src/persistence/durability/sync/layout.ts
835
+ var SYNC_KEY_NUMBER_WIDTH = 16;
836
+ function syncDirectoryPrefix(indexName) {
837
+ return `${indexName}/sync/`;
893
838
  }
894
- function padded(bytes) {
895
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
896
- const message = new Uint8Array(total);
897
- message.set(bytes);
898
- message[bytes.byteLength] = 128;
899
- const view = new DataView(message.buffer);
900
- const bitLength = bytes.byteLength * 8;
901
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
902
- view.setUint32(total - 4, bitLength >>> 0, false);
903
- return view;
839
+ function publishedSnapshotKey(indexName) {
840
+ return `${syncDirectoryPrefix(indexName)}published`;
904
841
  }
905
- function sha256(bytes) {
906
- const view = padded(bytes);
907
- const state = INITIAL_STATE.slice();
908
- const words = new Uint32Array(64);
909
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
910
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
911
- for (let i = 16; i < 64; i++) {
912
- const w15 = words[i - 15];
913
- const w2 = words[i - 2];
914
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
915
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
916
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
917
- }
918
- let [a, b, c, d, e, f, g, h] = state;
919
- for (let i = 0; i < 64; i++) {
920
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
921
- const choice = e & f ^ ~e & g;
922
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
923
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
924
- const majority = a & b ^ a & c ^ b & c;
925
- const t2 = s0 + majority >>> 0;
926
- h = g;
927
- g = f;
928
- f = e;
929
- e = d + t1 >>> 0;
930
- d = c;
931
- c = b;
932
- b = a;
933
- a = t1 + t2 >>> 0;
934
- }
935
- state[0] = state[0] + a >>> 0;
936
- state[1] = state[1] + b >>> 0;
937
- state[2] = state[2] + c >>> 0;
938
- state[3] = state[3] + d >>> 0;
939
- state[4] = state[4] + e >>> 0;
940
- state[5] = state[5] + f >>> 0;
941
- state[6] = state[6] + g >>> 0;
942
- state[7] = state[7] + h >>> 0;
943
- }
944
- const digest = new Uint8Array(32);
945
- const out = new DataView(digest.buffer);
946
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
947
- return digest;
842
+ function historyRoot(indexName) {
843
+ return `${indexName}/history/`;
948
844
  }
949
-
950
- // src/persistence/durability/restore-journal.ts
951
- var RESTORE_JOURNAL_PREFIX = ".swap-";
952
- var RESTORE_STAGING_PREFIX = ".restore-";
953
- var RETIRED_INDEX_PREFIX = ".retired-";
954
- var LOCK_FILE_KEY = ".narsil.lock";
955
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
956
- function digestOf(indexName) {
957
- let hex = "";
958
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
959
- return hex;
845
+ function historyPrefix(indexName, partitionId) {
846
+ return `${historyRoot(indexName)}${partitionId}/`;
960
847
  }
961
- function journalKey(indexName) {
962
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
848
+ function walSegmentPrefix(indexName, partitionId) {
849
+ return `${indexName}/wal/${partitionId}/`;
963
850
  }
964
- function randomSuffix() {
965
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
851
+ function formatSyncKeyNumber(value) {
852
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
853
+ }
854
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
855
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
856
+ }
857
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
858
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
859
+ }
860
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
861
+ function parseHistoryKey(indexName, key) {
862
+ const root = historyRoot(indexName);
863
+ if (!key.startsWith(root)) return null;
864
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
865
+ if (match === null) return null;
866
+ const partitionId = Number.parseInt(match[1], 10);
867
+ const startSeqNo = Number.parseInt(match[2], 10);
868
+ const movedAtMs = Number.parseInt(match[3], 10);
869
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
870
+ return null;
871
+ }
872
+ return { key, partitionId, startSeqNo, movedAtMs };
966
873
  }
967
- function newStagingPrefix() {
968
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
874
+ function parseWalSegmentStart(indexName, partitionId, key) {
875
+ const prefix = walSegmentPrefix(indexName, partitionId);
876
+ if (!key.startsWith(prefix)) return null;
877
+ const tail = key.slice(prefix.length);
878
+ if (!/^\d{16}$/.test(tail)) return null;
879
+ const value = Number.parseInt(tail, 10);
880
+ return Number.isSafeInteger(value) ? value : null;
969
881
  }
970
- function newRetiredPrefix() {
971
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
882
+
883
+ // src/persistence/durability/sync/history.ts
884
+ function compareHistorySegments(a, b) {
885
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
886
+ }
887
+ async function listHistorySegments(directory, indexName, partitionId) {
888
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
889
+ const segments = [];
890
+ for (const key of await directory.list(prefix)) {
891
+ const segment = parseHistoryKey(indexName, key);
892
+ if (segment !== null) segments.push(segment);
893
+ }
894
+ return segments.sort(compareHistorySegments);
895
+ }
896
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
897
+ await directory.move(
898
+ walSegmentKey(indexName, partitionId, startSeqNo),
899
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
900
+ );
972
901
  }
973
- function encodeJournal(indexName, stagingPrefix) {
974
- return new TextEncoder().encode(`${indexName}
975
- ${stagingPrefix}`);
902
+ async function removeHistory(directory, indexName, partitionId) {
903
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
904
+ await directory.remove(segment.key);
905
+ }
976
906
  }
977
- function isStagingPrefix(prefix) {
978
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
907
+
908
+ // src/persistence/durability/checkpoint.ts
909
+ function walPrefix(indexName, partitionId) {
910
+ return `${indexName}/wal/${partitionId}/`;
979
911
  }
980
- function corruptJournal(key, reason) {
981
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
912
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
913
+ for (const { partitionId, lastSeqNo } of checkpoint) {
914
+ const prefix = walPrefix(indexName, partitionId);
915
+ const markerBytes = await directory.read(`${prefix}commit`);
916
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
917
+ if (marker === null) {
918
+ continue;
919
+ }
920
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
921
+ const segments = await directory.list(prefix);
922
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
923
+ for (let i = 0; i < startSeqNos.length; i += 1) {
924
+ const start = startSeqNos[i];
925
+ if (start >= activeSegmentSeqNo) {
926
+ break;
927
+ }
928
+ const nextStart = startSeqNos[i + 1];
929
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
930
+ if (segmentCoversBeyondCheckpoint) {
931
+ break;
932
+ }
933
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
934
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
935
+ }
936
+ }
982
937
  }
983
- function readJournal(key, bytes) {
984
- const text = new TextDecoder().decode(bytes);
985
- const lineEnd = text.indexOf("\n");
986
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
987
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
988
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
989
- corruptJournal(key, "names an index that no index name can match");
938
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
939
+ for (const [partitionId, partition] of [...partitions]) {
940
+ if (partitionId < partitionCount) continue;
941
+ try {
942
+ await partition.walWriter.close();
943
+ } catch (err) {
944
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
945
+ }
946
+ partitions.delete(partitionId);
990
947
  }
991
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
992
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
993
- corruptJournal(key, "names no staging prefix that a restore writes");
948
+ const walRoot = `${indexName}/wal/`;
949
+ for (const key of await directory.list(walRoot)) {
950
+ const partitionId = walKeyPartitionId(key, walRoot);
951
+ if (partitionId === null || partitionId < partitionCount) continue;
952
+ await directory.remove(key);
994
953
  }
995
- return { kind: "journal", indexName, stagingPrefix };
996
954
  }
997
- async function listedKeys(storage, indexName, stagingPrefix) {
998
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
999
- if (manifestBytes === null) {
1000
- throw new NarsilError(
1001
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1002
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1003
- { indexName, stagingPrefix }
1004
- );
955
+ function walKeyPartitionId(key, walRoot) {
956
+ const tail = key.slice(walRoot.length);
957
+ const separator = tail.indexOf("/");
958
+ if (separator <= 0) {
959
+ return null;
1005
960
  }
1006
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1007
- if (decoded.kind !== "current") {
1008
- throw new NarsilError(
1009
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1010
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1011
- { indexName }
1012
- );
961
+ const segment = tail.slice(0, separator);
962
+ if (!/^\d+$/.test(segment)) {
963
+ return null;
1013
964
  }
1014
- const root = `${indexName}/`;
1015
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1016
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1017
- return keys;
965
+ const value = Number.parseInt(segment, 10);
966
+ return Number.isSafeInteger(value) ? value : null;
1018
967
  }
1019
- function withoutTrailingSlash(prefix) {
1020
- return prefix.slice(0, -1);
968
+ function parseSegmentStartSeqNo(key, prefix) {
969
+ if (!key.startsWith(prefix)) {
970
+ return null;
971
+ }
972
+ const tail = key.slice(prefix.length);
973
+ if (!/^\d{16}$/.test(tail)) {
974
+ return null;
975
+ }
976
+ const value = Number.parseInt(tail, 10);
977
+ return Number.isSafeInteger(value) ? value : null;
1021
978
  }
1022
- async function finishJournal(storage, indexName, stagingPrefix) {
1023
- const root = `${indexName}/`;
1024
- const listed = await listedKeys(storage, indexName, stagingPrefix);
1025
- const staged = await storage.list(stagingPrefix);
1026
- let retiredPrefix = null;
1027
- if (storage.moveTree !== void 0 && staged.length > 0) {
1028
- retiredPrefix = newRetiredPrefix();
1029
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1030
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1031
- } else {
1032
- const stagedKeys = new Set(staged);
1033
- for (const key of listed) {
1034
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
979
+ function formatSegmentStart(startSeqNo) {
980
+ return startSeqNo.toString().padStart(16, "0");
981
+ }
982
+
983
+ // src/persistence/durability/marker-writer.ts
984
+ function markerKey(indexName, partitionId) {
985
+ return `${indexName}/wal/${partitionId}/commit`;
986
+ }
987
+ async function createMarkerWriter(directory, indexName, partitionId) {
988
+ const key = markerKey(indexName, partitionId);
989
+ const handle = await directory.markerHandle(key);
990
+ const existing = await handle.read();
991
+ const current = readCommitMarker(existing);
992
+ const createdFresh = existing.length === 0;
993
+ let lastSlot = current?.slotIndex ?? null;
994
+ let writeSeq = current?.state.writeSeq ?? 0;
995
+ return {
996
+ created: createdFresh,
997
+ existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
998
+ async commit(state) {
999
+ const slotIndex = nextMarkerSlot(lastSlot);
1000
+ writeSeq += 1;
1001
+ const slot = encodeMarkerSlot({ ...state, writeSeq });
1002
+ await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1003
+ lastSlot = slotIndex;
1004
+ },
1005
+ async close() {
1006
+ await handle.close();
1007
+ }
1008
+ };
1009
+ }
1010
+
1011
+ // src/persistence/durability/recovery.ts
1012
+ async function removeLeftoverRewriteFiles(directory, names) {
1013
+ for (const indexName of names) {
1014
+ for (const place of ["rebuilt", "received"]) {
1015
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1035
1016
  }
1036
1017
  }
1037
- for (const key of await storage.list(root)) {
1038
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
1039
- }
1040
- await storage.remove(journalKey(indexName));
1041
- return retiredPrefix;
1042
1018
  }
1043
- async function removePrefix(storage, prefix) {
1044
- if (storage.removeTree !== void 0) {
1045
- await storage.removeTree(prefix);
1046
- return;
1019
+ async function listPersistedIndexes(directory) {
1020
+ const metaKeys = await directory.list("");
1021
+ const names = [];
1022
+ for (const key of metaKeys) {
1023
+ if (key.endsWith("/meta")) {
1024
+ names.push(key.slice(0, -"/meta".length));
1025
+ }
1047
1026
  }
1048
- for (const key of await storage.list(prefix)) await storage.remove(key);
1027
+ return names;
1049
1028
  }
1050
- async function commitJournal(storage, indexName, stagingPrefix) {
1051
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1029
+ async function loadMetadata(directory, indexName) {
1030
+ const bytes = await directory.read(`${indexName}/meta`);
1031
+ if (bytes === null) {
1032
+ return null;
1033
+ }
1034
+ const { metadata } = await readMetadataEnvelope(bytes);
1035
+ return metadata;
1052
1036
  }
1053
- function isTopLevelJournal(key) {
1054
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1037
+ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1038
+ await directory.remove(nextSegmentKey(indexName, partitionId));
1039
+ const read = await readWalSegments(directory, indexName, partitionId);
1040
+ if (read === null) {
1041
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1042
+ }
1043
+ const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1044
+ if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1045
+ throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1046
+ }
1047
+ let highestSeqNo = fromSeqNoExclusive;
1048
+ let highestPrimaryTerm = 0;
1049
+ const replay = async (entries) => {
1050
+ for (const entry of entries) {
1051
+ if (entry.seqNo <= highestSeqNo) {
1052
+ continue;
1053
+ }
1054
+ if (turnIsSpent()) await yieldTurn();
1055
+ applyEntry(entry, deps);
1056
+ highestSeqNo = entry.seqNo;
1057
+ if (entry.primaryTerm > highestPrimaryTerm) {
1058
+ highestPrimaryTerm = entry.primaryTerm;
1059
+ }
1060
+ }
1061
+ };
1062
+ await replay(durableEntries);
1063
+ if (activeTail !== null) {
1064
+ await replay(activeTail.entries);
1065
+ }
1066
+ await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1067
+ if (activeTail !== null) {
1068
+ await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1069
+ }
1070
+ return { highestSeqNo, highestPrimaryTerm };
1055
1071
  }
1056
- async function removeStagingPrefixes(storage) {
1057
- const prefixes = /* @__PURE__ */ new Set();
1058
- for (const key of await storage.list(".")) {
1059
- const slash = key.indexOf("/");
1060
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1072
+ async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1073
+ const lastReplayed = tail.entries.at(-1);
1074
+ if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1075
+ const handle = await directory.appendHandle(tail.key);
1076
+ try {
1077
+ if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1078
+ else await handle.sync();
1079
+ } finally {
1080
+ await handle.close();
1081
+ }
1082
+ if (lastReplayed === void 0) return;
1083
+ const writer = await createMarkerWriter(directory, indexName, partitionId);
1084
+ try {
1085
+ await writer.commit({
1086
+ activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1087
+ durableByteLength: tail.cleanEnd,
1088
+ highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1089
+ });
1090
+ } finally {
1091
+ await writer.close();
1061
1092
  }
1062
- for (const prefix of prefixes) await removePrefix(storage, prefix);
1063
1093
  }
1064
- async function recoverRestoreJournals(storage) {
1065
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1066
- if (!isTopLevelJournal(key)) continue;
1067
- const bytes = await storage.read(key);
1068
- if (bytes === null) continue;
1069
- const reading = readJournal(key, bytes);
1070
- if (reading.kind === "leftover") {
1071
- await storage.remove(key);
1072
- continue;
1094
+ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1095
+ for (const { key, startSeqNo } of segments) {
1096
+ if (startSeqNo > activeSegmentSeqNo) {
1097
+ await directory.remove(key);
1073
1098
  }
1074
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1075
1099
  }
1076
- await removeStagingPrefixes(storage);
1077
1100
  }
1078
-
1079
- // src/persistence/durability/restore-directory.ts
1080
- function missing(error) {
1081
- return error.code === "ENOENT";
1101
+ function applyEntry(entry, deps) {
1102
+ deps.writeRecords?.replayed(entry);
1103
+ if (entry.operation === "DELETE") {
1104
+ applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1105
+ return;
1106
+ }
1107
+ applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1082
1108
  }
1083
- function storageFailed(code, message, key, error) {
1084
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1109
+ function snapshotCheckpointFor(checkpoint, partitionId) {
1110
+ return checkpointLastSeqNo(checkpoint, partitionId);
1085
1111
  }
1086
- function directoryRestoreStorage(directory) {
1087
- return {
1088
- directory,
1089
- read: (key) => directory.read(key),
1090
- write: (key, chunks) => directory.atomicWrite(key, chunks),
1091
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1092
- remove: (key) => directory.remove(key),
1093
- list: (prefix) => directory.list(prefix),
1094
- moveKey: (from, to) => directory.move(from, to),
1095
- pathOf: (key) => directory.pathOf(key),
1096
- async moveTree(from, to) {
1097
- const source = await directory.pathOf(from);
1098
- const target = await directory.pathOf(to);
1099
- const fs = await getFs();
1100
- try {
1101
- await fs.rename(source, target);
1102
- } catch (error) {
1103
- if (missing(error)) return;
1104
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1105
- }
1106
- await directory.syncDirectoryOf(to);
1107
- },
1108
- async removeTree(prefix) {
1109
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1110
- const fs = await getFs();
1112
+
1113
+ // src/persistence/durability/checkpoint-capture.ts
1114
+ function whileNoMutationApplies(partitions, capture2) {
1115
+ return new Promise((resolve, reject) => {
1116
+ let release = () => void 0;
1117
+ const released = new Promise((settle) => {
1118
+ release = settle;
1119
+ });
1120
+ function captureNow() {
1111
1121
  try {
1112
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1122
+ resolve(capture2());
1113
1123
  } catch (error) {
1114
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1124
+ reject(error);
1125
+ } finally {
1126
+ release();
1115
1127
  }
1116
- await directory.syncDirectoryOf(key);
1117
1128
  }
1118
- };
1129
+ let waiting = partitions.length;
1130
+ if (waiting === 0) {
1131
+ captureNow();
1132
+ return;
1133
+ }
1134
+ for (const partition of partitions) {
1135
+ partition.appendChain = partition.appendChain.then(() => {
1136
+ waiting -= 1;
1137
+ if (waiting === 0) captureNow();
1138
+ return released;
1139
+ });
1140
+ }
1141
+ });
1119
1142
  }
1120
- function recoverDirectoryRestores(directory) {
1121
- return recoverRestoreJournals(directoryRestoreStorage(directory));
1143
+ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
1144
+ for (const [partitionId, partition] of [...indexState.partitions]) {
1145
+ await partition.appendChain;
1146
+ if (partition.failed !== null) continue;
1147
+ try {
1148
+ await partition.walWriter.commit();
1149
+ } catch (error) {
1150
+ failPartition2(partitionId, partition, error);
1151
+ }
1152
+ }
1122
1153
  }
1123
-
1124
- // src/persistence/durability/restore-staging.ts
1125
- function toError2(error) {
1126
- return error instanceof Error ? error : new Error(String(error));
1154
+ async function openNewLogSegments(indexState, failPartition2) {
1155
+ const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
1156
+ try {
1157
+ if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
1158
+ } catch (error) {
1159
+ failPartition2(partitionId, partition, error);
1160
+ return;
1161
+ }
1162
+ const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
1163
+ failPartition2(partitionId, partition, error);
1164
+ });
1165
+ partition.appendChain = rolled;
1166
+ await rolled;
1167
+ });
1168
+ await Promise.all(rolls);
1127
1169
  }
1128
- function pathUnder(path, root) {
1129
- if (!path.startsWith(root)) return false;
1130
- const next = path.charAt(root.length);
1131
- return next === "" || next === "/" || next === "\\";
1170
+ function aTargetFailed(indexState, targets) {
1171
+ return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
1132
1172
  }
1133
- function stagedDirectory(directory, staged, unstaged) {
1134
- const { readRange, byteLength } = directory;
1135
- return {
1136
- root: directory.root,
1137
- appendHandle: (key) => directory.appendHandle(staged(key)),
1138
- markerHandle: (key) => directory.markerHandle(staged(key)),
1139
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1140
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1141
- read: (key) => directory.read(staged(key)),
1142
- remove: (key) => directory.remove(staged(key)),
1143
- move: (from, to) => directory.move(staged(from), staged(to)),
1144
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1145
- pathOf: (key) => directory.pathOf(staged(key)),
1146
- ...readRange === void 0 ? {} : {
1147
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1148
- },
1149
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1150
- };
1173
+ function noWholePartitions() {
1174
+ return { contents: [], captured: [] };
1151
1175
  }
1152
- function createRestoreStaging(storage, indexName, options = {}) {
1153
- const stagingPrefix = newStagingPrefix();
1154
- const indexRoot = `${indexName}/`;
1155
- let committed = false;
1156
- const staged = (key) => {
1157
- if (!key.startsWith(indexRoot)) {
1158
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1159
- key
1160
- });
1161
- }
1162
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1163
- };
1164
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1165
- const store = {
1166
- read: (key) => storage.read(staged(key)),
1167
- write: (key, chunks) => storage.write(staged(key), chunks),
1168
- remove: (key) => storage.remove(staged(key)),
1169
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1170
- };
1171
- const pathOf = storage.pathOf;
1172
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1173
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1174
- let retiredPrefix = null;
1175
- async function finishOnStorage() {
1176
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1177
- }
1178
- return {
1179
- store,
1180
- async relocation() {
1181
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1182
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1183
- const to = await pathOf.call(storage, indexName);
1184
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1185
- },
1186
- async commit() {
1187
- try {
1188
- await commitJournal(storage, indexName, stagingPrefix);
1189
- } catch (error) {
1190
- try {
1191
- await storage.remove(journalKey(indexName));
1192
- } catch (removal) {
1193
- committed = true;
1194
- options.onStorageLost?.(toError2(removal));
1195
- }
1196
- throw error;
1197
- }
1198
- committed = true;
1199
- },
1200
- async finish() {
1201
- try {
1202
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1203
- } catch (error) {
1204
- options.onStorageLost?.(toError2(error));
1176
+ function wholePartitionsWhereMostChanged(manager, rules) {
1177
+ const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
1178
+ return (targets) => {
1179
+ const whole = { contents: [], captured: [] };
1180
+ const changed = targets.filter(
1181
+ (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1182
+ );
1183
+ const mostOfEachChanged = changed.every((target) => {
1184
+ const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
1185
+ const documents = manager.getPartition(target.partitionId).count();
1186
+ return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
1187
+ });
1188
+ const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
1189
+ if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
1190
+ for (const { partitionId } of changed) {
1191
+ const capture2 = captureRecords?.(partitionId);
1192
+ const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
1193
+ if (captured !== null) {
1194
+ whole.captured.push(captured);
1195
+ } else if (everyPartition || !aWorkerCanSerialise) {
1196
+ whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
1205
1197
  }
1206
- },
1207
- async removeRetired() {
1208
- const prefix = retiredPrefix;
1209
- retiredPrefix = null;
1210
- if (prefix !== null) await removePrefix(storage, prefix);
1211
- },
1212
- async discard() {
1213
- if (committed) return;
1214
- await removePrefix(storage, stagingPrefix);
1215
1198
  }
1199
+ return whole;
1216
1200
  };
1217
1201
  }
1218
- function adapterRestoreStorage(adapter) {
1219
- const store = adapterCheckpointStore(adapter);
1220
- return {
1221
- read: (key) => store.read(key),
1222
- write: (key, chunks) => store.write(key, chunks),
1223
- writeJournal: (key, bytes) => store.write(key, [bytes]),
1224
- remove: (key) => store.remove(key),
1225
- list: (prefix) => store.list(prefix),
1226
- async moveKey(from, to) {
1227
- const bytes = await store.read(from);
1228
- if (bytes === null) return;
1229
- await store.write(to, [bytes]);
1230
- await store.remove(from);
1202
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
1203
+ for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
1204
+ return whileNoMutationApplies([...indexState.partitions.values()], () => {
1205
+ onCaptured?.();
1206
+ const documentCount = manager.countDocuments();
1207
+ const targets = [];
1208
+ let recordsLeftInLog = 0;
1209
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
1210
+ const partition = indexState.partitions.get(partitionId);
1211
+ if (partition !== void 0 && partition.failed !== null) continue;
1212
+ const appliedSeqNo = partition?.appliedSeqNo ?? 0;
1213
+ const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
1214
+ appliedSeqNo,
1215
+ snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
1216
+ );
1217
+ recordsLeftInLog += appliedSeqNo - lastSeqNo;
1218
+ const target = {
1219
+ partitionId,
1220
+ lastSeqNo,
1221
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
1222
+ };
1223
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
1224
+ targets.push(target);
1231
1225
  }
1232
- };
1233
- }
1234
-
1235
- // src/persistence/durability/archive-checkpoint.ts
1236
- function toError3(error) {
1237
- return error instanceof Error ? error : new Error(String(error));
1238
- }
1239
- function stageDirectoryRestore(deps, indexName) {
1240
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1241
- async finish(run) {
1242
- try {
1243
- await run();
1244
- } catch (error) {
1245
- deps.markFatal(toError3(error));
1246
- throw error;
1226
+ const vectorPlans = /* @__PURE__ */ new Map();
1227
+ try {
1228
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
1229
+ const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
1230
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
1231
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
1247
1232
  }
1248
- },
1249
- onStorageLost: deps.markFatal
1233
+ const memoryMatchesTheTargets = recordsLeftInLog === 0;
1234
+ const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
1235
+ return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
1236
+ } catch (error) {
1237
+ for (const plan of vectorPlans.values()) plan.release();
1238
+ throw error;
1239
+ }
1250
1240
  });
1251
1241
  }
1252
- function releaser(held, indexName) {
1253
- let released = false;
1254
- return () => {
1255
- if (released) return;
1256
- released = true;
1257
- const count = (held.get(indexName) ?? 1) - 1;
1258
- if (count <= 0) held.delete(indexName);
1259
- else held.set(indexName, count);
1260
- };
1261
- }
1262
- function createArchiveCheckpointAccess(deps) {
1263
- const store = directoryCheckpointStore(deps.directory);
1264
- return {
1265
- async openArchiveCheckpoint(indexName) {
1266
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1267
- const release = releaser(deps.held, indexName);
1268
- try {
1269
- await deps.checkpoint(indexName, false);
1270
- await deps.checkpoint(indexName, true);
1271
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
1272
- if (manifestBytes === null) {
1273
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1274
- indexName
1275
- });
1276
- }
1277
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1278
- if (decoded.kind !== "current") {
1279
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1280
- indexName
1281
- });
1282
- }
1283
- return {
1284
- manifestBytes,
1285
- manifest: decoded.manifest,
1286
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1287
- readRange: async (ref, offset, length) => {
1288
- const readRange = deps.directory.readRange;
1289
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1290
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1291
- return whole.slice(offset, offset + length);
1292
- },
1293
- release,
1294
- directory: deps.directory
1295
- };
1296
- } catch (error) {
1297
- release();
1298
- throw error;
1299
- }
1300
- },
1301
- stageRestore(indexName) {
1302
- return stageDirectoryRestore(deps, indexName);
1303
- },
1304
- adoptRestoredCheckpoint(indexName, checkpoint) {
1305
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1306
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1307
- }
1308
- }
1309
- };
1242
+ async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
1243
+ if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
1244
+ const priorVectors = priorManifest?.vectors ?? [];
1245
+ const priorCheckpoint = priorManifest?.checkpoint ?? [];
1246
+ const noPartitionChanged = capture2.targets.every(
1247
+ (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1248
+ );
1249
+ const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
1250
+ (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
1251
+ );
1252
+ const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
1253
+ if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
1254
+ return { vectors: priorVectors, layouts: [], written: [] };
1255
+ }
1256
+ const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
1257
+ return { vectors: written.refs, layouts: written.layouts, written: written.written };
1310
1258
  }
1311
1259
 
1312
- // src/persistence/durability/checkpoint.ts
1313
- function walPrefix(indexName, partitionId) {
1314
- return `${indexName}/wal/${partitionId}/`;
1315
- }
1316
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
1317
- for (const { partitionId, lastSeqNo } of checkpoint) {
1318
- const prefix = walPrefix(indexName, partitionId);
1319
- const markerBytes = await directory.read(`${prefix}commit`);
1320
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1321
- if (marker === null) {
1322
- continue;
1260
+ // src/persistence/durability/checkpoint-worker-dispatch.ts
1261
+ var workerUsable = true;
1262
+ var pooledWorker = null;
1263
+ var workerBusy = false;
1264
+ var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
1265
+ var idleTimer = null;
1266
+ async function spawnWorker() {
1267
+ return null;
1268
+ }
1269
+ function discardWorker(worker) {
1270
+ if (pooledWorker === worker) {
1271
+ pooledWorker = null;
1272
+ }
1273
+ try {
1274
+ void worker.terminate();
1275
+ } catch {
1276
+ }
1277
+ }
1278
+ function unrefTimer(timer) {
1279
+ if (typeof timer.unref === "function") {
1280
+ timer.unref();
1281
+ }
1282
+ return timer;
1283
+ }
1284
+ function runWorker(worker, request) {
1285
+ return new Promise((resolve) => {
1286
+ let settled = false;
1287
+ let timeoutId = armSilenceTimeout();
1288
+ function armSilenceTimeout() {
1289
+ return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
1323
1290
  }
1324
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
1325
- const segments = await directory.list(prefix);
1326
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
1327
- for (let i = 0; i < startSeqNos.length; i += 1) {
1328
- const start = startSeqNos[i];
1329
- if (start >= activeSegmentSeqNo) {
1330
- break;
1291
+ const onMessage = (msg) => {
1292
+ const response = msg;
1293
+ if (response.type === "heartbeat") {
1294
+ clearTimeout(timeoutId);
1295
+ timeoutId = armSilenceTimeout();
1296
+ return;
1331
1297
  }
1332
- const nextStart = startSeqNos[i + 1];
1333
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
1334
- if (segmentCoversBeyondCheckpoint) {
1335
- break;
1298
+ if (response.type === "success") {
1299
+ settle({ written: response.segments, timedOut: false }, false);
1300
+ } else {
1301
+ settle({ written: null, timedOut: false }, true);
1302
+ }
1303
+ };
1304
+ const onError = () => {
1305
+ settle({ written: null, timedOut: false }, true);
1306
+ };
1307
+ const onExit = () => {
1308
+ settle({ written: null, timedOut: false }, true);
1309
+ };
1310
+ function settle(outcome, discard) {
1311
+ if (settled) {
1312
+ return;
1313
+ }
1314
+ settled = true;
1315
+ clearTimeout(timeoutId);
1316
+ worker.off("message", onMessage);
1317
+ worker.off("error", onError);
1318
+ worker.off("exit", onExit);
1319
+ if (discard) {
1320
+ discardWorker(worker);
1336
1321
  }
1337
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
1322
+ resolve(outcome);
1338
1323
  }
1339
- }
1340
- }
1341
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
1342
- for (const [partitionId, partition] of [...partitions]) {
1343
- if (partitionId < partitionCount) continue;
1324
+ worker.on("message", onMessage);
1325
+ worker.on("error", onError);
1326
+ worker.on("exit", onExit);
1344
1327
  try {
1345
- await partition.walWriter.close();
1346
- } catch (err) {
1347
- onCloseError(err instanceof Error ? err : new Error(String(err)));
1328
+ worker.postMessage(request);
1329
+ } catch {
1330
+ settle({ written: null, timedOut: false }, true);
1348
1331
  }
1349
- partitions.delete(partitionId);
1350
- }
1351
- const walRoot = `${indexName}/wal/`;
1352
- for (const key of await directory.list(walRoot)) {
1353
- const partitionId = walKeyPartitionId(key, walRoot);
1354
- if (partitionId === null || partitionId < partitionCount) continue;
1355
- await directory.remove(key);
1356
- }
1332
+ });
1357
1333
  }
1358
- function walKeyPartitionId(key, walRoot) {
1359
- const tail = key.slice(walRoot.length);
1360
- const separator = tail.indexOf("/");
1361
- if (separator <= 0) {
1362
- return null;
1363
- }
1364
- const segment = tail.slice(0, separator);
1365
- if (!/^\d+$/.test(segment)) {
1366
- return null;
1367
- }
1368
- const value = Number.parseInt(segment, 10);
1369
- return Number.isSafeInteger(value) ? value : null;
1334
+ function delay(ms) {
1335
+ return new Promise((resolve) => {
1336
+ unrefTimer(setTimeout(resolve, ms));
1337
+ });
1370
1338
  }
1371
- function parseSegmentStartSeqNo(key, prefix) {
1372
- if (!key.startsWith(prefix)) {
1339
+ function checkpointWorkerIsUsable() {
1340
+ const runtime = detectRuntime();
1341
+ return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
1342
+ }
1343
+ async function runCheckpointOnWorker(request) {
1344
+ const runtime = detectRuntime();
1345
+ if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
1373
1346
  return null;
1374
1347
  }
1375
- const tail = key.slice(prefix.length);
1376
- if (!/^\d{16}$/.test(tail)) {
1348
+ if (workerBusy) {
1377
1349
  return null;
1378
1350
  }
1379
- const value = Number.parseInt(tail, 10);
1380
- return Number.isSafeInteger(value) ? value : null;
1381
- }
1382
- function formatSegmentStart(startSeqNo) {
1383
- return startSeqNo.toString().padStart(16, "0");
1384
- }
1385
-
1386
- // src/persistence/durability/marker-writer.ts
1387
- function markerKey(indexName, partitionId) {
1388
- return `${indexName}/wal/${partitionId}/commit`;
1389
- }
1390
- async function createMarkerWriter(directory, indexName, partitionId) {
1391
- const key = markerKey(indexName, partitionId);
1392
- const handle = await directory.markerHandle(key);
1393
- const existing = await handle.read();
1394
- const current = readCommitMarker(existing);
1395
- const createdFresh = existing.length === 0;
1396
- let lastSlot = current?.slotIndex ?? null;
1397
- let writeSeq = current?.state.writeSeq ?? 0;
1398
- return {
1399
- created: createdFresh,
1400
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
1401
- async commit(state) {
1402
- const slotIndex = nextMarkerSlot(lastSlot);
1403
- writeSeq += 1;
1404
- const slot = encodeMarkerSlot({ ...state, writeSeq });
1405
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1406
- lastSlot = slotIndex;
1407
- },
1408
- async close() {
1409
- await handle.close();
1351
+ workerBusy = true;
1352
+ cancelIdleRetirement();
1353
+ try {
1354
+ if (pooledWorker === null) {
1355
+ pooledWorker = await spawnWorker();
1410
1356
  }
1411
- };
1412
- }
1413
-
1414
- // src/persistence/durability/recovery.ts
1415
- async function removeLeftoverRewriteFiles(directory, names) {
1416
- for (const indexName of names) {
1417
- for (const place of ["rebuilt", "received"]) {
1418
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1357
+ const worker = pooledWorker;
1358
+ if (worker === null) {
1359
+ workerUsable = false;
1360
+ return null;
1419
1361
  }
1420
- }
1421
- }
1422
- async function listPersistedIndexes(directory) {
1423
- const metaKeys = await directory.list("");
1424
- const names = [];
1425
- for (const key of metaKeys) {
1426
- if (key.endsWith("/meta")) {
1427
- names.push(key.slice(0, -"/meta".length));
1362
+ const outcome = await runWorker(worker, request);
1363
+ if (outcome.written !== null) {
1364
+ return outcome.written;
1365
+ }
1366
+ if (outcome.timedOut) {
1367
+ await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
1428
1368
  }
1429
- }
1430
- return names;
1431
- }
1432
- async function loadMetadata(directory, indexName) {
1433
- const bytes = await directory.read(`${indexName}/meta`);
1434
- if (bytes === null) {
1435
1369
  return null;
1370
+ } finally {
1371
+ workerBusy = false;
1372
+ retireWorkerOnceIdle();
1436
1373
  }
1437
- const { metadata } = await readMetadataEnvelope(bytes);
1438
- return metadata;
1439
1374
  }
1440
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1441
- await directory.remove(nextSegmentKey(indexName, partitionId));
1442
- const read = await readWalSegments(directory, indexName, partitionId);
1443
- if (read === null) {
1444
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1445
- }
1446
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1447
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1448
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1375
+ function cancelIdleRetirement() {
1376
+ if (idleTimer !== null) {
1377
+ clearTimeout(idleTimer);
1378
+ idleTimer = null;
1449
1379
  }
1450
- let highestSeqNo = fromSeqNoExclusive;
1451
- let highestPrimaryTerm = 0;
1452
- const replay = async (entries) => {
1453
- for (const entry of entries) {
1454
- if (entry.seqNo <= highestSeqNo) {
1455
- continue;
1456
- }
1457
- if (turnIsSpent()) await yieldTurn();
1458
- applyEntry(entry, deps);
1459
- highestSeqNo = entry.seqNo;
1460
- if (entry.primaryTerm > highestPrimaryTerm) {
1461
- highestPrimaryTerm = entry.primaryTerm;
1462
- }
1463
- }
1464
- };
1465
- await replay(durableEntries);
1466
- if (activeTail !== null) {
1467
- await replay(activeTail.entries);
1380
+ }
1381
+ function retireWorkerOnceIdle() {
1382
+ cancelIdleRetirement();
1383
+ idleTimer = unrefTimer(
1384
+ setTimeout(() => {
1385
+ idleTimer = null;
1386
+ if (!workerBusy) terminateCheckpointWorker();
1387
+ }, workerIdleMs)
1388
+ );
1389
+ }
1390
+ function terminateCheckpointWorker() {
1391
+ cancelIdleRetirement();
1392
+ const worker = pooledWorker;
1393
+ if (worker === null) {
1394
+ return;
1468
1395
  }
1469
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1470
- if (activeTail !== null) {
1471
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1396
+ pooledWorker = null;
1397
+ try {
1398
+ void worker.terminate();
1399
+ } catch {
1472
1400
  }
1473
- return { highestSeqNo, highestPrimaryTerm };
1474
1401
  }
1475
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1476
- const lastReplayed = tail.entries.at(-1);
1477
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1478
- const handle = await directory.appendHandle(tail.key);
1479
- try {
1480
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1481
- else await handle.sync();
1482
- } finally {
1483
- await handle.close();
1402
+
1403
+ // src/persistence/durability/checkpoint-write.ts
1404
+ function workerCanWriteEverySegment(input) {
1405
+ if (!input.canOffload || input.wholePartitions.length > 0) return false;
1406
+ const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
1407
+ if (!analysesTextInProcessOnly) return true;
1408
+ return input.targets.every(
1409
+ (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
1410
+ );
1411
+ }
1412
+ async function writeIndexCheckpointSegments(input) {
1413
+ const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
1414
+ const { keepsWriteRecords } = input;
1415
+ const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
1416
+ root: directory.root,
1417
+ metadata,
1418
+ targets,
1419
+ compactionThreshold,
1420
+ prior,
1421
+ capturedPartitions: [...capturedPartitions],
1422
+ keepsWriteRecords
1423
+ }) : null;
1424
+ if (offloaded !== null) {
1425
+ return offloaded;
1484
1426
  }
1485
- if (lastReplayed === void 0) return;
1486
- const writer = await createMarkerWriter(directory, indexName, partitionId);
1487
- try {
1488
- await writer.commit({
1489
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1490
- durableByteLength: tail.cleanEnd,
1491
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1427
+ return writeCheckpointSegments({
1428
+ store: directoryCheckpointStore(directory),
1429
+ directory,
1430
+ metadata,
1431
+ targets,
1432
+ compactionThreshold,
1433
+ prior,
1434
+ wholePartitions,
1435
+ capturedPartitions,
1436
+ keepsWriteRecords
1437
+ });
1438
+ }
1439
+
1440
+ // src/persistence/durability/segment/vector-adoption.ts
1441
+ function recordWrittenVectors(vectorIndexes, written) {
1442
+ for (const field of written) {
1443
+ vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
1444
+ }
1445
+ }
1446
+ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
1447
+ const pathOf = store.pathOf;
1448
+ if (pathOf === void 0) return;
1449
+ for (const layout of layouts) {
1450
+ const vecIndex = vectorIndexes.get(layout.fieldPath);
1451
+ if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
1452
+ const path = await pathOf.call(store, layout.key);
1453
+ await vecIndex.adoptDiskLayout({
1454
+ path,
1455
+ docIds: layout.docIds,
1456
+ dead: layout.dead,
1457
+ vectorsOffset: layout.vectorsOffset
1492
1458
  });
1493
- } finally {
1494
- await writer.close();
1495
1459
  }
1496
1460
  }
1497
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1498
- for (const { key, startSeqNo } of segments) {
1499
- if (startSeqNo > activeSegmentSeqNo) {
1500
- await directory.remove(key);
1501
- }
1461
+ async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
1462
+ const pathOf = store.pathOf;
1463
+ if (pathOf === void 0) return [...keys];
1464
+ const read = /* @__PURE__ */ new Set();
1465
+ for (const vectorIndex of vectorIndexes.values()) {
1466
+ for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
1467
+ }
1468
+ if (read.size === 0) return [...keys];
1469
+ const unread = [];
1470
+ for (const key of keys) {
1471
+ if (!read.has(await pathOf.call(store, key))) unread.push(key);
1502
1472
  }
1473
+ return unread;
1503
1474
  }
1504
- function applyEntry(entry, deps) {
1505
- deps.writeRecords?.replayed(entry);
1506
- if (entry.operation === "DELETE") {
1507
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1508
- return;
1475
+
1476
+ // src/persistence/durability/write-stall.ts
1477
+ function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1478
+ return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
1479
+ }
1480
+ async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1481
+ while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
1482
+ await waitOutsideCountedWrite(
1483
+ () => new Promise((resume) => {
1484
+ indexState.stalledWrites.push(resume);
1485
+ })
1486
+ );
1509
1487
  }
1510
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1511
1488
  }
1512
- function snapshotCheckpointFor(checkpoint, partitionId) {
1513
- return checkpointLastSeqNo(checkpoint, partitionId);
1489
+ function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
1490
+ const counted = indexState.mutationsSinceCheckpoint;
1491
+ const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
1492
+ indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
1493
+ indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
1494
+ releaseStalledWrites(indexState);
1495
+ }
1496
+ function releaseStalledWrites(indexState) {
1497
+ const waiting = indexState.stalledWrites.splice(0);
1498
+ for (const resume of waiting) resume();
1514
1499
  }
1515
1500
 
1516
- // src/persistence/durability/checkpoint-capture.ts
1517
- function whileNoMutationApplies(partitions, capture2) {
1518
- return new Promise((resolve, reject) => {
1519
- let release = () => void 0;
1520
- const released = new Promise((settle) => {
1521
- release = settle;
1522
- });
1523
- function captureNow() {
1524
- try {
1525
- resolve(capture2());
1526
- } catch (error) {
1527
- reject(error);
1528
- } finally {
1529
- release();
1530
- }
1531
- }
1532
- let waiting = partitions.length;
1533
- if (waiting === 0) {
1534
- captureNow();
1535
- return;
1536
- }
1537
- for (const partition of partitions) {
1538
- partition.appendChain = partition.appendChain.then(() => {
1539
- waiting -= 1;
1540
- if (waiting === 0) captureNow();
1541
- return released;
1542
- });
1543
- }
1544
- });
1501
+ // src/persistence/durability/checkpoint-run.ts
1502
+ async function bothOrTheFirstFailure(first, second) {
1503
+ const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
1504
+ if (firstSettled.status === "rejected") throw firstSettled.reason;
1505
+ if (secondSettled.status === "rejected") throw secondSettled.reason;
1506
+ return [firstSettled.value, secondSettled.value];
1545
1507
  }
1546
- async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
1547
- for (const [partitionId, partition] of [...indexState.partitions]) {
1548
- await partition.appendChain;
1549
- if (partition.failed !== null) continue;
1550
- try {
1551
- await partition.walWriter.commit();
1552
- } catch (error) {
1553
- failPartition2(partitionId, partition, error);
1508
+ async function runDurableCheckpoint(input) {
1509
+ try {
1510
+ let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1511
+ while (recordsLeftInLog > 0 && input.mayContinue()) {
1512
+ recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1554
1513
  }
1514
+ } catch (err) {
1515
+ if (!isSystemError(err)) throw err;
1516
+ const error = new NarsilError(
1517
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1518
+ `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
1519
+ {
1520
+ indexName: input.indexName,
1521
+ directory: input.directory.root,
1522
+ cause: err.message
1523
+ }
1524
+ );
1525
+ input.markFatal(error);
1526
+ throw error;
1555
1527
  }
1556
1528
  }
1557
- async function openNewLogSegments(indexState, failPartition2) {
1558
- const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
1559
- try {
1560
- if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
1561
- } catch (error) {
1562
- failPartition2(partitionId, partition, error);
1563
- return;
1529
+ function isSystemError(err) {
1530
+ return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
1531
+ }
1532
+ async function writeOneBoundedCheckpoint(input) {
1533
+ const manager = input.hooks.getManager(input.indexName);
1534
+ if (manager === void 0) {
1535
+ return 0;
1536
+ }
1537
+ const metadata = input.hooks.buildTermsMetadata(input.indexName);
1538
+ if (metadata === void 0) {
1539
+ return 0;
1540
+ }
1541
+ const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
1542
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1543
+ const store = directoryCheckpointStore(input.directory);
1544
+ const writeRecords = input.hooks.writeRecords ?? null;
1545
+ const { indexName } = input;
1546
+ const priorManifest = await readCurrentManifest(store, input.indexName);
1547
+ const textOnDisk = metadata.textStorage === "disk";
1548
+ const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1549
+ priorCheckpoint: priorManifest?.checkpoint ?? [],
1550
+ priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1551
+ everyPartition: fromMemory,
1552
+ aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1553
+ ...writeRecords === null ? {} : {
1554
+ captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1564
1555
  }
1565
- const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
1566
- failPartition2(partitionId, partition, error);
1567
- });
1568
- partition.appendChain = rolled;
1569
- await rolled;
1570
1556
  });
1571
- await Promise.all(rolls);
1572
- }
1573
- function aTargetFailed(indexState, targets) {
1574
- return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
1575
- }
1576
- function noWholePartitions() {
1577
- return { contents: [], captured: [] };
1578
- }
1579
- function wholePartitionsWhereMostChanged(manager, rules) {
1580
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
1581
- return (targets) => {
1582
- const whole = { contents: [], captured: [] };
1583
- const changed = targets.filter(
1584
- (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1585
- );
1586
- const mostOfEachChanged = changed.every((target) => {
1587
- const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
1588
- const documents = manager.getPartition(target.partitionId).count();
1589
- return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
1557
+ let changesSinceCapture = null;
1558
+ const capture2 = await captureCheckpoint(
1559
+ input.indexState,
1560
+ manager,
1561
+ vectorIndexes,
1562
+ textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1563
+ fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1564
+ priorManifest?.vectors ?? [],
1565
+ writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1566
+ textOnDisk ? () => {
1567
+ changesSinceCapture = manager.openCheckpointChangeLog();
1568
+ } : void 0,
1569
+ input.pinsRetiredVectors
1570
+ );
1571
+ try {
1572
+ await openNewLogSegments(input.indexState, input.failPartition);
1573
+ return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1574
+ return changesSinceCapture;
1590
1575
  });
1591
- const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
1592
- if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
1593
- for (const { partitionId } of changed) {
1594
- const capture2 = captureRecords?.(partitionId);
1595
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
1596
- if (captured !== null) {
1597
- whole.captured.push(captured);
1598
- } else if (everyPartition || !aWorkerCanSerialise) {
1599
- whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
1600
- }
1601
- }
1602
- return whole;
1603
- };
1576
+ } finally {
1577
+ for (const plan of capture2.vectorPlans.values()) plan.release();
1578
+ changesSinceCapture?.close();
1579
+ }
1604
1580
  }
1605
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
1606
- for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
1607
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
1608
- onCaptured?.();
1609
- const documentCount = manager.countDocuments();
1610
- const targets = [];
1611
- let recordsLeftInLog = 0;
1612
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
1613
- const partition = indexState.partitions.get(partitionId);
1614
- if (partition !== void 0 && partition.failed !== null) continue;
1615
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
1616
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
1617
- appliedSeqNo,
1618
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
1619
- );
1620
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
1621
- const target = {
1622
- partitionId,
1623
- lastSeqNo,
1624
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
1625
- };
1626
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
1627
- targets.push(target);
1628
- }
1629
- const vectorPlans = /* @__PURE__ */ new Map();
1630
- try {
1631
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
1632
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
1633
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
1634
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
1635
- }
1636
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
1637
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
1638
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
1639
- } catch (error) {
1640
- for (const plan of vectorPlans.values()) plan.release();
1641
- throw error;
1642
- }
1581
+ async function adoptWrittenSegments(input, manager, partitions, changes) {
1582
+ const { indexName, indexState, hooks } = input;
1583
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-QT5HZAY4.mjs');
1584
+ const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1585
+ if (adoption === null) return [];
1586
+ const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1587
+ if (changes.partitionsReplaced) return false;
1588
+ const changed = changes.takeWrittenDocIds();
1589
+ swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1590
+ hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1591
+ });
1592
+ hooks.queueDiskAdoption?.({
1593
+ type: "adoptDiskSegments",
1594
+ indexName,
1595
+ partitions: actionPartitionsOf(adoption),
1596
+ changed: [...changed],
1597
+ requestId: createRequestId()
1598
+ });
1599
+ return true;
1643
1600
  });
1644
- }
1645
- async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
1646
- if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
1647
- const priorVectors = priorManifest?.vectors ?? [];
1648
- const priorCheckpoint = priorManifest?.checkpoint ?? [];
1649
- const noPartitionChanged = capture2.targets.every(
1650
- (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1651
- );
1652
- const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
1653
- (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
1654
- );
1655
- const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
1656
- if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
1657
- return { vectors: priorVectors, layouts: [], written: [] };
1601
+ if (!adopted) {
1602
+ closeUnusedSegments(adoption);
1603
+ return [];
1658
1604
  }
1659
- const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
1660
- return { vectors: written.refs, layouts: written.layouts, written: written.written };
1661
- }
1662
-
1663
- // src/persistence/durability/checkpoint-worker-dispatch.ts
1664
- var workerUsable = true;
1665
- var pooledWorker = null;
1666
- var workerBusy = false;
1667
- var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
1668
- var idleTimer = null;
1669
- async function spawnWorker() {
1670
- return null;
1605
+ await hooks.awaitCopiesOfIndex?.(indexName);
1606
+ return adoption.droppedKeys;
1671
1607
  }
1672
- function discardWorker(worker) {
1673
- if (pooledWorker === worker) {
1674
- pooledWorker = null;
1675
- }
1676
- try {
1677
- void worker.terminate();
1678
- } catch {
1608
+ async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
1609
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1610
+ const writeRecords = input.hooks.writeRecords ?? null;
1611
+ const { targets, documentCount } = capture2;
1612
+ await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1613
+ if (aTargetFailed(input.indexState, targets)) return 0;
1614
+ const [liveVectors, segments] = await bothOrTheFirstFailure(
1615
+ writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
1616
+ writeIndexCheckpointSegments({
1617
+ directory: input.directory,
1618
+ metadata,
1619
+ targets,
1620
+ compactionThreshold: input.compactionThreshold,
1621
+ canOffload: input.canOffload,
1622
+ prior: priorManifest,
1623
+ wholePartitions: capture2.wholePartitions.contents,
1624
+ capturedPartitions: capture2.wholePartitions.captured,
1625
+ keepsWriteRecords: writeRecords !== null
1626
+ })
1627
+ );
1628
+ await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1629
+ if (aTargetFailed(input.indexState, targets)) return 0;
1630
+ const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
1631
+ input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
1632
+ recordWrittenVectors(vectorIndexes, liveVectors.written);
1633
+ await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
1634
+ const changes = changesSinceCapture();
1635
+ const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1636
+ if (!input.holdsCheckpointFiles()) {
1637
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1638
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-QT5HZAY4.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1639
+ const keptForSync = await input.keepsCheckpointKey();
1640
+ await removeCheckpointGarbage(
1641
+ store,
1642
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
1643
+ );
1679
1644
  }
1645
+ const checkpointDocumentCount = documentCount;
1646
+ await input.queueMetadataWrite(input.indexName, async () => {
1647
+ const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
1648
+ if (checkpointMetadata === void 0) {
1649
+ return;
1650
+ }
1651
+ const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
1652
+ await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1653
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1654
+ });
1655
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1656
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
1657
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
1658
+ await reclaimWalBeyondCount(
1659
+ input.directory,
1660
+ input.indexName,
1661
+ manager.partitionCount,
1662
+ input.indexState.partitions,
1663
+ input.markFatal
1664
+ );
1665
+ const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1666
+ noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1667
+ const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1668
+ return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
1680
1669
  }
1681
- function unrefTimer(timer) {
1682
- if (typeof timer.unref === "function") {
1683
- timer.unref();
1670
+ function recordsBeyond(indexState, targets) {
1671
+ let records = 0;
1672
+ for (const [partitionId, partition] of indexState.partitions) {
1673
+ if (partition.failed !== null) continue;
1674
+ records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
1684
1675
  }
1685
- return timer;
1676
+ return records;
1686
1677
  }
1687
- function runWorker(worker, request) {
1688
- return new Promise((resolve) => {
1689
- let settled = false;
1690
- let timeoutId = armSilenceTimeout();
1691
- function armSilenceTimeout() {
1692
- return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
1693
- }
1694
- const onMessage = (msg) => {
1695
- const response = msg;
1696
- if (response.type === "heartbeat") {
1697
- clearTimeout(timeoutId);
1698
- timeoutId = armSilenceTimeout();
1699
- return;
1700
- }
1701
- if (response.type === "success") {
1702
- settle({ written: response.segments, timedOut: false }, false);
1703
- } else {
1704
- settle({ written: null, timedOut: false }, true);
1705
- }
1706
- };
1707
- const onError = () => {
1708
- settle({ written: null, timedOut: false }, true);
1709
- };
1710
- const onExit = () => {
1711
- settle({ written: null, timedOut: false }, true);
1712
- };
1713
- function settle(outcome, discard) {
1714
- if (settled) {
1715
- return;
1716
- }
1717
- settled = true;
1718
- clearTimeout(timeoutId);
1719
- worker.off("message", onMessage);
1720
- worker.off("error", onError);
1721
- worker.off("exit", onExit);
1722
- if (discard) {
1723
- discardWorker(worker);
1724
- }
1725
- resolve(outcome);
1726
- }
1727
- worker.on("message", onMessage);
1728
- worker.on("error", onError);
1729
- worker.on("exit", onExit);
1730
- try {
1731
- worker.postMessage(request);
1732
- } catch {
1733
- settle({ written: null, timedOut: false }, true);
1678
+
1679
+ // src/persistence/durability/checkpoint-turn.ts
1680
+ function createIndexLoads(vectorIndexesOf2) {
1681
+ const loads = /* @__PURE__ */ new Map();
1682
+ const startGraphRepairs = (indexName) => {
1683
+ for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1684
+ vectorIndex.scheduleBuild();
1685
+ vectorIndex.startShuffledRebuildWhenDue();
1734
1686
  }
1735
- });
1687
+ };
1688
+ return {
1689
+ track(indexName, load, startsGraphRepairs = true) {
1690
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1691
+ loads.set(
1692
+ indexName,
1693
+ loaded.catch(() => void 0)
1694
+ );
1695
+ return loaded;
1696
+ },
1697
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1698
+ startGraphRepairs
1699
+ };
1736
1700
  }
1737
- function delay(ms) {
1738
- return new Promise((resolve) => {
1739
- unrefTimer(setTimeout(resolve, ms));
1701
+ async function switchInTurn(turns, turn) {
1702
+ let swapped = false;
1703
+ await checkpointInTurn(turns, {
1704
+ joinsInFlight: false,
1705
+ mayStart: turn.mayStart,
1706
+ perform: async () => {
1707
+ swapped = turn.swap();
1708
+ if (swapped) await turn.checkpoint();
1709
+ },
1710
+ afterSettling: turn.afterSettling
1740
1711
  });
1712
+ return swapped;
1741
1713
  }
1742
- function checkpointWorkerIsUsable() {
1743
- const runtime = detectRuntime();
1744
- return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
1745
- }
1746
- async function runCheckpointOnWorker(request) {
1747
- const runtime = detectRuntime();
1748
- if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
1749
- return null;
1750
- }
1751
- if (workerBusy) {
1752
- return null;
1753
- }
1754
- workerBusy = true;
1755
- cancelIdleRetirement();
1756
- try {
1757
- if (pooledWorker === null) {
1758
- pooledWorker = await spawnWorker();
1759
- }
1760
- const worker = pooledWorker;
1761
- if (worker === null) {
1762
- workerUsable = false;
1763
- return null;
1764
- }
1765
- const outcome = await runWorker(worker, request);
1766
- if (outcome.written !== null) {
1767
- return outcome.written;
1768
- }
1769
- if (outcome.timedOut) {
1770
- await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
1771
- }
1772
- return null;
1773
- } finally {
1774
- workerBusy = false;
1775
- retireWorkerOnceIdle();
1776
- }
1777
- }
1778
- function cancelIdleRetirement() {
1779
- if (idleTimer !== null) {
1780
- clearTimeout(idleTimer);
1781
- idleTimer = null;
1782
- }
1783
- }
1784
- function retireWorkerOnceIdle() {
1785
- cancelIdleRetirement();
1786
- idleTimer = unrefTimer(
1787
- setTimeout(() => {
1788
- idleTimer = null;
1789
- if (!workerBusy) terminateCheckpointWorker();
1790
- }, workerIdleMs)
1791
- );
1792
- }
1793
- function terminateCheckpointWorker() {
1794
- cancelIdleRetirement();
1795
- const worker = pooledWorker;
1796
- if (worker === null) {
1797
- return;
1798
- }
1799
- pooledWorker = null;
1800
- try {
1801
- void worker.terminate();
1802
- } catch {
1803
- }
1804
- }
1805
-
1806
- // src/persistence/durability/checkpoint-write.ts
1807
- function workerCanWriteEverySegment(input) {
1808
- if (!input.canOffload || input.wholePartitions.length > 0) return false;
1809
- const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
1810
- if (!analysesTextInProcessOnly) return true;
1811
- return input.targets.every(
1812
- (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
1813
- );
1814
- }
1815
- async function writeIndexCheckpointSegments(input) {
1816
- const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
1817
- const { keepsWriteRecords } = input;
1818
- const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
1819
- root: directory.root,
1820
- metadata,
1821
- targets,
1822
- compactionThreshold,
1823
- prior,
1824
- capturedPartitions: [...capturedPartitions],
1825
- keepsWriteRecords
1826
- }) : null;
1827
- if (offloaded !== null) {
1828
- return offloaded;
1829
- }
1830
- return writeCheckpointSegments({
1831
- store: directoryCheckpointStore(directory),
1832
- directory,
1833
- metadata,
1834
- targets,
1835
- compactionThreshold,
1836
- prior,
1837
- wholePartitions,
1838
- capturedPartitions,
1839
- keepsWriteRecords
1714
+ async function checkpointInTurn(turns, turn) {
1715
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1716
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1717
+ if (!turn.mayStart()) return;
1718
+ const run = turn.perform().finally(() => {
1719
+ turns.checkpointInFlight = null;
1720
+ turn.afterSettling();
1840
1721
  });
1722
+ turns.checkpointInFlight = run;
1723
+ return run;
1841
1724
  }
1842
1725
 
1843
- // src/persistence/durability/segment/vector-adoption.ts
1844
- function recordWrittenVectors(vectorIndexes, written) {
1845
- for (const field of written) {
1846
- vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
1847
- }
1848
- }
1849
- async function adoptVectorLayouts(store, vectorIndexes, layouts) {
1850
- const pathOf = store.pathOf;
1851
- if (pathOf === void 0) return;
1852
- for (const layout of layouts) {
1853
- const vecIndex = vectorIndexes.get(layout.fieldPath);
1854
- if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
1855
- const path = await pathOf.call(store, layout.key);
1856
- await vecIndex.adoptDiskLayout({
1857
- path,
1858
- docIds: layout.docIds,
1859
- dead: layout.dead,
1860
- vectorsOffset: layout.vectorsOffset
1861
- });
1862
- }
1863
- }
1864
- async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
1865
- const pathOf = store.pathOf;
1866
- if (pathOf === void 0) return [...keys];
1867
- const read = /* @__PURE__ */ new Set();
1868
- for (const vectorIndex of vectorIndexes.values()) {
1869
- for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
1870
- }
1871
- if (read.size === 0) return [...keys];
1872
- const unread = [];
1873
- for (const key of keys) {
1874
- if (!read.has(await pathOf.call(store, key))) unread.push(key);
1875
- }
1876
- return unread;
1877
- }
1878
-
1879
- // src/persistence/durability/write-stall.ts
1880
- function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1881
- return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
1726
+ // src/persistence/durability/held-checkpoints.ts
1727
+ function failedPartitionIds(indexState) {
1728
+ const failed = [];
1729
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1730
+ return failed;
1882
1731
  }
1883
- async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1884
- while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
1885
- await waitOutsideCountedWrite(
1886
- () => new Promise((resume) => {
1887
- indexState.stalledWrites.push(resume);
1888
- })
1889
- );
1732
+ function createHeldCheckpoints(hooks) {
1733
+ const reported = /* @__PURE__ */ new WeakSet();
1734
+ function heldReport(indexName, indexState) {
1735
+ const partitionIds = failedPartitionIds(indexState);
1736
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1737
+ return {
1738
+ indexName,
1739
+ partitionIds,
1740
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1741
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1742
+ };
1890
1743
  }
1891
- }
1892
- function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
1893
- const counted = indexState.mutationsSinceCheckpoint;
1894
- const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
1895
- indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
1896
- indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
1897
- releaseStalledWrites(indexState);
1898
- }
1899
- function releaseStalledWrites(indexState) {
1900
- const waiting = indexState.stalledWrites.splice(0);
1901
- for (const resume of waiting) resume();
1744
+ return {
1745
+ refuse(indexName, indexState) {
1746
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1747
+ if (partitionIds.length === 0) return;
1748
+ throw new NarsilError(
1749
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1750
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1751
+ { indexName, partitionIds }
1752
+ );
1753
+ },
1754
+ holds(indexName, indexState) {
1755
+ const held = heldReport(indexName, indexState);
1756
+ if (held === null) return false;
1757
+ const unreported = held.partitionIds.some((partitionId) => {
1758
+ const partition = indexState.partitions.get(partitionId);
1759
+ return partition !== void 0 && !reported.has(partition);
1760
+ });
1761
+ if (unreported) {
1762
+ for (const partitionId of held.partitionIds) {
1763
+ const partition = indexState.partitions.get(partitionId);
1764
+ if (partition !== void 0) reported.add(partition);
1765
+ }
1766
+ hooks.onCheckpointHeld?.(held);
1767
+ }
1768
+ return true;
1769
+ },
1770
+ report(indexes) {
1771
+ const held = [];
1772
+ for (const [indexName, indexState] of indexes) {
1773
+ const entry = heldReport(indexName, indexState);
1774
+ if (entry !== null) held.push(entry);
1775
+ }
1776
+ return held;
1777
+ }
1778
+ };
1902
1779
  }
1903
1780
 
1904
- // src/persistence/durability/checkpoint-run.ts
1905
- async function bothOrTheFirstFailure(first, second) {
1906
- const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
1907
- if (firstSettled.status === "rejected") throw firstSettled.reason;
1908
- if (secondSettled.status === "rejected") throw secondSettled.reason;
1909
- return [firstSettled.value, secondSettled.value];
1781
+ // src/persistence/durability/manager-state.ts
1782
+ function closePartitionWriter(partition) {
1783
+ return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1910
1784
  }
1911
- async function runDurableCheckpoint(input) {
1785
+ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1786
+ indexState.unloading = true;
1912
1787
  try {
1913
- let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1914
- while (recordsLeftInLog > 0 && input.mayContinue()) {
1915
- recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1788
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1789
+ await pendingMetadataWrite;
1790
+ for (const partition of indexState.partitions.values()) {
1791
+ await closeWalWriter(closePartitionWriter(partition));
1916
1792
  }
1917
- } catch (err) {
1918
- if (!isSystemError(err)) throw err;
1919
- const error = new NarsilError(
1920
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1921
- `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
1922
- {
1923
- indexName: input.indexName,
1924
- directory: input.directory.root,
1925
- cause: err.message
1926
- }
1927
- );
1928
- input.markFatal(error);
1793
+ } catch (error) {
1794
+ indexState.unloading = false;
1929
1795
  throw error;
1930
1796
  }
1931
1797
  }
1932
- function isSystemError(err) {
1933
- return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
1934
- }
1935
- async function writeOneBoundedCheckpoint(input) {
1936
- const manager = input.hooks.getManager(input.indexName);
1937
- if (manager === void 0) {
1938
- return 0;
1939
- }
1940
- const metadata = input.hooks.buildTermsMetadata(input.indexName);
1941
- if (metadata === void 0) {
1942
- return 0;
1943
- }
1944
- const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
1945
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1946
- const store = directoryCheckpointStore(input.directory);
1947
- const writeRecords = input.hooks.writeRecords ?? null;
1948
- const { indexName } = input;
1949
- const priorManifest = await readCurrentManifest(store, input.indexName);
1950
- const textOnDisk = metadata.textStorage === "disk";
1951
- const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1952
- priorCheckpoint: priorManifest?.checkpoint ?? [],
1953
- priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1954
- everyPartition: fromMemory,
1955
- aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1956
- ...writeRecords === null ? {} : {
1957
- captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1798
+
1799
+ // src/persistence/durability/sha256.ts
1800
+ var ROUND_CONSTANTS = new Uint32Array([
1801
+ 1116352408,
1802
+ 1899447441,
1803
+ 3049323471,
1804
+ 3921009573,
1805
+ 961987163,
1806
+ 1508970993,
1807
+ 2453635748,
1808
+ 2870763221,
1809
+ 3624381080,
1810
+ 310598401,
1811
+ 607225278,
1812
+ 1426881987,
1813
+ 1925078388,
1814
+ 2162078206,
1815
+ 2614888103,
1816
+ 3248222580,
1817
+ 3835390401,
1818
+ 4022224774,
1819
+ 264347078,
1820
+ 604807628,
1821
+ 770255983,
1822
+ 1249150122,
1823
+ 1555081692,
1824
+ 1996064986,
1825
+ 2554220882,
1826
+ 2821834349,
1827
+ 2952996808,
1828
+ 3210313671,
1829
+ 3336571891,
1830
+ 3584528711,
1831
+ 113926993,
1832
+ 338241895,
1833
+ 666307205,
1834
+ 773529912,
1835
+ 1294757372,
1836
+ 1396182291,
1837
+ 1695183700,
1838
+ 1986661051,
1839
+ 2177026350,
1840
+ 2456956037,
1841
+ 2730485921,
1842
+ 2820302411,
1843
+ 3259730800,
1844
+ 3345764771,
1845
+ 3516065817,
1846
+ 3600352804,
1847
+ 4094571909,
1848
+ 275423344,
1849
+ 430227734,
1850
+ 506948616,
1851
+ 659060556,
1852
+ 883997877,
1853
+ 958139571,
1854
+ 1322822218,
1855
+ 1537002063,
1856
+ 1747873779,
1857
+ 1955562222,
1858
+ 2024104815,
1859
+ 2227730452,
1860
+ 2361852424,
1861
+ 2428436474,
1862
+ 2756734187,
1863
+ 3204031479,
1864
+ 3329325298
1865
+ ]);
1866
+ var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
1867
+ var BLOCK_BYTES = 64;
1868
+ var LENGTH_BYTES = 8;
1869
+ function rotateRight(value, bits) {
1870
+ return value >>> bits | value << 32 - bits;
1871
+ }
1872
+ function padded(bytes) {
1873
+ const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
1874
+ const message = new Uint8Array(total);
1875
+ message.set(bytes);
1876
+ message[bytes.byteLength] = 128;
1877
+ const view = new DataView(message.buffer);
1878
+ const bitLength = bytes.byteLength * 8;
1879
+ view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
1880
+ view.setUint32(total - 4, bitLength >>> 0, false);
1881
+ return view;
1882
+ }
1883
+ function sha256(bytes) {
1884
+ const view = padded(bytes);
1885
+ const state = INITIAL_STATE.slice();
1886
+ const words = new Uint32Array(64);
1887
+ for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
1888
+ for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
1889
+ for (let i = 16; i < 64; i++) {
1890
+ const w15 = words[i - 15];
1891
+ const w2 = words[i - 2];
1892
+ const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
1893
+ const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
1894
+ words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
1958
1895
  }
1959
- });
1960
- let changesSinceCapture = null;
1961
- const capture2 = await captureCheckpoint(
1962
- input.indexState,
1963
- manager,
1964
- vectorIndexes,
1965
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1966
- fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1967
- priorManifest?.vectors ?? [],
1968
- writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1969
- textOnDisk ? () => {
1970
- changesSinceCapture = manager.openCheckpointChangeLog();
1971
- } : void 0,
1972
- input.pinsRetiredVectors
1973
- );
1974
- try {
1975
- await openNewLogSegments(input.indexState, input.failPartition);
1976
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1977
- return changesSinceCapture;
1978
- });
1979
- } finally {
1980
- for (const plan of capture2.vectorPlans.values()) plan.release();
1981
- changesSinceCapture?.close();
1896
+ let [a, b, c, d, e, f, g, h] = state;
1897
+ for (let i = 0; i < 64; i++) {
1898
+ const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
1899
+ const choice = e & f ^ ~e & g;
1900
+ const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
1901
+ const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
1902
+ const majority = a & b ^ a & c ^ b & c;
1903
+ const t2 = s0 + majority >>> 0;
1904
+ h = g;
1905
+ g = f;
1906
+ f = e;
1907
+ e = d + t1 >>> 0;
1908
+ d = c;
1909
+ c = b;
1910
+ b = a;
1911
+ a = t1 + t2 >>> 0;
1912
+ }
1913
+ state[0] = state[0] + a >>> 0;
1914
+ state[1] = state[1] + b >>> 0;
1915
+ state[2] = state[2] + c >>> 0;
1916
+ state[3] = state[3] + d >>> 0;
1917
+ state[4] = state[4] + e >>> 0;
1918
+ state[5] = state[5] + f >>> 0;
1919
+ state[6] = state[6] + g >>> 0;
1920
+ state[7] = state[7] + h >>> 0;
1921
+ }
1922
+ const digest = new Uint8Array(32);
1923
+ const out = new DataView(digest.buffer);
1924
+ for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
1925
+ return digest;
1926
+ }
1927
+
1928
+ // src/persistence/durability/restore-journal.ts
1929
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
1930
+ var RESTORE_STAGING_PREFIX = ".restore-";
1931
+ var RETIRED_INDEX_PREFIX = ".retired-";
1932
+ var LOCK_FILE_KEY = ".narsil.lock";
1933
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
1934
+ function digestOf(indexName) {
1935
+ let hex = "";
1936
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
1937
+ return hex;
1938
+ }
1939
+ function journalKey(indexName) {
1940
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
1941
+ }
1942
+ function randomSuffix() {
1943
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
1944
+ }
1945
+ function newStagingPrefix() {
1946
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
1947
+ }
1948
+ function newRetiredPrefix() {
1949
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
1950
+ }
1951
+ function encodeJournal(indexName, stagingPrefix) {
1952
+ return new TextEncoder().encode(`${indexName}
1953
+ ${stagingPrefix}`);
1954
+ }
1955
+ function isStagingPrefix(prefix) {
1956
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
1957
+ }
1958
+ function corruptJournal(key, reason) {
1959
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
1960
+ }
1961
+ function readJournal(key, bytes) {
1962
+ const text = new TextDecoder().decode(bytes);
1963
+ const lineEnd = text.indexOf("\n");
1964
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
1965
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
1966
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
1967
+ corruptJournal(key, "names an index that no index name can match");
1968
+ }
1969
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
1970
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
1971
+ corruptJournal(key, "names no staging prefix that a restore writes");
1972
+ }
1973
+ return { kind: "journal", indexName, stagingPrefix };
1974
+ }
1975
+ async function listedKeys(storage, indexName, stagingPrefix) {
1976
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
1977
+ if (manifestBytes === null) {
1978
+ throw new NarsilError(
1979
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1980
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1981
+ { indexName, stagingPrefix }
1982
+ );
1983
+ }
1984
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1985
+ if (decoded.kind !== "current") {
1986
+ throw new NarsilError(
1987
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1988
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1989
+ { indexName }
1990
+ );
1991
+ }
1992
+ const root = `${indexName}/`;
1993
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1994
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1995
+ return keys;
1996
+ }
1997
+ function withoutTrailingSlash(prefix) {
1998
+ return prefix.slice(0, -1);
1999
+ }
2000
+ async function finishJournal(storage, indexName, stagingPrefix) {
2001
+ const root = `${indexName}/`;
2002
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
2003
+ const staged = await storage.list(stagingPrefix);
2004
+ let retiredPrefix = null;
2005
+ if (storage.moveTree !== void 0 && staged.length > 0) {
2006
+ retiredPrefix = newRetiredPrefix();
2007
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
2008
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
2009
+ } else {
2010
+ const stagedKeys = new Set(staged);
2011
+ for (const key of listed) {
2012
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
2013
+ }
2014
+ }
2015
+ for (const key of await storage.list(root)) {
2016
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1982
2017
  }
2018
+ await storage.remove(journalKey(indexName));
2019
+ return retiredPrefix;
2020
+ }
2021
+ async function removePrefix(storage, prefix) {
2022
+ if (storage.removeTree !== void 0) {
2023
+ await storage.removeTree(prefix);
2024
+ return;
2025
+ }
2026
+ for (const key of await storage.list(prefix)) await storage.remove(key);
2027
+ }
2028
+ async function commitJournal(storage, indexName, stagingPrefix) {
2029
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
2030
+ }
2031
+ function isTopLevelJournal(key) {
2032
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
2033
+ }
2034
+ async function removeStagingPrefixes(storage) {
2035
+ const prefixes = /* @__PURE__ */ new Set();
2036
+ for (const key of await storage.list(".")) {
2037
+ const slash = key.indexOf("/");
2038
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
2039
+ }
2040
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
2041
+ }
2042
+ async function recoverRestoreJournals(storage) {
2043
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
2044
+ if (!isTopLevelJournal(key)) continue;
2045
+ const bytes = await storage.read(key);
2046
+ if (bytes === null) continue;
2047
+ const reading = readJournal(key, bytes);
2048
+ if (reading.kind === "leftover") {
2049
+ await storage.remove(key);
2050
+ continue;
2051
+ }
2052
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
2053
+ }
2054
+ await removeStagingPrefixes(storage);
2055
+ }
2056
+
2057
+ // src/persistence/durability/restore-directory.ts
2058
+ function missing(error) {
2059
+ return error.code === "ENOENT";
2060
+ }
2061
+ function storageFailed(code, message, key, error) {
2062
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
2063
+ }
2064
+ function directoryRestoreStorage(directory) {
2065
+ return {
2066
+ directory,
2067
+ read: (key) => directory.read(key),
2068
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
2069
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
2070
+ remove: (key) => directory.remove(key),
2071
+ list: (prefix) => directory.list(prefix),
2072
+ moveKey: (from, to) => directory.move(from, to),
2073
+ pathOf: (key) => directory.pathOf(key),
2074
+ async moveTree(from, to) {
2075
+ const source = await directory.pathOf(from);
2076
+ const target = await directory.pathOf(to);
2077
+ const fs = await getFs();
2078
+ try {
2079
+ await fs.rename(source, target);
2080
+ } catch (error) {
2081
+ if (missing(error)) return;
2082
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
2083
+ }
2084
+ await directory.syncDirectoryOf(to);
2085
+ },
2086
+ async removeTree(prefix) {
2087
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
2088
+ const fs = await getFs();
2089
+ try {
2090
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
2091
+ } catch (error) {
2092
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
2093
+ }
2094
+ await directory.syncDirectoryOf(key);
2095
+ }
2096
+ };
2097
+ }
2098
+ function recoverDirectoryRestores(directory) {
2099
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
2100
+ }
2101
+
2102
+ // src/persistence/durability/restore-staging.ts
2103
+ function toError2(error) {
2104
+ return error instanceof Error ? error : new Error(String(error));
1983
2105
  }
1984
- async function adoptWrittenSegments(input, manager, partitions, changes) {
1985
- const { indexName, indexState, hooks } = input;
1986
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-FHJ2GOBI.mjs');
1987
- const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1988
- if (adoption === null) return [];
1989
- const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1990
- if (changes.partitionsReplaced) return false;
1991
- const changed = changes.takeWrittenDocIds();
1992
- swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1993
- hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1994
- });
1995
- hooks.queueDiskAdoption?.({
1996
- type: "adoptDiskSegments",
1997
- indexName,
1998
- partitions: actionPartitionsOf(adoption),
1999
- changed: [...changed],
2000
- requestId: createRequestId()
2001
- });
2002
- return true;
2003
- });
2004
- if (!adopted) {
2005
- closeUnusedSegments(adoption);
2006
- return [];
2007
- }
2008
- await hooks.awaitCopiesOfIndex?.(indexName);
2009
- return adoption.droppedKeys;
2106
+ function pathUnder(path, root) {
2107
+ if (!path.startsWith(root)) return false;
2108
+ const next = path.charAt(root.length);
2109
+ return next === "" || next === "/" || next === "\\";
2010
2110
  }
2011
- async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
2012
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
2013
- const writeRecords = input.hooks.writeRecords ?? null;
2014
- const { targets, documentCount } = capture2;
2015
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
2016
- if (aTargetFailed(input.indexState, targets)) return 0;
2017
- const [liveVectors, segments] = await bothOrTheFirstFailure(
2018
- writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
2019
- writeIndexCheckpointSegments({
2020
- directory: input.directory,
2021
- metadata,
2022
- targets,
2023
- compactionThreshold: input.compactionThreshold,
2024
- canOffload: input.canOffload,
2025
- prior: priorManifest,
2026
- wholePartitions: capture2.wholePartitions.contents,
2027
- capturedPartitions: capture2.wholePartitions.captured,
2028
- keepsWriteRecords: writeRecords !== null
2029
- })
2030
- );
2031
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
2032
- if (aTargetFailed(input.indexState, targets)) return 0;
2033
- const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
2034
- input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
2035
- recordWrittenVectors(vectorIndexes, liveVectors.written);
2036
- await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
2037
- const changes = changesSinceCapture();
2038
- const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
2039
- if (!input.holdsCheckpointFiles()) {
2040
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
2041
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-FHJ2GOBI.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
2042
- await removeCheckpointGarbage(
2043
- store,
2044
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
2045
- );
2111
+ function stagedDirectory(directory, staged, unstaged) {
2112
+ const { readRange, byteLength } = directory;
2113
+ return {
2114
+ root: directory.root,
2115
+ appendHandle: (key) => directory.appendHandle(staged(key)),
2116
+ markerHandle: (key) => directory.markerHandle(staged(key)),
2117
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
2118
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
2119
+ read: (key) => directory.read(staged(key)),
2120
+ remove: (key) => directory.remove(staged(key)),
2121
+ move: (from, to) => directory.move(staged(from), staged(to)),
2122
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
2123
+ pathOf: (key) => directory.pathOf(staged(key)),
2124
+ ...readRange === void 0 ? {} : {
2125
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
2126
+ },
2127
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
2128
+ };
2129
+ }
2130
+ function createRestoreStaging(storage, indexName, options = {}) {
2131
+ const stagingPrefix = newStagingPrefix();
2132
+ const indexRoot = `${indexName}/`;
2133
+ let committed = false;
2134
+ const staged = (key) => {
2135
+ if (!key.startsWith(indexRoot)) {
2136
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
2137
+ key
2138
+ });
2139
+ }
2140
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
2141
+ };
2142
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
2143
+ const store = {
2144
+ read: (key) => storage.read(staged(key)),
2145
+ write: (key, chunks) => storage.write(staged(key), chunks),
2146
+ remove: (key) => storage.remove(staged(key)),
2147
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
2148
+ };
2149
+ const pathOf = storage.pathOf;
2150
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
2151
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
2152
+ let retiredPrefix = null;
2153
+ async function finishOnStorage() {
2154
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
2046
2155
  }
2047
- const checkpointDocumentCount = documentCount;
2048
- await input.queueMetadataWrite(input.indexName, async () => {
2049
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
2050
- if (checkpointMetadata === void 0) {
2051
- return;
2156
+ return {
2157
+ store,
2158
+ async relocation() {
2159
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
2160
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
2161
+ const to = await pathOf.call(storage, indexName);
2162
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
2163
+ },
2164
+ async commit() {
2165
+ try {
2166
+ await commitJournal(storage, indexName, stagingPrefix);
2167
+ } catch (error) {
2168
+ try {
2169
+ await storage.remove(journalKey(indexName));
2170
+ } catch (removal) {
2171
+ committed = true;
2172
+ options.onStorageLost?.(toError2(removal));
2173
+ }
2174
+ throw error;
2175
+ }
2176
+ committed = true;
2177
+ },
2178
+ async finish() {
2179
+ try {
2180
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
2181
+ } catch (error) {
2182
+ options.onStorageLost?.(toError2(error));
2183
+ }
2184
+ },
2185
+ async removeRetired() {
2186
+ const prefix = retiredPrefix;
2187
+ retiredPrefix = null;
2188
+ if (prefix !== null) await removePrefix(storage, prefix);
2189
+ },
2190
+ async discard() {
2191
+ if (committed) return;
2192
+ await removePrefix(storage, stagingPrefix);
2052
2193
  }
2053
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
2054
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
2055
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
2056
- });
2057
- await truncateCoveredSegments(input.directory, input.indexName, targets);
2058
- await reclaimWalBeyondCount(
2059
- input.directory,
2060
- input.indexName,
2061
- manager.partitionCount,
2062
- input.indexState.partitions,
2063
- input.markFatal
2064
- );
2065
- const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
2066
- noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
2067
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
2068
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
2194
+ };
2069
2195
  }
2070
- function recordsBeyond(indexState, targets) {
2071
- let records = 0;
2072
- for (const [partitionId, partition] of indexState.partitions) {
2073
- if (partition.failed !== null) continue;
2074
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
2075
- }
2076
- return records;
2196
+ function adapterRestoreStorage(adapter) {
2197
+ const store = adapterCheckpointStore(adapter);
2198
+ return {
2199
+ read: (key) => store.read(key),
2200
+ write: (key, chunks) => store.write(key, chunks),
2201
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
2202
+ remove: (key) => store.remove(key),
2203
+ list: (prefix) => store.list(prefix),
2204
+ async moveKey(from, to) {
2205
+ const bytes = await store.read(from);
2206
+ if (bytes === null) return;
2207
+ await store.write(to, [bytes]);
2208
+ await store.remove(from);
2209
+ }
2210
+ };
2077
2211
  }
2078
2212
 
2079
- // src/persistence/durability/checkpoint-turn.ts
2080
- function createIndexLoads(vectorIndexesOf2) {
2081
- const loads = /* @__PURE__ */ new Map();
2082
- const startGraphRepairs = (indexName) => {
2083
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
2084
- vectorIndex.scheduleBuild();
2085
- vectorIndex.startShuffledRebuildWhenDue();
2213
+ // src/persistence/durability/archive-checkpoint.ts
2214
+ function toError3(error) {
2215
+ return error instanceof Error ? error : new Error(String(error));
2216
+ }
2217
+ function stageDirectoryRestore(deps, indexName) {
2218
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
2219
+ async finish(run) {
2220
+ try {
2221
+ await run();
2222
+ } catch (error) {
2223
+ deps.markFatal(toError3(error));
2224
+ throw error;
2225
+ }
2226
+ },
2227
+ onStorageLost: deps.markFatal
2228
+ });
2229
+ }
2230
+ function releaser(held, indexName) {
2231
+ let released = false;
2232
+ return () => {
2233
+ if (released) return;
2234
+ released = true;
2235
+ const count = (held.get(indexName) ?? 1) - 1;
2236
+ if (count <= 0) held.delete(indexName);
2237
+ else held.set(indexName, count);
2238
+ };
2239
+ }
2240
+ function createArchiveCheckpointAccess(deps) {
2241
+ const store = directoryCheckpointStore(deps.directory);
2242
+ return {
2243
+ async openArchiveCheckpoint(indexName) {
2244
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
2245
+ const release = releaser(deps.held, indexName);
2246
+ try {
2247
+ await deps.checkpoint(indexName, false);
2248
+ await deps.checkpoint(indexName, true);
2249
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
2250
+ if (manifestBytes === null) {
2251
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
2252
+ indexName
2253
+ });
2254
+ }
2255
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2256
+ if (decoded.kind !== "current") {
2257
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
2258
+ indexName
2259
+ });
2260
+ }
2261
+ return {
2262
+ manifestBytes,
2263
+ manifest: decoded.manifest,
2264
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
2265
+ readRange: async (ref, offset, length) => {
2266
+ const readRange = deps.directory.readRange;
2267
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
2268
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
2269
+ return whole.slice(offset, offset + length);
2270
+ },
2271
+ release,
2272
+ directory: deps.directory
2273
+ };
2274
+ } catch (error) {
2275
+ release();
2276
+ throw error;
2277
+ }
2278
+ },
2279
+ stageRestore(indexName) {
2280
+ return stageDirectoryRestore(deps, indexName);
2281
+ },
2282
+ adoptRestoredCheckpoint(indexName, checkpoint) {
2283
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
2284
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
2285
+ }
2086
2286
  }
2087
2287
  };
2288
+ }
2289
+
2290
+ // src/persistence/durability/manager-unload.ts
2291
+ function createIndexUnloading(deps) {
2292
+ const { directory, hooks, indexes, metadataWrites } = deps;
2088
2293
  return {
2089
- track(indexName, load, startsGraphRepairs = true) {
2090
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
2091
- loads.set(
2092
- indexName,
2093
- loaded.catch(() => void 0)
2094
- );
2095
- return loaded;
2294
+ async removeIndex(indexName) {
2295
+ hooks.writeRecords?.dropIndex(indexName);
2296
+ const indexState = indexes.get(indexName);
2297
+ if (indexState !== void 0) {
2298
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
2299
+ indexes.delete(indexName);
2300
+ metadataWrites.forget(indexName);
2301
+ }
2302
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
2303
+ for (const key of await directory.list(`${indexName}/`)) {
2304
+ await directory.remove(key);
2305
+ }
2306
+ deps.forgetSyncState(indexName);
2096
2307
  },
2097
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
2098
- startGraphRepairs
2308
+ async unloadIndex(indexName) {
2309
+ const indexState = indexes.get(indexName);
2310
+ if (indexState === void 0) {
2311
+ return;
2312
+ }
2313
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
2314
+ indexes.delete(indexName);
2315
+ metadataWrites.forget(indexName);
2316
+ deps.forgetSyncState(indexName);
2317
+ }
2099
2318
  };
2100
2319
  }
2101
- async function switchInTurn(turns, turn) {
2102
- let swapped = false;
2103
- await checkpointInTurn(turns, {
2104
- joinsInFlight: false,
2105
- mayStart: turn.mayStart,
2106
- perform: async () => {
2107
- swapped = turn.swap();
2108
- if (swapped) await turn.checkpoint();
2109
- },
2110
- afterSettling: turn.afterSettling
2111
- });
2112
- return swapped;
2113
- }
2114
- async function checkpointInTurn(turns, turn) {
2115
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
2116
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
2117
- if (!turn.mayStart()) return;
2118
- const run = turn.perform().finally(() => {
2119
- turns.checkpointInFlight = null;
2120
- turn.afterSettling();
2121
- });
2122
- turns.checkpointInFlight = run;
2123
- return run;
2124
- }
2125
2320
 
2126
- // src/persistence/durability/directory-watch.ts
2127
- function unrefInterval(tick, intervalMs) {
2128
- const timer = setInterval(tick, intervalMs);
2129
- if (typeof timer.unref === "function") timer.unref();
2130
- return timer;
2131
- }
2132
- function createDirectoryWatch(directory, isStopped, onLost) {
2133
- let expected = null;
2134
- let timer = null;
2135
- let lock = null;
2136
- function cannotCheck(err) {
2137
- return new NarsilError(
2138
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2139
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
2140
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
2141
- );
2321
+ // src/persistence/durability/sync/frame-cursor.ts
2322
+ var SegmentFileMissing = class extends Error {
2323
+ };
2324
+ function corrupt(key, message, details) {
2325
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
2326
+ }
2327
+ function isMissingFile(error) {
2328
+ return error?.code === "ENOENT";
2329
+ }
2330
+ var FrameCursor = class {
2331
+ constructor(directory, key, limit) {
2332
+ this.directory = directory;
2333
+ this.key = key;
2334
+ this.limit = limit;
2335
+ }
2336
+ buffer = new Uint8Array(0);
2337
+ bufferStart = 0;
2338
+ whole = null;
2339
+ async load(offset, length) {
2340
+ const readRange = this.directory.readRange;
2341
+ if (readRange === void 0) {
2342
+ if (this.whole === null) {
2343
+ const bytes = await this.directory.read(this.key);
2344
+ if (bytes === null) throw new SegmentFileMissing(this.key);
2345
+ this.whole = bytes;
2346
+ }
2347
+ this.buffer = this.whole;
2348
+ this.bufferStart = 0;
2349
+ return;
2350
+ }
2351
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
2352
+ try {
2353
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
2354
+ } catch (error) {
2355
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
2356
+ throw error;
2357
+ }
2358
+ this.bufferStart = offset;
2359
+ }
2360
+ async bytesAt(offset, length) {
2361
+ const end = offset + length;
2362
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
2363
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
2364
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
2365
+ }
2366
+ async checkHeader() {
2367
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
2368
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
2369
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
2142
2370
  }
2143
- async function currentIdentity() {
2371
+ async frameLengthAt(offset) {
2372
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
2373
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
2374
+ 0,
2375
+ false
2376
+ );
2377
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
2378
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
2379
+ }
2380
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
2381
+ }
2382
+ async frameAt(offset, frameBytes) {
2383
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
2384
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
2385
+ const payload = body.subarray(0, recordLength);
2386
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
2387
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
2388
+ let seqNo;
2144
2389
  try {
2145
- return await directory.identity?.() ?? null;
2146
- } catch (err) {
2147
- return cannotCheck(err);
2390
+ seqNo = decodeWalEntry(payload).seqNo;
2391
+ } catch (error) {
2392
+ corrupt(this.key, "A log record fails to decode", {
2393
+ offset,
2394
+ cause: error instanceof Error ? error.message : String(error)
2395
+ });
2148
2396
  }
2397
+ return { seqNo, payload: payload.slice(), frameBytes };
2149
2398
  }
2150
- async function lockTakenOver() {
2151
- if (lock === null) return null;
2399
+ };
2400
+
2401
+ // src/persistence/durability/sync/log-reader.ts
2402
+ async function walSources(directory, indexName, partitionId) {
2403
+ const prefix = walSegmentPrefix(indexName, partitionId);
2404
+ const markerBytes = await directory.read(`${prefix}commit`);
2405
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
2406
+ if (marker === null) return { sources: [], head: 0 };
2407
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
2408
+ const sources = [];
2409
+ for (const key of await directory.list(prefix)) {
2410
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
2411
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
2412
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
2413
+ if (readableBytes === null || readableBytes === void 0) continue;
2414
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
2415
+ }
2416
+ return { sources, head: highestDurableSeqNo };
2417
+ }
2418
+ async function openPartitionLog(directory, indexName, partitionId) {
2419
+ const { sources, head } = await walSources(directory, indexName, partitionId);
2420
+ const byStart = /* @__PURE__ */ new Map();
2421
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
2422
+ const readableBytes = await directory.byteLength?.(segment.key);
2423
+ if (readableBytes === null || readableBytes === void 0) continue;
2424
+ byStart.set(segment.startSeqNo, {
2425
+ key: segment.key,
2426
+ startSeqNo: segment.startSeqNo,
2427
+ readableBytes,
2428
+ inHistory: true
2429
+ });
2430
+ }
2431
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
2432
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
2433
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
2434
+ }
2435
+ function sourceHolding(log, seqNo) {
2436
+ let found = -1;
2437
+ for (let i = 0; i < log.sources.length; i++) {
2438
+ if (log.sources[i].startSeqNo > seqNo) break;
2439
+ found = i;
2440
+ }
2441
+ return found;
2442
+ }
2443
+ async function offsetAfter(cursor, source, afterSeqNo) {
2444
+ let offset = SEGMENT_HEADER_SIZE;
2445
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
2446
+ offset += await cursor.frameLengthAt(offset);
2447
+ }
2448
+ return offset;
2449
+ }
2450
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
2451
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2452
+ await cursor.checkHeader();
2453
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
2454
+ let checksNumbering = true;
2455
+ while (offset < cursor.limit) {
2456
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
2457
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
2458
+ checksNumbering = false;
2459
+ if (frame.seqNo > afterSeqNo) {
2460
+ offset = SEGMENT_HEADER_SIZE;
2461
+ continue;
2462
+ }
2463
+ }
2464
+ checksNumbering = false;
2465
+ offset += frame.frameBytes;
2466
+ if (frame.seqNo <= afterSeqNo) continue;
2467
+ if (frame.seqNo > upToSeqNo) return true;
2468
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
2469
+ read.frames.push(frame);
2470
+ read.bytes += frame.payload.byteLength;
2471
+ if (frame.seqNo === upToSeqNo) return true;
2472
+ }
2473
+ return false;
2474
+ }
2475
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
2476
+ const read = { frames: [], bytes: 0, gone: false };
2477
+ if (afterSeqNo >= upToSeqNo) return read;
2478
+ for (let attempt = 0; attempt < 2; attempt++) {
2479
+ read.frames = [];
2480
+ read.bytes = 0;
2481
+ const log = await openPartitionLog(directory, indexName, partitionId);
2482
+ let index = sourceHolding(log, afterSeqNo + 1);
2483
+ if (index < 0) {
2484
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
2485
+ return read;
2486
+ }
2152
2487
  try {
2153
- if (await lock.holds()) return null;
2154
- } catch (err) {
2155
- return cannotCheck(err);
2488
+ for (; index < log.sources.length; index++) {
2489
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
2490
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
2491
+ }
2492
+ return read;
2493
+ } catch (error) {
2494
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
2156
2495
  }
2157
- return new NarsilError(
2158
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2159
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
2160
- { directory: directory.root }
2161
- );
2162
2496
  }
2163
- async function verify() {
2164
- if (directory.identity === void 0 || isStopped()) return true;
2165
- const current = await currentIdentity();
2166
- if (current instanceof Error) {
2167
- onLost(current);
2168
- return false;
2169
- }
2170
- if (expected === null) {
2171
- expected = current;
2172
- } else if (current !== expected) {
2173
- onLost(
2174
- new NarsilError(
2175
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2176
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
2177
- { directory: directory.root }
2178
- )
2179
- );
2180
- return false;
2497
+ return read;
2498
+ }
2499
+ async function logBytesAfter(directory, log, afterSeqNo) {
2500
+ const first = sourceHolding(log, afterSeqNo + 1);
2501
+ let bytes = 0;
2502
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
2503
+ const source = log.sources[i];
2504
+ if (i === first) {
2505
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2506
+ try {
2507
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
2508
+ } catch (error) {
2509
+ if (!(error instanceof SegmentFileMissing)) throw error;
2510
+ }
2511
+ continue;
2181
2512
  }
2182
- const takenOver = await lockTakenOver();
2183
- if (takenOver === null) return true;
2184
- onLost(takenOver);
2185
- return false;
2513
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
2186
2514
  }
2187
- function stop() {
2188
- if (timer !== null) {
2189
- clearInterval(timer);
2190
- timer = null;
2515
+ return bytes;
2516
+ }
2517
+
2518
+ // src/persistence/durability/sync/pins.ts
2519
+ function createFilePins(remove) {
2520
+ const counts = /* @__PURE__ */ new Map();
2521
+ const awaitingRemoval = /* @__PURE__ */ new Set();
2522
+ function unpin(key) {
2523
+ const count = (counts.get(key) ?? 1) - 1;
2524
+ if (count > 0) {
2525
+ counts.set(key, count);
2526
+ return;
2191
2527
  }
2528
+ counts.delete(key);
2529
+ if (!awaitingRemoval.delete(key)) return;
2530
+ void remove(key).catch(() => void 0);
2192
2531
  }
2193
2532
  return {
2194
- async claim() {
2195
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
2196
- },
2197
- async release() {
2198
- const held = lock;
2199
- lock = null;
2200
- await held?.release();
2201
- },
2202
- verify,
2203
- async verifyOnce() {
2204
- if (expected === null && directory.identity !== void 0) await verify();
2205
- },
2206
- start() {
2207
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
2208
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
2533
+ pin(keys) {
2534
+ const pinned = [...keys];
2535
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
2536
+ let released = false;
2537
+ return () => {
2538
+ if (released) return;
2539
+ released = true;
2540
+ for (const key of pinned) unpin(key);
2541
+ };
2209
2542
  },
2210
- stop
2543
+ isPinned: (key) => counts.has(key),
2544
+ removeWhenUnpinned(key) {
2545
+ if (!counts.has(key)) return false;
2546
+ awaitingRemoval.add(key);
2547
+ return true;
2548
+ }
2211
2549
  };
2212
2550
  }
2551
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2552
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2553
+ function hexOf(bytes) {
2554
+ let hex = "";
2555
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
2556
+ return hex;
2557
+ }
2558
+ function isBytes(value) {
2559
+ return value instanceof Uint8Array;
2560
+ }
2561
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
2562
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2563
+ if (decoded.kind !== "current") {
2564
+ throw new NarsilError(
2565
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2566
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
2567
+ { indexName }
2568
+ );
2569
+ }
2570
+ return {
2571
+ config,
2572
+ manifestBytes,
2573
+ manifest: decoded.manifest,
2574
+ entityTag: hexOf(sha256(payload)),
2575
+ keys: manifestReferencedKeys(decoded.manifest)
2576
+ };
2577
+ }
2578
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
2579
+ const payload = encode({ config, manifest: manifestBytes });
2580
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
2581
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
2582
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
2583
+ return published;
2584
+ }
2585
+ async function readPublishedSnapshot(directory, indexName) {
2586
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
2587
+ if (bytes === null) return null;
2588
+ const { payload } = await openChecksummedEnvelope(bytes, {
2589
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
2590
+ what: PUBLISHED_SNAPSHOT_WHAT,
2591
+ allowsCompression: false
2592
+ });
2593
+ const raw = decode(payload);
2594
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
2595
+ throw new NarsilError(
2596
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2597
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
2598
+ { indexName }
2599
+ );
2600
+ }
2601
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
2602
+ }
2213
2603
 
2214
- // src/persistence/durability/held-checkpoints.ts
2215
- function failedPartitionIds(indexState) {
2216
- const failed = [];
2217
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
2218
- return failed;
2604
+ // src/persistence/durability/sync/retention.ts
2605
+ async function firstLogStart(directory, indexName, partitionId) {
2606
+ let first = Number.POSITIVE_INFINITY;
2607
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
2608
+ const start = parseWalSegmentStart(indexName, partitionId, key);
2609
+ if (start !== null && start < first) first = start;
2610
+ }
2611
+ return first;
2612
+ }
2613
+ async function sizedQueues(retention) {
2614
+ const { directory, indexName } = retention;
2615
+ const queues = /* @__PURE__ */ new Map();
2616
+ for (const segment of await listHistorySegments(directory, indexName)) {
2617
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
2618
+ if (bytes === void 0 || bytes === null) continue;
2619
+ const queue = queues.get(segment.partitionId) ?? [];
2620
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
2621
+ queues.set(segment.partitionId, queue);
2622
+ }
2623
+ for (const [partitionId, queue] of queues) {
2624
+ const logStart = await firstLogStart(directory, indexName, partitionId);
2625
+ for (let i = 0; i < queue.length; i++) {
2626
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
2627
+ queue[i].lastSeqNo = nextStart - 1;
2628
+ }
2629
+ }
2630
+ return queues;
2631
+ }
2632
+ function nextToDelete(queues, totalBytes, retention) {
2633
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
2634
+ let oldest = null;
2635
+ for (const queue of queues.values()) {
2636
+ const head = queue[0];
2637
+ if (head === void 0) continue;
2638
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
2639
+ }
2640
+ if (oldest === null) return null;
2641
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
2642
+ return null;
2219
2643
  }
2220
- function createHeldCheckpoints(hooks) {
2221
- const reported = /* @__PURE__ */ new WeakSet();
2222
- function heldReport(indexName, indexState) {
2223
- const partitionIds = failedPartitionIds(indexState);
2224
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
2225
- return {
2226
- indexName,
2227
- partitionIds,
2228
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
2229
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
2230
- };
2644
+ async function enforceHistoryLimits(retention) {
2645
+ const queues = await sizedQueues(retention);
2646
+ let totalBytes = 0;
2647
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
2648
+ let published = false;
2649
+ let deleted = 0;
2650
+ let segment = nextToDelete(queues, totalBytes, retention);
2651
+ while (segment !== null) {
2652
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
2653
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
2654
+ published = true;
2655
+ await retention.publish();
2656
+ }
2657
+ await retention.directory.remove(segment.key);
2658
+ queues.get(segment.partitionId)?.shift();
2659
+ totalBytes -= segment.bytes;
2660
+ deleted += 1;
2661
+ segment = nextToDelete(queues, totalBytes, retention);
2662
+ }
2663
+ return deleted;
2664
+ }
2665
+
2666
+ // src/persistence/durability/sync/storage.ts
2667
+ function createSyncStorage(deps) {
2668
+ const { directory } = deps;
2669
+ const store = directoryCheckpointStore(directory);
2670
+ const cache = /* @__PURE__ */ new Map();
2671
+ const publishing = /* @__PURE__ */ new Map();
2672
+ const pins = createFilePins((key) => directory.remove(key));
2673
+ async function published(indexName) {
2674
+ const cached = cache.get(indexName);
2675
+ if (cached !== void 0) return cached;
2676
+ const read = await readPublishedSnapshot(directory, indexName);
2677
+ cache.set(indexName, read);
2678
+ return read;
2679
+ }
2680
+ async function removeUnlisted(indexName, previous, kept) {
2681
+ const manifest = await readCurrentManifest(store, indexName);
2682
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
2683
+ for (const key of previous) {
2684
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
2685
+ await directory.remove(key);
2686
+ }
2687
+ }
2688
+ async function publishNow(indexName, configurationOf) {
2689
+ const archive = await deps.openArchiveCheckpoint(indexName);
2690
+ try {
2691
+ const config = await configurationOf(archive.manifest);
2692
+ const previous = await published(indexName);
2693
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
2694
+ cache.set(indexName, next);
2695
+ if (previous !== null) {
2696
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
2697
+ }
2698
+ return next;
2699
+ } finally {
2700
+ archive.release();
2701
+ }
2231
2702
  }
2232
2703
  return {
2233
- refuse(indexName, indexState) {
2234
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
2235
- if (partitionIds.length === 0) return;
2236
- throw new NarsilError(
2237
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2238
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
2239
- { indexName, partitionIds }
2240
- );
2704
+ publish(indexName, configurationOf) {
2705
+ const inFlight = publishing.get(indexName);
2706
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
2707
+ const tracked = run.finally(() => {
2708
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
2709
+ });
2710
+ publishing.set(indexName, tracked);
2711
+ return tracked;
2241
2712
  },
2242
- holds(indexName, indexState) {
2243
- const held = heldReport(indexName, indexState);
2244
- if (held === null) return false;
2245
- const unreported = held.partitionIds.some((partitionId) => {
2246
- const partition = indexState.partitions.get(partitionId);
2247
- return partition !== void 0 && !reported.has(partition);
2713
+ published,
2714
+ pin: (snapshot) => pins.pin(snapshot.keys),
2715
+ async readFileRange(key, offset, length) {
2716
+ const readRange = directory.readRange;
2717
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
2718
+ const whole = await directory.read(key);
2719
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
2720
+ },
2721
+ async release(indexName) {
2722
+ await deps.inCheckpointTurn(indexName, async () => {
2723
+ const previous = await published(indexName);
2724
+ await directory.remove(publishedSnapshotKey(indexName));
2725
+ cache.set(indexName, null);
2726
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
2727
+ await removeHistory(directory, indexName);
2248
2728
  });
2249
- if (unreported) {
2250
- for (const partitionId of held.partitionIds) {
2251
- const partition = indexState.partitions.get(partitionId);
2252
- if (partition !== void 0) reported.add(partition);
2253
- }
2254
- hooks.onCheckpointHeld?.(held);
2255
- }
2256
- return true;
2257
2729
  },
2258
- report(indexes) {
2259
- const held = [];
2260
- for (const [indexName, indexState] of indexes) {
2261
- const entry = heldReport(indexName, indexState);
2262
- if (entry !== null) held.push(entry);
2730
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
2731
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
2732
+ const current = await published(indexName);
2733
+ return enforceHistoryLimits({
2734
+ directory,
2735
+ indexName,
2736
+ limits,
2737
+ nowMs,
2738
+ publish,
2739
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
2740
+ });
2741
+ },
2742
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
2743
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
2744
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
2745
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
2746
+ async keepsCheckpointKey(indexName) {
2747
+ const current = await published(indexName);
2748
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
2749
+ },
2750
+ async settleAtRecovery(indexName, active) {
2751
+ cache.delete(indexName);
2752
+ const publishedKey = publishedSnapshotKey(indexName);
2753
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
2754
+ if (key !== publishedKey) await directory.remove(key);
2263
2755
  }
2264
- return held;
2756
+ if (!active) {
2757
+ await directory.remove(publishedSnapshotKey(indexName));
2758
+ await removeHistory(directory, indexName);
2759
+ cache.set(indexName, null);
2760
+ return /* @__PURE__ */ new Set();
2761
+ }
2762
+ try {
2763
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
2764
+ } catch {
2765
+ await directory.remove(publishedSnapshotKey(indexName));
2766
+ cache.set(indexName, null);
2767
+ return /* @__PURE__ */ new Set();
2768
+ }
2769
+ },
2770
+ forget(indexName) {
2771
+ cache.delete(indexName);
2265
2772
  }
2266
2773
  };
2267
2774
  }
2268
2775
 
2269
- // src/persistence/durability/manager-state.ts
2270
- function closePartitionWriter(partition) {
2271
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
2272
- }
2273
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
2274
- indexState.unloading = true;
2275
- try {
2276
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2277
- await pendingMetadataWrite;
2278
- for (const partition of indexState.partitions.values()) {
2279
- await closeWalWriter(closePartitionWriter(partition));
2776
+ // src/persistence/durability/manager-storage.ts
2777
+ function createIndexStorageAccess(deps) {
2778
+ const { directory, indexes } = deps;
2779
+ const archiveAccess = createArchiveCheckpointAccess({
2780
+ directory,
2781
+ held: deps.heldCheckpointFiles,
2782
+ checkpoint: deps.checkpoint,
2783
+ registerPartition: deps.registerPartition,
2784
+ markFatal: deps.markFatal
2785
+ });
2786
+ const syncStorage = createSyncStorage({
2787
+ directory,
2788
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
2789
+ inCheckpointTurn: (indexName, work) => {
2790
+ const indexState = indexes.get(indexName);
2791
+ if (indexState === void 0) return work();
2792
+ return checkpointInTurn(indexState, {
2793
+ joinsInFlight: false,
2794
+ mayStart: () => true,
2795
+ perform: work,
2796
+ afterSettling: () => releaseStalledWrites(indexState)
2797
+ });
2280
2798
  }
2281
- } catch (error) {
2282
- indexState.unloading = false;
2283
- throw error;
2284
- }
2799
+ });
2800
+ const unloading = createIndexUnloading({
2801
+ directory,
2802
+ hooks: deps.hooks,
2803
+ indexes,
2804
+ metadataWrites: deps.metadataWrites,
2805
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
2806
+ forgetSyncState: syncStorage.forget
2807
+ });
2808
+ return { archiveAccess, syncStorage, unloading };
2285
2809
  }
2286
2810
 
2287
2811
  // src/persistence/durability/metadata-write-queue.ts
@@ -3254,7 +3778,7 @@ var decoder = new TextDecoder();
3254
3778
  async function countPagedPartition(store, partition) {
3255
3779
  const { directory } = store;
3256
3780
  if (directory === void 0 || partition.segments.length === 0) return null;
3257
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-FHJ2GOBI.mjs');
3781
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-QT5HZAY4.mjs');
3258
3782
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
3259
3783
  const scans = [];
3260
3784
  try {
@@ -3537,12 +4061,14 @@ async function recoverCheckpoint(deps) {
3537
4061
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3538
4062
  }
3539
4063
  if (decoded === null || decoded.kind !== "current") {
3540
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
4064
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3541
4065
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3542
4066
  }
3543
4067
  await deps.store.remove(snapshotBundleKey(deps.indexName));
3544
4068
  if (!deps.keepsUnlistedFiles) {
3545
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
4069
+ const referenced = manifestReferencedKeys(decoded.manifest);
4070
+ for (const key of deps.keptForSync) referenced.add(key);
4071
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
3546
4072
  }
3547
4073
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
3548
4074
  manager: deps.manager,
@@ -3663,6 +4189,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3663
4189
  }
3664
4190
  return replayed;
3665
4191
  };
4192
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
4193
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
3666
4194
  const recovered = await recoverCheckpoint({
3667
4195
  store,
3668
4196
  indexName,
@@ -3672,6 +4200,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3672
4200
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3673
4201
  compactionThreshold: deps.compactionThreshold,
3674
4202
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
4203
+ keptForSync,
3675
4204
  replay,
3676
4205
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3677
4206
  writeRecords
@@ -3787,6 +4316,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3787
4316
  mutationThreshold: checkpointMutationThreshold,
3788
4317
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3789
4318
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
4319
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3790
4320
  queueMetadataWrite,
3791
4321
  markFatal,
3792
4322
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -3840,6 +4370,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3840
4370
  startAsyncFlushTimer();
3841
4371
  return outcomes;
3842
4372
  }
4373
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
4374
+ directory,
4375
+ hooks,
4376
+ indexes,
4377
+ metadataWrites,
4378
+ heldCheckpointFiles,
4379
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
4380
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
4381
+ markFatal,
4382
+ closeWriterReportingFailure
4383
+ });
3843
4384
  const recoveryDeps = {
3844
4385
  directory,
3845
4386
  hooks,
@@ -3848,7 +4389,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3848
4389
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3849
4390
  },
3850
4391
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3851
- queueMetadataWrite
4392
+ queueMetadataWrite,
4393
+ syncStorage
3852
4394
  };
3853
4395
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3854
4396
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3936,35 +4478,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3936
4478
  await checkpointIndex(indexName);
3937
4479
  }
3938
4480
  },
3939
- async removeIndex(indexName) {
3940
- hooks.writeRecords?.dropIndex(indexName);
3941
- const indexState = indexes.get(indexName);
3942
- if (indexState !== void 0) {
3943
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3944
- indexes.delete(indexName);
3945
- metadataWrites.forget(indexName);
3946
- }
3947
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3948
- for (const key of await directory.list(`${indexName}/`)) {
3949
- await directory.remove(key);
3950
- }
3951
- },
3952
- async unloadIndex(indexName) {
3953
- const indexState = indexes.get(indexName);
3954
- if (indexState === void 0) {
3955
- return;
3956
- }
3957
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3958
- indexes.delete(indexName);
3959
- metadataWrites.forget(indexName);
3960
- },
3961
- ...createArchiveCheckpointAccess({
3962
- directory,
3963
- held: heldCheckpointFiles,
3964
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3965
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3966
- markFatal
3967
- }),
4481
+ ...unloading,
4482
+ syncStorage: () => syncStorage,
4483
+ ...archiveAccess,
3968
4484
  async shutdown() {
3969
4485
  shuttingDown = true;
3970
4486
  stopTimers();
@@ -4231,6 +4747,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4231
4747
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
4232
4748
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
4233
4749
  keepsUnlistedFiles: false,
4750
+ keptForSync: /* @__PURE__ */ new Set(),
4234
4751
  replay: async (checkpoint) => checkpoint,
4235
4752
  onLayoutUpgrade: hooks.onLayoutUpgrade
4236
4753
  });
@@ -4428,7 +4945,9 @@ function createDurabilityIntegration(tier, hooks) {
4428
4945
  recordCheckpoint: hooks.recordCheckpoint,
4429
4946
  onLayoutUpgrade: hooks.onLayoutUpgrade,
4430
4947
  onCheckpointHeld: hooks.onCheckpointHeld,
4431
- onFatalError: hooks.onFatalError
4948
+ onFatalError: hooks.onFatalError,
4949
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4950
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4432
4951
  };
4433
4952
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4434
4953
  function partitionFor(indexName, docId) {
@@ -4513,7 +5032,12 @@ function createDurabilityFromTier(tier, wiring) {
4513
5032
  recordCheckpoint: wiring.recordCheckpoint,
4514
5033
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
4515
5034
  onCheckpointHeld: wiring.emitCheckpointHeld,
4516
- onFatalError: wiring.emitFatalError
5035
+ onFatalError: wiring.emitFatalError,
5036
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
5037
+ onHistoryMoved(indexName) {
5038
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
5039
+ import('./retention-PPCTP2VY.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
5040
+ }
4517
5041
  });
4518
5042
  }
4519
5043