@delali/narsil 0.3.1-next.152 → 0.3.1-next.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-4GCRVRBW.mjs} +27 -18
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KYGSXW4K.mjs} +1 -1
  8. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-IXWPXXJJ.mjs} +17 -7
  9. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-FCSHFQSS.mjs} +1 -1
  10. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-2FX6ROWR.mjs} +15 -6
  11. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-2SHAKW2Z.mjs} +119 -1922
  12. package/dist/browser/{chunk-55WYRLDZ.mjs → chunk-3U3QR5YP.mjs} +2 -2
  13. package/dist/browser/chunk-4PD752F6.mjs +85 -0
  14. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  15. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-7MEYN7V6.mjs} +1 -151
  16. package/dist/browser/chunk-7TTWXUH6.mjs +1828 -0
  17. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-AJ4HHOVT.mjs} +4 -4
  18. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-AYY6PIKL.mjs} +2 -2
  19. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-C4KS2QNF.mjs} +9 -8
  20. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  21. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  22. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-EUNBNLKM.mjs} +6 -5
  23. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-G7LUMBSG.mjs} +137 -253
  24. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-H6QFBMVE.mjs} +3 -2
  25. package/dist/browser/chunk-J433A7P3.mjs +241 -0
  26. package/dist/browser/chunk-ML6HY7T7.mjs +96 -0
  27. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  28. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-MYITLCHT.mjs} +7 -1
  29. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-QXTPSX2H.mjs} +9 -150
  30. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  31. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SIR3LMTV.mjs} +2 -2
  32. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-SJB6VIZG.mjs} +1 -1
  33. package/dist/browser/chunk-TTD27VXX.mjs +298 -0
  34. package/dist/browser/chunk-U2E3HMYW.mjs +151 -0
  35. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-VCOYGVP7.mjs} +5 -31
  36. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  37. package/dist/browser/chunk-W5X2UFWQ.mjs +102 -0
  38. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-XRV72TJD.mjs} +2 -2
  39. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  40. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  41. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-ZPTA75ZC.mjs} +2 -79
  42. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-VTUGDYNY.mjs} +20 -14
  43. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-GL4A7KDV.mjs} +1886 -1364
  44. package/dist/browser/import-YI5GNB4H.mjs +17 -0
  45. package/dist/browser/methods-LMUTS7OI.mjs +30 -0
  46. package/dist/browser/{module-XQLEOYNM.mjs → module-EC3FYW7N.mjs} +9 -8
  47. package/dist/browser/{module-KAMRAJLE.mjs → module-M4JNBV6V.mjs} +16 -10
  48. package/dist/browser/operations-WVYLF72Q.mjs +5 -0
  49. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-NPYUVX75.mjs} +1 -1
  50. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-WCGDZHJP.mjs} +3 -2
  51. package/dist/{restore-queue-AOF5OTQV.mjs → browser/restore-queue-TYIGDL5L.mjs} +2 -2
  52. package/dist/browser/retention-6QMRS5VK.mjs +207 -0
  53. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-3EW2AGB6.mjs} +1 -1
  54. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-DN3CRVUQ.mjs} +38 -307
  55. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  56. package/dist/browser/{write-I5P5MW4O.mjs → write-WAWAFGKQ.mjs} +16 -11
  57. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-B3Brs1Rc.d.ts} +62 -7
  58. package/dist/{chunk-DGRM4EZZ.mjs → chunk-37IVJLP6.mjs} +1 -1
  59. package/dist/{chunk-KYVRZAYP.mjs → chunk-3A5KI24C.mjs} +2 -69
  60. package/dist/{chunk-POUYICOU.mjs → chunk-3OFU3NJT.mjs} +1 -1
  61. package/dist/{chunk-3TVDTACC.mjs → chunk-3Y3J4V2R.mjs} +1 -1
  62. package/dist/{chunk-FNBNZCGP.mjs → chunk-466PYR4P.mjs} +2 -2
  63. package/dist/{chunk-SEAJS664.mjs → chunk-4HWZV5MK.mjs} +5 -5
  64. package/dist/{chunk-UGVUW27S.mjs → chunk-5FUOVJKG.mjs} +13 -9
  65. package/dist/{chunk-ZHAXP7SD.mjs → chunk-5KGYQT6W.mjs} +6 -235
  66. package/dist/{chunk-NPXVQS33.mjs → chunk-62G63NMH.mjs} +4 -3
  67. package/dist/chunk-6MSZ543Y.mjs +32 -0
  68. package/dist/chunk-7GXJUJ24.mjs +49 -0
  69. package/dist/{chunk-3E4RRF5J.mjs → chunk-7HCZWC7B.mjs} +58 -8
  70. package/dist/{chunk-6RKSY3BX.mjs → chunk-7NTMUDC5.mjs} +5 -5
  71. package/dist/chunk-7QUGDVFC.mjs +101 -0
  72. package/dist/{chunk-G5KHSM6E.mjs → chunk-AO2FAYIZ.mjs} +1 -1
  73. package/dist/chunk-B7S6YEBP.mjs +102 -0
  74. package/dist/{chunk-AWNHDBKV.mjs → chunk-BPSKLCS7.mjs} +15 -7
  75. package/dist/chunk-BUMREPLU.mjs +15 -0
  76. package/dist/{chunk-T4CJDBBE.mjs → chunk-C6ASEFZI.mjs} +1 -1
  77. package/dist/{chunk-NITCPDFL.mjs → chunk-C73JLU4E.mjs} +2 -2
  78. package/dist/{chunk-NYSONUB7.mjs → chunk-CP5RRPJR.mjs} +1 -1
  79. package/dist/{chunk-TTCKT2RT.mjs → chunk-DMQPFLGI.mjs} +134 -11
  80. package/dist/{chunk-3SBJVQLL.mjs → chunk-DVWYODEB.mjs} +2 -2
  81. package/dist/chunk-ECX4SGX6.mjs +167 -0
  82. package/dist/chunk-GES7U2PM.mjs +68 -0
  83. package/dist/{chunk-NVTFQFUL.mjs → chunk-HVKXNBFL.mjs} +1 -1
  84. package/dist/chunk-HVYICDY3.mjs +214 -0
  85. package/dist/{chunk-3HV7M5OJ.mjs → chunk-HWEDFLB7.mjs} +9 -133
  86. package/dist/{chunk-52QQWEBS.mjs → chunk-IBNLW3AV.mjs} +21 -6
  87. package/dist/chunk-IJU7LL7D.mjs +242 -0
  88. package/dist/chunk-IPN7S4KX.mjs +130 -0
  89. package/dist/{chunk-5QMHMELB.mjs → chunk-IQ4R2I6N.mjs} +4 -1
  90. package/dist/chunk-JCLTESFN.mjs +96 -0
  91. package/dist/{chunk-J4BDAH2W.mjs → chunk-JFNKIBDT.mjs} +2 -2
  92. package/dist/{chunk-LDGGR7YT.mjs → chunk-K4B7OH65.mjs} +2 -2
  93. package/dist/{chunk-7TE2AJIH.mjs → chunk-K7KLL263.mjs} +13 -10
  94. package/dist/{chunk-CSHHU7PA.mjs → chunk-KCUIFVVW.mjs} +11 -30
  95. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-KUZKUONM.mjs} +11 -2
  96. package/dist/chunk-KYYABCSW.mjs +72 -0
  97. package/dist/{chunk-B2FTZNNF.mjs → chunk-LKUE5GXS.mjs} +34 -14
  98. package/dist/{chunk-2PNYKG2D.mjs → chunk-LSM2YTKM.mjs} +100 -335
  99. package/dist/{chunk-DMNPYF54.mjs → chunk-LTUGDMV4.mjs} +2 -2
  100. package/dist/{chunk-FDG4KXX5.mjs → chunk-MLHNSAPK.mjs} +5 -3
  101. package/dist/{chunk-VP6XRMWR.mjs → chunk-MMBEANB2.mjs} +6 -32
  102. package/dist/{chunk-LJ4ENXOD.mjs → chunk-MMEMEUOO.mjs} +2 -2
  103. package/dist/{chunk-2HJWQPR6.mjs → chunk-NTZOXVUZ.mjs} +6 -122
  104. package/dist/{chunk-5WB2A3XU.mjs → chunk-NZMAXR2S.mjs} +397 -24
  105. package/dist/{chunk-IZT3G24Q.mjs → chunk-O43Z4MS6.mjs} +2 -2
  106. package/dist/{chunk-U46IZNCW.mjs → chunk-OKX2BBNW.mjs} +3 -2
  107. package/dist/{chunk-RBEHLIKX.mjs → chunk-PFKM7VNZ.mjs} +14 -8
  108. package/dist/chunk-PGVPE7GJ.mjs +96 -0
  109. package/dist/chunk-Q3TCTQCC.mjs +85 -0
  110. package/dist/chunk-Q7MS6P2J.mjs +74 -0
  111. package/dist/chunk-QDZJM7NI.mjs +23 -0
  112. package/dist/{chunk-ZS6CMXAE.mjs → chunk-QKZHLADX.mjs} +89 -250
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-REL7LFLR.mjs} +3 -3
  114. package/dist/chunk-RVKFI3A2.mjs +237 -0
  115. package/dist/chunk-S2N75OTN.mjs +83 -0
  116. package/dist/chunk-TRISKD4Q.mjs +50 -0
  117. package/dist/{chunk-6F6USYII.mjs → chunk-TV2L3BN7.mjs} +13 -11
  118. package/dist/{chunk-6BWLDLS3.mjs → chunk-URPCYQVJ.mjs} +1 -1
  119. package/dist/chunk-USR7FQRZ.mjs +34 -0
  120. package/dist/chunk-V6EUBAIT.mjs +19 -0
  121. package/dist/{chunk-SOUZMPME.mjs → chunk-VMXSVVXG.mjs} +7 -194
  122. package/dist/{chunk-4QCSHLNM.mjs → chunk-VREK3MYG.mjs} +3 -3
  123. package/dist/{chunk-YGQOPB2Z.mjs → chunk-WCBA2LVZ.mjs} +1 -1
  124. package/dist/{chunk-KQJVNPRM.mjs → chunk-X6NSJYQ3.mjs} +19 -16
  125. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  126. package/dist/{chunk-IRWP524D.mjs → chunk-YBJDOA2T.mjs} +1 -1
  127. package/dist/{chunk-IJPK5PGM.mjs → chunk-YPUEV6ZC.mjs} +1 -1
  128. package/dist/{chunk-J7M73VY7.mjs → chunk-YROWNOMH.mjs} +1 -1
  129. package/dist/chunk-Z3TNWCI5.mjs +196 -0
  130. package/dist/{chunk-65WCV5JO.mjs → chunk-ZIFJXCKA.mjs} +1 -1
  131. package/dist/chunk-ZYAY26KG.mjs +29 -0
  132. package/dist/{client-Cn3gNxUU.d.ts → client-Df76GNKm.d.ts} +76 -6
  133. package/dist/client.d.ts +8 -7
  134. package/dist/client.mjs +56 -5
  135. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-YLWMTTZG.mjs} +38 -28
  136. package/dist/{config-CfhH1IWQ.d.ts → config-05G_hy-1.d.ts} +3 -3
  137. package/dist/config-B8DP2VwR.d.ts +463 -0
  138. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-FI5GJHWX.mjs} +14 -9
  139. package/dist/distribution/coordinator/etcd.d.ts +11 -2
  140. package/dist/distribution/coordinator/etcd.mjs +268 -107
  141. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  142. package/dist/distribution/transport/grpc.mjs +4 -4
  143. package/dist/distribution/transport/in-memory.mjs +2 -2
  144. package/dist/distribution/transport/tcp.mjs +3 -3
  145. package/dist/distribution.d.ts +41 -9
  146. package/dist/distribution.mjs +1296 -117
  147. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-RHLXDIVF.mjs} +1751 -1226
  148. package/dist/embeddings/openai.mjs +2 -2
  149. package/dist/{engine-CJRUHXgf.d.ts → engine-DYu-e0oR.d.ts} +50 -40
  150. package/dist/{import-GJOMB2HC.mjs → import-SRBNC6TE.mjs} +27 -19
  151. package/dist/{index-MsF7e5hL.d.ts → index-nRqDM6uS.d.ts} +1 -1
  152. package/dist/index.browser.mjs +8 -4
  153. package/dist/index.d.ts +7 -6
  154. package/dist/index.mjs +24 -16
  155. package/dist/index.react-native.mjs +17 -11
  156. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-EMYTGSJP.mjs} +28 -15
  157. package/dist/methods-ABM5ELBR.mjs +30 -0
  158. package/dist/{module-AVQEIB7O.mjs → module-3OMJZUFT.mjs} +12 -12
  159. package/dist/{module-6RLIKSZQ.mjs → module-MG7JMV2U.mjs} +33 -23
  160. package/dist/operations-PGOAY2KS.mjs +5 -0
  161. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-UOLHOEZJ.mjs} +37 -26
  162. package/dist/partial-words.mjs +6 -4
  163. package/dist/pattern.mjs +4 -3
  164. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -1
  165. package/dist/persistence/durability/checkpoint-worker.mjs +34 -23
  166. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-O3MDRQXP.mjs} +1 -1
  167. package/dist/{pool-UPV5PVPY.mjs → pool-CQXYPFLG.mjs} +2 -2
  168. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-EAYO3Y6Q.mjs} +27 -15
  169. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-TNLUC6IZ.mjs} +1 -1
  170. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-2CGSQM4O.mjs} +12 -10
  171. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  172. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-3O57C2ZO.mjs} +2 -2
  173. package/dist/react-native/chunks/chunk-3XKMRNRE.mjs +298 -0
  174. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-63YSZOXB.mjs} +97 -333
  175. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-75VWIY35.mjs} +1 -1
  176. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7GWPJDOA.mjs} +1 -1
  177. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-AUT7PMA4.mjs} +6 -190
  178. package/dist/react-native/chunks/chunk-C2H2QQFW.mjs +288 -0
  179. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  180. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-CUNGPR5R.mjs} +11 -2
  181. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-DCKMICOB.mjs} +2 -79
  182. package/dist/react-native/chunks/chunk-DH3TVWWK.mjs +102 -0
  183. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-EH5CFV6Z.mjs} +2 -2
  184. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  185. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-GFYFQLJD.mjs} +2 -2
  186. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-I5PGWQJF.mjs} +142 -257
  187. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-IOY2SM2P.mjs} +1 -1
  188. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  189. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-JPAJWO6D.mjs} +1 -1
  190. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  191. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-JX2RIAXH.mjs} +4 -4
  192. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  193. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-NCJS3WIY.mjs} +2 -2
  194. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-NKIZX26F.mjs} +2 -2
  195. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-NP7DD4G5.mjs} +5 -4
  196. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  197. package/dist/react-native/chunks/chunk-O32RB3BL.mjs +191 -0
  198. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-OQQVRWZ2.mjs} +2 -69
  199. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  200. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  201. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-RVAEBN74.mjs} +1 -1
  202. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-RYMDHGL4.mjs} +16 -8
  203. package/dist/react-native/chunks/chunk-SJWDBRNP.mjs +85 -0
  204. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  205. package/dist/react-native/chunks/chunk-UWDNZDWK.mjs +242 -0
  206. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-VQSKZ3EZ.mjs} +9 -152
  207. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-VVFYOADG.mjs} +7 -6
  208. package/dist/react-native/chunks/{chunk-VGUWIZFQ.mjs → chunk-YBEW6YTR.mjs} +1 -1
  209. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-YHKZSLOC.mjs} +7 -33
  210. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-ZBSWRSBG.mjs} +4 -30
  211. package/dist/react-native/chunks/chunk-ZYDKJXQJ.mjs +96 -0
  212. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-TJL6AFAD.mjs} +29 -21
  213. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-RRDYJLLM.mjs} +1882 -1358
  214. package/dist/react-native/chunks/gzip.react-native-DNB5YNFA.mjs +3 -0
  215. package/dist/react-native/chunks/import-JYBMEJPQ.mjs +29 -0
  216. package/dist/react-native/chunks/methods-X6F36SJE.mjs +30 -0
  217. package/dist/react-native/chunks/module-QT5HZAY4.mjs +33 -0
  218. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-S4DKSWMF.mjs} +10 -9
  219. package/dist/react-native/chunks/operations-RYKTSZQA.mjs +5 -0
  220. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-JZ37TB7N.mjs} +1 -1
  221. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-RTGXJXPU.mjs} +3 -2
  222. package/dist/{browser/restore-queue-UQ6AAUVX.mjs → react-native/chunks/restore-queue-ZBVIKJWI.mjs} +2 -2
  223. package/dist/react-native/chunks/retention-PPCTP2VY.mjs +219 -0
  224. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-KUATOLYB.mjs} +1 -1
  225. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-ZBIEOZTZ.mjs} +47 -314
  226. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  227. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-2W4FTX7J.mjs} +25 -18
  228. package/dist/react-native/partial-words.mjs +6 -4
  229. package/dist/react-native/pattern.mjs +4 -3
  230. package/dist/react-native/sync.mjs +2236 -0
  231. package/dist/react-native/synonyms.mjs +7 -4
  232. package/dist/react.d.ts +10 -9
  233. package/dist/react.mjs +7 -1
  234. package/dist/{rerank-3W54N6P2.mjs → rerank-IZOB6NII.mjs} +4 -4
  235. package/dist/rerankers/http.mjs +2 -2
  236. package/dist/{react-native/chunks/restore-queue-H5VCN453.mjs → restore-queue-COI2GSSK.mjs} +2 -2
  237. package/dist/retention-55WA7GPU.mjs +58 -0
  238. package/dist/{runtime-L2DZYY6N.mjs → runtime-UZOC3NPY.mjs} +5 -5
  239. package/dist/{schema-D33CyXCy.d.ts → schema-DDauvDOv.d.ts} +1 -1
  240. package/dist/server.d.ts +22 -19
  241. package/dist/server.mjs +868 -50
  242. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-NFX2EYN2.mjs} +62 -36
  243. package/dist/sync/filesystem.d.ts +30 -0
  244. package/dist/sync/filesystem.mjs +271 -0
  245. package/dist/sync/indexeddb.d.ts +30 -0
  246. package/dist/sync/indexeddb.mjs +237 -0
  247. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  248. package/dist/{config-aIsy7mcL.d.ts → sync-uXR7MmM7.d.ts} +53 -444
  249. package/dist/sync.browser.mjs +2231 -0
  250. package/dist/sync.d.ts +23 -0
  251. package/dist/sync.mjs +2206 -0
  252. package/dist/synonyms.mjs +5 -3
  253. package/dist/{tasks-C4ElXr4P.d.ts → tasks-BShx8pdK.d.ts} +4 -4
  254. package/dist/thread-P2G2FWM2.mjs +44 -0
  255. package/dist/types-SnO7_tAI.d.ts +228 -0
  256. package/dist/vector/search-worker.d.ts +1 -1
  257. package/dist/vector/search-worker.mjs +6 -6
  258. package/dist/workers/entry.mjs +7 -7
  259. package/dist/{write-SKJ6HW7O.mjs → write-FGB62BHQ.mjs} +31 -23
  260. package/openapi.json +119 -3
  261. package/package.json +28 -8
  262. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  263. package/dist/chunk-7LDMWKP6.mjs +0 -209
  264. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  265. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  266. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  267. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  268. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,20 +1,29 @@
1
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, SEGMENT_HEADER_SIZE, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-GJKXW3SF.mjs';
1
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-G7LUMBSG.mjs';
2
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-ML6HY7T7.mjs';
3
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-J433A7P3.mjs';
2
4
  import './chunk-JNOAKDUM.mjs';
3
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QJCO3TJU.mjs';
4
- import { applyDeleteEntry, applyIndexEntry } from './chunk-B3JM6Z74.mjs';
5
- import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
6
- import './chunk-3I27KXQD.mjs';
7
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-55WYRLDZ.mjs';
8
- import { encodeDocument, toError, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-KWYQ6KX4.mjs';
9
- import { fnv1a, createRequestId } from './chunk-MPK76GVS.mjs';
5
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-QXTPSX2H.mjs';
6
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-H6QFBMVE.mjs';
7
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-ZI2V4VD7.mjs';
8
+ import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-2SHAKW2Z.mjs';
9
+ import { createRequestId } from './chunk-VI7BQR4U.mjs';
10
10
  import './chunk-JPTQZWJQ.mjs';
11
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-4PD752F6.mjs';
12
+ import { encodeDocument, openChecksummedEnvelope, crc32, requireValidFieldTypes, HEADER_SIZE } from './chunk-7TTWXUH6.mjs';
13
+ import { fnv1a } from './chunk-RA6LSBCM.mjs';
11
14
  import './chunk-26A5JFT2.mjs';
12
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, compareCodePoints, waitOutsideCountedWrite, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-IAXZAVYL.mjs';
15
+ import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
16
+ import './chunk-3I27KXQD.mjs';
17
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-3U3QR5YP.mjs';
18
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime, resolveWorkerEntry } from './chunk-U2E3HMYW.mjs';
13
19
  import './chunk-TY7L6OXO.mjs';
14
20
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
21
+ import { compareCodePoints, MIN_M, MAX_FILE_BYTES } from './chunk-7MEYN7V6.mjs';
15
22
  import './chunk-WXLBZJMT.mjs';
16
- import { NarsilError, ErrorCodes } from './chunk-PIS7ZAP7.mjs';
17
- import { decode } from '@msgpack/msgpack';
23
+ import './chunk-YJHGOJAC.mjs';
24
+ import './chunk-ML7THVLJ.mjs';
25
+ import { NarsilError, ErrorCodes } from './chunk-MYITLCHT.mjs';
26
+ import { decode, encode } from '@msgpack/msgpack';
18
27
 
19
28
  // src/persistence/durability/node-fs.browser.ts
20
29
  function filesystemUnavailable() {
@@ -516,7 +525,9 @@ function createDurableDirectory(root) {
516
525
  const fs = await getFs();
517
526
  const pathMod = await getPath();
518
527
  const resolvedBase = pathMod.resolve(root);
519
- const all = await listRecursive(resolvedBase, resolvedBase, pathMod, fs);
528
+ const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
529
+ const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
530
+ const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
520
531
  return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
521
532
  },
522
533
  async pathOf(key) {
@@ -526,1496 +537,2007 @@ function createDurableDirectory(root) {
526
537
  };
527
538
  }
528
539
 
529
- // src/persistence/durability/sha256.ts
530
- var ROUND_CONSTANTS = new Uint32Array([
531
- 1116352408,
532
- 1899447441,
533
- 3049323471,
534
- 3921009573,
535
- 961987163,
536
- 1508970993,
537
- 2453635748,
538
- 2870763221,
539
- 3624381080,
540
- 310598401,
541
- 607225278,
542
- 1426881987,
543
- 1925078388,
544
- 2162078206,
545
- 2614888103,
546
- 3248222580,
547
- 3835390401,
548
- 4022224774,
549
- 264347078,
550
- 604807628,
551
- 770255983,
552
- 1249150122,
553
- 1555081692,
554
- 1996064986,
555
- 2554220882,
556
- 2821834349,
557
- 2952996808,
558
- 3210313671,
559
- 3336571891,
560
- 3584528711,
561
- 113926993,
562
- 338241895,
563
- 666307205,
564
- 773529912,
565
- 1294757372,
566
- 1396182291,
567
- 1695183700,
568
- 1986661051,
569
- 2177026350,
570
- 2456956037,
571
- 2730485921,
572
- 2820302411,
573
- 3259730800,
574
- 3345764771,
575
- 3516065817,
576
- 3600352804,
577
- 4094571909,
578
- 275423344,
579
- 430227734,
580
- 506948616,
581
- 659060556,
582
- 883997877,
583
- 958139571,
584
- 1322822218,
585
- 1537002063,
586
- 1747873779,
587
- 1955562222,
588
- 2024104815,
589
- 2227730452,
590
- 2361852424,
591
- 2428436474,
592
- 2756734187,
593
- 3204031479,
594
- 3329325298
595
- ]);
596
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
597
- var BLOCK_BYTES = 64;
598
- var LENGTH_BYTES = 8;
599
- function rotateRight(value, bits) {
600
- return value >>> bits | value << 32 - bits;
540
+ // src/persistence/durability/sync/layout.ts
541
+ var SYNC_KEY_NUMBER_WIDTH = 16;
542
+ function syncDirectoryPrefix(indexName) {
543
+ return `${indexName}/sync/`;
601
544
  }
602
- function padded(bytes) {
603
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
604
- const message = new Uint8Array(total);
605
- message.set(bytes);
606
- message[bytes.byteLength] = 128;
607
- const view = new DataView(message.buffer);
608
- const bitLength = bytes.byteLength * 8;
609
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
610
- view.setUint32(total - 4, bitLength >>> 0, false);
611
- return view;
545
+ function publishedSnapshotKey(indexName) {
546
+ return `${syncDirectoryPrefix(indexName)}published`;
612
547
  }
613
- function sha256(bytes) {
614
- const view = padded(bytes);
615
- const state = INITIAL_STATE.slice();
616
- const words = new Uint32Array(64);
617
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
618
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
619
- for (let i = 16; i < 64; i++) {
620
- const w15 = words[i - 15];
621
- const w2 = words[i - 2];
622
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
623
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
624
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
625
- }
626
- let [a, b, c, d, e, f, g, h] = state;
627
- for (let i = 0; i < 64; i++) {
628
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
629
- const choice = e & f ^ ~e & g;
630
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
631
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
632
- const majority = a & b ^ a & c ^ b & c;
633
- const t2 = s0 + majority >>> 0;
634
- h = g;
635
- g = f;
636
- f = e;
637
- e = d + t1 >>> 0;
638
- d = c;
639
- c = b;
640
- b = a;
641
- a = t1 + t2 >>> 0;
642
- }
643
- state[0] = state[0] + a >>> 0;
644
- state[1] = state[1] + b >>> 0;
645
- state[2] = state[2] + c >>> 0;
646
- state[3] = state[3] + d >>> 0;
647
- state[4] = state[4] + e >>> 0;
648
- state[5] = state[5] + f >>> 0;
649
- state[6] = state[6] + g >>> 0;
650
- state[7] = state[7] + h >>> 0;
651
- }
652
- const digest = new Uint8Array(32);
653
- const out = new DataView(digest.buffer);
654
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
655
- return digest;
548
+ function historyRoot(indexName) {
549
+ return `${indexName}/history/`;
656
550
  }
657
-
658
- // src/persistence/durability/restore-journal.ts
659
- var RESTORE_JOURNAL_PREFIX = ".swap-";
660
- var RESTORE_STAGING_PREFIX = ".restore-";
661
- var RETIRED_INDEX_PREFIX = ".retired-";
662
- var LOCK_FILE_KEY = ".narsil.lock";
663
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
664
- function digestOf(indexName) {
665
- let hex = "";
666
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
667
- return hex;
551
+ function historyPrefix(indexName, partitionId) {
552
+ return `${historyRoot(indexName)}${partitionId}/`;
668
553
  }
669
- function journalKey(indexName) {
670
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
554
+ function walSegmentPrefix(indexName, partitionId) {
555
+ return `${indexName}/wal/${partitionId}/`;
671
556
  }
672
- function randomSuffix() {
673
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
557
+ function formatSyncKeyNumber(value) {
558
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
559
+ }
560
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
561
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
562
+ }
563
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
564
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
565
+ }
566
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
567
+ function parseHistoryKey(indexName, key) {
568
+ const root = historyRoot(indexName);
569
+ if (!key.startsWith(root)) return null;
570
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
571
+ if (match === null) return null;
572
+ const partitionId = Number.parseInt(match[1], 10);
573
+ const startSeqNo = Number.parseInt(match[2], 10);
574
+ const movedAtMs = Number.parseInt(match[3], 10);
575
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
576
+ return null;
577
+ }
578
+ return { key, partitionId, startSeqNo, movedAtMs };
674
579
  }
675
- function newStagingPrefix() {
676
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
580
+ function parseWalSegmentStart(indexName, partitionId, key) {
581
+ const prefix = walSegmentPrefix(indexName, partitionId);
582
+ if (!key.startsWith(prefix)) return null;
583
+ const tail = key.slice(prefix.length);
584
+ if (!/^\d{16}$/.test(tail)) return null;
585
+ const value = Number.parseInt(tail, 10);
586
+ return Number.isSafeInteger(value) ? value : null;
677
587
  }
678
- function newRetiredPrefix() {
679
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
588
+
589
+ // src/persistence/durability/sync/history.ts
590
+ function compareHistorySegments(a, b) {
591
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
592
+ }
593
+ async function listHistorySegments(directory, indexName, partitionId) {
594
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
595
+ const segments = [];
596
+ for (const key of await directory.list(prefix)) {
597
+ const segment = parseHistoryKey(indexName, key);
598
+ if (segment !== null) segments.push(segment);
599
+ }
600
+ return segments.sort(compareHistorySegments);
601
+ }
602
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
603
+ await directory.move(
604
+ walSegmentKey(indexName, partitionId, startSeqNo),
605
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
606
+ );
680
607
  }
681
- function encodeJournal(indexName, stagingPrefix) {
682
- return new TextEncoder().encode(`${indexName}
683
- ${stagingPrefix}`);
608
+ async function removeHistory(directory, indexName, partitionId) {
609
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
610
+ await directory.remove(segment.key);
611
+ }
684
612
  }
685
- function isStagingPrefix(prefix) {
686
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
613
+
614
+ // src/persistence/durability/checkpoint.ts
615
+ function walPrefix(indexName, partitionId) {
616
+ return `${indexName}/wal/${partitionId}/`;
687
617
  }
688
- function corruptJournal(key, reason) {
689
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
618
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
619
+ for (const { partitionId, lastSeqNo } of checkpoint) {
620
+ const prefix = walPrefix(indexName, partitionId);
621
+ const markerBytes = await directory.read(`${prefix}commit`);
622
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
623
+ if (marker === null) {
624
+ continue;
625
+ }
626
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
627
+ const segments = await directory.list(prefix);
628
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
629
+ for (let i = 0; i < startSeqNos.length; i += 1) {
630
+ const start = startSeqNos[i];
631
+ if (start >= activeSegmentSeqNo) {
632
+ break;
633
+ }
634
+ const nextStart = startSeqNos[i + 1];
635
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
636
+ if (segmentCoversBeyondCheckpoint) {
637
+ break;
638
+ }
639
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
640
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
641
+ }
642
+ }
690
643
  }
691
- function readJournal(key, bytes) {
692
- const text = new TextDecoder().decode(bytes);
693
- const lineEnd = text.indexOf("\n");
694
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
695
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
696
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
697
- corruptJournal(key, "names an index that no index name can match");
644
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
645
+ for (const [partitionId, partition] of [...partitions]) {
646
+ if (partitionId < partitionCount) continue;
647
+ try {
648
+ await partition.walWriter.close();
649
+ } catch (err) {
650
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
651
+ }
652
+ partitions.delete(partitionId);
698
653
  }
699
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
700
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
701
- corruptJournal(key, "names no staging prefix that a restore writes");
654
+ const walRoot = `${indexName}/wal/`;
655
+ for (const key of await directory.list(walRoot)) {
656
+ const partitionId = walKeyPartitionId(key, walRoot);
657
+ if (partitionId === null || partitionId < partitionCount) continue;
658
+ await directory.remove(key);
702
659
  }
703
- return { kind: "journal", indexName, stagingPrefix };
704
660
  }
705
- async function listedKeys(storage, indexName, stagingPrefix) {
706
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
707
- if (manifestBytes === null) {
708
- throw new NarsilError(
709
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
710
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
711
- { indexName, stagingPrefix }
712
- );
661
+ function walKeyPartitionId(key, walRoot) {
662
+ const tail = key.slice(walRoot.length);
663
+ const separator = tail.indexOf("/");
664
+ if (separator <= 0) {
665
+ return null;
713
666
  }
714
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
715
- if (decoded.kind !== "current") {
716
- throw new NarsilError(
717
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
718
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
719
- { indexName }
720
- );
667
+ const segment = tail.slice(0, separator);
668
+ if (!/^\d+$/.test(segment)) {
669
+ return null;
721
670
  }
722
- const root = `${indexName}/`;
723
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
724
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
725
- return keys;
671
+ const value = Number.parseInt(segment, 10);
672
+ return Number.isSafeInteger(value) ? value : null;
726
673
  }
727
- function withoutTrailingSlash(prefix) {
728
- return prefix.slice(0, -1);
674
+ function parseSegmentStartSeqNo(key, prefix) {
675
+ if (!key.startsWith(prefix)) {
676
+ return null;
677
+ }
678
+ const tail = key.slice(prefix.length);
679
+ if (!/^\d{16}$/.test(tail)) {
680
+ return null;
681
+ }
682
+ const value = Number.parseInt(tail, 10);
683
+ return Number.isSafeInteger(value) ? value : null;
729
684
  }
730
- async function finishJournal(storage, indexName, stagingPrefix) {
731
- const root = `${indexName}/`;
732
- const listed = await listedKeys(storage, indexName, stagingPrefix);
733
- const staged = await storage.list(stagingPrefix);
734
- let retiredPrefix = null;
735
- if (storage.moveTree !== void 0 && staged.length > 0) {
736
- retiredPrefix = newRetiredPrefix();
737
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
738
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
739
- } else {
740
- const stagedKeys = new Set(staged);
741
- for (const key of listed) {
742
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
685
+ function formatSegmentStart(startSeqNo) {
686
+ return startSeqNo.toString().padStart(16, "0");
687
+ }
688
+
689
+ // src/persistence/durability/marker-writer.ts
690
+ function markerKey(indexName, partitionId) {
691
+ return `${indexName}/wal/${partitionId}/commit`;
692
+ }
693
+ async function createMarkerWriter(directory, indexName, partitionId) {
694
+ const key = markerKey(indexName, partitionId);
695
+ const handle = await directory.markerHandle(key);
696
+ const existing = await handle.read();
697
+ const current = readCommitMarker(existing);
698
+ const createdFresh = existing.length === 0;
699
+ let lastSlot = current?.slotIndex ?? null;
700
+ let writeSeq = current?.state.writeSeq ?? 0;
701
+ return {
702
+ created: createdFresh,
703
+ existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
704
+ async commit(state) {
705
+ const slotIndex = nextMarkerSlot(lastSlot);
706
+ writeSeq += 1;
707
+ const slot = encodeMarkerSlot({ ...state, writeSeq });
708
+ await handle.writeSlot(markerSlotOffset(slotIndex), slot);
709
+ lastSlot = slotIndex;
710
+ },
711
+ async close() {
712
+ await handle.close();
713
+ }
714
+ };
715
+ }
716
+
717
+ // src/persistence/durability/recovery.ts
718
+ async function removeLeftoverRewriteFiles(directory, names) {
719
+ for (const indexName of names) {
720
+ for (const place of ["rebuilt", "received"]) {
721
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
743
722
  }
744
723
  }
745
- for (const key of await storage.list(root)) {
746
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
747
- }
748
- await storage.remove(journalKey(indexName));
749
- return retiredPrefix;
750
724
  }
751
- async function removePrefix(storage, prefix) {
752
- if (storage.removeTree !== void 0) {
753
- await storage.removeTree(prefix);
754
- return;
725
+ async function listPersistedIndexes(directory) {
726
+ const metaKeys = await directory.list("");
727
+ const names = [];
728
+ for (const key of metaKeys) {
729
+ if (key.endsWith("/meta")) {
730
+ names.push(key.slice(0, -"/meta".length));
731
+ }
755
732
  }
756
- for (const key of await storage.list(prefix)) await storage.remove(key);
733
+ return names;
757
734
  }
758
- async function commitJournal(storage, indexName, stagingPrefix) {
759
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
735
+ async function loadMetadata(directory, indexName) {
736
+ const bytes = await directory.read(`${indexName}/meta`);
737
+ if (bytes === null) {
738
+ return null;
739
+ }
740
+ const { metadata } = await readMetadataEnvelope(bytes);
741
+ return metadata;
760
742
  }
761
- function isTopLevelJournal(key) {
762
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
743
+ async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
744
+ await directory.remove(nextSegmentKey(indexName, partitionId));
745
+ const read = await readWalSegments(directory, indexName, partitionId);
746
+ if (read === null) {
747
+ return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
748
+ }
749
+ const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
750
+ if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
751
+ throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
752
+ }
753
+ let highestSeqNo = fromSeqNoExclusive;
754
+ let highestPrimaryTerm = 0;
755
+ const replay = async (entries) => {
756
+ for (const entry of entries) {
757
+ if (entry.seqNo <= highestSeqNo) {
758
+ continue;
759
+ }
760
+ if (turnIsSpent()) await yieldTurn();
761
+ applyEntry(entry, deps);
762
+ highestSeqNo = entry.seqNo;
763
+ if (entry.primaryTerm > highestPrimaryTerm) {
764
+ highestPrimaryTerm = entry.primaryTerm;
765
+ }
766
+ }
767
+ };
768
+ await replay(durableEntries);
769
+ if (activeTail !== null) {
770
+ await replay(activeTail.entries);
771
+ }
772
+ await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
773
+ if (activeTail !== null) {
774
+ await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
775
+ }
776
+ return { highestSeqNo, highestPrimaryTerm };
763
777
  }
764
- async function removeStagingPrefixes(storage) {
765
- const prefixes = /* @__PURE__ */ new Set();
766
- for (const key of await storage.list(".")) {
767
- const slash = key.indexOf("/");
768
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
778
+ async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
779
+ const lastReplayed = tail.entries.at(-1);
780
+ if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
781
+ const handle = await directory.appendHandle(tail.key);
782
+ try {
783
+ if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
784
+ else await handle.sync();
785
+ } finally {
786
+ await handle.close();
787
+ }
788
+ if (lastReplayed === void 0) return;
789
+ const writer = await createMarkerWriter(directory, indexName, partitionId);
790
+ try {
791
+ await writer.commit({
792
+ activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
793
+ durableByteLength: tail.cleanEnd,
794
+ highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
795
+ });
796
+ } finally {
797
+ await writer.close();
769
798
  }
770
- for (const prefix of prefixes) await removePrefix(storage, prefix);
771
799
  }
772
- async function recoverRestoreJournals(storage) {
773
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
774
- if (!isTopLevelJournal(key)) continue;
775
- const bytes = await storage.read(key);
776
- if (bytes === null) continue;
777
- const reading = readJournal(key, bytes);
778
- if (reading.kind === "leftover") {
779
- await storage.remove(key);
780
- continue;
800
+ async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
801
+ for (const { key, startSeqNo } of segments) {
802
+ if (startSeqNo > activeSegmentSeqNo) {
803
+ await directory.remove(key);
781
804
  }
782
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
783
805
  }
784
- await removeStagingPrefixes(storage);
785
806
  }
786
-
787
- // src/persistence/durability/restore-directory.ts
788
- function missing(error) {
789
- return error.code === "ENOENT";
807
+ function applyEntry(entry, deps) {
808
+ deps.writeRecords?.replayed(entry);
809
+ if (entry.operation === "DELETE") {
810
+ applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
811
+ return;
812
+ }
813
+ applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
790
814
  }
791
- function storageFailed(code, message, key, error) {
792
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
815
+ function snapshotCheckpointFor(checkpoint, partitionId) {
816
+ return checkpointLastSeqNo(checkpoint, partitionId);
793
817
  }
794
- function directoryRestoreStorage(directory) {
795
- return {
796
- directory,
797
- read: (key) => directory.read(key),
798
- write: (key, chunks) => directory.atomicWrite(key, chunks),
799
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
800
- remove: (key) => directory.remove(key),
801
- list: (prefix) => directory.list(prefix),
802
- moveKey: (from, to) => directory.move(from, to),
803
- pathOf: (key) => directory.pathOf(key),
804
- async moveTree(from, to) {
805
- const source = await directory.pathOf(from);
806
- const target = await directory.pathOf(to);
807
- const fs = await getFs();
808
- try {
809
- await fs.rename(source, target);
810
- } catch (error) {
811
- if (missing(error)) return;
812
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
813
- }
814
- await directory.syncDirectoryOf(to);
815
- },
816
- async removeTree(prefix) {
817
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
818
- const fs = await getFs();
818
+
819
+ // src/persistence/durability/checkpoint-capture.ts
820
+ function whileNoMutationApplies(partitions, capture2) {
821
+ return new Promise((resolve, reject) => {
822
+ let release = () => void 0;
823
+ const released = new Promise((settle) => {
824
+ release = settle;
825
+ });
826
+ function captureNow() {
819
827
  try {
820
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
828
+ resolve(capture2());
821
829
  } catch (error) {
822
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
830
+ reject(error);
831
+ } finally {
832
+ release();
823
833
  }
824
- await directory.syncDirectoryOf(key);
825
834
  }
826
- };
835
+ let waiting = partitions.length;
836
+ if (waiting === 0) {
837
+ captureNow();
838
+ return;
839
+ }
840
+ for (const partition of partitions) {
841
+ partition.appendChain = partition.appendChain.then(() => {
842
+ waiting -= 1;
843
+ if (waiting === 0) captureNow();
844
+ return released;
845
+ });
846
+ }
847
+ });
827
848
  }
828
- function recoverDirectoryRestores(directory) {
829
- return recoverRestoreJournals(directoryRestoreStorage(directory));
849
+ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
850
+ for (const [partitionId, partition] of [...indexState.partitions]) {
851
+ await partition.appendChain;
852
+ if (partition.failed !== null) continue;
853
+ try {
854
+ await partition.walWriter.commit();
855
+ } catch (error) {
856
+ failPartition2(partitionId, partition, error);
857
+ }
858
+ }
830
859
  }
831
-
832
- // src/persistence/durability/restore-staging.ts
833
- function toError2(error) {
834
- return error instanceof Error ? error : new Error(String(error));
860
+ async function openNewLogSegments(indexState, failPartition2) {
861
+ const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
862
+ try {
863
+ if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
864
+ } catch (error) {
865
+ failPartition2(partitionId, partition, error);
866
+ return;
867
+ }
868
+ const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
869
+ failPartition2(partitionId, partition, error);
870
+ });
871
+ partition.appendChain = rolled;
872
+ await rolled;
873
+ });
874
+ await Promise.all(rolls);
835
875
  }
836
- function pathUnder(path, root) {
837
- if (!path.startsWith(root)) return false;
838
- const next = path.charAt(root.length);
839
- return next === "" || next === "/" || next === "\\";
876
+ function aTargetFailed(indexState, targets) {
877
+ return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
840
878
  }
841
- function stagedDirectory(directory, staged, unstaged) {
842
- const { readRange, byteLength } = directory;
843
- return {
844
- root: directory.root,
845
- appendHandle: (key) => directory.appendHandle(staged(key)),
846
- markerHandle: (key) => directory.markerHandle(staged(key)),
847
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
848
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
849
- read: (key) => directory.read(staged(key)),
850
- remove: (key) => directory.remove(staged(key)),
851
- move: (from, to) => directory.move(staged(from), staged(to)),
852
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
853
- pathOf: (key) => directory.pathOf(staged(key)),
854
- ...readRange === void 0 ? {} : {
855
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
856
- },
857
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
858
- };
879
+ function noWholePartitions() {
880
+ return { contents: [], captured: [] };
859
881
  }
860
- function createRestoreStaging(storage, indexName, options = {}) {
861
- const stagingPrefix = newStagingPrefix();
862
- const indexRoot = `${indexName}/`;
863
- let committed = false;
864
- const staged = (key) => {
865
- if (!key.startsWith(indexRoot)) {
866
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
867
- key
868
- });
869
- }
870
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
871
- };
872
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
873
- const store = {
874
- read: (key) => storage.read(staged(key)),
875
- write: (key, chunks) => storage.write(staged(key), chunks),
876
- remove: (key) => storage.remove(staged(key)),
877
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
878
- };
879
- const pathOf = storage.pathOf;
880
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
881
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
882
- let retiredPrefix = null;
883
- async function finishOnStorage() {
884
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
885
- }
886
- return {
887
- store,
888
- async relocation() {
889
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
890
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
891
- const to = await pathOf.call(storage, indexName);
892
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
893
- },
894
- async commit() {
895
- try {
896
- await commitJournal(storage, indexName, stagingPrefix);
897
- } catch (error) {
898
- try {
899
- await storage.remove(journalKey(indexName));
900
- } catch (removal) {
901
- committed = true;
902
- options.onStorageLost?.(toError2(removal));
903
- }
904
- throw error;
905
- }
906
- committed = true;
907
- },
908
- async finish() {
909
- try {
910
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
911
- } catch (error) {
912
- options.onStorageLost?.(toError2(error));
882
+ function wholePartitionsWhereMostChanged(manager, rules) {
883
+ const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
884
+ return (targets) => {
885
+ const whole = { contents: [], captured: [] };
886
+ const changed = targets.filter(
887
+ (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
888
+ );
889
+ const mostOfEachChanged = changed.every((target) => {
890
+ const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
891
+ const documents = manager.getPartition(target.partitionId).count();
892
+ return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
893
+ });
894
+ const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
895
+ if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
896
+ for (const { partitionId } of changed) {
897
+ const capture2 = captureRecords?.(partitionId);
898
+ const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
899
+ if (captured !== null) {
900
+ whole.captured.push(captured);
901
+ } else if (everyPartition || !aWorkerCanSerialise) {
902
+ whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
913
903
  }
914
- },
915
- async removeRetired() {
916
- const prefix = retiredPrefix;
917
- retiredPrefix = null;
918
- if (prefix !== null) await removePrefix(storage, prefix);
919
- },
920
- async discard() {
921
- if (committed) return;
922
- await removePrefix(storage, stagingPrefix);
923
904
  }
905
+ return whole;
924
906
  };
925
907
  }
926
- function adapterRestoreStorage(adapter) {
927
- const store = adapterCheckpointStore(adapter);
928
- return {
929
- read: (key) => store.read(key),
930
- write: (key, chunks) => store.write(key, chunks),
931
- writeJournal: (key, bytes) => store.write(key, [bytes]),
932
- remove: (key) => store.remove(key),
933
- list: (prefix) => store.list(prefix),
934
- async moveKey(from, to) {
935
- const bytes = await store.read(from);
936
- if (bytes === null) return;
937
- await store.write(to, [bytes]);
938
- await store.remove(from);
908
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
909
+ for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
910
+ return whileNoMutationApplies([...indexState.partitions.values()], () => {
911
+ onCaptured?.();
912
+ const documentCount = manager.countDocuments();
913
+ const targets = [];
914
+ let recordsLeftInLog = 0;
915
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
916
+ const partition = indexState.partitions.get(partitionId);
917
+ if (partition !== void 0 && partition.failed !== null) continue;
918
+ const appliedSeqNo = partition?.appliedSeqNo ?? 0;
919
+ const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
920
+ appliedSeqNo,
921
+ snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
922
+ );
923
+ recordsLeftInLog += appliedSeqNo - lastSeqNo;
924
+ const target = {
925
+ partitionId,
926
+ lastSeqNo,
927
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
928
+ };
929
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
930
+ targets.push(target);
939
931
  }
940
- };
941
- }
942
-
943
- // src/persistence/durability/archive-checkpoint.ts
944
- function toError3(error) {
945
- return error instanceof Error ? error : new Error(String(error));
946
- }
947
- function stageDirectoryRestore(deps, indexName) {
948
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
949
- async finish(run) {
950
- try {
951
- await run();
952
- } catch (error) {
953
- deps.markFatal(toError3(error));
954
- throw error;
932
+ const vectorPlans = /* @__PURE__ */ new Map();
933
+ try {
934
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
935
+ const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
936
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
937
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
955
938
  }
956
- },
957
- onStorageLost: deps.markFatal
939
+ const memoryMatchesTheTargets = recordsLeftInLog === 0;
940
+ const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
941
+ return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
942
+ } catch (error) {
943
+ for (const plan of vectorPlans.values()) plan.release();
944
+ throw error;
945
+ }
958
946
  });
959
947
  }
960
- function releaser(held, indexName) {
961
- let released = false;
962
- return () => {
963
- if (released) return;
964
- released = true;
965
- const count = (held.get(indexName) ?? 1) - 1;
966
- if (count <= 0) held.delete(indexName);
967
- else held.set(indexName, count);
968
- };
948
+ async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
949
+ if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
950
+ const priorVectors = priorManifest?.vectors ?? [];
951
+ const priorCheckpoint = priorManifest?.checkpoint ?? [];
952
+ const noPartitionChanged = capture2.targets.every(
953
+ (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
954
+ );
955
+ const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
956
+ (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
957
+ );
958
+ const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
959
+ if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
960
+ return { vectors: priorVectors, layouts: [], written: [] };
961
+ }
962
+ const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
963
+ return { vectors: written.refs, layouts: written.layouts, written: written.written };
969
964
  }
970
- function createArchiveCheckpointAccess(deps) {
971
- const store = directoryCheckpointStore(deps.directory);
972
- return {
973
- async openArchiveCheckpoint(indexName) {
974
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
975
- const release = releaser(deps.held, indexName);
976
- try {
977
- await deps.checkpoint(indexName, false);
978
- await deps.checkpoint(indexName, true);
979
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
980
- if (manifestBytes === null) {
981
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
982
- indexName
983
- });
984
- }
985
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
986
- if (decoded.kind !== "current") {
987
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
988
- indexName
989
- });
990
- }
991
- return {
992
- manifestBytes,
993
- manifest: decoded.manifest,
994
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
995
- readRange: async (ref, offset, length) => {
996
- const readRange = deps.directory.readRange;
997
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
998
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
999
- return whole.slice(offset, offset + length);
1000
- },
1001
- release,
1002
- directory: deps.directory
1003
- };
1004
- } catch (error) {
1005
- release();
1006
- throw error;
1007
- }
1008
- },
1009
- stageRestore(indexName) {
1010
- return stageDirectoryRestore(deps, indexName);
1011
- },
1012
- adoptRestoredCheckpoint(indexName, checkpoint) {
1013
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1014
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1015
- }
1016
- }
1017
- };
965
+
966
+ // src/persistence/durability/checkpoint-worker-entry.ts
967
+ function resolveCheckpointWorkerEntry() {
968
+ return resolveWorkerEntry(
969
+ import.meta.url,
970
+ /\/src\/persistence\/durability\/[^/]+$/,
971
+ "persistence/durability/checkpoint-worker.mjs"
972
+ );
1018
973
  }
1019
974
 
1020
- // src/persistence/durability/checkpoint.ts
1021
- function walPrefix(indexName, partitionId) {
1022
- return `${indexName}/wal/${partitionId}/`;
1023
- }
1024
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
1025
- for (const { partitionId, lastSeqNo } of checkpoint) {
1026
- const prefix = walPrefix(indexName, partitionId);
1027
- const markerBytes = await directory.read(`${prefix}commit`);
1028
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1029
- if (marker === null) {
1030
- continue;
975
+ // src/persistence/durability/checkpoint-worker-dispatch.ts
976
+ var workerUsable = true;
977
+ var pooledWorker = null;
978
+ var workerBusy = false;
979
+ var spawnedWorkerCount = 0;
980
+ var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
981
+ var idleTimer = null;
982
+ async function spawnWorker() {
983
+ const entryPoint = resolveCheckpointWorkerEntry();
984
+ if (entryPoint === null) return null;
985
+ try {
986
+ const worker = await spawnNodeWorker(new URL(entryPoint));
987
+ if (worker === null) {
988
+ return null;
1031
989
  }
1032
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
1033
- const segments = await directory.list(prefix);
1034
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
1035
- for (let i = 0; i < startSeqNos.length; i += 1) {
1036
- const start = startSeqNos[i];
1037
- if (start >= activeSegmentSeqNo) {
1038
- break;
1039
- }
1040
- const nextStart = startSeqNos[i + 1];
1041
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
1042
- if (segmentCoversBeyondCheckpoint) {
1043
- break;
1044
- }
1045
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
990
+ if (typeof worker.unref === "function") {
991
+ worker.unref();
1046
992
  }
993
+ worker.on("error", () => discardWorker(worker));
994
+ worker.on("exit", () => discardWorker(worker));
995
+ spawnedWorkerCount += 1;
996
+ return worker;
997
+ } catch {
998
+ return null;
1047
999
  }
1048
1000
  }
1049
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
1050
- for (const [partitionId, partition] of [...partitions]) {
1051
- if (partitionId < partitionCount) continue;
1052
- try {
1053
- await partition.walWriter.close();
1054
- } catch (err) {
1055
- onCloseError(err instanceof Error ? err : new Error(String(err)));
1056
- }
1057
- partitions.delete(partitionId);
1001
+ function discardWorker(worker) {
1002
+ if (pooledWorker === worker) {
1003
+ pooledWorker = null;
1058
1004
  }
1059
- const walRoot = `${indexName}/wal/`;
1060
- for (const key of await directory.list(walRoot)) {
1061
- const partitionId = walKeyPartitionId(key, walRoot);
1062
- if (partitionId === null || partitionId < partitionCount) continue;
1063
- await directory.remove(key);
1005
+ try {
1006
+ void worker.terminate();
1007
+ } catch {
1064
1008
  }
1065
1009
  }
1066
- function walKeyPartitionId(key, walRoot) {
1067
- const tail = key.slice(walRoot.length);
1068
- const separator = tail.indexOf("/");
1069
- if (separator <= 0) {
1070
- return null;
1071
- }
1072
- const segment = tail.slice(0, separator);
1073
- if (!/^\d+$/.test(segment)) {
1074
- return null;
1010
+ function unrefTimer(timer) {
1011
+ if (typeof timer.unref === "function") {
1012
+ timer.unref();
1075
1013
  }
1076
- const value = Number.parseInt(segment, 10);
1077
- return Number.isSafeInteger(value) ? value : null;
1014
+ return timer;
1078
1015
  }
1079
- function parseSegmentStartSeqNo(key, prefix) {
1080
- if (!key.startsWith(prefix)) {
1016
+ function runWorker(worker, request) {
1017
+ return new Promise((resolve) => {
1018
+ let settled = false;
1019
+ let timeoutId = armSilenceTimeout();
1020
+ function armSilenceTimeout() {
1021
+ return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
1022
+ }
1023
+ const onMessage = (msg) => {
1024
+ const response = msg;
1025
+ if (response.type === "heartbeat") {
1026
+ clearTimeout(timeoutId);
1027
+ timeoutId = armSilenceTimeout();
1028
+ return;
1029
+ }
1030
+ if (response.type === "success") {
1031
+ settle({ written: response.segments, timedOut: false }, false);
1032
+ } else {
1033
+ settle({ written: null, timedOut: false }, true);
1034
+ }
1035
+ };
1036
+ const onError = () => {
1037
+ settle({ written: null, timedOut: false }, true);
1038
+ };
1039
+ const onExit = () => {
1040
+ settle({ written: null, timedOut: false }, true);
1041
+ };
1042
+ function settle(outcome, discard) {
1043
+ if (settled) {
1044
+ return;
1045
+ }
1046
+ settled = true;
1047
+ clearTimeout(timeoutId);
1048
+ worker.off("message", onMessage);
1049
+ worker.off("error", onError);
1050
+ worker.off("exit", onExit);
1051
+ if (discard) {
1052
+ discardWorker(worker);
1053
+ }
1054
+ resolve(outcome);
1055
+ }
1056
+ worker.on("message", onMessage);
1057
+ worker.on("error", onError);
1058
+ worker.on("exit", onExit);
1059
+ try {
1060
+ worker.postMessage(request);
1061
+ } catch {
1062
+ settle({ written: null, timedOut: false }, true);
1063
+ }
1064
+ });
1065
+ }
1066
+ function delay(ms) {
1067
+ return new Promise((resolve) => {
1068
+ unrefTimer(setTimeout(resolve, ms));
1069
+ });
1070
+ }
1071
+ function checkpointWorkerIsUsable() {
1072
+ const runtime = detectRuntime();
1073
+ return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
1074
+ }
1075
+ async function runCheckpointOnWorker(request) {
1076
+ const runtime = detectRuntime();
1077
+ if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
1081
1078
  return null;
1082
1079
  }
1083
- const tail = key.slice(prefix.length);
1084
- if (!/^\d{16}$/.test(tail)) {
1080
+ if (workerBusy) {
1085
1081
  return null;
1086
1082
  }
1087
- const value = Number.parseInt(tail, 10);
1088
- return Number.isSafeInteger(value) ? value : null;
1089
- }
1090
- function formatSegmentStart(startSeqNo) {
1091
- return startSeqNo.toString().padStart(16, "0");
1092
- }
1093
-
1094
- // src/persistence/durability/marker-writer.ts
1095
- function markerKey(indexName, partitionId) {
1096
- return `${indexName}/wal/${partitionId}/commit`;
1097
- }
1098
- async function createMarkerWriter(directory, indexName, partitionId) {
1099
- const key = markerKey(indexName, partitionId);
1100
- const handle = await directory.markerHandle(key);
1101
- const existing = await handle.read();
1102
- const current = readCommitMarker(existing);
1103
- const createdFresh = existing.length === 0;
1104
- let lastSlot = current?.slotIndex ?? null;
1105
- let writeSeq = current?.state.writeSeq ?? 0;
1106
- return {
1107
- created: createdFresh,
1108
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
1109
- async commit(state) {
1110
- const slotIndex = nextMarkerSlot(lastSlot);
1111
- writeSeq += 1;
1112
- const slot = encodeMarkerSlot({ ...state, writeSeq });
1113
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
1114
- lastSlot = slotIndex;
1115
- },
1116
- async close() {
1117
- await handle.close();
1083
+ workerBusy = true;
1084
+ cancelIdleRetirement();
1085
+ try {
1086
+ if (pooledWorker === null) {
1087
+ pooledWorker = await spawnWorker();
1118
1088
  }
1119
- };
1120
- }
1121
-
1122
- // src/persistence/durability/recovery.ts
1123
- async function removeLeftoverRewriteFiles(directory, names) {
1124
- for (const indexName of names) {
1125
- for (const place of ["rebuilt", "received"]) {
1126
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1089
+ const worker = pooledWorker;
1090
+ if (worker === null) {
1091
+ workerUsable = false;
1092
+ return null;
1127
1093
  }
1128
- }
1129
- }
1130
- async function listPersistedIndexes(directory) {
1131
- const metaKeys = await directory.list("");
1132
- const names = [];
1133
- for (const key of metaKeys) {
1134
- if (key.endsWith("/meta")) {
1135
- names.push(key.slice(0, -"/meta".length));
1094
+ const outcome = await runWorker(worker, request);
1095
+ if (outcome.written !== null) {
1096
+ return outcome.written;
1097
+ }
1098
+ if (outcome.timedOut) {
1099
+ await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
1136
1100
  }
1101
+ return null;
1102
+ } finally {
1103
+ workerBusy = false;
1104
+ retireWorkerOnceIdle();
1137
1105
  }
1138
- return names;
1139
1106
  }
1140
- async function loadMetadata(directory, indexName) {
1141
- const bytes = await directory.read(`${indexName}/meta`);
1142
- if (bytes === null) {
1143
- return null;
1107
+ function cancelIdleRetirement() {
1108
+ if (idleTimer !== null) {
1109
+ clearTimeout(idleTimer);
1110
+ idleTimer = null;
1144
1111
  }
1145
- const { metadata } = await readMetadataEnvelope(bytes);
1146
- return metadata;
1147
1112
  }
1148
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1149
- await directory.remove(nextSegmentKey(indexName, partitionId));
1150
- const read = await readWalSegments(directory, indexName, partitionId);
1151
- if (read === null) {
1152
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
1113
+ function retireWorkerOnceIdle() {
1114
+ cancelIdleRetirement();
1115
+ idleTimer = unrefTimer(
1116
+ setTimeout(() => {
1117
+ idleTimer = null;
1118
+ if (!workerBusy) terminateCheckpointWorker();
1119
+ }, workerIdleMs)
1120
+ );
1121
+ }
1122
+ function terminateCheckpointWorker() {
1123
+ cancelIdleRetirement();
1124
+ const worker = pooledWorker;
1125
+ if (worker === null) {
1126
+ return;
1153
1127
  }
1154
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
1155
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
1156
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
1128
+ pooledWorker = null;
1129
+ try {
1130
+ void worker.terminate();
1131
+ } catch {
1157
1132
  }
1158
- let highestSeqNo = fromSeqNoExclusive;
1159
- let highestPrimaryTerm = 0;
1160
- const replay = async (entries) => {
1161
- for (const entry of entries) {
1162
- if (entry.seqNo <= highestSeqNo) {
1163
- continue;
1164
- }
1165
- if (turnIsSpent()) await yieldTurn();
1166
- applyEntry(entry, deps);
1167
- highestSeqNo = entry.seqNo;
1168
- if (entry.primaryTerm > highestPrimaryTerm) {
1169
- highestPrimaryTerm = entry.primaryTerm;
1170
- }
1171
- }
1172
- };
1173
- await replay(durableEntries);
1174
- if (activeTail !== null) {
1175
- await replay(activeTail.entries);
1176
- }
1177
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
1178
- if (activeTail !== null) {
1179
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
1133
+ }
1134
+
1135
+ // src/persistence/durability/checkpoint-write.ts
1136
+ function workerCanWriteEverySegment(input) {
1137
+ if (!input.canOffload || input.wholePartitions.length > 0) return false;
1138
+ const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
1139
+ if (!analysesTextInProcessOnly) return true;
1140
+ return input.targets.every(
1141
+ (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
1142
+ );
1143
+ }
1144
+ async function writeIndexCheckpointSegments(input) {
1145
+ const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
1146
+ const { keepsWriteRecords } = input;
1147
+ const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
1148
+ root: directory.root,
1149
+ metadata,
1150
+ targets,
1151
+ compactionThreshold,
1152
+ prior,
1153
+ capturedPartitions: [...capturedPartitions],
1154
+ keepsWriteRecords
1155
+ }) : null;
1156
+ if (offloaded !== null) {
1157
+ return offloaded;
1180
1158
  }
1181
- return { highestSeqNo, highestPrimaryTerm };
1159
+ return writeCheckpointSegments({
1160
+ store: directoryCheckpointStore(directory),
1161
+ directory,
1162
+ metadata,
1163
+ targets,
1164
+ compactionThreshold,
1165
+ prior,
1166
+ wholePartitions,
1167
+ capturedPartitions,
1168
+ keepsWriteRecords
1169
+ });
1182
1170
  }
1183
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
1184
- const lastReplayed = tail.entries.at(-1);
1185
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
1186
- const handle = await directory.appendHandle(tail.key);
1187
- try {
1188
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
1189
- else await handle.sync();
1190
- } finally {
1191
- await handle.close();
1171
+
1172
+ // src/persistence/durability/segment/vector-adoption.ts
1173
+ function recordWrittenVectors(vectorIndexes, written) {
1174
+ for (const field of written) {
1175
+ vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
1192
1176
  }
1193
- if (lastReplayed === void 0) return;
1194
- const writer = await createMarkerWriter(directory, indexName, partitionId);
1195
- try {
1196
- await writer.commit({
1197
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
1198
- durableByteLength: tail.cleanEnd,
1199
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
1177
+ }
1178
+ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
1179
+ const pathOf = store.pathOf;
1180
+ if (pathOf === void 0) return;
1181
+ for (const layout of layouts) {
1182
+ const vecIndex = vectorIndexes.get(layout.fieldPath);
1183
+ if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
1184
+ const path = await pathOf.call(store, layout.key);
1185
+ await vecIndex.adoptDiskLayout({
1186
+ path,
1187
+ docIds: layout.docIds,
1188
+ dead: layout.dead,
1189
+ vectorsOffset: layout.vectorsOffset
1200
1190
  });
1201
- } finally {
1202
- await writer.close();
1203
1191
  }
1204
1192
  }
1205
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
1206
- for (const { key, startSeqNo } of segments) {
1207
- if (startSeqNo > activeSegmentSeqNo) {
1208
- await directory.remove(key);
1209
- }
1193
+ async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
1194
+ const pathOf = store.pathOf;
1195
+ if (pathOf === void 0) return [...keys];
1196
+ const read = /* @__PURE__ */ new Set();
1197
+ for (const vectorIndex of vectorIndexes.values()) {
1198
+ for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
1210
1199
  }
1211
- }
1212
- function applyEntry(entry, deps) {
1213
- deps.writeRecords?.replayed(entry);
1214
- if (entry.operation === "DELETE") {
1215
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
1216
- return;
1200
+ if (read.size === 0) return [...keys];
1201
+ const unread = [];
1202
+ for (const key of keys) {
1203
+ if (!read.has(await pathOf.call(store, key))) unread.push(key);
1217
1204
  }
1218
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
1219
- }
1220
- function snapshotCheckpointFor(checkpoint, partitionId) {
1221
- return checkpointLastSeqNo(checkpoint, partitionId);
1205
+ return unread;
1222
1206
  }
1223
1207
 
1224
- // src/persistence/durability/checkpoint-capture.ts
1225
- function whileNoMutationApplies(partitions, capture2) {
1226
- return new Promise((resolve, reject) => {
1227
- let release = () => void 0;
1228
- const released = new Promise((settle) => {
1229
- release = settle;
1230
- });
1231
- function captureNow() {
1232
- try {
1233
- resolve(capture2());
1234
- } catch (error) {
1235
- reject(error);
1236
- } finally {
1237
- release();
1238
- }
1239
- }
1240
- let waiting = partitions.length;
1241
- if (waiting === 0) {
1242
- captureNow();
1243
- return;
1244
- }
1245
- for (const partition of partitions) {
1246
- partition.appendChain = partition.appendChain.then(() => {
1247
- waiting -= 1;
1248
- if (waiting === 0) captureNow();
1249
- return released;
1250
- });
1251
- }
1252
- });
1208
+ // src/persistence/durability/write-stall.ts
1209
+ function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1210
+ return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
1253
1211
  }
1254
- async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
1255
- for (const [partitionId, partition] of [...indexState.partitions]) {
1256
- await partition.appendChain;
1257
- if (partition.failed !== null) continue;
1258
- try {
1259
- await partition.walWriter.commit();
1260
- } catch (error) {
1261
- failPartition2(partitionId, partition, error);
1262
- }
1212
+ async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1213
+ while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
1214
+ await waitOutsideCountedWrite(
1215
+ () => new Promise((resume) => {
1216
+ indexState.stalledWrites.push(resume);
1217
+ })
1218
+ );
1263
1219
  }
1264
1220
  }
1265
- async function openNewLogSegments(indexState, failPartition2) {
1266
- const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
1267
- try {
1268
- if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
1269
- } catch (error) {
1270
- failPartition2(partitionId, partition, error);
1271
- return;
1272
- }
1273
- const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
1274
- failPartition2(partitionId, partition, error);
1275
- });
1276
- partition.appendChain = rolled;
1277
- await rolled;
1278
- });
1279
- await Promise.all(rolls);
1221
+ function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
1222
+ const counted = indexState.mutationsSinceCheckpoint;
1223
+ const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
1224
+ indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
1225
+ indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
1226
+ releaseStalledWrites(indexState);
1280
1227
  }
1281
- function aTargetFailed(indexState, targets) {
1282
- return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
1228
+ function releaseStalledWrites(indexState) {
1229
+ const waiting = indexState.stalledWrites.splice(0);
1230
+ for (const resume of waiting) resume();
1283
1231
  }
1284
- function noWholePartitions() {
1285
- return { contents: [], captured: [] };
1232
+
1233
+ // src/persistence/durability/checkpoint-run.ts
1234
+ async function bothOrTheFirstFailure(first, second) {
1235
+ const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
1236
+ if (firstSettled.status === "rejected") throw firstSettled.reason;
1237
+ if (secondSettled.status === "rejected") throw secondSettled.reason;
1238
+ return [firstSettled.value, secondSettled.value];
1286
1239
  }
1287
- function wholePartitionsWhereMostChanged(manager, rules) {
1288
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
1289
- return (targets) => {
1290
- const whole = { contents: [], captured: [] };
1291
- const changed = targets.filter(
1292
- (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1293
- );
1294
- const mostOfEachChanged = changed.every((target) => {
1295
- const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
1296
- const documents = manager.getPartition(target.partitionId).count();
1297
- return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
1298
- });
1299
- const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
1300
- if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
1301
- for (const { partitionId } of changed) {
1302
- const capture2 = captureRecords?.(partitionId);
1303
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
1304
- if (captured !== null) {
1305
- whole.captured.push(captured);
1306
- } else if (everyPartition || !aWorkerCanSerialise) {
1307
- whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
1308
- }
1240
+ async function runDurableCheckpoint(input) {
1241
+ try {
1242
+ let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1243
+ while (recordsLeftInLog > 0 && input.mayContinue()) {
1244
+ recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1309
1245
  }
1310
- return whole;
1311
- };
1246
+ } catch (err) {
1247
+ if (!isSystemError(err)) throw err;
1248
+ const error = new NarsilError(
1249
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1250
+ `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
1251
+ {
1252
+ indexName: input.indexName,
1253
+ directory: input.directory.root,
1254
+ cause: err.message
1255
+ }
1256
+ );
1257
+ input.markFatal(error);
1258
+ throw error;
1259
+ }
1312
1260
  }
1313
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
1314
- for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
1315
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
1316
- onCaptured?.();
1317
- const documentCount = manager.countDocuments();
1318
- const targets = [];
1319
- let recordsLeftInLog = 0;
1320
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
1321
- const partition = indexState.partitions.get(partitionId);
1322
- if (partition !== void 0 && partition.failed !== null) continue;
1323
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
1324
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
1325
- appliedSeqNo,
1326
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
1327
- );
1328
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
1329
- const target = {
1330
- partitionId,
1331
- lastSeqNo,
1332
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
1333
- };
1334
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
1335
- targets.push(target);
1336
- }
1337
- const vectorPlans = /* @__PURE__ */ new Map();
1338
- try {
1339
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
1340
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
1341
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
1342
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
1343
- }
1344
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
1345
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
1346
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
1347
- } catch (error) {
1348
- for (const plan of vectorPlans.values()) plan.release();
1349
- throw error;
1350
- }
1351
- });
1261
+ function isSystemError(err) {
1262
+ return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
1352
1263
  }
1353
- async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
1354
- if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
1355
- const priorVectors = priorManifest?.vectors ?? [];
1356
- const priorCheckpoint = priorManifest?.checkpoint ?? [];
1357
- const noPartitionChanged = capture2.targets.every(
1358
- (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
1359
- );
1360
- const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
1361
- (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
1362
- );
1363
- const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
1364
- if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
1365
- return { vectors: priorVectors, layouts: [], written: [] };
1264
+ async function writeOneBoundedCheckpoint(input) {
1265
+ const manager = input.hooks.getManager(input.indexName);
1266
+ if (manager === void 0) {
1267
+ return 0;
1366
1268
  }
1367
- const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
1368
- return { vectors: written.refs, layouts: written.layouts, written: written.written };
1369
- }
1370
-
1371
- // src/persistence/durability/checkpoint-worker-entry.ts
1372
- function resolveCheckpointWorkerEntry() {
1373
- return resolveWorkerEntry(
1374
- import.meta.url,
1375
- /\/src\/persistence\/durability\/[^/]+$/,
1376
- "persistence/durability/checkpoint-worker.mjs"
1269
+ const metadata = input.hooks.buildTermsMetadata(input.indexName);
1270
+ if (metadata === void 0) {
1271
+ return 0;
1272
+ }
1273
+ const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
1274
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1275
+ const store = directoryCheckpointStore(input.directory);
1276
+ const writeRecords = input.hooks.writeRecords ?? null;
1277
+ const { indexName } = input;
1278
+ const priorManifest = await readCurrentManifest(store, input.indexName);
1279
+ const textOnDisk = metadata.textStorage === "disk";
1280
+ const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1281
+ priorCheckpoint: priorManifest?.checkpoint ?? [],
1282
+ priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1283
+ everyPartition: fromMemory,
1284
+ aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1285
+ ...writeRecords === null ? {} : {
1286
+ captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1287
+ }
1288
+ });
1289
+ let changesSinceCapture = null;
1290
+ const capture2 = await captureCheckpoint(
1291
+ input.indexState,
1292
+ manager,
1293
+ vectorIndexes,
1294
+ textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1295
+ fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1296
+ priorManifest?.vectors ?? [],
1297
+ writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1298
+ textOnDisk ? () => {
1299
+ changesSinceCapture = manager.openCheckpointChangeLog();
1300
+ } : void 0,
1301
+ input.pinsRetiredVectors
1377
1302
  );
1378
- }
1379
-
1380
- // src/persistence/durability/checkpoint-worker-dispatch.ts
1381
- var workerUsable = true;
1382
- var pooledWorker = null;
1383
- var workerBusy = false;
1384
- var spawnedWorkerCount = 0;
1385
- var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
1386
- var idleTimer = null;
1387
- async function spawnWorker() {
1388
- const entryPoint = resolveCheckpointWorkerEntry();
1389
- if (entryPoint === null) return null;
1390
1303
  try {
1391
- const worker = await spawnNodeWorker(new URL(entryPoint));
1392
- if (worker === null) {
1393
- return null;
1394
- }
1395
- if (typeof worker.unref === "function") {
1396
- worker.unref();
1397
- }
1398
- worker.on("error", () => discardWorker(worker));
1399
- worker.on("exit", () => discardWorker(worker));
1400
- spawnedWorkerCount += 1;
1401
- return worker;
1402
- } catch {
1403
- return null;
1304
+ await openNewLogSegments(input.indexState, input.failPartition);
1305
+ return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1306
+ return changesSinceCapture;
1307
+ });
1308
+ } finally {
1309
+ for (const plan of capture2.vectorPlans.values()) plan.release();
1310
+ changesSinceCapture?.close();
1404
1311
  }
1405
1312
  }
1406
- function discardWorker(worker) {
1407
- if (pooledWorker === worker) {
1408
- pooledWorker = null;
1409
- }
1410
- try {
1411
- void worker.terminate();
1412
- } catch {
1313
+ async function adoptWrittenSegments(input, manager, partitions, changes) {
1314
+ const { indexName, indexState, hooks } = input;
1315
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-M4JNBV6V.mjs');
1316
+ const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1317
+ if (adoption === null) return [];
1318
+ const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1319
+ if (changes.partitionsReplaced) return false;
1320
+ const changed = changes.takeWrittenDocIds();
1321
+ swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1322
+ hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1323
+ });
1324
+ hooks.queueDiskAdoption?.({
1325
+ type: "adoptDiskSegments",
1326
+ indexName,
1327
+ partitions: actionPartitionsOf(adoption),
1328
+ changed: [...changed],
1329
+ requestId: createRequestId()
1330
+ });
1331
+ return true;
1332
+ });
1333
+ if (!adopted) {
1334
+ closeUnusedSegments(adoption);
1335
+ return [];
1413
1336
  }
1337
+ await hooks.awaitCopiesOfIndex?.(indexName);
1338
+ return adoption.droppedKeys;
1414
1339
  }
1415
- function unrefTimer(timer) {
1416
- if (typeof timer.unref === "function") {
1417
- timer.unref();
1340
+ async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
1341
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1342
+ const writeRecords = input.hooks.writeRecords ?? null;
1343
+ const { targets, documentCount } = capture2;
1344
+ await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1345
+ if (aTargetFailed(input.indexState, targets)) return 0;
1346
+ const [liveVectors, segments] = await bothOrTheFirstFailure(
1347
+ writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
1348
+ writeIndexCheckpointSegments({
1349
+ directory: input.directory,
1350
+ metadata,
1351
+ targets,
1352
+ compactionThreshold: input.compactionThreshold,
1353
+ canOffload: input.canOffload,
1354
+ prior: priorManifest,
1355
+ wholePartitions: capture2.wholePartitions.contents,
1356
+ capturedPartitions: capture2.wholePartitions.captured,
1357
+ keepsWriteRecords: writeRecords !== null
1358
+ })
1359
+ );
1360
+ await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1361
+ if (aTargetFailed(input.indexState, targets)) return 0;
1362
+ const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
1363
+ input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
1364
+ recordWrittenVectors(vectorIndexes, liveVectors.written);
1365
+ await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
1366
+ const changes = changesSinceCapture();
1367
+ const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1368
+ if (!input.holdsCheckpointFiles()) {
1369
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1370
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-M4JNBV6V.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1371
+ const keptForSync = await input.keepsCheckpointKey();
1372
+ await removeCheckpointGarbage(
1373
+ store,
1374
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
1375
+ );
1418
1376
  }
1419
- return timer;
1420
- }
1421
- function runWorker(worker, request) {
1422
- return new Promise((resolve) => {
1423
- let settled = false;
1424
- let timeoutId = armSilenceTimeout();
1425
- function armSilenceTimeout() {
1426
- return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
1427
- }
1428
- const onMessage = (msg) => {
1429
- const response = msg;
1430
- if (response.type === "heartbeat") {
1431
- clearTimeout(timeoutId);
1432
- timeoutId = armSilenceTimeout();
1433
- return;
1434
- }
1435
- if (response.type === "success") {
1436
- settle({ written: response.segments, timedOut: false }, false);
1437
- } else {
1438
- settle({ written: null, timedOut: false }, true);
1439
- }
1440
- };
1441
- const onError = () => {
1442
- settle({ written: null, timedOut: false }, true);
1443
- };
1444
- const onExit = () => {
1445
- settle({ written: null, timedOut: false }, true);
1446
- };
1447
- function settle(outcome, discard) {
1448
- if (settled) {
1449
- return;
1450
- }
1451
- settled = true;
1452
- clearTimeout(timeoutId);
1453
- worker.off("message", onMessage);
1454
- worker.off("error", onError);
1455
- worker.off("exit", onExit);
1456
- if (discard) {
1457
- discardWorker(worker);
1458
- }
1459
- resolve(outcome);
1460
- }
1461
- worker.on("message", onMessage);
1462
- worker.on("error", onError);
1463
- worker.on("exit", onExit);
1464
- try {
1465
- worker.postMessage(request);
1466
- } catch {
1467
- settle({ written: null, timedOut: false }, true);
1377
+ const checkpointDocumentCount = documentCount;
1378
+ await input.queueMetadataWrite(input.indexName, async () => {
1379
+ const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
1380
+ if (checkpointMetadata === void 0) {
1381
+ return;
1468
1382
  }
1383
+ const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
1384
+ await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1385
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1469
1386
  });
1387
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1388
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
1389
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
1390
+ await reclaimWalBeyondCount(
1391
+ input.directory,
1392
+ input.indexName,
1393
+ manager.partitionCount,
1394
+ input.indexState.partitions,
1395
+ input.markFatal
1396
+ );
1397
+ const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1398
+ noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1399
+ const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1400
+ return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
1470
1401
  }
1471
- function delay(ms) {
1472
- return new Promise((resolve) => {
1473
- unrefTimer(setTimeout(resolve, ms));
1474
- });
1475
- }
1476
- function checkpointWorkerIsUsable() {
1477
- const runtime = detectRuntime();
1478
- return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
1479
- }
1480
- async function runCheckpointOnWorker(request) {
1481
- const runtime = detectRuntime();
1482
- if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
1483
- return null;
1484
- }
1485
- if (workerBusy) {
1486
- return null;
1487
- }
1488
- workerBusy = true;
1489
- cancelIdleRetirement();
1490
- try {
1491
- if (pooledWorker === null) {
1492
- pooledWorker = await spawnWorker();
1493
- }
1494
- const worker = pooledWorker;
1495
- if (worker === null) {
1496
- workerUsable = false;
1497
- return null;
1498
- }
1499
- const outcome = await runWorker(worker, request);
1500
- if (outcome.written !== null) {
1501
- return outcome.written;
1502
- }
1503
- if (outcome.timedOut) {
1504
- await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
1505
- }
1506
- return null;
1507
- } finally {
1508
- workerBusy = false;
1509
- retireWorkerOnceIdle();
1510
- }
1511
- }
1512
- function cancelIdleRetirement() {
1513
- if (idleTimer !== null) {
1514
- clearTimeout(idleTimer);
1515
- idleTimer = null;
1516
- }
1517
- }
1518
- function retireWorkerOnceIdle() {
1519
- cancelIdleRetirement();
1520
- idleTimer = unrefTimer(
1521
- setTimeout(() => {
1522
- idleTimer = null;
1523
- if (!workerBusy) terminateCheckpointWorker();
1524
- }, workerIdleMs)
1525
- );
1526
- }
1527
- function terminateCheckpointWorker() {
1528
- cancelIdleRetirement();
1529
- const worker = pooledWorker;
1530
- if (worker === null) {
1531
- return;
1532
- }
1533
- pooledWorker = null;
1534
- try {
1535
- void worker.terminate();
1536
- } catch {
1402
+ function recordsBeyond(indexState, targets) {
1403
+ let records = 0;
1404
+ for (const [partitionId, partition] of indexState.partitions) {
1405
+ if (partition.failed !== null) continue;
1406
+ records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
1537
1407
  }
1408
+ return records;
1538
1409
  }
1539
1410
 
1540
- // src/persistence/durability/checkpoint-write.ts
1541
- function workerCanWriteEverySegment(input) {
1542
- if (!input.canOffload || input.wholePartitions.length > 0) return false;
1543
- const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
1544
- if (!analysesTextInProcessOnly) return true;
1545
- return input.targets.every(
1546
- (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
1547
- );
1411
+ // src/persistence/durability/checkpoint-turn.ts
1412
+ function createIndexLoads(vectorIndexesOf2) {
1413
+ const loads = /* @__PURE__ */ new Map();
1414
+ const startGraphRepairs = (indexName) => {
1415
+ for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1416
+ vectorIndex.scheduleBuild();
1417
+ vectorIndex.startShuffledRebuildWhenDue();
1418
+ }
1419
+ };
1420
+ return {
1421
+ track(indexName, load, startsGraphRepairs = true) {
1422
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1423
+ loads.set(
1424
+ indexName,
1425
+ loaded.catch(() => void 0)
1426
+ );
1427
+ return loaded;
1428
+ },
1429
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1430
+ startGraphRepairs
1431
+ };
1548
1432
  }
1549
- async function writeIndexCheckpointSegments(input) {
1550
- const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
1551
- const { keepsWriteRecords } = input;
1552
- const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
1553
- root: directory.root,
1554
- metadata,
1555
- targets,
1556
- compactionThreshold,
1557
- prior,
1558
- capturedPartitions: [...capturedPartitions],
1559
- keepsWriteRecords
1560
- }) : null;
1561
- if (offloaded !== null) {
1562
- return offloaded;
1563
- }
1564
- return writeCheckpointSegments({
1565
- store: directoryCheckpointStore(directory),
1566
- directory,
1567
- metadata,
1568
- targets,
1569
- compactionThreshold,
1570
- prior,
1571
- wholePartitions,
1572
- capturedPartitions,
1573
- keepsWriteRecords
1433
+ async function switchInTurn(turns, turn) {
1434
+ let swapped = false;
1435
+ await checkpointInTurn(turns, {
1436
+ joinsInFlight: false,
1437
+ mayStart: turn.mayStart,
1438
+ perform: async () => {
1439
+ swapped = turn.swap();
1440
+ if (swapped) await turn.checkpoint();
1441
+ },
1442
+ afterSettling: turn.afterSettling
1574
1443
  });
1444
+ return swapped;
1575
1445
  }
1576
-
1577
- // src/persistence/durability/segment/vector-adoption.ts
1578
- function recordWrittenVectors(vectorIndexes, written) {
1579
- for (const field of written) {
1580
- vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
1581
- }
1446
+ async function checkpointInTurn(turns, turn) {
1447
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1448
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1449
+ if (!turn.mayStart()) return;
1450
+ const run = turn.perform().finally(() => {
1451
+ turns.checkpointInFlight = null;
1452
+ turn.afterSettling();
1453
+ });
1454
+ turns.checkpointInFlight = run;
1455
+ return run;
1582
1456
  }
1583
- async function adoptVectorLayouts(store, vectorIndexes, layouts) {
1584
- const pathOf = store.pathOf;
1585
- if (pathOf === void 0) return;
1586
- for (const layout of layouts) {
1587
- const vecIndex = vectorIndexes.get(layout.fieldPath);
1588
- if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
1589
- const path = await pathOf.call(store, layout.key);
1590
- await vecIndex.adoptDiskLayout({
1591
- path,
1592
- docIds: layout.docIds,
1593
- dead: layout.dead,
1594
- vectorsOffset: layout.vectorsOffset
1595
- });
1596
- }
1457
+
1458
+ // src/persistence/durability/held-checkpoints.ts
1459
+ function failedPartitionIds(indexState) {
1460
+ const failed = [];
1461
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1462
+ return failed;
1597
1463
  }
1598
- async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
1599
- const pathOf = store.pathOf;
1600
- if (pathOf === void 0) return [...keys];
1601
- const read = /* @__PURE__ */ new Set();
1602
- for (const vectorIndex of vectorIndexes.values()) {
1603
- for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
1604
- }
1605
- if (read.size === 0) return [...keys];
1606
- const unread = [];
1607
- for (const key of keys) {
1608
- if (!read.has(await pathOf.call(store, key))) unread.push(key);
1464
+ function createHeldCheckpoints(hooks) {
1465
+ const reported = /* @__PURE__ */ new WeakSet();
1466
+ function heldReport(indexName, indexState) {
1467
+ const partitionIds = failedPartitionIds(indexState);
1468
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1469
+ return {
1470
+ indexName,
1471
+ partitionIds,
1472
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1473
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1474
+ };
1609
1475
  }
1610
- return unread;
1476
+ return {
1477
+ refuse(indexName, indexState) {
1478
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1479
+ if (partitionIds.length === 0) return;
1480
+ throw new NarsilError(
1481
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
1482
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1483
+ { indexName, partitionIds }
1484
+ );
1485
+ },
1486
+ holds(indexName, indexState) {
1487
+ const held = heldReport(indexName, indexState);
1488
+ if (held === null) return false;
1489
+ const unreported = held.partitionIds.some((partitionId) => {
1490
+ const partition = indexState.partitions.get(partitionId);
1491
+ return partition !== void 0 && !reported.has(partition);
1492
+ });
1493
+ if (unreported) {
1494
+ for (const partitionId of held.partitionIds) {
1495
+ const partition = indexState.partitions.get(partitionId);
1496
+ if (partition !== void 0) reported.add(partition);
1497
+ }
1498
+ hooks.onCheckpointHeld?.(held);
1499
+ }
1500
+ return true;
1501
+ },
1502
+ report(indexes) {
1503
+ const held = [];
1504
+ for (const [indexName, indexState] of indexes) {
1505
+ const entry = heldReport(indexName, indexState);
1506
+ if (entry !== null) held.push(entry);
1507
+ }
1508
+ return held;
1509
+ }
1510
+ };
1611
1511
  }
1612
1512
 
1613
- // src/persistence/durability/write-stall.ts
1614
- function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1615
- return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
1513
+ // src/persistence/durability/manager-state.ts
1514
+ function closePartitionWriter(partition) {
1515
+ return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1616
1516
  }
1617
- async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1618
- while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
1619
- await waitOutsideCountedWrite(
1620
- () => new Promise((resume) => {
1621
- indexState.stalledWrites.push(resume);
1622
- })
1623
- );
1517
+ async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1518
+ indexState.unloading = true;
1519
+ try {
1520
+ if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1521
+ await pendingMetadataWrite;
1522
+ for (const partition of indexState.partitions.values()) {
1523
+ await closeWalWriter(closePartitionWriter(partition));
1524
+ }
1525
+ } catch (error) {
1526
+ indexState.unloading = false;
1527
+ throw error;
1624
1528
  }
1625
1529
  }
1626
- function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
1627
- const counted = indexState.mutationsSinceCheckpoint;
1628
- const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
1629
- indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
1630
- indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
1631
- releaseStalledWrites(indexState);
1530
+
1531
+ // src/persistence/durability/sha256.ts
1532
+ var ROUND_CONSTANTS = new Uint32Array([
1533
+ 1116352408,
1534
+ 1899447441,
1535
+ 3049323471,
1536
+ 3921009573,
1537
+ 961987163,
1538
+ 1508970993,
1539
+ 2453635748,
1540
+ 2870763221,
1541
+ 3624381080,
1542
+ 310598401,
1543
+ 607225278,
1544
+ 1426881987,
1545
+ 1925078388,
1546
+ 2162078206,
1547
+ 2614888103,
1548
+ 3248222580,
1549
+ 3835390401,
1550
+ 4022224774,
1551
+ 264347078,
1552
+ 604807628,
1553
+ 770255983,
1554
+ 1249150122,
1555
+ 1555081692,
1556
+ 1996064986,
1557
+ 2554220882,
1558
+ 2821834349,
1559
+ 2952996808,
1560
+ 3210313671,
1561
+ 3336571891,
1562
+ 3584528711,
1563
+ 113926993,
1564
+ 338241895,
1565
+ 666307205,
1566
+ 773529912,
1567
+ 1294757372,
1568
+ 1396182291,
1569
+ 1695183700,
1570
+ 1986661051,
1571
+ 2177026350,
1572
+ 2456956037,
1573
+ 2730485921,
1574
+ 2820302411,
1575
+ 3259730800,
1576
+ 3345764771,
1577
+ 3516065817,
1578
+ 3600352804,
1579
+ 4094571909,
1580
+ 275423344,
1581
+ 430227734,
1582
+ 506948616,
1583
+ 659060556,
1584
+ 883997877,
1585
+ 958139571,
1586
+ 1322822218,
1587
+ 1537002063,
1588
+ 1747873779,
1589
+ 1955562222,
1590
+ 2024104815,
1591
+ 2227730452,
1592
+ 2361852424,
1593
+ 2428436474,
1594
+ 2756734187,
1595
+ 3204031479,
1596
+ 3329325298
1597
+ ]);
1598
+ var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
1599
+ var BLOCK_BYTES = 64;
1600
+ var LENGTH_BYTES = 8;
1601
+ function rotateRight(value, bits) {
1602
+ return value >>> bits | value << 32 - bits;
1632
1603
  }
1633
- function releaseStalledWrites(indexState) {
1634
- const waiting = indexState.stalledWrites.splice(0);
1635
- for (const resume of waiting) resume();
1604
+ function padded(bytes) {
1605
+ const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
1606
+ const message = new Uint8Array(total);
1607
+ message.set(bytes);
1608
+ message[bytes.byteLength] = 128;
1609
+ const view = new DataView(message.buffer);
1610
+ const bitLength = bytes.byteLength * 8;
1611
+ view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
1612
+ view.setUint32(total - 4, bitLength >>> 0, false);
1613
+ return view;
1614
+ }
1615
+ function sha256(bytes) {
1616
+ const view = padded(bytes);
1617
+ const state = INITIAL_STATE.slice();
1618
+ const words = new Uint32Array(64);
1619
+ for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
1620
+ for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
1621
+ for (let i = 16; i < 64; i++) {
1622
+ const w15 = words[i - 15];
1623
+ const w2 = words[i - 2];
1624
+ const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
1625
+ const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
1626
+ words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
1627
+ }
1628
+ let [a, b, c, d, e, f, g, h] = state;
1629
+ for (let i = 0; i < 64; i++) {
1630
+ const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
1631
+ const choice = e & f ^ ~e & g;
1632
+ const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
1633
+ const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
1634
+ const majority = a & b ^ a & c ^ b & c;
1635
+ const t2 = s0 + majority >>> 0;
1636
+ h = g;
1637
+ g = f;
1638
+ f = e;
1639
+ e = d + t1 >>> 0;
1640
+ d = c;
1641
+ c = b;
1642
+ b = a;
1643
+ a = t1 + t2 >>> 0;
1644
+ }
1645
+ state[0] = state[0] + a >>> 0;
1646
+ state[1] = state[1] + b >>> 0;
1647
+ state[2] = state[2] + c >>> 0;
1648
+ state[3] = state[3] + d >>> 0;
1649
+ state[4] = state[4] + e >>> 0;
1650
+ state[5] = state[5] + f >>> 0;
1651
+ state[6] = state[6] + g >>> 0;
1652
+ state[7] = state[7] + h >>> 0;
1653
+ }
1654
+ const digest = new Uint8Array(32);
1655
+ const out = new DataView(digest.buffer);
1656
+ for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
1657
+ return digest;
1636
1658
  }
1637
1659
 
1638
- // src/persistence/durability/checkpoint-run.ts
1639
- async function bothOrTheFirstFailure(first, second) {
1640
- const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
1641
- if (firstSettled.status === "rejected") throw firstSettled.reason;
1642
- if (secondSettled.status === "rejected") throw secondSettled.reason;
1643
- return [firstSettled.value, secondSettled.value];
1660
+ // src/persistence/durability/restore-journal.ts
1661
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
1662
+ var RESTORE_STAGING_PREFIX = ".restore-";
1663
+ var RETIRED_INDEX_PREFIX = ".retired-";
1664
+ var LOCK_FILE_KEY = ".narsil.lock";
1665
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
1666
+ function digestOf(indexName) {
1667
+ let hex = "";
1668
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
1669
+ return hex;
1644
1670
  }
1645
- async function runDurableCheckpoint(input) {
1646
- try {
1647
- let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1648
- while (recordsLeftInLog > 0 && input.mayContinue()) {
1649
- recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1650
- }
1651
- } catch (err) {
1652
- if (!isSystemError(err)) throw err;
1653
- const error = new NarsilError(
1654
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1655
- `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
1656
- {
1657
- indexName: input.indexName,
1658
- directory: input.directory.root,
1659
- cause: err.message
1660
- }
1671
+ function journalKey(indexName) {
1672
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
1673
+ }
1674
+ function randomSuffix() {
1675
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
1676
+ }
1677
+ function newStagingPrefix() {
1678
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
1679
+ }
1680
+ function newRetiredPrefix() {
1681
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
1682
+ }
1683
+ function encodeJournal(indexName, stagingPrefix) {
1684
+ return new TextEncoder().encode(`${indexName}
1685
+ ${stagingPrefix}`);
1686
+ }
1687
+ function isStagingPrefix(prefix) {
1688
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
1689
+ }
1690
+ function corruptJournal(key, reason) {
1691
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
1692
+ }
1693
+ function readJournal(key, bytes) {
1694
+ const text = new TextDecoder().decode(bytes);
1695
+ const lineEnd = text.indexOf("\n");
1696
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
1697
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
1698
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
1699
+ corruptJournal(key, "names an index that no index name can match");
1700
+ }
1701
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
1702
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
1703
+ corruptJournal(key, "names no staging prefix that a restore writes");
1704
+ }
1705
+ return { kind: "journal", indexName, stagingPrefix };
1706
+ }
1707
+ async function listedKeys(storage, indexName, stagingPrefix) {
1708
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
1709
+ if (manifestBytes === null) {
1710
+ throw new NarsilError(
1711
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1712
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1713
+ { indexName, stagingPrefix }
1714
+ );
1715
+ }
1716
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1717
+ if (decoded.kind !== "current") {
1718
+ throw new NarsilError(
1719
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1720
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1721
+ { indexName }
1661
1722
  );
1662
- input.markFatal(error);
1663
- throw error;
1664
1723
  }
1724
+ const root = `${indexName}/`;
1725
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1726
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1727
+ return keys;
1665
1728
  }
1666
- function isSystemError(err) {
1667
- return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
1729
+ function withoutTrailingSlash(prefix) {
1730
+ return prefix.slice(0, -1);
1668
1731
  }
1669
- async function writeOneBoundedCheckpoint(input) {
1670
- const manager = input.hooks.getManager(input.indexName);
1671
- if (manager === void 0) {
1672
- return 0;
1732
+ async function finishJournal(storage, indexName, stagingPrefix) {
1733
+ const root = `${indexName}/`;
1734
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
1735
+ const staged = await storage.list(stagingPrefix);
1736
+ let retiredPrefix = null;
1737
+ if (storage.moveTree !== void 0 && staged.length > 0) {
1738
+ retiredPrefix = newRetiredPrefix();
1739
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1740
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1741
+ } else {
1742
+ const stagedKeys = new Set(staged);
1743
+ for (const key of listed) {
1744
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1745
+ }
1673
1746
  }
1674
- const metadata = input.hooks.buildTermsMetadata(input.indexName);
1675
- if (metadata === void 0) {
1676
- return 0;
1747
+ for (const key of await storage.list(root)) {
1748
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1677
1749
  }
1678
- const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
1679
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1680
- const store = directoryCheckpointStore(input.directory);
1681
- const writeRecords = input.hooks.writeRecords ?? null;
1682
- const { indexName } = input;
1683
- const priorManifest = await readCurrentManifest(store, input.indexName);
1684
- const textOnDisk = metadata.textStorage === "disk";
1685
- const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1686
- priorCheckpoint: priorManifest?.checkpoint ?? [],
1687
- priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1688
- everyPartition: fromMemory,
1689
- aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1690
- ...writeRecords === null ? {} : {
1691
- captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1692
- }
1693
- });
1694
- let changesSinceCapture = null;
1695
- const capture2 = await captureCheckpoint(
1696
- input.indexState,
1697
- manager,
1698
- vectorIndexes,
1699
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1700
- fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1701
- priorManifest?.vectors ?? [],
1702
- writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1703
- textOnDisk ? () => {
1704
- changesSinceCapture = manager.openCheckpointChangeLog();
1705
- } : void 0,
1706
- input.pinsRetiredVectors
1707
- );
1708
- try {
1709
- await openNewLogSegments(input.indexState, input.failPartition);
1710
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1711
- return changesSinceCapture;
1712
- });
1713
- } finally {
1714
- for (const plan of capture2.vectorPlans.values()) plan.release();
1715
- changesSinceCapture?.close();
1750
+ await storage.remove(journalKey(indexName));
1751
+ return retiredPrefix;
1752
+ }
1753
+ async function removePrefix(storage, prefix) {
1754
+ if (storage.removeTree !== void 0) {
1755
+ await storage.removeTree(prefix);
1756
+ return;
1716
1757
  }
1758
+ for (const key of await storage.list(prefix)) await storage.remove(key);
1717
1759
  }
1718
- async function adoptWrittenSegments(input, manager, partitions, changes) {
1719
- const { indexName, indexState, hooks } = input;
1720
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-KAMRAJLE.mjs');
1721
- const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1722
- if (adoption === null) return [];
1723
- const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1724
- if (changes.partitionsReplaced) return false;
1725
- const changed = changes.takeWrittenDocIds();
1726
- swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1727
- hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1728
- });
1729
- hooks.queueDiskAdoption?.({
1730
- type: "adoptDiskSegments",
1731
- indexName,
1732
- partitions: actionPartitionsOf(adoption),
1733
- changed: [...changed],
1734
- requestId: createRequestId()
1735
- });
1736
- return true;
1737
- });
1738
- if (!adopted) {
1739
- closeUnusedSegments(adoption);
1740
- return [];
1760
+ async function commitJournal(storage, indexName, stagingPrefix) {
1761
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1762
+ }
1763
+ function isTopLevelJournal(key) {
1764
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1765
+ }
1766
+ async function removeStagingPrefixes(storage) {
1767
+ const prefixes = /* @__PURE__ */ new Set();
1768
+ for (const key of await storage.list(".")) {
1769
+ const slash = key.indexOf("/");
1770
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1741
1771
  }
1742
- await hooks.awaitCopiesOfIndex?.(indexName);
1743
- return adoption.droppedKeys;
1772
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
1744
1773
  }
1745
- async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
1746
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1747
- const writeRecords = input.hooks.writeRecords ?? null;
1748
- const { targets, documentCount } = capture2;
1749
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1750
- if (aTargetFailed(input.indexState, targets)) return 0;
1751
- const [liveVectors, segments] = await bothOrTheFirstFailure(
1752
- writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
1753
- writeIndexCheckpointSegments({
1754
- directory: input.directory,
1755
- metadata,
1756
- targets,
1757
- compactionThreshold: input.compactionThreshold,
1758
- canOffload: input.canOffload,
1759
- prior: priorManifest,
1760
- wholePartitions: capture2.wholePartitions.contents,
1761
- capturedPartitions: capture2.wholePartitions.captured,
1762
- keepsWriteRecords: writeRecords !== null
1763
- })
1764
- );
1765
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1766
- if (aTargetFailed(input.indexState, targets)) return 0;
1767
- const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
1768
- input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
1769
- recordWrittenVectors(vectorIndexes, liveVectors.written);
1770
- await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
1771
- const changes = changesSinceCapture();
1772
- const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1773
- if (!input.holdsCheckpointFiles()) {
1774
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1775
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-KAMRAJLE.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1776
- await removeCheckpointGarbage(
1777
- store,
1778
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
1779
- );
1774
+ async function recoverRestoreJournals(storage) {
1775
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1776
+ if (!isTopLevelJournal(key)) continue;
1777
+ const bytes = await storage.read(key);
1778
+ if (bytes === null) continue;
1779
+ const reading = readJournal(key, bytes);
1780
+ if (reading.kind === "leftover") {
1781
+ await storage.remove(key);
1782
+ continue;
1783
+ }
1784
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1780
1785
  }
1781
- const checkpointDocumentCount = documentCount;
1782
- await input.queueMetadataWrite(input.indexName, async () => {
1783
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
1784
- if (checkpointMetadata === void 0) {
1785
- return;
1786
+ await removeStagingPrefixes(storage);
1787
+ }
1788
+
1789
+ // src/persistence/durability/restore-directory.ts
1790
+ function missing(error) {
1791
+ return error.code === "ENOENT";
1792
+ }
1793
+ function storageFailed(code, message, key, error) {
1794
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1795
+ }
1796
+ function directoryRestoreStorage(directory) {
1797
+ return {
1798
+ directory,
1799
+ read: (key) => directory.read(key),
1800
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
1801
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1802
+ remove: (key) => directory.remove(key),
1803
+ list: (prefix) => directory.list(prefix),
1804
+ moveKey: (from, to) => directory.move(from, to),
1805
+ pathOf: (key) => directory.pathOf(key),
1806
+ async moveTree(from, to) {
1807
+ const source = await directory.pathOf(from);
1808
+ const target = await directory.pathOf(to);
1809
+ const fs = await getFs();
1810
+ try {
1811
+ await fs.rename(source, target);
1812
+ } catch (error) {
1813
+ if (missing(error)) return;
1814
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1815
+ }
1816
+ await directory.syncDirectoryOf(to);
1817
+ },
1818
+ async removeTree(prefix) {
1819
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1820
+ const fs = await getFs();
1821
+ try {
1822
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1823
+ } catch (error) {
1824
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1825
+ }
1826
+ await directory.syncDirectoryOf(key);
1786
1827
  }
1787
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
1788
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1789
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1790
- });
1791
- await truncateCoveredSegments(input.directory, input.indexName, targets);
1792
- await reclaimWalBeyondCount(
1793
- input.directory,
1794
- input.indexName,
1795
- manager.partitionCount,
1796
- input.indexState.partitions,
1797
- input.markFatal
1798
- );
1799
- const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1800
- noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1801
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1802
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
1828
+ };
1803
1829
  }
1804
- function recordsBeyond(indexState, targets) {
1805
- let records = 0;
1806
- for (const [partitionId, partition] of indexState.partitions) {
1807
- if (partition.failed !== null) continue;
1808
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
1830
+ function recoverDirectoryRestores(directory) {
1831
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
1832
+ }
1833
+
1834
+ // src/persistence/durability/restore-staging.ts
1835
+ function toError2(error) {
1836
+ return error instanceof Error ? error : new Error(String(error));
1837
+ }
1838
+ function pathUnder(path, root) {
1839
+ if (!path.startsWith(root)) return false;
1840
+ const next = path.charAt(root.length);
1841
+ return next === "" || next === "/" || next === "\\";
1842
+ }
1843
+ function stagedDirectory(directory, staged, unstaged) {
1844
+ const { readRange, byteLength } = directory;
1845
+ return {
1846
+ root: directory.root,
1847
+ appendHandle: (key) => directory.appendHandle(staged(key)),
1848
+ markerHandle: (key) => directory.markerHandle(staged(key)),
1849
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1850
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1851
+ read: (key) => directory.read(staged(key)),
1852
+ remove: (key) => directory.remove(staged(key)),
1853
+ move: (from, to) => directory.move(staged(from), staged(to)),
1854
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1855
+ pathOf: (key) => directory.pathOf(staged(key)),
1856
+ ...readRange === void 0 ? {} : {
1857
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1858
+ },
1859
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1860
+ };
1861
+ }
1862
+ function createRestoreStaging(storage, indexName, options = {}) {
1863
+ const stagingPrefix = newStagingPrefix();
1864
+ const indexRoot = `${indexName}/`;
1865
+ let committed = false;
1866
+ const staged = (key) => {
1867
+ if (!key.startsWith(indexRoot)) {
1868
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1869
+ key
1870
+ });
1871
+ }
1872
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1873
+ };
1874
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1875
+ const store = {
1876
+ read: (key) => storage.read(staged(key)),
1877
+ write: (key, chunks) => storage.write(staged(key), chunks),
1878
+ remove: (key) => storage.remove(staged(key)),
1879
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1880
+ };
1881
+ const pathOf = storage.pathOf;
1882
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1883
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1884
+ let retiredPrefix = null;
1885
+ async function finishOnStorage() {
1886
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1809
1887
  }
1810
- return records;
1888
+ return {
1889
+ store,
1890
+ async relocation() {
1891
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1892
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1893
+ const to = await pathOf.call(storage, indexName);
1894
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1895
+ },
1896
+ async commit() {
1897
+ try {
1898
+ await commitJournal(storage, indexName, stagingPrefix);
1899
+ } catch (error) {
1900
+ try {
1901
+ await storage.remove(journalKey(indexName));
1902
+ } catch (removal) {
1903
+ committed = true;
1904
+ options.onStorageLost?.(toError2(removal));
1905
+ }
1906
+ throw error;
1907
+ }
1908
+ committed = true;
1909
+ },
1910
+ async finish() {
1911
+ try {
1912
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1913
+ } catch (error) {
1914
+ options.onStorageLost?.(toError2(error));
1915
+ }
1916
+ },
1917
+ async removeRetired() {
1918
+ const prefix = retiredPrefix;
1919
+ retiredPrefix = null;
1920
+ if (prefix !== null) await removePrefix(storage, prefix);
1921
+ },
1922
+ async discard() {
1923
+ if (committed) return;
1924
+ await removePrefix(storage, stagingPrefix);
1925
+ }
1926
+ };
1927
+ }
1928
+ function adapterRestoreStorage(adapter) {
1929
+ const store = adapterCheckpointStore(adapter);
1930
+ return {
1931
+ read: (key) => store.read(key),
1932
+ write: (key, chunks) => store.write(key, chunks),
1933
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
1934
+ remove: (key) => store.remove(key),
1935
+ list: (prefix) => store.list(prefix),
1936
+ async moveKey(from, to) {
1937
+ const bytes = await store.read(from);
1938
+ if (bytes === null) return;
1939
+ await store.write(to, [bytes]);
1940
+ await store.remove(from);
1941
+ }
1942
+ };
1811
1943
  }
1812
1944
 
1813
- // src/persistence/durability/checkpoint-turn.ts
1814
- function createIndexLoads(vectorIndexesOf2) {
1815
- const loads = /* @__PURE__ */ new Map();
1816
- const startGraphRepairs = (indexName) => {
1817
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1818
- vectorIndex.scheduleBuild();
1819
- vectorIndex.startShuffledRebuildWhenDue();
1945
+ // src/persistence/durability/archive-checkpoint.ts
1946
+ function toError3(error) {
1947
+ return error instanceof Error ? error : new Error(String(error));
1948
+ }
1949
+ function stageDirectoryRestore(deps, indexName) {
1950
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1951
+ async finish(run) {
1952
+ try {
1953
+ await run();
1954
+ } catch (error) {
1955
+ deps.markFatal(toError3(error));
1956
+ throw error;
1957
+ }
1958
+ },
1959
+ onStorageLost: deps.markFatal
1960
+ });
1961
+ }
1962
+ function releaser(held, indexName) {
1963
+ let released = false;
1964
+ return () => {
1965
+ if (released) return;
1966
+ released = true;
1967
+ const count = (held.get(indexName) ?? 1) - 1;
1968
+ if (count <= 0) held.delete(indexName);
1969
+ else held.set(indexName, count);
1970
+ };
1971
+ }
1972
+ function createArchiveCheckpointAccess(deps) {
1973
+ const store = directoryCheckpointStore(deps.directory);
1974
+ return {
1975
+ async openArchiveCheckpoint(indexName) {
1976
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1977
+ const release = releaser(deps.held, indexName);
1978
+ try {
1979
+ await deps.checkpoint(indexName, false);
1980
+ await deps.checkpoint(indexName, true);
1981
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
1982
+ if (manifestBytes === null) {
1983
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1984
+ indexName
1985
+ });
1986
+ }
1987
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1988
+ if (decoded.kind !== "current") {
1989
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1990
+ indexName
1991
+ });
1992
+ }
1993
+ return {
1994
+ manifestBytes,
1995
+ manifest: decoded.manifest,
1996
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1997
+ readRange: async (ref, offset, length) => {
1998
+ const readRange = deps.directory.readRange;
1999
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
2000
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
2001
+ return whole.slice(offset, offset + length);
2002
+ },
2003
+ release,
2004
+ directory: deps.directory
2005
+ };
2006
+ } catch (error) {
2007
+ release();
2008
+ throw error;
2009
+ }
2010
+ },
2011
+ stageRestore(indexName) {
2012
+ return stageDirectoryRestore(deps, indexName);
2013
+ },
2014
+ adoptRestoredCheckpoint(indexName, checkpoint) {
2015
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
2016
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
2017
+ }
1820
2018
  }
1821
2019
  };
2020
+ }
2021
+
2022
+ // src/persistence/durability/manager-unload.ts
2023
+ function createIndexUnloading(deps) {
2024
+ const { directory, hooks, indexes, metadataWrites } = deps;
1822
2025
  return {
1823
- track(indexName, load, startsGraphRepairs = true) {
1824
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1825
- loads.set(
1826
- indexName,
1827
- loaded.catch(() => void 0)
1828
- );
1829
- return loaded;
2026
+ async removeIndex(indexName) {
2027
+ hooks.writeRecords?.dropIndex(indexName);
2028
+ const indexState = indexes.get(indexName);
2029
+ if (indexState !== void 0) {
2030
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
2031
+ indexes.delete(indexName);
2032
+ metadataWrites.forget(indexName);
2033
+ }
2034
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
2035
+ for (const key of await directory.list(`${indexName}/`)) {
2036
+ await directory.remove(key);
2037
+ }
2038
+ deps.forgetSyncState(indexName);
1830
2039
  },
1831
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1832
- startGraphRepairs
2040
+ async unloadIndex(indexName) {
2041
+ const indexState = indexes.get(indexName);
2042
+ if (indexState === void 0) {
2043
+ return;
2044
+ }
2045
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
2046
+ indexes.delete(indexName);
2047
+ metadataWrites.forget(indexName);
2048
+ deps.forgetSyncState(indexName);
2049
+ }
1833
2050
  };
1834
2051
  }
1835
- async function switchInTurn(turns, turn) {
1836
- let swapped = false;
1837
- await checkpointInTurn(turns, {
1838
- joinsInFlight: false,
1839
- mayStart: turn.mayStart,
1840
- perform: async () => {
1841
- swapped = turn.swap();
1842
- if (swapped) await turn.checkpoint();
1843
- },
1844
- afterSettling: turn.afterSettling
1845
- });
1846
- return swapped;
1847
- }
1848
- async function checkpointInTurn(turns, turn) {
1849
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1850
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1851
- if (!turn.mayStart()) return;
1852
- const run = turn.perform().finally(() => {
1853
- turns.checkpointInFlight = null;
1854
- turn.afterSettling();
1855
- });
1856
- turns.checkpointInFlight = run;
1857
- return run;
1858
- }
1859
2052
 
1860
- // src/persistence/durability/directory-watch.ts
1861
- function unrefInterval(tick, intervalMs) {
1862
- const timer = setInterval(tick, intervalMs);
1863
- if (typeof timer.unref === "function") timer.unref();
1864
- return timer;
1865
- }
1866
- function createDirectoryWatch(directory, isStopped, onLost) {
1867
- let expected = null;
1868
- let timer = null;
1869
- let lock = null;
1870
- function cannotCheck(err) {
1871
- return new NarsilError(
1872
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1873
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
1874
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
1875
- );
2053
+ // src/persistence/durability/sync/frame-cursor.ts
2054
+ var SegmentFileMissing = class extends Error {
2055
+ };
2056
+ function corrupt(key, message, details) {
2057
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
2058
+ }
2059
+ function isMissingFile(error) {
2060
+ return error?.code === "ENOENT";
2061
+ }
2062
+ var FrameCursor = class {
2063
+ constructor(directory, key, limit) {
2064
+ this.directory = directory;
2065
+ this.key = key;
2066
+ this.limit = limit;
2067
+ }
2068
+ buffer = new Uint8Array(0);
2069
+ bufferStart = 0;
2070
+ whole = null;
2071
+ async load(offset, length) {
2072
+ const readRange = this.directory.readRange;
2073
+ if (readRange === void 0) {
2074
+ if (this.whole === null) {
2075
+ const bytes = await this.directory.read(this.key);
2076
+ if (bytes === null) throw new SegmentFileMissing(this.key);
2077
+ this.whole = bytes;
2078
+ }
2079
+ this.buffer = this.whole;
2080
+ this.bufferStart = 0;
2081
+ return;
2082
+ }
2083
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
2084
+ try {
2085
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
2086
+ } catch (error) {
2087
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
2088
+ throw error;
2089
+ }
2090
+ this.bufferStart = offset;
1876
2091
  }
1877
- async function currentIdentity() {
2092
+ async bytesAt(offset, length) {
2093
+ const end = offset + length;
2094
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
2095
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
2096
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
2097
+ }
2098
+ async checkHeader() {
2099
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
2100
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
2101
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
2102
+ }
2103
+ async frameLengthAt(offset) {
2104
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
2105
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
2106
+ 0,
2107
+ false
2108
+ );
2109
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
2110
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
2111
+ }
2112
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
2113
+ }
2114
+ async frameAt(offset, frameBytes) {
2115
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
2116
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
2117
+ const payload = body.subarray(0, recordLength);
2118
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
2119
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
2120
+ let seqNo;
1878
2121
  try {
1879
- return await directory.identity?.() ?? null;
1880
- } catch (err) {
1881
- return cannotCheck(err);
2122
+ seqNo = decodeWalEntry(payload).seqNo;
2123
+ } catch (error) {
2124
+ corrupt(this.key, "A log record fails to decode", {
2125
+ offset,
2126
+ cause: error instanceof Error ? error.message : String(error)
2127
+ });
1882
2128
  }
2129
+ return { seqNo, payload: payload.slice(), frameBytes };
2130
+ }
2131
+ };
2132
+
2133
+ // src/persistence/durability/sync/log-reader.ts
2134
+ async function walSources(directory, indexName, partitionId) {
2135
+ const prefix = walSegmentPrefix(indexName, partitionId);
2136
+ const markerBytes = await directory.read(`${prefix}commit`);
2137
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
2138
+ if (marker === null) return { sources: [], head: 0 };
2139
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
2140
+ const sources = [];
2141
+ for (const key of await directory.list(prefix)) {
2142
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
2143
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
2144
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
2145
+ if (readableBytes === null || readableBytes === void 0) continue;
2146
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
2147
+ }
2148
+ return { sources, head: highestDurableSeqNo };
2149
+ }
2150
+ async function openPartitionLog(directory, indexName, partitionId) {
2151
+ const { sources, head } = await walSources(directory, indexName, partitionId);
2152
+ const byStart = /* @__PURE__ */ new Map();
2153
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
2154
+ const readableBytes = await directory.byteLength?.(segment.key);
2155
+ if (readableBytes === null || readableBytes === void 0) continue;
2156
+ byStart.set(segment.startSeqNo, {
2157
+ key: segment.key,
2158
+ startSeqNo: segment.startSeqNo,
2159
+ readableBytes,
2160
+ inHistory: true
2161
+ });
1883
2162
  }
1884
- async function lockTakenOver() {
1885
- if (lock === null) return null;
2163
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
2164
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
2165
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
2166
+ }
2167
+ function sourceHolding(log, seqNo) {
2168
+ let found = -1;
2169
+ for (let i = 0; i < log.sources.length; i++) {
2170
+ if (log.sources[i].startSeqNo > seqNo) break;
2171
+ found = i;
2172
+ }
2173
+ return found;
2174
+ }
2175
+ async function offsetAfter(cursor, source, afterSeqNo) {
2176
+ let offset = SEGMENT_HEADER_SIZE;
2177
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
2178
+ offset += await cursor.frameLengthAt(offset);
2179
+ }
2180
+ return offset;
2181
+ }
2182
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
2183
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2184
+ await cursor.checkHeader();
2185
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
2186
+ let checksNumbering = true;
2187
+ while (offset < cursor.limit) {
2188
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
2189
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
2190
+ checksNumbering = false;
2191
+ if (frame.seqNo > afterSeqNo) {
2192
+ offset = SEGMENT_HEADER_SIZE;
2193
+ continue;
2194
+ }
2195
+ }
2196
+ checksNumbering = false;
2197
+ offset += frame.frameBytes;
2198
+ if (frame.seqNo <= afterSeqNo) continue;
2199
+ if (frame.seqNo > upToSeqNo) return true;
2200
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
2201
+ read.frames.push(frame);
2202
+ read.bytes += frame.payload.byteLength;
2203
+ if (frame.seqNo === upToSeqNo) return true;
2204
+ }
2205
+ return false;
2206
+ }
2207
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
2208
+ const read = { frames: [], bytes: 0, gone: false };
2209
+ if (afterSeqNo >= upToSeqNo) return read;
2210
+ for (let attempt = 0; attempt < 2; attempt++) {
2211
+ read.frames = [];
2212
+ read.bytes = 0;
2213
+ const log = await openPartitionLog(directory, indexName, partitionId);
2214
+ let index = sourceHolding(log, afterSeqNo + 1);
2215
+ if (index < 0) {
2216
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
2217
+ return read;
2218
+ }
1886
2219
  try {
1887
- if (await lock.holds()) return null;
1888
- } catch (err) {
1889
- return cannotCheck(err);
2220
+ for (; index < log.sources.length; index++) {
2221
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
2222
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
2223
+ }
2224
+ return read;
2225
+ } catch (error) {
2226
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
1890
2227
  }
1891
- return new NarsilError(
1892
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1893
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
1894
- { directory: directory.root }
1895
- );
1896
2228
  }
1897
- async function verify() {
1898
- if (directory.identity === void 0 || isStopped()) return true;
1899
- const current = await currentIdentity();
1900
- if (current instanceof Error) {
1901
- onLost(current);
1902
- return false;
1903
- }
1904
- if (expected === null) {
1905
- expected = current;
1906
- } else if (current !== expected) {
1907
- onLost(
1908
- new NarsilError(
1909
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1910
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
1911
- { directory: directory.root }
1912
- )
1913
- );
1914
- return false;
2229
+ return read;
2230
+ }
2231
+ async function logBytesAfter(directory, log, afterSeqNo) {
2232
+ const first = sourceHolding(log, afterSeqNo + 1);
2233
+ let bytes = 0;
2234
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
2235
+ const source = log.sources[i];
2236
+ if (i === first) {
2237
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
2238
+ try {
2239
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
2240
+ } catch (error) {
2241
+ if (!(error instanceof SegmentFileMissing)) throw error;
2242
+ }
2243
+ continue;
1915
2244
  }
1916
- const takenOver = await lockTakenOver();
1917
- if (takenOver === null) return true;
1918
- onLost(takenOver);
1919
- return false;
2245
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
1920
2246
  }
1921
- function stop() {
1922
- if (timer !== null) {
1923
- clearInterval(timer);
1924
- timer = null;
2247
+ return bytes;
2248
+ }
2249
+
2250
+ // src/persistence/durability/sync/pins.ts
2251
+ function createFilePins(remove) {
2252
+ const counts = /* @__PURE__ */ new Map();
2253
+ const awaitingRemoval = /* @__PURE__ */ new Set();
2254
+ function unpin(key) {
2255
+ const count = (counts.get(key) ?? 1) - 1;
2256
+ if (count > 0) {
2257
+ counts.set(key, count);
2258
+ return;
1925
2259
  }
2260
+ counts.delete(key);
2261
+ if (!awaitingRemoval.delete(key)) return;
2262
+ void remove(key).catch(() => void 0);
1926
2263
  }
1927
2264
  return {
1928
- async claim() {
1929
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
1930
- },
1931
- async release() {
1932
- const held = lock;
1933
- lock = null;
1934
- await held?.release();
1935
- },
1936
- verify,
1937
- async verifyOnce() {
1938
- if (expected === null && directory.identity !== void 0) await verify();
1939
- },
1940
- start() {
1941
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
1942
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
2265
+ pin(keys) {
2266
+ const pinned = [...keys];
2267
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
2268
+ let released = false;
2269
+ return () => {
2270
+ if (released) return;
2271
+ released = true;
2272
+ for (const key of pinned) unpin(key);
2273
+ };
1943
2274
  },
1944
- stop
2275
+ isPinned: (key) => counts.has(key),
2276
+ removeWhenUnpinned(key) {
2277
+ if (!counts.has(key)) return false;
2278
+ awaitingRemoval.add(key);
2279
+ return true;
2280
+ }
2281
+ };
2282
+ }
2283
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2284
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2285
+ function hexOf(bytes) {
2286
+ let hex = "";
2287
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
2288
+ return hex;
2289
+ }
2290
+ function isBytes(value) {
2291
+ return value instanceof Uint8Array;
2292
+ }
2293
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
2294
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
2295
+ if (decoded.kind !== "current") {
2296
+ throw new NarsilError(
2297
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2298
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
2299
+ { indexName }
2300
+ );
2301
+ }
2302
+ return {
2303
+ config,
2304
+ manifestBytes,
2305
+ manifest: decoded.manifest,
2306
+ entityTag: hexOf(sha256(payload)),
2307
+ keys: manifestReferencedKeys(decoded.manifest)
1945
2308
  };
1946
2309
  }
2310
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
2311
+ const payload = encode({ config, manifest: manifestBytes });
2312
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
2313
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
2314
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
2315
+ return published;
2316
+ }
2317
+ async function readPublishedSnapshot(directory, indexName) {
2318
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
2319
+ if (bytes === null) return null;
2320
+ const { payload } = await openChecksummedEnvelope(bytes, {
2321
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
2322
+ what: PUBLISHED_SNAPSHOT_WHAT,
2323
+ allowsCompression: false
2324
+ });
2325
+ const raw = decode(payload);
2326
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
2327
+ throw new NarsilError(
2328
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
2329
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
2330
+ { indexName }
2331
+ );
2332
+ }
2333
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
2334
+ }
1947
2335
 
1948
- // src/persistence/durability/held-checkpoints.ts
1949
- function failedPartitionIds(indexState) {
1950
- const failed = [];
1951
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1952
- return failed;
2336
+ // src/persistence/durability/sync/retention.ts
2337
+ async function firstLogStart(directory, indexName, partitionId) {
2338
+ let first = Number.POSITIVE_INFINITY;
2339
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
2340
+ const start = parseWalSegmentStart(indexName, partitionId, key);
2341
+ if (start !== null && start < first) first = start;
2342
+ }
2343
+ return first;
2344
+ }
2345
+ async function sizedQueues(retention) {
2346
+ const { directory, indexName } = retention;
2347
+ const queues = /* @__PURE__ */ new Map();
2348
+ for (const segment of await listHistorySegments(directory, indexName)) {
2349
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
2350
+ if (bytes === void 0 || bytes === null) continue;
2351
+ const queue = queues.get(segment.partitionId) ?? [];
2352
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
2353
+ queues.set(segment.partitionId, queue);
2354
+ }
2355
+ for (const [partitionId, queue] of queues) {
2356
+ const logStart = await firstLogStart(directory, indexName, partitionId);
2357
+ for (let i = 0; i < queue.length; i++) {
2358
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
2359
+ queue[i].lastSeqNo = nextStart - 1;
2360
+ }
2361
+ }
2362
+ return queues;
2363
+ }
2364
+ function nextToDelete(queues, totalBytes, retention) {
2365
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
2366
+ let oldest = null;
2367
+ for (const queue of queues.values()) {
2368
+ const head = queue[0];
2369
+ if (head === void 0) continue;
2370
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
2371
+ }
2372
+ if (oldest === null) return null;
2373
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
2374
+ return null;
2375
+ }
2376
+ async function enforceHistoryLimits(retention) {
2377
+ const queues = await sizedQueues(retention);
2378
+ let totalBytes = 0;
2379
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
2380
+ let published = false;
2381
+ let deleted = 0;
2382
+ let segment = nextToDelete(queues, totalBytes, retention);
2383
+ while (segment !== null) {
2384
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
2385
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
2386
+ published = true;
2387
+ await retention.publish();
2388
+ }
2389
+ await retention.directory.remove(segment.key);
2390
+ queues.get(segment.partitionId)?.shift();
2391
+ totalBytes -= segment.bytes;
2392
+ deleted += 1;
2393
+ segment = nextToDelete(queues, totalBytes, retention);
2394
+ }
2395
+ return deleted;
1953
2396
  }
1954
- function createHeldCheckpoints(hooks) {
1955
- const reported = /* @__PURE__ */ new WeakSet();
1956
- function heldReport(indexName, indexState) {
1957
- const partitionIds = failedPartitionIds(indexState);
1958
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1959
- return {
1960
- indexName,
1961
- partitionIds,
1962
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1963
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1964
- };
2397
+
2398
+ // src/persistence/durability/sync/storage.ts
2399
+ function createSyncStorage(deps) {
2400
+ const { directory } = deps;
2401
+ const store = directoryCheckpointStore(directory);
2402
+ const cache = /* @__PURE__ */ new Map();
2403
+ const publishing = /* @__PURE__ */ new Map();
2404
+ const pins = createFilePins((key) => directory.remove(key));
2405
+ async function published(indexName) {
2406
+ const cached = cache.get(indexName);
2407
+ if (cached !== void 0) return cached;
2408
+ const read = await readPublishedSnapshot(directory, indexName);
2409
+ cache.set(indexName, read);
2410
+ return read;
2411
+ }
2412
+ async function removeUnlisted(indexName, previous, kept) {
2413
+ const manifest = await readCurrentManifest(store, indexName);
2414
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
2415
+ for (const key of previous) {
2416
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
2417
+ await directory.remove(key);
2418
+ }
2419
+ }
2420
+ async function publishNow(indexName, configurationOf) {
2421
+ const archive = await deps.openArchiveCheckpoint(indexName);
2422
+ try {
2423
+ const config = await configurationOf(archive.manifest);
2424
+ const previous = await published(indexName);
2425
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
2426
+ cache.set(indexName, next);
2427
+ if (previous !== null) {
2428
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
2429
+ }
2430
+ return next;
2431
+ } finally {
2432
+ archive.release();
2433
+ }
1965
2434
  }
1966
2435
  return {
1967
- refuse(indexName, indexState) {
1968
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1969
- if (partitionIds.length === 0) return;
1970
- throw new NarsilError(
1971
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1972
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1973
- { indexName, partitionIds }
1974
- );
2436
+ publish(indexName, configurationOf) {
2437
+ const inFlight = publishing.get(indexName);
2438
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
2439
+ const tracked = run.finally(() => {
2440
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
2441
+ });
2442
+ publishing.set(indexName, tracked);
2443
+ return tracked;
1975
2444
  },
1976
- holds(indexName, indexState) {
1977
- const held = heldReport(indexName, indexState);
1978
- if (held === null) return false;
1979
- const unreported = held.partitionIds.some((partitionId) => {
1980
- const partition = indexState.partitions.get(partitionId);
1981
- return partition !== void 0 && !reported.has(partition);
2445
+ published,
2446
+ pin: (snapshot) => pins.pin(snapshot.keys),
2447
+ async readFileRange(key, offset, length) {
2448
+ const readRange = directory.readRange;
2449
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
2450
+ const whole = await directory.read(key);
2451
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
2452
+ },
2453
+ async release(indexName) {
2454
+ await deps.inCheckpointTurn(indexName, async () => {
2455
+ const previous = await published(indexName);
2456
+ await directory.remove(publishedSnapshotKey(indexName));
2457
+ cache.set(indexName, null);
2458
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
2459
+ await removeHistory(directory, indexName);
1982
2460
  });
1983
- if (unreported) {
1984
- for (const partitionId of held.partitionIds) {
1985
- const partition = indexState.partitions.get(partitionId);
1986
- if (partition !== void 0) reported.add(partition);
1987
- }
1988
- hooks.onCheckpointHeld?.(held);
1989
- }
1990
- return true;
1991
2461
  },
1992
- report(indexes) {
1993
- const held = [];
1994
- for (const [indexName, indexState] of indexes) {
1995
- const entry = heldReport(indexName, indexState);
1996
- if (entry !== null) held.push(entry);
2462
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
2463
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
2464
+ const current = await published(indexName);
2465
+ return enforceHistoryLimits({
2466
+ directory,
2467
+ indexName,
2468
+ limits,
2469
+ nowMs,
2470
+ publish,
2471
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
2472
+ });
2473
+ },
2474
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
2475
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
2476
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
2477
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
2478
+ async keepsCheckpointKey(indexName) {
2479
+ const current = await published(indexName);
2480
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
2481
+ },
2482
+ async settleAtRecovery(indexName, active) {
2483
+ cache.delete(indexName);
2484
+ const publishedKey = publishedSnapshotKey(indexName);
2485
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
2486
+ if (key !== publishedKey) await directory.remove(key);
1997
2487
  }
1998
- return held;
2488
+ if (!active) {
2489
+ await directory.remove(publishedSnapshotKey(indexName));
2490
+ await removeHistory(directory, indexName);
2491
+ cache.set(indexName, null);
2492
+ return /* @__PURE__ */ new Set();
2493
+ }
2494
+ try {
2495
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
2496
+ } catch {
2497
+ await directory.remove(publishedSnapshotKey(indexName));
2498
+ cache.set(indexName, null);
2499
+ return /* @__PURE__ */ new Set();
2500
+ }
2501
+ },
2502
+ forget(indexName) {
2503
+ cache.delete(indexName);
1999
2504
  }
2000
2505
  };
2001
2506
  }
2002
2507
 
2003
- // src/persistence/durability/manager-state.ts
2004
- function closePartitionWriter(partition) {
2005
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
2006
- }
2007
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
2008
- indexState.unloading = true;
2009
- try {
2010
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
2011
- await pendingMetadataWrite;
2012
- for (const partition of indexState.partitions.values()) {
2013
- await closeWalWriter(closePartitionWriter(partition));
2508
+ // src/persistence/durability/manager-storage.ts
2509
+ function createIndexStorageAccess(deps) {
2510
+ const { directory, indexes } = deps;
2511
+ const archiveAccess = createArchiveCheckpointAccess({
2512
+ directory,
2513
+ held: deps.heldCheckpointFiles,
2514
+ checkpoint: deps.checkpoint,
2515
+ registerPartition: deps.registerPartition,
2516
+ markFatal: deps.markFatal
2517
+ });
2518
+ const syncStorage = createSyncStorage({
2519
+ directory,
2520
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
2521
+ inCheckpointTurn: (indexName, work) => {
2522
+ const indexState = indexes.get(indexName);
2523
+ if (indexState === void 0) return work();
2524
+ return checkpointInTurn(indexState, {
2525
+ joinsInFlight: false,
2526
+ mayStart: () => true,
2527
+ perform: work,
2528
+ afterSettling: () => releaseStalledWrites(indexState)
2529
+ });
2014
2530
  }
2015
- } catch (error) {
2016
- indexState.unloading = false;
2017
- throw error;
2018
- }
2531
+ });
2532
+ const unloading = createIndexUnloading({
2533
+ directory,
2534
+ hooks: deps.hooks,
2535
+ indexes,
2536
+ metadataWrites: deps.metadataWrites,
2537
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
2538
+ forgetSyncState: syncStorage.forget
2539
+ });
2540
+ return { archiveAccess, syncStorage, unloading };
2019
2541
  }
2020
2542
 
2021
2543
  // src/persistence/durability/metadata-write-queue.ts
@@ -2988,7 +3510,7 @@ var decoder = new TextDecoder();
2988
3510
  async function countPagedPartition(store, partition) {
2989
3511
  const { directory } = store;
2990
3512
  if (directory === void 0 || partition.segments.length === 0) return null;
2991
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-KAMRAJLE.mjs');
3513
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-M4JNBV6V.mjs');
2992
3514
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2993
3515
  const scans = [];
2994
3516
  try {
@@ -3271,12 +3793,14 @@ async function recoverCheckpoint(deps) {
3271
3793
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3272
3794
  }
3273
3795
  if (decoded === null || decoded.kind !== "current") {
3274
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
3796
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3275
3797
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3276
3798
  }
3277
3799
  await deps.store.remove(snapshotBundleKey(deps.indexName));
3278
3800
  if (!deps.keepsUnlistedFiles) {
3279
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
3801
+ const referenced = manifestReferencedKeys(decoded.manifest);
3802
+ for (const key of deps.keptForSync) referenced.add(key);
3803
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
3280
3804
  }
3281
3805
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
3282
3806
  manager: deps.manager,
@@ -3397,6 +3921,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3397
3921
  }
3398
3922
  return replayed;
3399
3923
  };
3924
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
3925
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
3400
3926
  const recovered = await recoverCheckpoint({
3401
3927
  store,
3402
3928
  indexName,
@@ -3406,6 +3932,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3406
3932
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3407
3933
  compactionThreshold: deps.compactionThreshold,
3408
3934
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3935
+ keptForSync,
3409
3936
  replay,
3410
3937
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3411
3938
  writeRecords
@@ -3521,6 +4048,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3521
4048
  mutationThreshold: checkpointMutationThreshold,
3522
4049
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3523
4050
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
4051
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3524
4052
  queueMetadataWrite,
3525
4053
  markFatal,
3526
4054
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -3574,6 +4102,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3574
4102
  startAsyncFlushTimer();
3575
4103
  return outcomes;
3576
4104
  }
4105
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
4106
+ directory,
4107
+ hooks,
4108
+ indexes,
4109
+ metadataWrites,
4110
+ heldCheckpointFiles,
4111
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
4112
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
4113
+ markFatal,
4114
+ closeWriterReportingFailure
4115
+ });
3577
4116
  const recoveryDeps = {
3578
4117
  directory,
3579
4118
  hooks,
@@ -3582,7 +4121,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3582
4121
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3583
4122
  },
3584
4123
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3585
- queueMetadataWrite
4124
+ queueMetadataWrite,
4125
+ syncStorage
3586
4126
  };
3587
4127
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3588
4128
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3670,35 +4210,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3670
4210
  await checkpointIndex(indexName);
3671
4211
  }
3672
4212
  },
3673
- async removeIndex(indexName) {
3674
- hooks.writeRecords?.dropIndex(indexName);
3675
- const indexState = indexes.get(indexName);
3676
- if (indexState !== void 0) {
3677
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3678
- indexes.delete(indexName);
3679
- metadataWrites.forget(indexName);
3680
- }
3681
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3682
- for (const key of await directory.list(`${indexName}/`)) {
3683
- await directory.remove(key);
3684
- }
3685
- },
3686
- async unloadIndex(indexName) {
3687
- const indexState = indexes.get(indexName);
3688
- if (indexState === void 0) {
3689
- return;
3690
- }
3691
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3692
- indexes.delete(indexName);
3693
- metadataWrites.forget(indexName);
3694
- },
3695
- ...createArchiveCheckpointAccess({
3696
- directory,
3697
- held: heldCheckpointFiles,
3698
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3699
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3700
- markFatal
3701
- }),
4213
+ ...unloading,
4214
+ syncStorage: () => syncStorage,
4215
+ ...archiveAccess,
3702
4216
  async shutdown() {
3703
4217
  shuttingDown = true;
3704
4218
  stopTimers();
@@ -3965,6 +4479,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3965
4479
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3966
4480
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3967
4481
  keepsUnlistedFiles: false,
4482
+ keptForSync: /* @__PURE__ */ new Set(),
3968
4483
  replay: async (checkpoint) => checkpoint,
3969
4484
  onLayoutUpgrade: hooks.onLayoutUpgrade
3970
4485
  });
@@ -4162,7 +4677,9 @@ function createDurabilityIntegration(tier, hooks) {
4162
4677
  recordCheckpoint: hooks.recordCheckpoint,
4163
4678
  onLayoutUpgrade: hooks.onLayoutUpgrade,
4164
4679
  onCheckpointHeld: hooks.onCheckpointHeld,
4165
- onFatalError: hooks.onFatalError
4680
+ onFatalError: hooks.onFatalError,
4681
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4682
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4166
4683
  };
4167
4684
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4168
4685
  function partitionFor(indexName, docId) {
@@ -4247,7 +4764,12 @@ function createDurabilityFromTier(tier, wiring) {
4247
4764
  recordCheckpoint: wiring.recordCheckpoint,
4248
4765
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
4249
4766
  onCheckpointHeld: wiring.emitCheckpointHeld,
4250
- onFatalError: wiring.emitFatalError
4767
+ onFatalError: wiring.emitFatalError,
4768
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4769
+ onHistoryMoved(indexName) {
4770
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4771
+ import('./retention-6QMRS5VK.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4772
+ }
4251
4773
  });
4252
4774
  }
4253
4775