@delali/narsil 0.3.1-next.152 → 0.3.1-next.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-4GCRVRBW.mjs} +27 -18
  7. package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KYGSXW4K.mjs} +1 -1
  8. package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-IXWPXXJJ.mjs} +17 -7
  9. package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-FCSHFQSS.mjs} +1 -1
  10. package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-2FX6ROWR.mjs} +15 -6
  11. package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-2SHAKW2Z.mjs} +119 -1922
  12. package/dist/browser/{chunk-55WYRLDZ.mjs → chunk-3U3QR5YP.mjs} +2 -2
  13. package/dist/browser/chunk-4PD752F6.mjs +85 -0
  14. package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
  15. package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-7MEYN7V6.mjs} +1 -151
  16. package/dist/browser/chunk-7TTWXUH6.mjs +1828 -0
  17. package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-AJ4HHOVT.mjs} +4 -4
  18. package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-AYY6PIKL.mjs} +2 -2
  19. package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-C4KS2QNF.mjs} +9 -8
  20. package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
  21. package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
  22. package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-EUNBNLKM.mjs} +6 -5
  23. package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-G7LUMBSG.mjs} +137 -253
  24. package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-H6QFBMVE.mjs} +3 -2
  25. package/dist/browser/chunk-J433A7P3.mjs +241 -0
  26. package/dist/browser/chunk-ML6HY7T7.mjs +96 -0
  27. package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
  28. package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-MYITLCHT.mjs} +7 -1
  29. package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-QXTPSX2H.mjs} +9 -150
  30. package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
  31. package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SIR3LMTV.mjs} +2 -2
  32. package/dist/browser/{chunk-LURXVCHG.mjs → chunk-SJB6VIZG.mjs} +1 -1
  33. package/dist/browser/chunk-TTD27VXX.mjs +298 -0
  34. package/dist/browser/chunk-U2E3HMYW.mjs +151 -0
  35. package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-VCOYGVP7.mjs} +5 -31
  36. package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
  37. package/dist/browser/chunk-W5X2UFWQ.mjs +102 -0
  38. package/dist/browser/{chunk-HT7J73QL.mjs → chunk-XRV72TJD.mjs} +2 -2
  39. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  40. package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
  41. package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-ZPTA75ZC.mjs} +2 -79
  42. package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-VTUGDYNY.mjs} +20 -14
  43. package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-GL4A7KDV.mjs} +1886 -1364
  44. package/dist/browser/import-YI5GNB4H.mjs +17 -0
  45. package/dist/browser/methods-LMUTS7OI.mjs +30 -0
  46. package/dist/browser/{module-XQLEOYNM.mjs → module-EC3FYW7N.mjs} +9 -8
  47. package/dist/browser/{module-KAMRAJLE.mjs → module-M4JNBV6V.mjs} +16 -10
  48. package/dist/browser/operations-WVYLF72Q.mjs +5 -0
  49. package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-NPYUVX75.mjs} +1 -1
  50. package/dist/browser/{pool-DTYSPBMQ.mjs → pool-WCGDZHJP.mjs} +3 -2
  51. package/dist/{restore-queue-AOF5OTQV.mjs → browser/restore-queue-TYIGDL5L.mjs} +2 -2
  52. package/dist/browser/retention-6QMRS5VK.mjs +207 -0
  53. package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-3EW2AGB6.mjs} +1 -1
  54. package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-DN3CRVUQ.mjs} +38 -307
  55. package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
  56. package/dist/browser/{write-I5P5MW4O.mjs → write-WAWAFGKQ.mjs} +16 -11
  57. package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-B3Brs1Rc.d.ts} +62 -7
  58. package/dist/{chunk-DGRM4EZZ.mjs → chunk-37IVJLP6.mjs} +1 -1
  59. package/dist/{chunk-KYVRZAYP.mjs → chunk-3A5KI24C.mjs} +2 -69
  60. package/dist/{chunk-POUYICOU.mjs → chunk-3OFU3NJT.mjs} +1 -1
  61. package/dist/{chunk-3TVDTACC.mjs → chunk-3Y3J4V2R.mjs} +1 -1
  62. package/dist/{chunk-FNBNZCGP.mjs → chunk-466PYR4P.mjs} +2 -2
  63. package/dist/{chunk-SEAJS664.mjs → chunk-4HWZV5MK.mjs} +5 -5
  64. package/dist/{chunk-UGVUW27S.mjs → chunk-5FUOVJKG.mjs} +13 -9
  65. package/dist/{chunk-ZHAXP7SD.mjs → chunk-5KGYQT6W.mjs} +6 -235
  66. package/dist/{chunk-NPXVQS33.mjs → chunk-62G63NMH.mjs} +4 -3
  67. package/dist/chunk-6MSZ543Y.mjs +32 -0
  68. package/dist/chunk-7GXJUJ24.mjs +49 -0
  69. package/dist/{chunk-3E4RRF5J.mjs → chunk-7HCZWC7B.mjs} +58 -8
  70. package/dist/{chunk-6RKSY3BX.mjs → chunk-7NTMUDC5.mjs} +5 -5
  71. package/dist/chunk-7QUGDVFC.mjs +101 -0
  72. package/dist/{chunk-G5KHSM6E.mjs → chunk-AO2FAYIZ.mjs} +1 -1
  73. package/dist/chunk-B7S6YEBP.mjs +102 -0
  74. package/dist/{chunk-AWNHDBKV.mjs → chunk-BPSKLCS7.mjs} +15 -7
  75. package/dist/chunk-BUMREPLU.mjs +15 -0
  76. package/dist/{chunk-T4CJDBBE.mjs → chunk-C6ASEFZI.mjs} +1 -1
  77. package/dist/{chunk-NITCPDFL.mjs → chunk-C73JLU4E.mjs} +2 -2
  78. package/dist/{chunk-NYSONUB7.mjs → chunk-CP5RRPJR.mjs} +1 -1
  79. package/dist/{chunk-TTCKT2RT.mjs → chunk-DMQPFLGI.mjs} +134 -11
  80. package/dist/{chunk-3SBJVQLL.mjs → chunk-DVWYODEB.mjs} +2 -2
  81. package/dist/chunk-ECX4SGX6.mjs +167 -0
  82. package/dist/chunk-GES7U2PM.mjs +68 -0
  83. package/dist/{chunk-NVTFQFUL.mjs → chunk-HVKXNBFL.mjs} +1 -1
  84. package/dist/chunk-HVYICDY3.mjs +214 -0
  85. package/dist/{chunk-3HV7M5OJ.mjs → chunk-HWEDFLB7.mjs} +9 -133
  86. package/dist/{chunk-52QQWEBS.mjs → chunk-IBNLW3AV.mjs} +21 -6
  87. package/dist/chunk-IJU7LL7D.mjs +242 -0
  88. package/dist/chunk-IPN7S4KX.mjs +130 -0
  89. package/dist/{chunk-5QMHMELB.mjs → chunk-IQ4R2I6N.mjs} +4 -1
  90. package/dist/chunk-JCLTESFN.mjs +96 -0
  91. package/dist/{chunk-J4BDAH2W.mjs → chunk-JFNKIBDT.mjs} +2 -2
  92. package/dist/{chunk-LDGGR7YT.mjs → chunk-K4B7OH65.mjs} +2 -2
  93. package/dist/{chunk-7TE2AJIH.mjs → chunk-K7KLL263.mjs} +13 -10
  94. package/dist/{chunk-CSHHU7PA.mjs → chunk-KCUIFVVW.mjs} +11 -30
  95. package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-KUZKUONM.mjs} +11 -2
  96. package/dist/chunk-KYYABCSW.mjs +72 -0
  97. package/dist/{chunk-B2FTZNNF.mjs → chunk-LKUE5GXS.mjs} +34 -14
  98. package/dist/{chunk-2PNYKG2D.mjs → chunk-LSM2YTKM.mjs} +100 -335
  99. package/dist/{chunk-DMNPYF54.mjs → chunk-LTUGDMV4.mjs} +2 -2
  100. package/dist/{chunk-FDG4KXX5.mjs → chunk-MLHNSAPK.mjs} +5 -3
  101. package/dist/{chunk-VP6XRMWR.mjs → chunk-MMBEANB2.mjs} +6 -32
  102. package/dist/{chunk-LJ4ENXOD.mjs → chunk-MMEMEUOO.mjs} +2 -2
  103. package/dist/{chunk-2HJWQPR6.mjs → chunk-NTZOXVUZ.mjs} +6 -122
  104. package/dist/{chunk-5WB2A3XU.mjs → chunk-NZMAXR2S.mjs} +397 -24
  105. package/dist/{chunk-IZT3G24Q.mjs → chunk-O43Z4MS6.mjs} +2 -2
  106. package/dist/{chunk-U46IZNCW.mjs → chunk-OKX2BBNW.mjs} +3 -2
  107. package/dist/{chunk-RBEHLIKX.mjs → chunk-PFKM7VNZ.mjs} +14 -8
  108. package/dist/chunk-PGVPE7GJ.mjs +96 -0
  109. package/dist/chunk-Q3TCTQCC.mjs +85 -0
  110. package/dist/chunk-Q7MS6P2J.mjs +74 -0
  111. package/dist/chunk-QDZJM7NI.mjs +23 -0
  112. package/dist/{chunk-ZS6CMXAE.mjs → chunk-QKZHLADX.mjs} +89 -250
  113. package/dist/{chunk-IJSFWGVW.mjs → chunk-REL7LFLR.mjs} +3 -3
  114. package/dist/chunk-RVKFI3A2.mjs +237 -0
  115. package/dist/chunk-S2N75OTN.mjs +83 -0
  116. package/dist/chunk-TRISKD4Q.mjs +50 -0
  117. package/dist/{chunk-6F6USYII.mjs → chunk-TV2L3BN7.mjs} +13 -11
  118. package/dist/{chunk-6BWLDLS3.mjs → chunk-URPCYQVJ.mjs} +1 -1
  119. package/dist/chunk-USR7FQRZ.mjs +34 -0
  120. package/dist/chunk-V6EUBAIT.mjs +19 -0
  121. package/dist/{chunk-SOUZMPME.mjs → chunk-VMXSVVXG.mjs} +7 -194
  122. package/dist/{chunk-4QCSHLNM.mjs → chunk-VREK3MYG.mjs} +3 -3
  123. package/dist/{chunk-YGQOPB2Z.mjs → chunk-WCBA2LVZ.mjs} +1 -1
  124. package/dist/{chunk-KQJVNPRM.mjs → chunk-X6NSJYQ3.mjs} +19 -16
  125. package/dist/chunk-Y4WPNHMC.mjs +50 -0
  126. package/dist/{chunk-IRWP524D.mjs → chunk-YBJDOA2T.mjs} +1 -1
  127. package/dist/{chunk-IJPK5PGM.mjs → chunk-YPUEV6ZC.mjs} +1 -1
  128. package/dist/{chunk-J7M73VY7.mjs → chunk-YROWNOMH.mjs} +1 -1
  129. package/dist/chunk-Z3TNWCI5.mjs +196 -0
  130. package/dist/{chunk-65WCV5JO.mjs → chunk-ZIFJXCKA.mjs} +1 -1
  131. package/dist/chunk-ZYAY26KG.mjs +29 -0
  132. package/dist/{client-Cn3gNxUU.d.ts → client-Df76GNKm.d.ts} +76 -6
  133. package/dist/client.d.ts +8 -7
  134. package/dist/client.mjs +56 -5
  135. package/dist/{cold-module-PYSSBIL7.mjs → cold-module-YLWMTTZG.mjs} +38 -28
  136. package/dist/{config-CfhH1IWQ.d.ts → config-05G_hy-1.d.ts} +3 -3
  137. package/dist/config-B8DP2VwR.d.ts +463 -0
  138. package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-FI5GJHWX.mjs} +14 -9
  139. package/dist/distribution/coordinator/etcd.d.ts +11 -2
  140. package/dist/distribution/coordinator/etcd.mjs +268 -107
  141. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  142. package/dist/distribution/transport/grpc.mjs +4 -4
  143. package/dist/distribution/transport/in-memory.mjs +2 -2
  144. package/dist/distribution/transport/tcp.mjs +3 -3
  145. package/dist/distribution.d.ts +41 -9
  146. package/dist/distribution.mjs +1296 -117
  147. package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-RHLXDIVF.mjs} +1751 -1226
  148. package/dist/embeddings/openai.mjs +2 -2
  149. package/dist/{engine-CJRUHXgf.d.ts → engine-DYu-e0oR.d.ts} +50 -40
  150. package/dist/{import-GJOMB2HC.mjs → import-SRBNC6TE.mjs} +27 -19
  151. package/dist/{index-MsF7e5hL.d.ts → index-nRqDM6uS.d.ts} +1 -1
  152. package/dist/index.browser.mjs +8 -4
  153. package/dist/index.d.ts +7 -6
  154. package/dist/index.mjs +24 -16
  155. package/dist/index.react-native.mjs +17 -11
  156. package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-EMYTGSJP.mjs} +28 -15
  157. package/dist/methods-ABM5ELBR.mjs +30 -0
  158. package/dist/{module-AVQEIB7O.mjs → module-3OMJZUFT.mjs} +12 -12
  159. package/dist/{module-6RLIKSZQ.mjs → module-MG7JMV2U.mjs} +33 -23
  160. package/dist/operations-PGOAY2KS.mjs +5 -0
  161. package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-UOLHOEZJ.mjs} +37 -26
  162. package/dist/partial-words.mjs +6 -4
  163. package/dist/pattern.mjs +4 -3
  164. package/dist/persistence/durability/checkpoint-worker.d.ts +9 -1
  165. package/dist/persistence/durability/checkpoint-worker.mjs +34 -23
  166. package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-O3MDRQXP.mjs} +1 -1
  167. package/dist/{pool-UPV5PVPY.mjs → pool-CQXYPFLG.mjs} +2 -2
  168. package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-EAYO3Y6Q.mjs} +27 -15
  169. package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-TNLUC6IZ.mjs} +1 -1
  170. package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-2CGSQM4O.mjs} +12 -10
  171. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
  172. package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-3O57C2ZO.mjs} +2 -2
  173. package/dist/react-native/chunks/chunk-3XKMRNRE.mjs +298 -0
  174. package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-63YSZOXB.mjs} +97 -333
  175. package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-75VWIY35.mjs} +1 -1
  176. package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7GWPJDOA.mjs} +1 -1
  177. package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-AUT7PMA4.mjs} +6 -190
  178. package/dist/react-native/chunks/chunk-C2H2QQFW.mjs +288 -0
  179. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
  180. package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-CUNGPR5R.mjs} +11 -2
  181. package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-DCKMICOB.mjs} +2 -79
  182. package/dist/react-native/chunks/chunk-DH3TVWWK.mjs +102 -0
  183. package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-EH5CFV6Z.mjs} +2 -2
  184. package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
  185. package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-GFYFQLJD.mjs} +2 -2
  186. package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-I5PGWQJF.mjs} +142 -257
  187. package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-IOY2SM2P.mjs} +1 -1
  188. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
  189. package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-JPAJWO6D.mjs} +1 -1
  190. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
  191. package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-JX2RIAXH.mjs} +4 -4
  192. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
  193. package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-NCJS3WIY.mjs} +2 -2
  194. package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-NKIZX26F.mjs} +2 -2
  195. package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-NP7DD4G5.mjs} +5 -4
  196. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
  197. package/dist/react-native/chunks/chunk-O32RB3BL.mjs +191 -0
  198. package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-OQQVRWZ2.mjs} +2 -69
  199. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
  200. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  201. package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-RVAEBN74.mjs} +1 -1
  202. package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-RYMDHGL4.mjs} +16 -8
  203. package/dist/react-native/chunks/chunk-SJWDBRNP.mjs +85 -0
  204. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
  205. package/dist/react-native/chunks/chunk-UWDNZDWK.mjs +242 -0
  206. package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-VQSKZ3EZ.mjs} +9 -152
  207. package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-VVFYOADG.mjs} +7 -6
  208. package/dist/react-native/chunks/{chunk-VGUWIZFQ.mjs → chunk-YBEW6YTR.mjs} +1 -1
  209. package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-YHKZSLOC.mjs} +7 -33
  210. package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-ZBSWRSBG.mjs} +4 -30
  211. package/dist/react-native/chunks/chunk-ZYDKJXQJ.mjs +96 -0
  212. package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-TJL6AFAD.mjs} +29 -21
  213. package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-RRDYJLLM.mjs} +1882 -1358
  214. package/dist/react-native/chunks/gzip.react-native-DNB5YNFA.mjs +3 -0
  215. package/dist/react-native/chunks/import-JYBMEJPQ.mjs +29 -0
  216. package/dist/react-native/chunks/methods-X6F36SJE.mjs +30 -0
  217. package/dist/react-native/chunks/module-QT5HZAY4.mjs +33 -0
  218. package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-S4DKSWMF.mjs} +10 -9
  219. package/dist/react-native/chunks/operations-RYKTSZQA.mjs +5 -0
  220. package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-JZ37TB7N.mjs} +1 -1
  221. package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-RTGXJXPU.mjs} +3 -2
  222. package/dist/{browser/restore-queue-UQ6AAUVX.mjs → react-native/chunks/restore-queue-ZBVIKJWI.mjs} +2 -2
  223. package/dist/react-native/chunks/retention-PPCTP2VY.mjs +219 -0
  224. package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-KUATOLYB.mjs} +1 -1
  225. package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-ZBIEOZTZ.mjs} +47 -314
  226. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
  227. package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-2W4FTX7J.mjs} +25 -18
  228. package/dist/react-native/partial-words.mjs +6 -4
  229. package/dist/react-native/pattern.mjs +4 -3
  230. package/dist/react-native/sync.mjs +2236 -0
  231. package/dist/react-native/synonyms.mjs +7 -4
  232. package/dist/react.d.ts +10 -9
  233. package/dist/react.mjs +7 -1
  234. package/dist/{rerank-3W54N6P2.mjs → rerank-IZOB6NII.mjs} +4 -4
  235. package/dist/rerankers/http.mjs +2 -2
  236. package/dist/{react-native/chunks/restore-queue-H5VCN453.mjs → restore-queue-COI2GSSK.mjs} +2 -2
  237. package/dist/retention-55WA7GPU.mjs +58 -0
  238. package/dist/{runtime-L2DZYY6N.mjs → runtime-UZOC3NPY.mjs} +5 -5
  239. package/dist/{schema-D33CyXCy.d.ts → schema-DDauvDOv.d.ts} +1 -1
  240. package/dist/server.d.ts +22 -19
  241. package/dist/server.mjs +868 -50
  242. package/dist/{snapshot-WMX6JCLP.mjs → snapshot-NFX2EYN2.mjs} +62 -36
  243. package/dist/sync/filesystem.d.ts +30 -0
  244. package/dist/sync/filesystem.mjs +271 -0
  245. package/dist/sync/indexeddb.d.ts +30 -0
  246. package/dist/sync/indexeddb.mjs +237 -0
  247. package/dist/sync-state-ZWHLAFLE.mjs +15 -0
  248. package/dist/{config-aIsy7mcL.d.ts → sync-uXR7MmM7.d.ts} +53 -444
  249. package/dist/sync.browser.mjs +2231 -0
  250. package/dist/sync.d.ts +23 -0
  251. package/dist/sync.mjs +2206 -0
  252. package/dist/synonyms.mjs +5 -3
  253. package/dist/{tasks-C4ElXr4P.d.ts → tasks-BShx8pdK.d.ts} +4 -4
  254. package/dist/thread-P2G2FWM2.mjs +44 -0
  255. package/dist/types-SnO7_tAI.d.ts +228 -0
  256. package/dist/vector/search-worker.d.ts +1 -1
  257. package/dist/vector/search-worker.mjs +6 -6
  258. package/dist/workers/entry.mjs +7 -7
  259. package/dist/{write-SKJ6HW7O.mjs → write-FGB62BHQ.mjs} +31 -23
  260. package/openapi.json +119 -3
  261. package/package.json +28 -8
  262. package/dist/browser/import-QOY6ZH6X.mjs +0 -12
  263. package/dist/chunk-7LDMWKP6.mjs +0 -209
  264. package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
  265. package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
  266. package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
  267. package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
  268. package/dist/thread-I7DVHHW7.mjs +0 -31
@@ -1,1123 +1,721 @@
1
- import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-FNBNZCGP.mjs';
2
- import { closePartitionWriter, drainIndexStateForUnload, listPersistedIndexes, removeLeftoverRewriteFiles, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-4QCSHLNM.mjs';
3
- import { writeMetadataEnvelope, decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, isRecord, invalidManifest, unpackEnvelopeBytes, isId, isUint32, isSequenceNumber, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf, readCurrentManifest } from './chunk-ZS6CMXAE.mjs';
4
- import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
1
+ import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-466PYR4P.mjs';
2
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-PGVPE7GJ.mjs';
3
+ import { closePartitionWriter, listPersistedIndexes, removeLeftoverRewriteFiles, drainIndexStateForUnload, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-VREK3MYG.mjs';
4
+ import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, unpackEnvelopeBytes, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf } from './chunk-QKZHLADX.mjs';
5
+ import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-7NTMUDC5.mjs';
5
6
  import './chunk-ZXNNR73R.mjs';
6
- import './chunk-DMNPYF54.mjs';
7
7
  import './chunk-MATMVH65.mjs';
8
8
  import './chunk-VD7J552I.mjs';
9
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-G5KHSM6E.mjs';
9
+ import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-IJU7LL7D.mjs';
10
+ import './chunk-BUMREPLU.mjs';
11
+ import './chunk-LTUGDMV4.mjs';
12
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-AO2FAYIZ.mjs';
10
13
  import './chunk-37CD3OKS.mjs';
11
- import { createDurableDirectory } from './chunk-FDG4KXX5.mjs';
14
+ import { createDurableDirectory } from './chunk-MLHNSAPK.mjs';
12
15
  import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
13
- import { getFs } from './chunk-HLPRHOSO.mjs';
14
- import { nextSegmentKey, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, SEGMENT_HEADER_SIZE, buildEntry, captureWholePartition, checkpointLastSeqNo, readCommitMarker } from './chunk-KQJVNPRM.mjs';
16
+ import { SEGMENT_HEADER_SIZE, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, buildEntry, captureWholePartition, checkpointLastSeqNo } from './chunk-X6NSJYQ3.mjs';
17
+ import './chunk-TRISKD4Q.mjs';
15
18
  import './chunk-JDDXV6UF.mjs';
16
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, readSegmentFile, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-CSHHU7PA.mjs';
17
- import './chunk-NPXVQS33.mjs';
18
- import { directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-7LDMWKP6.mjs';
19
- import './chunk-5QMHMELB.mjs';
20
- import { toError, openChecksummedEnvelope, HEADER_SIZE } from './chunk-2PNYKG2D.mjs';
21
- import './chunk-6F6USYII.mjs';
22
- import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-UGVUW27S.mjs';
23
- import './chunk-3HV7M5OJ.mjs';
24
- import { fnv1a } from './chunk-KAKQ74LS.mjs';
19
+ import { getFs } from './chunk-HLPRHOSO.mjs';
20
+ import { readSegmentFile } from './chunk-KCUIFVVW.mjs';
21
+ import './chunk-62G63NMH.mjs';
22
+ import { adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-S2N75OTN.mjs';
23
+ import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-Q3TCTQCC.mjs';
24
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
25
+ import './chunk-IQ4R2I6N.mjs';
26
+ import { toError, packChecksummedEnvelope } from './chunk-LSM2YTKM.mjs';
27
+ import './chunk-TV2L3BN7.mjs';
25
28
  import './chunk-HJSN7UYG.mjs';
26
- import { detectRuntime, resolveWorkerEntry } from './chunk-6BWLDLS3.mjs';
29
+ import { detectRuntime, resolveWorkerEntry } from './chunk-URPCYQVJ.mjs';
27
30
  import './chunk-XIHUI4KB.mjs';
28
- import './chunk-Q2UL2TPC.mjs';
29
31
  import './chunk-3MXV3VHR.mjs';
30
32
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
31
- import './chunk-2HJWQPR6.mjs';
33
+ import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-5FUOVJKG.mjs';
34
+ import './chunk-HWEDFLB7.mjs';
35
+ import './chunk-USR7FQRZ.mjs';
36
+ import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-HVYICDY3.mjs';
37
+ import './chunk-IPN7S4KX.mjs';
38
+ import { fnv1a } from './chunk-KAKQ74LS.mjs';
39
+ import { crc32 } from './chunk-7GXJUJ24.mjs';
40
+ import './chunk-Q2UL2TPC.mjs';
41
+ import './chunk-NTZOXVUZ.mjs';
32
42
  import './chunk-X5CV2E2B.mjs';
43
+ import './chunk-Y4WPNHMC.mjs';
44
+ import './chunk-JCLTESFN.mjs';
33
45
  import './chunk-TGP5M5R7.mjs';
34
46
  import './chunk-U2BKYEC7.mjs';
35
- import './chunk-DGRM4EZZ.mjs';
36
47
  import { createRequestId } from './chunk-QHPYS427.mjs';
37
48
  import './chunk-QYRNLLLI.mjs';
49
+ import './chunk-37IVJLP6.mjs';
38
50
  import './chunk-ZXNQEW43.mjs';
39
51
  import './chunk-KOKM4PQN.mjs';
40
- import './chunk-NVTFQFUL.mjs';
41
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
52
+ import './chunk-HVKXNBFL.mjs';
42
53
  import './chunk-BTPGQC6Z.mjs';
54
+ import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
43
55
  import './chunk-RFBLQLLZ.mjs';
44
- import './chunk-U46IZNCW.mjs';
56
+ import './chunk-OKX2BBNW.mjs';
45
57
  import './chunk-JUKYYXSP.mjs';
46
- import { encodeDocument, requireValidFieldTypes } from './chunk-SOUZMPME.mjs';
58
+ import './chunk-VMXSVVXG.mjs';
59
+ import { encodeDocument, requireValidFieldTypes } from './chunk-Z3TNWCI5.mjs';
47
60
  import './chunk-FQBFZAWG.mjs';
48
- import './chunk-65WCV5JO.mjs';
61
+ import './chunk-ZIFJXCKA.mjs';
49
62
  import './chunk-4UMZ7KQL.mjs';
50
63
  import { MIN_M } from './chunk-ZXX5GZEX.mjs';
51
- import './chunk-KYVRZAYP.mjs';
64
+ import './chunk-GES7U2PM.mjs';
65
+ import './chunk-3A5KI24C.mjs';
52
66
  import './chunk-L5ORED7V.mjs';
53
67
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
54
- import { NarsilError, ErrorCodes } from './chunk-MDF6PDIA.mjs';
68
+ import { NarsilError, ErrorCodes } from './chunk-KUZKUONM.mjs';
55
69
  import './chunk-R6A773WK.mjs';
56
70
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
57
- import { decode } from '@msgpack/msgpack';
71
+ import { decode, encode } from '@msgpack/msgpack';
58
72
 
59
- // src/persistence/durability/sha256.ts
60
- var ROUND_CONSTANTS = new Uint32Array([
61
- 1116352408,
62
- 1899447441,
63
- 3049323471,
64
- 3921009573,
65
- 961987163,
66
- 1508970993,
67
- 2453635748,
68
- 2870763221,
69
- 3624381080,
70
- 310598401,
71
- 607225278,
72
- 1426881987,
73
- 1925078388,
74
- 2162078206,
75
- 2614888103,
76
- 3248222580,
77
- 3835390401,
78
- 4022224774,
79
- 264347078,
80
- 604807628,
81
- 770255983,
82
- 1249150122,
83
- 1555081692,
84
- 1996064986,
85
- 2554220882,
86
- 2821834349,
87
- 2952996808,
88
- 3210313671,
89
- 3336571891,
90
- 3584528711,
91
- 113926993,
92
- 338241895,
93
- 666307205,
94
- 773529912,
95
- 1294757372,
96
- 1396182291,
97
- 1695183700,
98
- 1986661051,
99
- 2177026350,
100
- 2456956037,
101
- 2730485921,
102
- 2820302411,
103
- 3259730800,
104
- 3345764771,
105
- 3516065817,
106
- 3600352804,
107
- 4094571909,
108
- 275423344,
109
- 430227734,
110
- 506948616,
111
- 659060556,
112
- 883997877,
113
- 958139571,
114
- 1322822218,
115
- 1537002063,
116
- 1747873779,
117
- 1955562222,
118
- 2024104815,
119
- 2227730452,
120
- 2361852424,
121
- 2428436474,
122
- 2756734187,
123
- 3204031479,
124
- 3329325298
125
- ]);
126
- var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
127
- var BLOCK_BYTES = 64;
128
- var LENGTH_BYTES = 8;
129
- function rotateRight(value, bits) {
130
- return value >>> bits | value << 32 - bits;
73
+ // src/persistence/durability/sync/layout.ts
74
+ var SYNC_KEY_NUMBER_WIDTH = 16;
75
+ function syncDirectoryPrefix(indexName) {
76
+ return `${indexName}/sync/`;
131
77
  }
132
- function padded(bytes) {
133
- const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
134
- const message = new Uint8Array(total);
135
- message.set(bytes);
136
- message[bytes.byteLength] = 128;
137
- const view = new DataView(message.buffer);
138
- const bitLength = bytes.byteLength * 8;
139
- view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
140
- view.setUint32(total - 4, bitLength >>> 0, false);
141
- return view;
78
+ function publishedSnapshotKey(indexName) {
79
+ return `${syncDirectoryPrefix(indexName)}published`;
142
80
  }
143
- function sha256(bytes) {
144
- const view = padded(bytes);
145
- const state = INITIAL_STATE.slice();
146
- const words = new Uint32Array(64);
147
- for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
148
- for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
149
- for (let i = 16; i < 64; i++) {
150
- const w15 = words[i - 15];
151
- const w2 = words[i - 2];
152
- const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
153
- const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
154
- words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
155
- }
156
- let [a, b, c, d, e, f, g, h] = state;
157
- for (let i = 0; i < 64; i++) {
158
- const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
159
- const choice = e & f ^ ~e & g;
160
- const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
161
- const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
162
- const majority = a & b ^ a & c ^ b & c;
163
- const t2 = s0 + majority >>> 0;
164
- h = g;
165
- g = f;
166
- f = e;
167
- e = d + t1 >>> 0;
168
- d = c;
169
- c = b;
170
- b = a;
171
- a = t1 + t2 >>> 0;
172
- }
173
- state[0] = state[0] + a >>> 0;
174
- state[1] = state[1] + b >>> 0;
175
- state[2] = state[2] + c >>> 0;
176
- state[3] = state[3] + d >>> 0;
177
- state[4] = state[4] + e >>> 0;
178
- state[5] = state[5] + f >>> 0;
179
- state[6] = state[6] + g >>> 0;
180
- state[7] = state[7] + h >>> 0;
181
- }
182
- const digest = new Uint8Array(32);
183
- const out = new DataView(digest.buffer);
184
- for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
185
- return digest;
81
+ function historyRoot(indexName) {
82
+ return `${indexName}/history/`;
186
83
  }
187
-
188
- // src/persistence/durability/restore-journal.ts
189
- var RESTORE_JOURNAL_PREFIX = ".swap-";
190
- var RESTORE_STAGING_PREFIX = ".restore-";
191
- var RETIRED_INDEX_PREFIX = ".retired-";
192
- var LOCK_FILE_KEY = ".narsil.lock";
193
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
194
- function digestOf(indexName) {
195
- let hex = "";
196
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
197
- return hex;
84
+ function historyPrefix(indexName, partitionId) {
85
+ return `${historyRoot(indexName)}${partitionId}/`;
198
86
  }
199
- function journalKey(indexName) {
200
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
87
+ function walSegmentPrefix(indexName, partitionId) {
88
+ return `${indexName}/wal/${partitionId}/`;
201
89
  }
202
- function randomSuffix() {
203
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
90
+ function formatSyncKeyNumber(value) {
91
+ return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
92
+ }
93
+ function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
94
+ return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
95
+ }
96
+ function walSegmentKey(indexName, partitionId, startSeqNo) {
97
+ return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
98
+ }
99
+ var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
100
+ function parseHistoryKey(indexName, key) {
101
+ const root = historyRoot(indexName);
102
+ if (!key.startsWith(root)) return null;
103
+ const match = HISTORY_TAIL.exec(key.slice(root.length));
104
+ if (match === null) return null;
105
+ const partitionId = Number.parseInt(match[1], 10);
106
+ const startSeqNo = Number.parseInt(match[2], 10);
107
+ const movedAtMs = Number.parseInt(match[3], 10);
108
+ if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
109
+ return null;
110
+ }
111
+ return { key, partitionId, startSeqNo, movedAtMs };
204
112
  }
205
- function newStagingPrefix() {
206
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
113
+ function parseWalSegmentStart(indexName, partitionId, key) {
114
+ const prefix = walSegmentPrefix(indexName, partitionId);
115
+ if (!key.startsWith(prefix)) return null;
116
+ const tail = key.slice(prefix.length);
117
+ if (!/^\d{16}$/.test(tail)) return null;
118
+ const value = Number.parseInt(tail, 10);
119
+ return Number.isSafeInteger(value) ? value : null;
207
120
  }
208
- function newRetiredPrefix() {
209
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
121
+
122
+ // src/persistence/durability/sync/history.ts
123
+ function compareHistorySegments(a, b) {
124
+ return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
125
+ }
126
+ async function listHistorySegments(directory, indexName, partitionId) {
127
+ const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
128
+ const segments = [];
129
+ for (const key of await directory.list(prefix)) {
130
+ const segment = parseHistoryKey(indexName, key);
131
+ if (segment !== null) segments.push(segment);
132
+ }
133
+ return segments.sort(compareHistorySegments);
134
+ }
135
+ async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
136
+ await directory.move(
137
+ walSegmentKey(indexName, partitionId, startSeqNo),
138
+ historyKey(indexName, partitionId, startSeqNo, movedAtMs)
139
+ );
210
140
  }
211
- function encodeJournal(indexName, stagingPrefix) {
212
- return new TextEncoder().encode(`${indexName}
213
- ${stagingPrefix}`);
141
+ async function removeHistory(directory, indexName, partitionId) {
142
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
143
+ await directory.remove(segment.key);
144
+ }
214
145
  }
215
- function isStagingPrefix(prefix) {
216
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
146
+
147
+ // src/persistence/durability/checkpoint.ts
148
+ function walPrefix(indexName, partitionId) {
149
+ return `${indexName}/wal/${partitionId}/`;
217
150
  }
218
- function corruptJournal(key, reason) {
219
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
151
+ async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
152
+ for (const { partitionId, lastSeqNo } of checkpoint) {
153
+ const prefix = walPrefix(indexName, partitionId);
154
+ const markerBytes = await directory.read(`${prefix}commit`);
155
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
156
+ if (marker === null) {
157
+ continue;
158
+ }
159
+ const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
160
+ const segments = await directory.list(prefix);
161
+ const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
162
+ for (let i = 0; i < startSeqNos.length; i += 1) {
163
+ const start = startSeqNos[i];
164
+ if (start >= activeSegmentSeqNo) {
165
+ break;
166
+ }
167
+ const nextStart = startSeqNos[i + 1];
168
+ const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
169
+ if (segmentCoversBeyondCheckpoint) {
170
+ break;
171
+ }
172
+ if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
173
+ else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
174
+ }
175
+ }
220
176
  }
221
- function readJournal(key, bytes) {
222
- const text = new TextDecoder().decode(bytes);
223
- const lineEnd = text.indexOf("\n");
224
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
225
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
226
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
227
- corruptJournal(key, "names an index that no index name can match");
177
+ async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
178
+ for (const [partitionId, partition] of [...partitions]) {
179
+ if (partitionId < partitionCount) continue;
180
+ try {
181
+ await partition.walWriter.close();
182
+ } catch (err) {
183
+ onCloseError(err instanceof Error ? err : new Error(String(err)));
184
+ }
185
+ partitions.delete(partitionId);
228
186
  }
229
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
230
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
231
- corruptJournal(key, "names no staging prefix that a restore writes");
187
+ const walRoot = `${indexName}/wal/`;
188
+ for (const key of await directory.list(walRoot)) {
189
+ const partitionId = walKeyPartitionId(key, walRoot);
190
+ if (partitionId === null || partitionId < partitionCount) continue;
191
+ await directory.remove(key);
232
192
  }
233
- return { kind: "journal", indexName, stagingPrefix };
234
193
  }
235
- async function listedKeys(storage, indexName, stagingPrefix) {
236
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
237
- if (manifestBytes === null) {
238
- throw new NarsilError(
239
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
240
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
241
- { indexName, stagingPrefix }
242
- );
194
+ function walKeyPartitionId(key, walRoot) {
195
+ const tail = key.slice(walRoot.length);
196
+ const separator = tail.indexOf("/");
197
+ if (separator <= 0) {
198
+ return null;
243
199
  }
244
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
245
- if (decoded.kind !== "current") {
246
- throw new NarsilError(
247
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
248
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
249
- { indexName }
250
- );
200
+ const segment = tail.slice(0, separator);
201
+ if (!/^\d+$/.test(segment)) {
202
+ return null;
251
203
  }
252
- const root = `${indexName}/`;
253
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
254
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
255
- return keys;
256
- }
257
- function withoutTrailingSlash(prefix) {
258
- return prefix.slice(0, -1);
204
+ const value = Number.parseInt(segment, 10);
205
+ return Number.isSafeInteger(value) ? value : null;
259
206
  }
260
- async function finishJournal(storage, indexName, stagingPrefix) {
261
- const root = `${indexName}/`;
262
- const listed = await listedKeys(storage, indexName, stagingPrefix);
263
- const staged = await storage.list(stagingPrefix);
264
- let retiredPrefix = null;
265
- if (storage.moveTree !== void 0 && staged.length > 0) {
266
- retiredPrefix = newRetiredPrefix();
267
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
268
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
269
- } else {
270
- const stagedKeys = new Set(staged);
271
- for (const key of listed) {
272
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
273
- }
274
- }
275
- for (const key of await storage.list(root)) {
276
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
207
+ function parseSegmentStartSeqNo(key, prefix) {
208
+ if (!key.startsWith(prefix)) {
209
+ return null;
277
210
  }
278
- await storage.remove(journalKey(indexName));
279
- return retiredPrefix;
280
- }
281
- async function removePrefix(storage, prefix) {
282
- if (storage.removeTree !== void 0) {
283
- await storage.removeTree(prefix);
284
- return;
211
+ const tail = key.slice(prefix.length);
212
+ if (!/^\d{16}$/.test(tail)) {
213
+ return null;
285
214
  }
286
- for (const key of await storage.list(prefix)) await storage.remove(key);
215
+ const value = Number.parseInt(tail, 10);
216
+ return Number.isSafeInteger(value) ? value : null;
287
217
  }
288
- async function commitJournal(storage, indexName, stagingPrefix) {
289
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
218
+ function formatSegmentStart(startSeqNo) {
219
+ return startSeqNo.toString().padStart(16, "0");
290
220
  }
291
- function isTopLevelJournal(key) {
292
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
221
+
222
+ // src/persistence/durability/checkpoint-capture.ts
223
+ function whileNoMutationApplies(partitions, capture2) {
224
+ return new Promise((resolve, reject) => {
225
+ let release = () => void 0;
226
+ const released = new Promise((settle) => {
227
+ release = settle;
228
+ });
229
+ function captureNow() {
230
+ try {
231
+ resolve(capture2());
232
+ } catch (error) {
233
+ reject(error);
234
+ } finally {
235
+ release();
236
+ }
237
+ }
238
+ let waiting = partitions.length;
239
+ if (waiting === 0) {
240
+ captureNow();
241
+ return;
242
+ }
243
+ for (const partition of partitions) {
244
+ partition.appendChain = partition.appendChain.then(() => {
245
+ waiting -= 1;
246
+ if (waiting === 0) captureNow();
247
+ return released;
248
+ });
249
+ }
250
+ });
293
251
  }
294
- async function removeStagingPrefixes(storage) {
295
- const prefixes = /* @__PURE__ */ new Set();
296
- for (const key of await storage.list(".")) {
297
- const slash = key.indexOf("/");
298
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
252
+ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
253
+ for (const [partitionId, partition] of [...indexState.partitions]) {
254
+ await partition.appendChain;
255
+ if (partition.failed !== null) continue;
256
+ try {
257
+ await partition.walWriter.commit();
258
+ } catch (error) {
259
+ failPartition2(partitionId, partition, error);
260
+ }
299
261
  }
300
- for (const prefix of prefixes) await removePrefix(storage, prefix);
301
262
  }
302
- async function recoverRestoreJournals(storage) {
303
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
304
- if (!isTopLevelJournal(key)) continue;
305
- const bytes = await storage.read(key);
306
- if (bytes === null) continue;
307
- const reading = readJournal(key, bytes);
308
- if (reading.kind === "leftover") {
309
- await storage.remove(key);
310
- continue;
263
+ async function openNewLogSegments(indexState, failPartition2) {
264
+ const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
265
+ try {
266
+ if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
267
+ } catch (error) {
268
+ failPartition2(partitionId, partition, error);
269
+ return;
311
270
  }
312
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
313
- }
314
- await removeStagingPrefixes(storage);
271
+ const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
272
+ failPartition2(partitionId, partition, error);
273
+ });
274
+ partition.appendChain = rolled;
275
+ await rolled;
276
+ });
277
+ await Promise.all(rolls);
315
278
  }
316
-
317
- // src/persistence/durability/restore-directory.ts
318
- function missing(error) {
319
- return error.code === "ENOENT";
279
+ function aTargetFailed(indexState, targets) {
280
+ return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
320
281
  }
321
- function storageFailed(code, message, key, error) {
322
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
282
+ function noWholePartitions() {
283
+ return { contents: [], captured: [] };
323
284
  }
324
- function directoryRestoreStorage(directory) {
325
- return {
326
- directory,
327
- read: (key) => directory.read(key),
328
- write: (key, chunks) => directory.atomicWrite(key, chunks),
329
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
330
- remove: (key) => directory.remove(key),
331
- list: (prefix) => directory.list(prefix),
332
- moveKey: (from, to) => directory.move(from, to),
333
- pathOf: (key) => directory.pathOf(key),
334
- async moveTree(from, to) {
335
- const source = await directory.pathOf(from);
336
- const target = await directory.pathOf(to);
337
- const fs = await getFs();
338
- try {
339
- await fs.rename(source, target);
340
- } catch (error) {
341
- if (missing(error)) return;
342
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
343
- }
344
- await directory.syncDirectoryOf(to);
345
- },
346
- async removeTree(prefix) {
347
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
348
- const fs = await getFs();
349
- try {
350
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
351
- } catch (error) {
352
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
285
+ function wholePartitionsWhereMostChanged(manager, rules) {
286
+ const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
287
+ return (targets) => {
288
+ const whole = { contents: [], captured: [] };
289
+ const changed = targets.filter(
290
+ (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
291
+ );
292
+ const mostOfEachChanged = changed.every((target) => {
293
+ const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
294
+ const documents = manager.getPartition(target.partitionId).count();
295
+ return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
296
+ });
297
+ const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
298
+ if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
299
+ for (const { partitionId } of changed) {
300
+ const capture2 = captureRecords?.(partitionId);
301
+ const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
302
+ if (captured !== null) {
303
+ whole.captured.push(captured);
304
+ } else if (everyPartition || !aWorkerCanSerialise) {
305
+ whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
353
306
  }
354
- await directory.syncDirectoryOf(key);
355
307
  }
308
+ return whole;
356
309
  };
357
310
  }
358
- function recoverDirectoryRestores(directory) {
359
- return recoverRestoreJournals(directoryRestoreStorage(directory));
311
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
312
+ for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
313
+ return whileNoMutationApplies([...indexState.partitions.values()], () => {
314
+ onCaptured?.();
315
+ const documentCount = manager.countDocuments();
316
+ const targets = [];
317
+ let recordsLeftInLog = 0;
318
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
319
+ const partition = indexState.partitions.get(partitionId);
320
+ if (partition !== void 0 && partition.failed !== null) continue;
321
+ const appliedSeqNo = partition?.appliedSeqNo ?? 0;
322
+ const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
323
+ appliedSeqNo,
324
+ snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
325
+ );
326
+ recordsLeftInLog += appliedSeqNo - lastSeqNo;
327
+ const target = {
328
+ partitionId,
329
+ lastSeqNo,
330
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
331
+ };
332
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
333
+ targets.push(target);
334
+ }
335
+ const vectorPlans = /* @__PURE__ */ new Map();
336
+ try {
337
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
338
+ const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
339
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
340
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
341
+ }
342
+ const memoryMatchesTheTargets = recordsLeftInLog === 0;
343
+ const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
344
+ return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
345
+ } catch (error) {
346
+ for (const plan of vectorPlans.values()) plan.release();
347
+ throw error;
348
+ }
349
+ });
360
350
  }
361
-
362
- // src/persistence/durability/restore-staging.ts
363
- function toError2(error) {
364
- return error instanceof Error ? error : new Error(String(error));
351
+ async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
352
+ if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
353
+ const priorVectors = priorManifest?.vectors ?? [];
354
+ const priorCheckpoint = priorManifest?.checkpoint ?? [];
355
+ const noPartitionChanged = capture2.targets.every(
356
+ (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
357
+ );
358
+ const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
359
+ (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
360
+ );
361
+ const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
362
+ if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
363
+ return { vectors: priorVectors, layouts: [], written: [] };
364
+ }
365
+ const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
366
+ return { vectors: written.refs, layouts: written.layouts, written: written.written };
365
367
  }
366
- function pathUnder(path, root) {
367
- if (!path.startsWith(root)) return false;
368
- const next = path.charAt(root.length);
369
- return next === "" || next === "/" || next === "\\";
368
+
369
+ // src/persistence/durability/checkpoint-worker-entry.ts
370
+ function resolveCheckpointWorkerEntry() {
371
+ return resolveWorkerEntry(
372
+ import.meta.url,
373
+ /\/src\/persistence\/durability\/[^/]+$/,
374
+ "persistence/durability/checkpoint-worker.mjs"
375
+ );
370
376
  }
371
- function stagedDirectory(directory, staged, unstaged) {
372
- const { readRange, byteLength } = directory;
373
- return {
374
- root: directory.root,
375
- appendHandle: (key) => directory.appendHandle(staged(key)),
376
- markerHandle: (key) => directory.markerHandle(staged(key)),
377
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
378
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
379
- read: (key) => directory.read(staged(key)),
380
- remove: (key) => directory.remove(staged(key)),
381
- move: (from, to) => directory.move(staged(from), staged(to)),
382
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
383
- pathOf: (key) => directory.pathOf(staged(key)),
384
- ...readRange === void 0 ? {} : {
385
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
386
- },
387
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
388
- };
389
- }
390
- function createRestoreStaging(storage, indexName, options = {}) {
391
- const stagingPrefix = newStagingPrefix();
392
- const indexRoot = `${indexName}/`;
393
- let committed = false;
394
- const staged = (key) => {
395
- if (!key.startsWith(indexRoot)) {
396
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
397
- key
398
- });
377
+
378
+ // src/persistence/durability/checkpoint-worker-dispatch.ts
379
+ var workerUsable = true;
380
+ var pooledWorker = null;
381
+ var workerBusy = false;
382
+ var spawnedWorkerCount = 0;
383
+ var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
384
+ var idleTimer = null;
385
+ async function spawnWorker() {
386
+ const entryPoint = resolveCheckpointWorkerEntry();
387
+ if (entryPoint === null) return null;
388
+ try {
389
+ const worker = await spawnNodeWorker(new URL(entryPoint));
390
+ if (worker === null) {
391
+ return null;
399
392
  }
400
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
401
- };
402
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
403
- const store = {
404
- read: (key) => storage.read(staged(key)),
405
- write: (key, chunks) => storage.write(staged(key), chunks),
406
- remove: (key) => storage.remove(staged(key)),
407
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
408
- };
409
- const pathOf = storage.pathOf;
410
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
411
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
412
- let retiredPrefix = null;
413
- async function finishOnStorage() {
414
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
415
- }
416
- return {
417
- store,
418
- async relocation() {
419
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
420
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
421
- const to = await pathOf.call(storage, indexName);
422
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
423
- },
424
- async commit() {
425
- try {
426
- await commitJournal(storage, indexName, stagingPrefix);
427
- } catch (error) {
428
- try {
429
- await storage.remove(journalKey(indexName));
430
- } catch (removal) {
431
- committed = true;
432
- options.onStorageLost?.(toError2(removal));
433
- }
434
- throw error;
435
- }
436
- committed = true;
437
- },
438
- async finish() {
439
- try {
440
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
441
- } catch (error) {
442
- options.onStorageLost?.(toError2(error));
443
- }
444
- },
445
- async removeRetired() {
446
- const prefix = retiredPrefix;
447
- retiredPrefix = null;
448
- if (prefix !== null) await removePrefix(storage, prefix);
449
- },
450
- async discard() {
451
- if (committed) return;
452
- await removePrefix(storage, stagingPrefix);
393
+ if (typeof worker.unref === "function") {
394
+ worker.unref();
453
395
  }
454
- };
396
+ worker.on("error", () => discardWorker(worker));
397
+ worker.on("exit", () => discardWorker(worker));
398
+ spawnedWorkerCount += 1;
399
+ return worker;
400
+ } catch {
401
+ return null;
402
+ }
455
403
  }
456
- function adapterRestoreStorage(adapter) {
457
- const store = adapterCheckpointStore(adapter);
458
- return {
459
- read: (key) => store.read(key),
460
- write: (key, chunks) => store.write(key, chunks),
461
- writeJournal: (key, bytes) => store.write(key, [bytes]),
462
- remove: (key) => store.remove(key),
463
- list: (prefix) => store.list(prefix),
464
- async moveKey(from, to) {
465
- const bytes = await store.read(from);
466
- if (bytes === null) return;
467
- await store.write(to, [bytes]);
468
- await store.remove(from);
469
- }
470
- };
404
+ function discardWorker(worker) {
405
+ if (pooledWorker === worker) {
406
+ pooledWorker = null;
407
+ }
408
+ try {
409
+ void worker.terminate();
410
+ } catch {
411
+ }
471
412
  }
472
-
473
- // src/persistence/durability/archive-checkpoint.ts
474
- function toError3(error) {
475
- return error instanceof Error ? error : new Error(String(error));
413
+ function unrefTimer(timer) {
414
+ if (typeof timer.unref === "function") {
415
+ timer.unref();
416
+ }
417
+ return timer;
476
418
  }
477
- function stageDirectoryRestore(deps, indexName) {
478
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
479
- async finish(run) {
480
- try {
481
- await run();
482
- } catch (error) {
483
- deps.markFatal(toError3(error));
484
- throw error;
419
+ function runWorker(worker, request) {
420
+ return new Promise((resolve) => {
421
+ let settled = false;
422
+ let timeoutId = armSilenceTimeout();
423
+ function armSilenceTimeout() {
424
+ return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
425
+ }
426
+ const onMessage = (msg) => {
427
+ const response = msg;
428
+ if (response.type === "heartbeat") {
429
+ clearTimeout(timeoutId);
430
+ timeoutId = armSilenceTimeout();
431
+ return;
485
432
  }
486
- },
487
- onStorageLost: deps.markFatal
488
- });
489
- }
490
- function releaser(held, indexName) {
491
- let released = false;
492
- return () => {
493
- if (released) return;
494
- released = true;
495
- const count = (held.get(indexName) ?? 1) - 1;
496
- if (count <= 0) held.delete(indexName);
497
- else held.set(indexName, count);
498
- };
499
- }
500
- function createArchiveCheckpointAccess(deps) {
501
- const store = directoryCheckpointStore(deps.directory);
502
- return {
503
- async openArchiveCheckpoint(indexName) {
504
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
505
- const release = releaser(deps.held, indexName);
506
- try {
507
- await deps.checkpoint(indexName, false);
508
- await deps.checkpoint(indexName, true);
509
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
510
- if (manifestBytes === null) {
511
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
512
- indexName
513
- });
514
- }
515
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
516
- if (decoded.kind !== "current") {
517
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
518
- indexName
519
- });
520
- }
521
- return {
522
- manifestBytes,
523
- manifest: decoded.manifest,
524
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
525
- readRange: async (ref, offset, length) => {
526
- const readRange = deps.directory.readRange;
527
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
528
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
529
- return whole.slice(offset, offset + length);
530
- },
531
- release,
532
- directory: deps.directory
533
- };
534
- } catch (error) {
535
- release();
536
- throw error;
433
+ if (response.type === "success") {
434
+ settle({ written: response.segments, timedOut: false }, false);
435
+ } else {
436
+ settle({ written: null, timedOut: false }, true);
537
437
  }
538
- },
539
- stageRestore(indexName) {
540
- return stageDirectoryRestore(deps, indexName);
541
- },
542
- adoptRestoredCheckpoint(indexName, checkpoint) {
543
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
544
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
438
+ };
439
+ const onError = () => {
440
+ settle({ written: null, timedOut: false }, true);
441
+ };
442
+ const onExit = () => {
443
+ settle({ written: null, timedOut: false }, true);
444
+ };
445
+ function settle(outcome, discard) {
446
+ if (settled) {
447
+ return;
448
+ }
449
+ settled = true;
450
+ clearTimeout(timeoutId);
451
+ worker.off("message", onMessage);
452
+ worker.off("error", onError);
453
+ worker.off("exit", onExit);
454
+ if (discard) {
455
+ discardWorker(worker);
545
456
  }
457
+ resolve(outcome);
546
458
  }
547
- };
459
+ worker.on("message", onMessage);
460
+ worker.on("error", onError);
461
+ worker.on("exit", onExit);
462
+ try {
463
+ worker.postMessage(request);
464
+ } catch {
465
+ settle({ written: null, timedOut: false }, true);
466
+ }
467
+ });
548
468
  }
549
-
550
- // src/persistence/durability/checkpoint.ts
551
- function walPrefix(indexName, partitionId) {
552
- return `${indexName}/wal/${partitionId}/`;
469
+ function delay(ms) {
470
+ return new Promise((resolve) => {
471
+ unrefTimer(setTimeout(resolve, ms));
472
+ });
553
473
  }
554
- async function truncateCoveredSegments(directory, indexName, checkpoint) {
555
- for (const { partitionId, lastSeqNo } of checkpoint) {
556
- const prefix = walPrefix(indexName, partitionId);
557
- const markerBytes = await directory.read(`${prefix}commit`);
558
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
559
- if (marker === null) {
560
- continue;
561
- }
562
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
563
- const segments = await directory.list(prefix);
564
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
565
- for (let i = 0; i < startSeqNos.length; i += 1) {
566
- const start = startSeqNos[i];
567
- if (start >= activeSegmentSeqNo) {
568
- break;
569
- }
570
- const nextStart = startSeqNos[i + 1];
571
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
572
- if (segmentCoversBeyondCheckpoint) {
573
- break;
574
- }
575
- await directory.remove(`${prefix}${formatSegmentStart(start)}`);
576
- }
577
- }
578
- }
579
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
580
- for (const [partitionId, partition] of [...partitions]) {
581
- if (partitionId < partitionCount) continue;
582
- try {
583
- await partition.walWriter.close();
584
- } catch (err) {
585
- onCloseError(err instanceof Error ? err : new Error(String(err)));
586
- }
587
- partitions.delete(partitionId);
588
- }
589
- const walRoot = `${indexName}/wal/`;
590
- for (const key of await directory.list(walRoot)) {
591
- const partitionId = walKeyPartitionId(key, walRoot);
592
- if (partitionId === null || partitionId < partitionCount) continue;
593
- await directory.remove(key);
594
- }
474
+ function checkpointWorkerIsUsable() {
475
+ const runtime = detectRuntime();
476
+ return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
595
477
  }
596
- function walKeyPartitionId(key, walRoot) {
597
- const tail = key.slice(walRoot.length);
598
- const separator = tail.indexOf("/");
599
- if (separator <= 0) {
478
+ async function runCheckpointOnWorker(request) {
479
+ const runtime = detectRuntime();
480
+ if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
600
481
  return null;
601
482
  }
602
- const segment = tail.slice(0, separator);
603
- if (!/^\d+$/.test(segment)) {
483
+ if (workerBusy) {
604
484
  return null;
605
485
  }
606
- const value = Number.parseInt(segment, 10);
607
- return Number.isSafeInteger(value) ? value : null;
608
- }
609
- function parseSegmentStartSeqNo(key, prefix) {
610
- if (!key.startsWith(prefix)) {
486
+ workerBusy = true;
487
+ cancelIdleRetirement();
488
+ try {
489
+ if (pooledWorker === null) {
490
+ pooledWorker = await spawnWorker();
491
+ }
492
+ const worker = pooledWorker;
493
+ if (worker === null) {
494
+ workerUsable = false;
495
+ return null;
496
+ }
497
+ const outcome = await runWorker(worker, request);
498
+ if (outcome.written !== null) {
499
+ return outcome.written;
500
+ }
501
+ if (outcome.timedOut) {
502
+ await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
503
+ }
611
504
  return null;
505
+ } finally {
506
+ workerBusy = false;
507
+ retireWorkerOnceIdle();
612
508
  }
613
- const tail = key.slice(prefix.length);
614
- if (!/^\d{16}$/.test(tail)) {
615
- return null;
509
+ }
510
+ function cancelIdleRetirement() {
511
+ if (idleTimer !== null) {
512
+ clearTimeout(idleTimer);
513
+ idleTimer = null;
616
514
  }
617
- const value = Number.parseInt(tail, 10);
618
- return Number.isSafeInteger(value) ? value : null;
619
515
  }
620
- function formatSegmentStart(startSeqNo) {
621
- return startSeqNo.toString().padStart(16, "0");
516
+ function retireWorkerOnceIdle() {
517
+ cancelIdleRetirement();
518
+ idleTimer = unrefTimer(
519
+ setTimeout(() => {
520
+ idleTimer = null;
521
+ if (!workerBusy) terminateCheckpointWorker();
522
+ }, workerIdleMs)
523
+ );
524
+ }
525
+ function terminateCheckpointWorker() {
526
+ cancelIdleRetirement();
527
+ const worker = pooledWorker;
528
+ if (worker === null) {
529
+ return;
530
+ }
531
+ pooledWorker = null;
532
+ try {
533
+ void worker.terminate();
534
+ } catch {
535
+ }
622
536
  }
623
537
 
624
- // src/persistence/durability/checkpoint-capture.ts
625
- function whileNoMutationApplies(partitions, capture2) {
626
- return new Promise((resolve, reject) => {
627
- let release = () => void 0;
628
- const released = new Promise((settle) => {
629
- release = settle;
630
- });
631
- function captureNow() {
632
- try {
633
- resolve(capture2());
634
- } catch (error) {
635
- reject(error);
636
- } finally {
637
- release();
638
- }
639
- }
640
- let waiting = partitions.length;
641
- if (waiting === 0) {
642
- captureNow();
643
- return;
644
- }
645
- for (const partition of partitions) {
646
- partition.appendChain = partition.appendChain.then(() => {
647
- waiting -= 1;
648
- if (waiting === 0) captureNow();
649
- return released;
650
- });
651
- }
538
+ // src/persistence/durability/checkpoint-write.ts
539
+ function workerCanWriteEverySegment(input) {
540
+ if (!input.canOffload || input.wholePartitions.length > 0) return false;
541
+ const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
542
+ if (!analysesTextInProcessOnly) return true;
543
+ return input.targets.every(
544
+ (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
545
+ );
546
+ }
547
+ async function writeIndexCheckpointSegments(input) {
548
+ const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
549
+ const { keepsWriteRecords } = input;
550
+ const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
551
+ root: directory.root,
552
+ metadata,
553
+ targets,
554
+ compactionThreshold,
555
+ prior,
556
+ capturedPartitions: [...capturedPartitions],
557
+ keepsWriteRecords
558
+ }) : null;
559
+ if (offloaded !== null) {
560
+ return offloaded;
561
+ }
562
+ return writeCheckpointSegments({
563
+ store: directoryCheckpointStore(directory),
564
+ directory,
565
+ metadata,
566
+ targets,
567
+ compactionThreshold,
568
+ prior,
569
+ wholePartitions,
570
+ capturedPartitions,
571
+ keepsWriteRecords
652
572
  });
653
573
  }
654
- async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
655
- for (const [partitionId, partition] of [...indexState.partitions]) {
656
- await partition.appendChain;
657
- if (partition.failed !== null) continue;
658
- try {
659
- await partition.walWriter.commit();
660
- } catch (error) {
661
- failPartition2(partitionId, partition, error);
662
- }
574
+
575
+ // src/persistence/durability/segment/vector-adoption.ts
576
+ function recordWrittenVectors(vectorIndexes, written) {
577
+ for (const field of written) {
578
+ vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
663
579
  }
664
580
  }
665
- async function openNewLogSegments(indexState, failPartition2) {
666
- const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
667
- try {
668
- if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
669
- } catch (error) {
670
- failPartition2(partitionId, partition, error);
671
- return;
672
- }
673
- const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
674
- failPartition2(partitionId, partition, error);
581
+ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
582
+ const pathOf = store.pathOf;
583
+ if (pathOf === void 0) return;
584
+ for (const layout of layouts) {
585
+ const vecIndex = vectorIndexes.get(layout.fieldPath);
586
+ if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
587
+ const path = await pathOf.call(store, layout.key);
588
+ await vecIndex.adoptDiskLayout({
589
+ path,
590
+ docIds: layout.docIds,
591
+ dead: layout.dead,
592
+ vectorsOffset: layout.vectorsOffset
675
593
  });
676
- partition.appendChain = rolled;
677
- await rolled;
678
- });
679
- await Promise.all(rolls);
594
+ }
680
595
  }
681
- function aTargetFailed(indexState, targets) {
682
- return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
596
+ async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
597
+ const pathOf = store.pathOf;
598
+ if (pathOf === void 0) return [...keys];
599
+ const read = /* @__PURE__ */ new Set();
600
+ for (const vectorIndex of vectorIndexes.values()) {
601
+ for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
602
+ }
603
+ if (read.size === 0) return [...keys];
604
+ const unread = [];
605
+ for (const key of keys) {
606
+ if (!read.has(await pathOf.call(store, key))) unread.push(key);
607
+ }
608
+ return unread;
683
609
  }
684
- function noWholePartitions() {
685
- return { contents: [], captured: [] };
610
+
611
+ // src/persistence/durability/write-stall.ts
612
+ function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
613
+ return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
686
614
  }
687
- function wholePartitionsWhereMostChanged(manager, rules) {
688
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
689
- return (targets) => {
690
- const whole = { contents: [], captured: [] };
691
- const changed = targets.filter(
692
- (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
615
+ async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
616
+ while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
617
+ await waitOutsideCountedWrite(
618
+ () => new Promise((resume) => {
619
+ indexState.stalledWrites.push(resume);
620
+ })
693
621
  );
694
- const mostOfEachChanged = changed.every((target) => {
695
- const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
696
- const documents = manager.getPartition(target.partitionId).count();
697
- return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
698
- });
699
- const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
700
- if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
701
- for (const { partitionId } of changed) {
702
- const capture2 = captureRecords?.(partitionId);
703
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
704
- if (captured !== null) {
705
- whole.captured.push(captured);
706
- } else if (everyPartition || !aWorkerCanSerialise) {
707
- whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
708
- }
709
- }
710
- return whole;
711
- };
622
+ }
712
623
  }
713
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
714
- for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
715
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
716
- onCaptured?.();
717
- const documentCount = manager.countDocuments();
718
- const targets = [];
719
- let recordsLeftInLog = 0;
720
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
721
- const partition = indexState.partitions.get(partitionId);
722
- if (partition !== void 0 && partition.failed !== null) continue;
723
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
724
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
725
- appliedSeqNo,
726
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
727
- );
728
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
729
- const target = {
730
- partitionId,
731
- lastSeqNo,
732
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
733
- };
734
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
735
- targets.push(target);
736
- }
737
- const vectorPlans = /* @__PURE__ */ new Map();
738
- try {
739
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
740
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
741
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
742
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
743
- }
744
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
745
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
746
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
747
- } catch (error) {
748
- for (const plan of vectorPlans.values()) plan.release();
749
- throw error;
750
- }
751
- });
624
+ function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
625
+ const counted = indexState.mutationsSinceCheckpoint;
626
+ const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
627
+ indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
628
+ indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
629
+ releaseStalledWrites(indexState);
752
630
  }
753
- async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
754
- if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
755
- const priorVectors = priorManifest?.vectors ?? [];
756
- const priorCheckpoint = priorManifest?.checkpoint ?? [];
757
- const noPartitionChanged = capture2.targets.every(
758
- (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
759
- );
760
- const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
761
- (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
762
- );
763
- const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
764
- if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
765
- return { vectors: priorVectors, layouts: [], written: [] };
766
- }
767
- const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
768
- return { vectors: written.refs, layouts: written.layouts, written: written.written };
631
+ function releaseStalledWrites(indexState) {
632
+ const waiting = indexState.stalledWrites.splice(0);
633
+ for (const resume of waiting) resume();
769
634
  }
770
635
 
771
- // src/persistence/durability/checkpoint-worker-entry.ts
772
- function resolveCheckpointWorkerEntry() {
773
- return resolveWorkerEntry(
774
- import.meta.url,
775
- /\/src\/persistence\/durability\/[^/]+$/,
776
- "persistence/durability/checkpoint-worker.mjs"
777
- );
636
+ // src/persistence/durability/checkpoint-run.ts
637
+ async function bothOrTheFirstFailure(first, second) {
638
+ const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
639
+ if (firstSettled.status === "rejected") throw firstSettled.reason;
640
+ if (secondSettled.status === "rejected") throw secondSettled.reason;
641
+ return [firstSettled.value, secondSettled.value];
778
642
  }
779
-
780
- // src/persistence/durability/checkpoint-worker-dispatch.ts
781
- var workerUsable = true;
782
- var pooledWorker = null;
783
- var workerBusy = false;
784
- var spawnedWorkerCount = 0;
785
- var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
786
- var idleTimer = null;
787
- async function spawnWorker() {
788
- const entryPoint = resolveCheckpointWorkerEntry();
789
- if (entryPoint === null) return null;
643
+ async function runDurableCheckpoint(input) {
790
644
  try {
791
- const worker = await spawnNodeWorker(new URL(entryPoint));
792
- if (worker === null) {
793
- return null;
794
- }
795
- if (typeof worker.unref === "function") {
796
- worker.unref();
645
+ let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
646
+ while (recordsLeftInLog > 0 && input.mayContinue()) {
647
+ recordsLeftInLog = await writeOneBoundedCheckpoint(input);
797
648
  }
798
- worker.on("error", () => discardWorker(worker));
799
- worker.on("exit", () => discardWorker(worker));
800
- spawnedWorkerCount += 1;
801
- return worker;
802
- } catch {
803
- return null;
649
+ } catch (err) {
650
+ if (!isSystemError(err)) throw err;
651
+ const error = new NarsilError(
652
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
653
+ `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
654
+ {
655
+ indexName: input.indexName,
656
+ directory: input.directory.root,
657
+ cause: err.message
658
+ }
659
+ );
660
+ input.markFatal(error);
661
+ throw error;
804
662
  }
805
663
  }
806
- function discardWorker(worker) {
807
- if (pooledWorker === worker) {
808
- pooledWorker = null;
809
- }
810
- try {
811
- void worker.terminate();
812
- } catch {
813
- }
664
+ function isSystemError(err) {
665
+ return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
814
666
  }
815
- function unrefTimer(timer) {
816
- if (typeof timer.unref === "function") {
817
- timer.unref();
667
+ async function writeOneBoundedCheckpoint(input) {
668
+ const manager = input.hooks.getManager(input.indexName);
669
+ if (manager === void 0) {
670
+ return 0;
818
671
  }
819
- return timer;
820
- }
821
- function runWorker(worker, request) {
822
- return new Promise((resolve) => {
823
- let settled = false;
824
- let timeoutId = armSilenceTimeout();
825
- function armSilenceTimeout() {
826
- return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
827
- }
828
- const onMessage = (msg) => {
829
- const response = msg;
830
- if (response.type === "heartbeat") {
831
- clearTimeout(timeoutId);
832
- timeoutId = armSilenceTimeout();
833
- return;
834
- }
835
- if (response.type === "success") {
836
- settle({ written: response.segments, timedOut: false }, false);
837
- } else {
838
- settle({ written: null, timedOut: false }, true);
839
- }
840
- };
841
- const onError = () => {
842
- settle({ written: null, timedOut: false }, true);
843
- };
844
- const onExit = () => {
845
- settle({ written: null, timedOut: false }, true);
846
- };
847
- function settle(outcome, discard) {
848
- if (settled) {
849
- return;
850
- }
851
- settled = true;
852
- clearTimeout(timeoutId);
853
- worker.off("message", onMessage);
854
- worker.off("error", onError);
855
- worker.off("exit", onExit);
856
- if (discard) {
857
- discardWorker(worker);
858
- }
859
- resolve(outcome);
860
- }
861
- worker.on("message", onMessage);
862
- worker.on("error", onError);
863
- worker.on("exit", onExit);
864
- try {
865
- worker.postMessage(request);
866
- } catch {
867
- settle({ written: null, timedOut: false }, true);
672
+ const metadata = input.hooks.buildTermsMetadata(input.indexName);
673
+ if (metadata === void 0) {
674
+ return 0;
675
+ }
676
+ const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
677
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
678
+ const store = directoryCheckpointStore(input.directory);
679
+ const writeRecords = input.hooks.writeRecords ?? null;
680
+ const { indexName } = input;
681
+ const priorManifest = await readCurrentManifest(store, input.indexName);
682
+ const textOnDisk = metadata.textStorage === "disk";
683
+ const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
684
+ priorCheckpoint: priorManifest?.checkpoint ?? [],
685
+ priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
686
+ everyPartition: fromMemory,
687
+ aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
688
+ ...writeRecords === null ? {} : {
689
+ captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
868
690
  }
869
691
  });
870
- }
871
- function delay(ms) {
872
- return new Promise((resolve) => {
873
- unrefTimer(setTimeout(resolve, ms));
874
- });
875
- }
876
- function checkpointWorkerIsUsable() {
877
- const runtime = detectRuntime();
878
- return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
879
- }
880
- async function runCheckpointOnWorker(request) {
881
- const runtime = detectRuntime();
882
- if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
883
- return null;
884
- }
885
- if (workerBusy) {
886
- return null;
887
- }
888
- workerBusy = true;
889
- cancelIdleRetirement();
692
+ let changesSinceCapture = null;
693
+ const capture2 = await captureCheckpoint(
694
+ input.indexState,
695
+ manager,
696
+ vectorIndexes,
697
+ textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
698
+ fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
699
+ priorManifest?.vectors ?? [],
700
+ writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
701
+ textOnDisk ? () => {
702
+ changesSinceCapture = manager.openCheckpointChangeLog();
703
+ } : void 0,
704
+ input.pinsRetiredVectors
705
+ );
890
706
  try {
891
- if (pooledWorker === null) {
892
- pooledWorker = await spawnWorker();
893
- }
894
- const worker = pooledWorker;
895
- if (worker === null) {
896
- workerUsable = false;
897
- return null;
898
- }
899
- const outcome = await runWorker(worker, request);
900
- if (outcome.written !== null) {
901
- return outcome.written;
902
- }
903
- if (outcome.timedOut) {
904
- await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
905
- }
906
- return null;
707
+ await openNewLogSegments(input.indexState, input.failPartition);
708
+ return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
709
+ return changesSinceCapture;
710
+ });
907
711
  } finally {
908
- workerBusy = false;
909
- retireWorkerOnceIdle();
910
- }
911
- }
912
- function cancelIdleRetirement() {
913
- if (idleTimer !== null) {
914
- clearTimeout(idleTimer);
915
- idleTimer = null;
916
- }
917
- }
918
- function retireWorkerOnceIdle() {
919
- cancelIdleRetirement();
920
- idleTimer = unrefTimer(
921
- setTimeout(() => {
922
- idleTimer = null;
923
- if (!workerBusy) terminateCheckpointWorker();
924
- }, workerIdleMs)
925
- );
926
- }
927
- function terminateCheckpointWorker() {
928
- cancelIdleRetirement();
929
- const worker = pooledWorker;
930
- if (worker === null) {
931
- return;
932
- }
933
- pooledWorker = null;
934
- try {
935
- void worker.terminate();
936
- } catch {
937
- }
938
- }
939
-
940
- // src/persistence/durability/checkpoint-write.ts
941
- function workerCanWriteEverySegment(input) {
942
- if (!input.canOffload || input.wholePartitions.length > 0) return false;
943
- const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
944
- if (!analysesTextInProcessOnly) return true;
945
- return input.targets.every(
946
- (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
947
- );
948
- }
949
- async function writeIndexCheckpointSegments(input) {
950
- const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
951
- const { keepsWriteRecords } = input;
952
- const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
953
- root: directory.root,
954
- metadata,
955
- targets,
956
- compactionThreshold,
957
- prior,
958
- capturedPartitions: [...capturedPartitions],
959
- keepsWriteRecords
960
- }) : null;
961
- if (offloaded !== null) {
962
- return offloaded;
963
- }
964
- return writeCheckpointSegments({
965
- store: directoryCheckpointStore(directory),
966
- directory,
967
- metadata,
968
- targets,
969
- compactionThreshold,
970
- prior,
971
- wholePartitions,
972
- capturedPartitions,
973
- keepsWriteRecords
974
- });
975
- }
976
-
977
- // src/persistence/durability/segment/vector-adoption.ts
978
- function recordWrittenVectors(vectorIndexes, written) {
979
- for (const field of written) {
980
- vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
981
- }
982
- }
983
- async function adoptVectorLayouts(store, vectorIndexes, layouts) {
984
- const pathOf = store.pathOf;
985
- if (pathOf === void 0) return;
986
- for (const layout of layouts) {
987
- const vecIndex = vectorIndexes.get(layout.fieldPath);
988
- if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
989
- const path = await pathOf.call(store, layout.key);
990
- await vecIndex.adoptDiskLayout({
991
- path,
992
- docIds: layout.docIds,
993
- dead: layout.dead,
994
- vectorsOffset: layout.vectorsOffset
995
- });
996
- }
997
- }
998
- async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
999
- const pathOf = store.pathOf;
1000
- if (pathOf === void 0) return [...keys];
1001
- const read = /* @__PURE__ */ new Set();
1002
- for (const vectorIndex of vectorIndexes.values()) {
1003
- for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
1004
- }
1005
- if (read.size === 0) return [...keys];
1006
- const unread = [];
1007
- for (const key of keys) {
1008
- if (!read.has(await pathOf.call(store, key))) unread.push(key);
1009
- }
1010
- return unread;
1011
- }
1012
-
1013
- // src/persistence/durability/write-stall.ts
1014
- function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1015
- return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
1016
- }
1017
- async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
1018
- while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
1019
- await waitOutsideCountedWrite(
1020
- () => new Promise((resume) => {
1021
- indexState.stalledWrites.push(resume);
1022
- })
1023
- );
1024
- }
1025
- }
1026
- function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
1027
- const counted = indexState.mutationsSinceCheckpoint;
1028
- const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
1029
- indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
1030
- indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
1031
- releaseStalledWrites(indexState);
1032
- }
1033
- function releaseStalledWrites(indexState) {
1034
- const waiting = indexState.stalledWrites.splice(0);
1035
- for (const resume of waiting) resume();
1036
- }
1037
-
1038
- // src/persistence/durability/checkpoint-run.ts
1039
- async function bothOrTheFirstFailure(first, second) {
1040
- const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
1041
- if (firstSettled.status === "rejected") throw firstSettled.reason;
1042
- if (secondSettled.status === "rejected") throw secondSettled.reason;
1043
- return [firstSettled.value, secondSettled.value];
1044
- }
1045
- async function runDurableCheckpoint(input) {
1046
- try {
1047
- let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1048
- while (recordsLeftInLog > 0 && input.mayContinue()) {
1049
- recordsLeftInLog = await writeOneBoundedCheckpoint(input);
1050
- }
1051
- } catch (err) {
1052
- if (!isSystemError(err)) throw err;
1053
- const error = new NarsilError(
1054
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1055
- `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
1056
- {
1057
- indexName: input.indexName,
1058
- directory: input.directory.root,
1059
- cause: err.message
1060
- }
1061
- );
1062
- input.markFatal(error);
1063
- throw error;
1064
- }
1065
- }
1066
- function isSystemError(err) {
1067
- return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
1068
- }
1069
- async function writeOneBoundedCheckpoint(input) {
1070
- const manager = input.hooks.getManager(input.indexName);
1071
- if (manager === void 0) {
1072
- return 0;
1073
- }
1074
- const metadata = input.hooks.buildTermsMetadata(input.indexName);
1075
- if (metadata === void 0) {
1076
- return 0;
1077
- }
1078
- const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
1079
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1080
- const store = directoryCheckpointStore(input.directory);
1081
- const writeRecords = input.hooks.writeRecords ?? null;
1082
- const { indexName } = input;
1083
- const priorManifest = await readCurrentManifest(store, input.indexName);
1084
- const textOnDisk = metadata.textStorage === "disk";
1085
- const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1086
- priorCheckpoint: priorManifest?.checkpoint ?? [],
1087
- priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1088
- everyPartition: fromMemory,
1089
- aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1090
- ...writeRecords === null ? {} : {
1091
- captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1092
- }
1093
- });
1094
- let changesSinceCapture = null;
1095
- const capture2 = await captureCheckpoint(
1096
- input.indexState,
1097
- manager,
1098
- vectorIndexes,
1099
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1100
- fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1101
- priorManifest?.vectors ?? [],
1102
- writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1103
- textOnDisk ? () => {
1104
- changesSinceCapture = manager.openCheckpointChangeLog();
1105
- } : void 0,
1106
- input.pinsRetiredVectors
1107
- );
1108
- try {
1109
- await openNewLogSegments(input.indexState, input.failPartition);
1110
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1111
- return changesSinceCapture;
1112
- });
1113
- } finally {
1114
- for (const plan of capture2.vectorPlans.values()) plan.release();
1115
- changesSinceCapture?.close();
712
+ for (const plan of capture2.vectorPlans.values()) plan.release();
713
+ changesSinceCapture?.close();
1116
714
  }
1117
715
  }
1118
716
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1119
717
  const { indexName, indexState, hooks } = input;
1120
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-6RLIKSZQ.mjs');
718
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-MG7JMV2U.mjs');
1121
719
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1122
720
  if (adoption === null) return [];
1123
721
  const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
@@ -1172,10 +770,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1172
770
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1173
771
  if (!input.holdsCheckpointFiles()) {
1174
772
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1175
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-6RLIKSZQ.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
773
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-MG7JMV2U.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
774
+ const keptForSync = await input.keepsCheckpointKey();
1176
775
  await removeCheckpointGarbage(
1177
776
  store,
1178
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
777
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
1179
778
  );
1180
779
  }
1181
780
  const checkpointDocumentCount = documentCount;
@@ -1184,222 +783,1148 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1184
783
  if (checkpointMetadata === void 0) {
1185
784
  return;
1186
785
  }
1187
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
1188
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1189
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
1190
- });
1191
- await truncateCoveredSegments(input.directory, input.indexName, targets);
1192
- await reclaimWalBeyondCount(
1193
- input.directory,
1194
- input.indexName,
1195
- manager.partitionCount,
1196
- input.indexState.partitions,
1197
- input.markFatal
1198
- );
1199
- const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1200
- noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1201
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1202
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
1203
- }
1204
- function recordsBeyond(indexState, targets) {
1205
- let records = 0;
1206
- for (const [partitionId, partition] of indexState.partitions) {
1207
- if (partition.failed !== null) continue;
1208
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
1209
- }
1210
- return records;
786
+ const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
787
+ await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
788
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
789
+ });
790
+ const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
791
+ await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
792
+ if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
793
+ await reclaimWalBeyondCount(
794
+ input.directory,
795
+ input.indexName,
796
+ manager.partitionCount,
797
+ input.indexState.partitions,
798
+ input.markFatal
799
+ );
800
+ const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
801
+ noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
802
+ const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
803
+ return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
804
+ }
805
+ function recordsBeyond(indexState, targets) {
806
+ let records = 0;
807
+ for (const [partitionId, partition] of indexState.partitions) {
808
+ if (partition.failed !== null) continue;
809
+ records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
810
+ }
811
+ return records;
812
+ }
813
+
814
+ // src/persistence/durability/checkpoint-turn.ts
815
+ function createIndexLoads(vectorIndexesOf2) {
816
+ const loads = /* @__PURE__ */ new Map();
817
+ const startGraphRepairs = (indexName) => {
818
+ for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
819
+ vectorIndex.scheduleBuild();
820
+ vectorIndex.startShuffledRebuildWhenDue();
821
+ }
822
+ };
823
+ return {
824
+ track(indexName, load, startsGraphRepairs = true) {
825
+ const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
826
+ loads.set(
827
+ indexName,
828
+ loaded.catch(() => void 0)
829
+ );
830
+ return loaded;
831
+ },
832
+ settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
833
+ startGraphRepairs
834
+ };
835
+ }
836
+ async function switchInTurn(turns, turn) {
837
+ let swapped = false;
838
+ await checkpointInTurn(turns, {
839
+ joinsInFlight: false,
840
+ mayStart: turn.mayStart,
841
+ perform: async () => {
842
+ swapped = turn.swap();
843
+ if (swapped) await turn.checkpoint();
844
+ },
845
+ afterSettling: turn.afterSettling
846
+ });
847
+ return swapped;
848
+ }
849
+ async function checkpointInTurn(turns, turn) {
850
+ if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
851
+ while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
852
+ if (!turn.mayStart()) return;
853
+ const run = turn.perform().finally(() => {
854
+ turns.checkpointInFlight = null;
855
+ turn.afterSettling();
856
+ });
857
+ turns.checkpointInFlight = run;
858
+ return run;
859
+ }
860
+
861
+ // src/persistence/durability/held-checkpoints.ts
862
+ function failedPartitionIds(indexState) {
863
+ const failed = [];
864
+ for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
865
+ return failed;
866
+ }
867
+ function createHeldCheckpoints(hooks) {
868
+ const reported = /* @__PURE__ */ new WeakSet();
869
+ function heldReport(indexName, indexState) {
870
+ const partitionIds = failedPartitionIds(indexState);
871
+ if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
872
+ return {
873
+ indexName,
874
+ partitionIds,
875
+ recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
876
+ documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
877
+ };
878
+ }
879
+ return {
880
+ refuse(indexName, indexState) {
881
+ const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
882
+ if (partitionIds.length === 0) return;
883
+ throw new NarsilError(
884
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
885
+ `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
886
+ { indexName, partitionIds }
887
+ );
888
+ },
889
+ holds(indexName, indexState) {
890
+ const held = heldReport(indexName, indexState);
891
+ if (held === null) return false;
892
+ const unreported = held.partitionIds.some((partitionId) => {
893
+ const partition = indexState.partitions.get(partitionId);
894
+ return partition !== void 0 && !reported.has(partition);
895
+ });
896
+ if (unreported) {
897
+ for (const partitionId of held.partitionIds) {
898
+ const partition = indexState.partitions.get(partitionId);
899
+ if (partition !== void 0) reported.add(partition);
900
+ }
901
+ hooks.onCheckpointHeld?.(held);
902
+ }
903
+ return true;
904
+ },
905
+ report(indexes) {
906
+ const held = [];
907
+ for (const [indexName, indexState] of indexes) {
908
+ const entry = heldReport(indexName, indexState);
909
+ if (entry !== null) held.push(entry);
910
+ }
911
+ return held;
912
+ }
913
+ };
914
+ }
915
+
916
+ // src/persistence/durability/sha256.ts
917
+ var ROUND_CONSTANTS = new Uint32Array([
918
+ 1116352408,
919
+ 1899447441,
920
+ 3049323471,
921
+ 3921009573,
922
+ 961987163,
923
+ 1508970993,
924
+ 2453635748,
925
+ 2870763221,
926
+ 3624381080,
927
+ 310598401,
928
+ 607225278,
929
+ 1426881987,
930
+ 1925078388,
931
+ 2162078206,
932
+ 2614888103,
933
+ 3248222580,
934
+ 3835390401,
935
+ 4022224774,
936
+ 264347078,
937
+ 604807628,
938
+ 770255983,
939
+ 1249150122,
940
+ 1555081692,
941
+ 1996064986,
942
+ 2554220882,
943
+ 2821834349,
944
+ 2952996808,
945
+ 3210313671,
946
+ 3336571891,
947
+ 3584528711,
948
+ 113926993,
949
+ 338241895,
950
+ 666307205,
951
+ 773529912,
952
+ 1294757372,
953
+ 1396182291,
954
+ 1695183700,
955
+ 1986661051,
956
+ 2177026350,
957
+ 2456956037,
958
+ 2730485921,
959
+ 2820302411,
960
+ 3259730800,
961
+ 3345764771,
962
+ 3516065817,
963
+ 3600352804,
964
+ 4094571909,
965
+ 275423344,
966
+ 430227734,
967
+ 506948616,
968
+ 659060556,
969
+ 883997877,
970
+ 958139571,
971
+ 1322822218,
972
+ 1537002063,
973
+ 1747873779,
974
+ 1955562222,
975
+ 2024104815,
976
+ 2227730452,
977
+ 2361852424,
978
+ 2428436474,
979
+ 2756734187,
980
+ 3204031479,
981
+ 3329325298
982
+ ]);
983
+ var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
984
+ var BLOCK_BYTES = 64;
985
+ var LENGTH_BYTES = 8;
986
+ function rotateRight(value, bits) {
987
+ return value >>> bits | value << 32 - bits;
988
+ }
989
+ function padded(bytes) {
990
+ const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
991
+ const message = new Uint8Array(total);
992
+ message.set(bytes);
993
+ message[bytes.byteLength] = 128;
994
+ const view = new DataView(message.buffer);
995
+ const bitLength = bytes.byteLength * 8;
996
+ view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
997
+ view.setUint32(total - 4, bitLength >>> 0, false);
998
+ return view;
999
+ }
1000
+ function sha256(bytes) {
1001
+ const view = padded(bytes);
1002
+ const state = INITIAL_STATE.slice();
1003
+ const words = new Uint32Array(64);
1004
+ for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
1005
+ for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
1006
+ for (let i = 16; i < 64; i++) {
1007
+ const w15 = words[i - 15];
1008
+ const w2 = words[i - 2];
1009
+ const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
1010
+ const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
1011
+ words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
1012
+ }
1013
+ let [a, b, c, d, e, f, g, h] = state;
1014
+ for (let i = 0; i < 64; i++) {
1015
+ const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
1016
+ const choice = e & f ^ ~e & g;
1017
+ const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
1018
+ const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
1019
+ const majority = a & b ^ a & c ^ b & c;
1020
+ const t2 = s0 + majority >>> 0;
1021
+ h = g;
1022
+ g = f;
1023
+ f = e;
1024
+ e = d + t1 >>> 0;
1025
+ d = c;
1026
+ c = b;
1027
+ b = a;
1028
+ a = t1 + t2 >>> 0;
1029
+ }
1030
+ state[0] = state[0] + a >>> 0;
1031
+ state[1] = state[1] + b >>> 0;
1032
+ state[2] = state[2] + c >>> 0;
1033
+ state[3] = state[3] + d >>> 0;
1034
+ state[4] = state[4] + e >>> 0;
1035
+ state[5] = state[5] + f >>> 0;
1036
+ state[6] = state[6] + g >>> 0;
1037
+ state[7] = state[7] + h >>> 0;
1038
+ }
1039
+ const digest = new Uint8Array(32);
1040
+ const out = new DataView(digest.buffer);
1041
+ for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
1042
+ return digest;
1043
+ }
1044
+
1045
+ // src/persistence/durability/restore-journal.ts
1046
+ var RESTORE_JOURNAL_PREFIX = ".swap-";
1047
+ var RESTORE_STAGING_PREFIX = ".restore-";
1048
+ var RETIRED_INDEX_PREFIX = ".retired-";
1049
+ var LOCK_FILE_KEY = ".narsil.lock";
1050
+ var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
1051
+ function digestOf(indexName) {
1052
+ let hex = "";
1053
+ for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
1054
+ return hex;
1055
+ }
1056
+ function journalKey(indexName) {
1057
+ return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
1058
+ }
1059
+ function randomSuffix() {
1060
+ return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
1061
+ }
1062
+ function newStagingPrefix() {
1063
+ return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
1064
+ }
1065
+ function newRetiredPrefix() {
1066
+ return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
1067
+ }
1068
+ function encodeJournal(indexName, stagingPrefix) {
1069
+ return new TextEncoder().encode(`${indexName}
1070
+ ${stagingPrefix}`);
1071
+ }
1072
+ function isStagingPrefix(prefix) {
1073
+ return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
1074
+ }
1075
+ function corruptJournal(key, reason) {
1076
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
1077
+ }
1078
+ function readJournal(key, bytes) {
1079
+ const text = new TextDecoder().decode(bytes);
1080
+ const lineEnd = text.indexOf("\n");
1081
+ const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
1082
+ if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
1083
+ if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
1084
+ corruptJournal(key, "names an index that no index name can match");
1085
+ }
1086
+ const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
1087
+ if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
1088
+ corruptJournal(key, "names no staging prefix that a restore writes");
1089
+ }
1090
+ return { kind: "journal", indexName, stagingPrefix };
1091
+ }
1092
+ async function listedKeys(storage, indexName, stagingPrefix) {
1093
+ const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
1094
+ if (manifestBytes === null) {
1095
+ throw new NarsilError(
1096
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1097
+ `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
1098
+ { indexName, stagingPrefix }
1099
+ );
1100
+ }
1101
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1102
+ if (decoded.kind !== "current") {
1103
+ throw new NarsilError(
1104
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1105
+ `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
1106
+ { indexName }
1107
+ );
1108
+ }
1109
+ const root = `${indexName}/`;
1110
+ const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
1111
+ for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
1112
+ return keys;
1113
+ }
1114
+ function withoutTrailingSlash(prefix) {
1115
+ return prefix.slice(0, -1);
1116
+ }
1117
+ async function finishJournal(storage, indexName, stagingPrefix) {
1118
+ const root = `${indexName}/`;
1119
+ const listed = await listedKeys(storage, indexName, stagingPrefix);
1120
+ const staged = await storage.list(stagingPrefix);
1121
+ let retiredPrefix = null;
1122
+ if (storage.moveTree !== void 0 && staged.length > 0) {
1123
+ retiredPrefix = newRetiredPrefix();
1124
+ await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1125
+ await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1126
+ } else {
1127
+ const stagedKeys = new Set(staged);
1128
+ for (const key of listed) {
1129
+ if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1130
+ }
1131
+ }
1132
+ for (const key of await storage.list(root)) {
1133
+ if (!listed.has(key.slice(root.length))) await storage.remove(key);
1134
+ }
1135
+ await storage.remove(journalKey(indexName));
1136
+ return retiredPrefix;
1137
+ }
1138
+ async function removePrefix(storage, prefix) {
1139
+ if (storage.removeTree !== void 0) {
1140
+ await storage.removeTree(prefix);
1141
+ return;
1142
+ }
1143
+ for (const key of await storage.list(prefix)) await storage.remove(key);
1144
+ }
1145
+ async function commitJournal(storage, indexName, stagingPrefix) {
1146
+ await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1147
+ }
1148
+ function isTopLevelJournal(key) {
1149
+ return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1150
+ }
1151
+ async function removeStagingPrefixes(storage) {
1152
+ const prefixes = /* @__PURE__ */ new Set();
1153
+ for (const key of await storage.list(".")) {
1154
+ const slash = key.indexOf("/");
1155
+ if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1156
+ }
1157
+ for (const prefix of prefixes) await removePrefix(storage, prefix);
1158
+ }
1159
+ async function recoverRestoreJournals(storage) {
1160
+ for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1161
+ if (!isTopLevelJournal(key)) continue;
1162
+ const bytes = await storage.read(key);
1163
+ if (bytes === null) continue;
1164
+ const reading = readJournal(key, bytes);
1165
+ if (reading.kind === "leftover") {
1166
+ await storage.remove(key);
1167
+ continue;
1168
+ }
1169
+ await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1170
+ }
1171
+ await removeStagingPrefixes(storage);
1172
+ }
1173
+
1174
+ // src/persistence/durability/restore-directory.ts
1175
+ function missing(error) {
1176
+ return error.code === "ENOENT";
1177
+ }
1178
+ function storageFailed(code, message, key, error) {
1179
+ return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1180
+ }
1181
+ function directoryRestoreStorage(directory) {
1182
+ return {
1183
+ directory,
1184
+ read: (key) => directory.read(key),
1185
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
1186
+ writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1187
+ remove: (key) => directory.remove(key),
1188
+ list: (prefix) => directory.list(prefix),
1189
+ moveKey: (from, to) => directory.move(from, to),
1190
+ pathOf: (key) => directory.pathOf(key),
1191
+ async moveTree(from, to) {
1192
+ const source = await directory.pathOf(from);
1193
+ const target = await directory.pathOf(to);
1194
+ const fs = await getFs();
1195
+ try {
1196
+ await fs.rename(source, target);
1197
+ } catch (error) {
1198
+ if (missing(error)) return;
1199
+ throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1200
+ }
1201
+ await directory.syncDirectoryOf(to);
1202
+ },
1203
+ async removeTree(prefix) {
1204
+ const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1205
+ const fs = await getFs();
1206
+ try {
1207
+ await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1208
+ } catch (error) {
1209
+ throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1210
+ }
1211
+ await directory.syncDirectoryOf(key);
1212
+ }
1213
+ };
1214
+ }
1215
+ function recoverDirectoryRestores(directory) {
1216
+ return recoverRestoreJournals(directoryRestoreStorage(directory));
1217
+ }
1218
+
1219
+ // src/persistence/durability/restore-staging.ts
1220
+ function toError2(error) {
1221
+ return error instanceof Error ? error : new Error(String(error));
1222
+ }
1223
+ function pathUnder(path, root) {
1224
+ if (!path.startsWith(root)) return false;
1225
+ const next = path.charAt(root.length);
1226
+ return next === "" || next === "/" || next === "\\";
1227
+ }
1228
+ function stagedDirectory(directory, staged, unstaged) {
1229
+ const { readRange, byteLength } = directory;
1230
+ return {
1231
+ root: directory.root,
1232
+ appendHandle: (key) => directory.appendHandle(staged(key)),
1233
+ markerHandle: (key) => directory.markerHandle(staged(key)),
1234
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1235
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1236
+ read: (key) => directory.read(staged(key)),
1237
+ remove: (key) => directory.remove(staged(key)),
1238
+ move: (from, to) => directory.move(staged(from), staged(to)),
1239
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1240
+ pathOf: (key) => directory.pathOf(staged(key)),
1241
+ ...readRange === void 0 ? {} : {
1242
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1243
+ },
1244
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1245
+ };
1246
+ }
1247
+ function createRestoreStaging(storage, indexName, options = {}) {
1248
+ const stagingPrefix = newStagingPrefix();
1249
+ const indexRoot = `${indexName}/`;
1250
+ let committed = false;
1251
+ const staged = (key) => {
1252
+ if (!key.startsWith(indexRoot)) {
1253
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1254
+ key
1255
+ });
1256
+ }
1257
+ return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1258
+ };
1259
+ const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1260
+ const store = {
1261
+ read: (key) => storage.read(staged(key)),
1262
+ write: (key, chunks) => storage.write(staged(key), chunks),
1263
+ remove: (key) => storage.remove(staged(key)),
1264
+ list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1265
+ };
1266
+ const pathOf = storage.pathOf;
1267
+ if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1268
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1269
+ let retiredPrefix = null;
1270
+ async function finishOnStorage() {
1271
+ retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1272
+ }
1273
+ return {
1274
+ store,
1275
+ async relocation() {
1276
+ if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1277
+ const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1278
+ const to = await pathOf.call(storage, indexName);
1279
+ return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1280
+ },
1281
+ async commit() {
1282
+ try {
1283
+ await commitJournal(storage, indexName, stagingPrefix);
1284
+ } catch (error) {
1285
+ try {
1286
+ await storage.remove(journalKey(indexName));
1287
+ } catch (removal) {
1288
+ committed = true;
1289
+ options.onStorageLost?.(toError2(removal));
1290
+ }
1291
+ throw error;
1292
+ }
1293
+ committed = true;
1294
+ },
1295
+ async finish() {
1296
+ try {
1297
+ await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1298
+ } catch (error) {
1299
+ options.onStorageLost?.(toError2(error));
1300
+ }
1301
+ },
1302
+ async removeRetired() {
1303
+ const prefix = retiredPrefix;
1304
+ retiredPrefix = null;
1305
+ if (prefix !== null) await removePrefix(storage, prefix);
1306
+ },
1307
+ async discard() {
1308
+ if (committed) return;
1309
+ await removePrefix(storage, stagingPrefix);
1310
+ }
1311
+ };
1312
+ }
1313
+ function adapterRestoreStorage(adapter) {
1314
+ const store = adapterCheckpointStore(adapter);
1315
+ return {
1316
+ read: (key) => store.read(key),
1317
+ write: (key, chunks) => store.write(key, chunks),
1318
+ writeJournal: (key, bytes) => store.write(key, [bytes]),
1319
+ remove: (key) => store.remove(key),
1320
+ list: (prefix) => store.list(prefix),
1321
+ async moveKey(from, to) {
1322
+ const bytes = await store.read(from);
1323
+ if (bytes === null) return;
1324
+ await store.write(to, [bytes]);
1325
+ await store.remove(from);
1326
+ }
1327
+ };
1328
+ }
1329
+
1330
+ // src/persistence/durability/archive-checkpoint.ts
1331
+ function toError3(error) {
1332
+ return error instanceof Error ? error : new Error(String(error));
1333
+ }
1334
+ function stageDirectoryRestore(deps, indexName) {
1335
+ return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1336
+ async finish(run) {
1337
+ try {
1338
+ await run();
1339
+ } catch (error) {
1340
+ deps.markFatal(toError3(error));
1341
+ throw error;
1342
+ }
1343
+ },
1344
+ onStorageLost: deps.markFatal
1345
+ });
1346
+ }
1347
+ function releaser(held, indexName) {
1348
+ let released = false;
1349
+ return () => {
1350
+ if (released) return;
1351
+ released = true;
1352
+ const count = (held.get(indexName) ?? 1) - 1;
1353
+ if (count <= 0) held.delete(indexName);
1354
+ else held.set(indexName, count);
1355
+ };
1356
+ }
1357
+ function createArchiveCheckpointAccess(deps) {
1358
+ const store = directoryCheckpointStore(deps.directory);
1359
+ return {
1360
+ async openArchiveCheckpoint(indexName) {
1361
+ deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1362
+ const release = releaser(deps.held, indexName);
1363
+ try {
1364
+ await deps.checkpoint(indexName, false);
1365
+ await deps.checkpoint(indexName, true);
1366
+ const manifestBytes = await deps.directory.read(manifestKey(indexName));
1367
+ if (manifestBytes === null) {
1368
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1369
+ indexName
1370
+ });
1371
+ }
1372
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1373
+ if (decoded.kind !== "current") {
1374
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1375
+ indexName
1376
+ });
1377
+ }
1378
+ return {
1379
+ manifestBytes,
1380
+ manifest: decoded.manifest,
1381
+ readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1382
+ readRange: async (ref, offset, length) => {
1383
+ const readRange = deps.directory.readRange;
1384
+ if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1385
+ const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1386
+ return whole.slice(offset, offset + length);
1387
+ },
1388
+ release,
1389
+ directory: deps.directory
1390
+ };
1391
+ } catch (error) {
1392
+ release();
1393
+ throw error;
1394
+ }
1395
+ },
1396
+ stageRestore(indexName) {
1397
+ return stageDirectoryRestore(deps, indexName);
1398
+ },
1399
+ adoptRestoredCheckpoint(indexName, checkpoint) {
1400
+ for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1401
+ deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1402
+ }
1403
+ }
1404
+ };
1211
1405
  }
1212
1406
 
1213
- // src/persistence/durability/checkpoint-turn.ts
1214
- function createIndexLoads(vectorIndexesOf2) {
1215
- const loads = /* @__PURE__ */ new Map();
1216
- const startGraphRepairs = (indexName) => {
1217
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1218
- vectorIndex.scheduleBuild();
1219
- vectorIndex.startShuffledRebuildWhenDue();
1220
- }
1221
- };
1407
+ // src/persistence/durability/manager-unload.ts
1408
+ function createIndexUnloading(deps) {
1409
+ const { directory, hooks, indexes, metadataWrites } = deps;
1222
1410
  return {
1223
- track(indexName, load, startsGraphRepairs = true) {
1224
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1225
- loads.set(
1226
- indexName,
1227
- loaded.catch(() => void 0)
1228
- );
1229
- return loaded;
1411
+ async removeIndex(indexName) {
1412
+ hooks.writeRecords?.dropIndex(indexName);
1413
+ const indexState = indexes.get(indexName);
1414
+ if (indexState !== void 0) {
1415
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
1416
+ indexes.delete(indexName);
1417
+ metadataWrites.forget(indexName);
1418
+ }
1419
+ for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
1420
+ for (const key of await directory.list(`${indexName}/`)) {
1421
+ await directory.remove(key);
1422
+ }
1423
+ deps.forgetSyncState(indexName);
1230
1424
  },
1231
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1232
- startGraphRepairs
1425
+ async unloadIndex(indexName) {
1426
+ const indexState = indexes.get(indexName);
1427
+ if (indexState === void 0) {
1428
+ return;
1429
+ }
1430
+ await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
1431
+ indexes.delete(indexName);
1432
+ metadataWrites.forget(indexName);
1433
+ deps.forgetSyncState(indexName);
1434
+ }
1233
1435
  };
1234
1436
  }
1235
- async function switchInTurn(turns, turn) {
1236
- let swapped = false;
1237
- await checkpointInTurn(turns, {
1238
- joinsInFlight: false,
1239
- mayStart: turn.mayStart,
1240
- perform: async () => {
1241
- swapped = turn.swap();
1242
- if (swapped) await turn.checkpoint();
1243
- },
1244
- afterSettling: turn.afterSettling
1245
- });
1246
- return swapped;
1247
- }
1248
- async function checkpointInTurn(turns, turn) {
1249
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1250
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1251
- if (!turn.mayStart()) return;
1252
- const run = turn.perform().finally(() => {
1253
- turns.checkpointInFlight = null;
1254
- turn.afterSettling();
1255
- });
1256
- turns.checkpointInFlight = run;
1257
- return run;
1258
- }
1259
1437
 
1260
- // src/persistence/durability/directory-watch.ts
1261
- function unrefInterval(tick, intervalMs) {
1262
- const timer = setInterval(tick, intervalMs);
1263
- if (typeof timer.unref === "function") timer.unref();
1264
- return timer;
1265
- }
1266
- function createDirectoryWatch(directory, isStopped, onLost) {
1267
- let expected = null;
1268
- let timer = null;
1269
- let lock = null;
1270
- function cannotCheck(err) {
1271
- return new NarsilError(
1272
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1273
- `The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
1274
- { directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
1275
- );
1438
+ // src/persistence/durability/sync/frame-cursor.ts
1439
+ var SegmentFileMissing = class extends Error {
1440
+ };
1441
+ function corrupt(key, message, details) {
1442
+ throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
1443
+ }
1444
+ function isMissingFile(error) {
1445
+ return error?.code === "ENOENT";
1446
+ }
1447
+ var FrameCursor = class {
1448
+ constructor(directory, key, limit) {
1449
+ this.directory = directory;
1450
+ this.key = key;
1451
+ this.limit = limit;
1452
+ }
1453
+ buffer = new Uint8Array(0);
1454
+ bufferStart = 0;
1455
+ whole = null;
1456
+ async load(offset, length) {
1457
+ const readRange = this.directory.readRange;
1458
+ if (readRange === void 0) {
1459
+ if (this.whole === null) {
1460
+ const bytes = await this.directory.read(this.key);
1461
+ if (bytes === null) throw new SegmentFileMissing(this.key);
1462
+ this.whole = bytes;
1463
+ }
1464
+ this.buffer = this.whole;
1465
+ this.bufferStart = 0;
1466
+ return;
1467
+ }
1468
+ const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
1469
+ try {
1470
+ this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
1471
+ } catch (error) {
1472
+ if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
1473
+ throw error;
1474
+ }
1475
+ this.bufferStart = offset;
1476
+ }
1477
+ async bytesAt(offset, length) {
1478
+ const end = offset + length;
1479
+ if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
1480
+ if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
1481
+ return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
1276
1482
  }
1277
- async function currentIdentity() {
1483
+ async checkHeader() {
1484
+ if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
1485
+ const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
1486
+ if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
1487
+ }
1488
+ async frameLengthAt(offset) {
1489
+ const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
1490
+ const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
1491
+ 0,
1492
+ false
1493
+ );
1494
+ if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
1495
+ corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
1496
+ }
1497
+ return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
1498
+ }
1499
+ async frameAt(offset, frameBytes) {
1500
+ const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
1501
+ const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
1502
+ const payload = body.subarray(0, recordLength);
1503
+ const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
1504
+ if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
1505
+ let seqNo;
1278
1506
  try {
1279
- return await directory.identity?.() ?? null;
1280
- } catch (err) {
1281
- return cannotCheck(err);
1507
+ seqNo = decodeWalEntry(payload).seqNo;
1508
+ } catch (error) {
1509
+ corrupt(this.key, "A log record fails to decode", {
1510
+ offset,
1511
+ cause: error instanceof Error ? error.message : String(error)
1512
+ });
1282
1513
  }
1514
+ return { seqNo, payload: payload.slice(), frameBytes };
1515
+ }
1516
+ };
1517
+
1518
+ // src/persistence/durability/sync/log-reader.ts
1519
+ async function walSources(directory, indexName, partitionId) {
1520
+ const prefix = walSegmentPrefix(indexName, partitionId);
1521
+ const markerBytes = await directory.read(`${prefix}commit`);
1522
+ const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1523
+ if (marker === null) return { sources: [], head: 0 };
1524
+ const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
1525
+ const sources = [];
1526
+ for (const key of await directory.list(prefix)) {
1527
+ const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
1528
+ if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
1529
+ const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
1530
+ if (readableBytes === null || readableBytes === void 0) continue;
1531
+ sources.push({ key, startSeqNo, readableBytes, inHistory: false });
1532
+ }
1533
+ return { sources, head: highestDurableSeqNo };
1534
+ }
1535
+ async function openPartitionLog(directory, indexName, partitionId) {
1536
+ const { sources, head } = await walSources(directory, indexName, partitionId);
1537
+ const byStart = /* @__PURE__ */ new Map();
1538
+ for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
1539
+ const readableBytes = await directory.byteLength?.(segment.key);
1540
+ if (readableBytes === null || readableBytes === void 0) continue;
1541
+ byStart.set(segment.startSeqNo, {
1542
+ key: segment.key,
1543
+ startSeqNo: segment.startSeqNo,
1544
+ readableBytes,
1545
+ inHistory: true
1546
+ });
1283
1547
  }
1284
- async function lockTakenOver() {
1285
- if (lock === null) return null;
1548
+ for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
1549
+ const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
1550
+ return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
1551
+ }
1552
+ function sourceHolding(log, seqNo) {
1553
+ let found = -1;
1554
+ for (let i = 0; i < log.sources.length; i++) {
1555
+ if (log.sources[i].startSeqNo > seqNo) break;
1556
+ found = i;
1557
+ }
1558
+ return found;
1559
+ }
1560
+ async function offsetAfter(cursor, source, afterSeqNo) {
1561
+ let offset = SEGMENT_HEADER_SIZE;
1562
+ for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
1563
+ offset += await cursor.frameLengthAt(offset);
1564
+ }
1565
+ return offset;
1566
+ }
1567
+ async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
1568
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1569
+ await cursor.checkHeader();
1570
+ let offset = await offsetAfter(cursor, source, afterSeqNo);
1571
+ let checksNumbering = true;
1572
+ while (offset < cursor.limit) {
1573
+ const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
1574
+ if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
1575
+ checksNumbering = false;
1576
+ if (frame.seqNo > afterSeqNo) {
1577
+ offset = SEGMENT_HEADER_SIZE;
1578
+ continue;
1579
+ }
1580
+ }
1581
+ checksNumbering = false;
1582
+ offset += frame.frameBytes;
1583
+ if (frame.seqNo <= afterSeqNo) continue;
1584
+ if (frame.seqNo > upToSeqNo) return true;
1585
+ if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
1586
+ read.frames.push(frame);
1587
+ read.bytes += frame.payload.byteLength;
1588
+ if (frame.seqNo === upToSeqNo) return true;
1589
+ }
1590
+ return false;
1591
+ }
1592
+ async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
1593
+ const read = { frames: [], bytes: 0, gone: false };
1594
+ if (afterSeqNo >= upToSeqNo) return read;
1595
+ for (let attempt = 0; attempt < 2; attempt++) {
1596
+ read.frames = [];
1597
+ read.bytes = 0;
1598
+ const log = await openPartitionLog(directory, indexName, partitionId);
1599
+ let index = sourceHolding(log, afterSeqNo + 1);
1600
+ if (index < 0) {
1601
+ read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
1602
+ return read;
1603
+ }
1286
1604
  try {
1287
- if (await lock.holds()) return null;
1288
- } catch (err) {
1289
- return cannotCheck(err);
1605
+ for (; index < log.sources.length; index++) {
1606
+ const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
1607
+ if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
1608
+ }
1609
+ return read;
1610
+ } catch (error) {
1611
+ if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
1290
1612
  }
1291
- return new NarsilError(
1292
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1293
- `The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
1294
- { directory: directory.root }
1295
- );
1296
1613
  }
1297
- async function verify() {
1298
- if (directory.identity === void 0 || isStopped()) return true;
1299
- const current = await currentIdentity();
1300
- if (current instanceof Error) {
1301
- onLost(current);
1302
- return false;
1303
- }
1304
- if (expected === null) {
1305
- expected = current;
1306
- } else if (current !== expected) {
1307
- onLost(
1308
- new NarsilError(
1309
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1310
- `The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
1311
- { directory: directory.root }
1312
- )
1313
- );
1314
- return false;
1614
+ return read;
1615
+ }
1616
+ async function logBytesAfter(directory, log, afterSeqNo) {
1617
+ const first = sourceHolding(log, afterSeqNo + 1);
1618
+ let bytes = 0;
1619
+ for (let i = Math.max(first, 0); i < log.sources.length; i++) {
1620
+ const source = log.sources[i];
1621
+ if (i === first) {
1622
+ const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1623
+ try {
1624
+ bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
1625
+ } catch (error) {
1626
+ if (!(error instanceof SegmentFileMissing)) throw error;
1627
+ }
1628
+ continue;
1315
1629
  }
1316
- const takenOver = await lockTakenOver();
1317
- if (takenOver === null) return true;
1318
- onLost(takenOver);
1319
- return false;
1630
+ bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
1320
1631
  }
1321
- function stop() {
1322
- if (timer !== null) {
1323
- clearInterval(timer);
1324
- timer = null;
1632
+ return bytes;
1633
+ }
1634
+
1635
+ // src/persistence/durability/sync/pins.ts
1636
+ function createFilePins(remove) {
1637
+ const counts = /* @__PURE__ */ new Map();
1638
+ const awaitingRemoval = /* @__PURE__ */ new Set();
1639
+ function unpin(key) {
1640
+ const count = (counts.get(key) ?? 1) - 1;
1641
+ if (count > 0) {
1642
+ counts.set(key, count);
1643
+ return;
1325
1644
  }
1645
+ counts.delete(key);
1646
+ if (!awaitingRemoval.delete(key)) return;
1647
+ void remove(key).catch(() => void 0);
1326
1648
  }
1327
1649
  return {
1328
- async claim() {
1329
- if (lock === null && directory.lock !== void 0) lock = await directory.lock();
1330
- },
1331
- async release() {
1332
- const held = lock;
1333
- lock = null;
1334
- await held?.release();
1335
- },
1336
- verify,
1337
- async verifyOnce() {
1338
- if (expected === null && directory.identity !== void 0) await verify();
1339
- },
1340
- start() {
1341
- if (directory.identity === void 0 || timer !== null || isStopped()) return;
1342
- timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
1650
+ pin(keys) {
1651
+ const pinned = [...keys];
1652
+ for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
1653
+ let released = false;
1654
+ return () => {
1655
+ if (released) return;
1656
+ released = true;
1657
+ for (const key of pinned) unpin(key);
1658
+ };
1343
1659
  },
1344
- stop
1660
+ isPinned: (key) => counts.has(key),
1661
+ removeWhenUnpinned(key) {
1662
+ if (!counts.has(key)) return false;
1663
+ awaitingRemoval.add(key);
1664
+ return true;
1665
+ }
1666
+ };
1667
+ }
1668
+ var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
1669
+ var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
1670
+ function hexOf(bytes) {
1671
+ let hex = "";
1672
+ for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
1673
+ return hex;
1674
+ }
1675
+ function isBytes(value) {
1676
+ return value instanceof Uint8Array;
1677
+ }
1678
+ async function publishedFrom(indexName, payload, config, manifestBytes) {
1679
+ const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1680
+ if (decoded.kind !== "current") {
1681
+ throw new NarsilError(
1682
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1683
+ `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
1684
+ { indexName }
1685
+ );
1686
+ }
1687
+ return {
1688
+ config,
1689
+ manifestBytes,
1690
+ manifest: decoded.manifest,
1691
+ entityTag: hexOf(sha256(payload)),
1692
+ keys: manifestReferencedKeys(decoded.manifest)
1345
1693
  };
1346
1694
  }
1695
+ async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
1696
+ const payload = encode({ config, manifest: manifestBytes });
1697
+ const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
1698
+ const published = await publishedFrom(indexName, payload, config, manifestBytes);
1699
+ await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
1700
+ return published;
1701
+ }
1702
+ async function readPublishedSnapshot(directory, indexName) {
1703
+ const bytes = await directory.read(publishedSnapshotKey(indexName));
1704
+ if (bytes === null) return null;
1705
+ const { payload } = await openChecksummedEnvelope(bytes, {
1706
+ envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
1707
+ what: PUBLISHED_SNAPSHOT_WHAT,
1708
+ allowsCompression: false
1709
+ });
1710
+ const raw = decode(payload);
1711
+ if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
1712
+ throw new NarsilError(
1713
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1714
+ `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
1715
+ { indexName }
1716
+ );
1717
+ }
1718
+ return publishedFrom(indexName, payload, raw.config, raw.manifest);
1719
+ }
1347
1720
 
1348
- // src/persistence/durability/held-checkpoints.ts
1349
- function failedPartitionIds(indexState) {
1350
- const failed = [];
1351
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
1352
- return failed;
1721
+ // src/persistence/durability/sync/retention.ts
1722
+ async function firstLogStart(directory, indexName, partitionId) {
1723
+ let first = Number.POSITIVE_INFINITY;
1724
+ for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
1725
+ const start = parseWalSegmentStart(indexName, partitionId, key);
1726
+ if (start !== null && start < first) first = start;
1727
+ }
1728
+ return first;
1729
+ }
1730
+ async function sizedQueues(retention) {
1731
+ const { directory, indexName } = retention;
1732
+ const queues = /* @__PURE__ */ new Map();
1733
+ for (const segment of await listHistorySegments(directory, indexName)) {
1734
+ const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
1735
+ if (bytes === void 0 || bytes === null) continue;
1736
+ const queue = queues.get(segment.partitionId) ?? [];
1737
+ queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
1738
+ queues.set(segment.partitionId, queue);
1739
+ }
1740
+ for (const [partitionId, queue] of queues) {
1741
+ const logStart = await firstLogStart(directory, indexName, partitionId);
1742
+ for (let i = 0; i < queue.length; i++) {
1743
+ const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
1744
+ queue[i].lastSeqNo = nextStart - 1;
1745
+ }
1746
+ }
1747
+ return queues;
1748
+ }
1749
+ function nextToDelete(queues, totalBytes, retention) {
1750
+ const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
1751
+ let oldest = null;
1752
+ for (const queue of queues.values()) {
1753
+ const head = queue[0];
1754
+ if (head === void 0) continue;
1755
+ if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
1756
+ }
1757
+ if (oldest === null) return null;
1758
+ if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
1759
+ return null;
1760
+ }
1761
+ async function enforceHistoryLimits(retention) {
1762
+ const queues = await sizedQueues(retention);
1763
+ let totalBytes = 0;
1764
+ for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
1765
+ let published = false;
1766
+ let deleted = 0;
1767
+ let segment = nextToDelete(queues, totalBytes, retention);
1768
+ while (segment !== null) {
1769
+ const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
1770
+ if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
1771
+ published = true;
1772
+ await retention.publish();
1773
+ }
1774
+ await retention.directory.remove(segment.key);
1775
+ queues.get(segment.partitionId)?.shift();
1776
+ totalBytes -= segment.bytes;
1777
+ deleted += 1;
1778
+ segment = nextToDelete(queues, totalBytes, retention);
1779
+ }
1780
+ return deleted;
1353
1781
  }
1354
- function createHeldCheckpoints(hooks) {
1355
- const reported = /* @__PURE__ */ new WeakSet();
1356
- function heldReport(indexName, indexState) {
1357
- const partitionIds = failedPartitionIds(indexState);
1358
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
1359
- return {
1360
- indexName,
1361
- partitionIds,
1362
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
1363
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
1364
- };
1782
+
1783
+ // src/persistence/durability/sync/storage.ts
1784
+ function createSyncStorage(deps) {
1785
+ const { directory } = deps;
1786
+ const store = directoryCheckpointStore(directory);
1787
+ const cache = /* @__PURE__ */ new Map();
1788
+ const publishing = /* @__PURE__ */ new Map();
1789
+ const pins = createFilePins((key) => directory.remove(key));
1790
+ async function published(indexName) {
1791
+ const cached = cache.get(indexName);
1792
+ if (cached !== void 0) return cached;
1793
+ const read = await readPublishedSnapshot(directory, indexName);
1794
+ cache.set(indexName, read);
1795
+ return read;
1796
+ }
1797
+ async function removeUnlisted(indexName, previous, kept) {
1798
+ const manifest = await readCurrentManifest(store, indexName);
1799
+ const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
1800
+ for (const key of previous) {
1801
+ if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
1802
+ await directory.remove(key);
1803
+ }
1804
+ }
1805
+ async function publishNow(indexName, configurationOf) {
1806
+ const archive = await deps.openArchiveCheckpoint(indexName);
1807
+ try {
1808
+ const config = await configurationOf(archive.manifest);
1809
+ const previous = await published(indexName);
1810
+ const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
1811
+ cache.set(indexName, next);
1812
+ if (previous !== null) {
1813
+ await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
1814
+ }
1815
+ return next;
1816
+ } finally {
1817
+ archive.release();
1818
+ }
1365
1819
  }
1366
1820
  return {
1367
- refuse(indexName, indexState) {
1368
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
1369
- if (partitionIds.length === 0) return;
1370
- throw new NarsilError(
1371
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1372
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
1373
- { indexName, partitionIds }
1374
- );
1821
+ publish(indexName, configurationOf) {
1822
+ const inFlight = publishing.get(indexName);
1823
+ const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
1824
+ const tracked = run.finally(() => {
1825
+ if (publishing.get(indexName) === tracked) publishing.delete(indexName);
1826
+ });
1827
+ publishing.set(indexName, tracked);
1828
+ return tracked;
1375
1829
  },
1376
- holds(indexName, indexState) {
1377
- const held = heldReport(indexName, indexState);
1378
- if (held === null) return false;
1379
- const unreported = held.partitionIds.some((partitionId) => {
1380
- const partition = indexState.partitions.get(partitionId);
1381
- return partition !== void 0 && !reported.has(partition);
1830
+ published,
1831
+ pin: (snapshot) => pins.pin(snapshot.keys),
1832
+ async readFileRange(key, offset, length) {
1833
+ const readRange = directory.readRange;
1834
+ if (readRange !== void 0) return readRange.call(directory, key, offset, length);
1835
+ const whole = await directory.read(key);
1836
+ return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
1837
+ },
1838
+ async release(indexName) {
1839
+ await deps.inCheckpointTurn(indexName, async () => {
1840
+ const previous = await published(indexName);
1841
+ await directory.remove(publishedSnapshotKey(indexName));
1842
+ cache.set(indexName, null);
1843
+ if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
1844
+ await removeHistory(directory, indexName);
1382
1845
  });
1383
- if (unreported) {
1384
- for (const partitionId of held.partitionIds) {
1385
- const partition = indexState.partitions.get(partitionId);
1386
- if (partition !== void 0) reported.add(partition);
1387
- }
1388
- hooks.onCheckpointHeld?.(held);
1389
- }
1390
- return true;
1391
1846
  },
1392
- report(indexes) {
1393
- const held = [];
1394
- for (const [indexName, indexState] of indexes) {
1395
- const entry = heldReport(indexName, indexState);
1396
- if (entry !== null) held.push(entry);
1847
+ removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
1848
+ async enforceHistoryLimits(indexName, limits, publish, nowMs) {
1849
+ const current = await published(indexName);
1850
+ return enforceHistoryLimits({
1851
+ directory,
1852
+ indexName,
1853
+ limits,
1854
+ nowMs,
1855
+ publish,
1856
+ publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
1857
+ });
1858
+ },
1859
+ openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
1860
+ readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
1861
+ logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
1862
+ currentManifest: (indexName) => readCurrentManifest(store, indexName),
1863
+ async keepsCheckpointKey(indexName) {
1864
+ const current = await published(indexName);
1865
+ return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
1866
+ },
1867
+ async settleAtRecovery(indexName, active) {
1868
+ cache.delete(indexName);
1869
+ const publishedKey = publishedSnapshotKey(indexName);
1870
+ for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
1871
+ if (key !== publishedKey) await directory.remove(key);
1397
1872
  }
1398
- return held;
1873
+ if (!active) {
1874
+ await directory.remove(publishedSnapshotKey(indexName));
1875
+ await removeHistory(directory, indexName);
1876
+ cache.set(indexName, null);
1877
+ return /* @__PURE__ */ new Set();
1878
+ }
1879
+ try {
1880
+ return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
1881
+ } catch {
1882
+ await directory.remove(publishedSnapshotKey(indexName));
1883
+ cache.set(indexName, null);
1884
+ return /* @__PURE__ */ new Set();
1885
+ }
1886
+ },
1887
+ forget(indexName) {
1888
+ cache.delete(indexName);
1399
1889
  }
1400
1890
  };
1401
1891
  }
1402
1892
 
1893
+ // src/persistence/durability/manager-storage.ts
1894
+ function createIndexStorageAccess(deps) {
1895
+ const { directory, indexes } = deps;
1896
+ const archiveAccess = createArchiveCheckpointAccess({
1897
+ directory,
1898
+ held: deps.heldCheckpointFiles,
1899
+ checkpoint: deps.checkpoint,
1900
+ registerPartition: deps.registerPartition,
1901
+ markFatal: deps.markFatal
1902
+ });
1903
+ const syncStorage = createSyncStorage({
1904
+ directory,
1905
+ openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
1906
+ inCheckpointTurn: (indexName, work) => {
1907
+ const indexState = indexes.get(indexName);
1908
+ if (indexState === void 0) return work();
1909
+ return checkpointInTurn(indexState, {
1910
+ joinsInFlight: false,
1911
+ mayStart: () => true,
1912
+ perform: work,
1913
+ afterSettling: () => releaseStalledWrites(indexState)
1914
+ });
1915
+ }
1916
+ });
1917
+ const unloading = createIndexUnloading({
1918
+ directory,
1919
+ hooks: deps.hooks,
1920
+ indexes,
1921
+ metadataWrites: deps.metadataWrites,
1922
+ closeWriterReportingFailure: deps.closeWriterReportingFailure,
1923
+ forgetSyncState: syncStorage.forget
1924
+ });
1925
+ return { archiveAccess, syncStorage, unloading };
1926
+ }
1927
+
1403
1928
  // src/persistence/durability/metadata-write-queue.ts
1404
1929
  function createMetadataWriteQueue() {
1405
1930
  const writes = /* @__PURE__ */ new Map();
@@ -2370,7 +2895,7 @@ var decoder = new TextDecoder();
2370
2895
  async function countPagedPartition(store, partition) {
2371
2896
  const { directory } = store;
2372
2897
  if (directory === void 0 || partition.segments.length === 0) return null;
2373
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-6RLIKSZQ.mjs');
2898
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-MG7JMV2U.mjs');
2374
2899
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2375
2900
  const scans = [];
2376
2901
  try {
@@ -2653,12 +3178,14 @@ async function recoverCheckpoint(deps) {
2653
3178
  return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
2654
3179
  }
2655
3180
  if (decoded === null || decoded.kind !== "current") {
2656
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
3181
+ if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
2657
3182
  return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
2658
3183
  }
2659
3184
  await deps.store.remove(snapshotBundleKey(deps.indexName));
2660
3185
  if (!deps.keepsUnlistedFiles) {
2661
- await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
3186
+ const referenced = manifestReferencedKeys(decoded.manifest);
3187
+ for (const key of deps.keptForSync) referenced.add(key);
3188
+ await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
2662
3189
  }
2663
3190
  const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
2664
3191
  manager: deps.manager,
@@ -2779,6 +3306,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
2779
3306
  }
2780
3307
  return replayed;
2781
3308
  };
3309
+ const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
3310
+ const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
2782
3311
  const recovered = await recoverCheckpoint({
2783
3312
  store,
2784
3313
  indexName,
@@ -2788,6 +3317,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
2788
3317
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
2789
3318
  compactionThreshold: deps.compactionThreshold,
2790
3319
  keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3320
+ keptForSync,
2791
3321
  replay,
2792
3322
  onLayoutUpgrade: hooks.onLayoutUpgrade,
2793
3323
  writeRecords
@@ -2903,6 +3433,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2903
3433
  mutationThreshold: checkpointMutationThreshold,
2904
3434
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
2905
3435
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
3436
+ keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
2906
3437
  queueMetadataWrite,
2907
3438
  markFatal,
2908
3439
  failPartition: partitionFailureFor(indexName, partitionBatchDeps)
@@ -2956,6 +3487,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2956
3487
  startAsyncFlushTimer();
2957
3488
  return outcomes;
2958
3489
  }
3490
+ const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
3491
+ directory,
3492
+ hooks,
3493
+ indexes,
3494
+ metadataWrites,
3495
+ heldCheckpointFiles,
3496
+ checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3497
+ registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3498
+ markFatal,
3499
+ closeWriterReportingFailure
3500
+ });
2959
3501
  const recoveryDeps = {
2960
3502
  directory,
2961
3503
  hooks,
@@ -2964,7 +3506,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
2964
3506
  getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
2965
3507
  },
2966
3508
  holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
2967
- queueMetadataWrite
3509
+ queueMetadataWrite,
3510
+ syncStorage
2968
3511
  };
2969
3512
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
2970
3513
  return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
@@ -3052,35 +3595,9 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3052
3595
  await checkpointIndex(indexName);
3053
3596
  }
3054
3597
  },
3055
- async removeIndex(indexName) {
3056
- hooks.writeRecords?.dropIndex(indexName);
3057
- const indexState = indexes.get(indexName);
3058
- if (indexState !== void 0) {
3059
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
3060
- indexes.delete(indexName);
3061
- metadataWrites.forget(indexName);
3062
- }
3063
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
3064
- for (const key of await directory.list(`${indexName}/`)) {
3065
- await directory.remove(key);
3066
- }
3067
- },
3068
- async unloadIndex(indexName) {
3069
- const indexState = indexes.get(indexName);
3070
- if (indexState === void 0) {
3071
- return;
3072
- }
3073
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
3074
- indexes.delete(indexName);
3075
- metadataWrites.forget(indexName);
3076
- },
3077
- ...createArchiveCheckpointAccess({
3078
- directory,
3079
- held: heldCheckpointFiles,
3080
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3081
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3082
- markFatal
3083
- }),
3598
+ ...unloading,
3599
+ syncStorage: () => syncStorage,
3600
+ ...archiveAccess,
3084
3601
  async shutdown() {
3085
3602
  shuttingDown = true;
3086
3603
  stopTimers();
@@ -3347,6 +3864,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3347
3864
  storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3348
3865
  compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3349
3866
  keepsUnlistedFiles: false,
3867
+ keptForSync: /* @__PURE__ */ new Set(),
3350
3868
  replay: async (checkpoint) => checkpoint,
3351
3869
  onLayoutUpgrade: hooks.onLayoutUpgrade
3352
3870
  });
@@ -3544,7 +4062,9 @@ function createDurabilityIntegration(tier, hooks) {
3544
4062
  recordCheckpoint: hooks.recordCheckpoint,
3545
4063
  onLayoutUpgrade: hooks.onLayoutUpgrade,
3546
4064
  onCheckpointHeld: hooks.onCheckpointHeld,
3547
- onFatalError: hooks.onFatalError
4065
+ onFatalError: hooks.onFatalError,
4066
+ ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
4067
+ ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
3548
4068
  };
3549
4069
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
3550
4070
  function partitionFor(indexName, docId) {
@@ -3629,7 +4149,12 @@ function createDurabilityFromTier(tier, wiring) {
3629
4149
  recordCheckpoint: wiring.recordCheckpoint,
3630
4150
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
3631
4151
  onCheckpointHeld: wiring.emitCheckpointHeld,
3632
- onFatalError: wiring.emitFatalError
4152
+ onFatalError: wiring.emitFatalError,
4153
+ keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4154
+ onHistoryMoved(indexName) {
4155
+ if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4156
+ import('./retention-55WA7GPU.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4157
+ }
3633
4158
  });
3634
4159
  }
3635
4160