@delali/narsil 0.3.1-next.157 → 0.3.1-next.161

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (565) hide show
  1. package/dist/adapters/filesystem.mjs +3 -3
  2. package/dist/adapters/gcs.mjs +3 -3
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +3 -3
  5. package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
  6. package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
  7. package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
  8. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  9. package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
  10. package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
  11. package/dist/browser/embed-RF2MPGAO.mjs +222 -0
  12. package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
  13. package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
  14. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  15. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  16. package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
  17. package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
  18. package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
  19. package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
  20. package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
  21. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  22. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  23. package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
  24. package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
  25. package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
  26. package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
  27. package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
  28. package/dist/chunk-2JRERUNZ.mjs +217 -0
  29. package/dist/chunk-2RIS76RJ.mjs +212 -0
  30. package/dist/chunk-3B775F3C.mjs +252 -0
  31. package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
  32. package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
  33. package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
  34. package/dist/chunk-6EZW2GQL.mjs +978 -0
  35. package/dist/chunk-75UNINFA.mjs +271 -0
  36. package/dist/chunk-77AUVDQ5.mjs +7 -0
  37. package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
  38. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  39. package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
  40. package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
  41. package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
  42. package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
  43. package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
  44. package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
  45. package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
  46. package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
  47. package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
  48. package/dist/chunk-GXCDWLQI.mjs +193 -0
  49. package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
  50. package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
  51. package/dist/chunk-IB4BYLE3.mjs +127 -0
  52. package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
  53. package/dist/chunk-IH3G6SJO.mjs +40 -0
  54. package/dist/chunk-JV74RKCY.mjs +3041 -0
  55. package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
  56. package/dist/chunk-KLVMK3R5.mjs +47 -0
  57. package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
  58. package/dist/chunk-LF4GB4PO.mjs +281 -0
  59. package/dist/chunk-NC4QLCZC.mjs +59 -0
  60. package/dist/chunk-OOO54OVC.mjs +627 -0
  61. package/dist/chunk-OUWZKNO3.mjs +207 -0
  62. package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
  63. package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
  64. package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
  65. package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
  66. package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
  67. package/dist/chunk-S2POFTLN.mjs +10 -0
  68. package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
  69. package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
  70. package/dist/chunk-SXIRM4MX.mjs +3549 -0
  71. package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
  72. package/dist/chunk-TINW4BSH.mjs +31301 -0
  73. package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
  74. package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
  75. package/dist/chunk-U6B7DUQW.mjs +788 -0
  76. package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
  77. package/dist/chunk-W34VJXY7.mjs +4723 -0
  78. package/dist/chunk-YETKPX5M.mjs +5 -0
  79. package/dist/chunk-ZQ6QY42L.mjs +59 -0
  80. package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
  81. package/dist/client.d.ts +8 -8
  82. package/dist/client.mjs +66 -18
  83. package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
  84. package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
  85. package/dist/core-leaf-KXFGP74A.mjs +7 -0
  86. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  87. package/dist/direct-executor-MKTZBR26.mjs +16 -0
  88. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  89. package/dist/distribution/coordinator/etcd.mjs +7 -4
  90. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  91. package/dist/distribution/coordinator/in-memory.mjs +1 -2
  92. package/dist/distribution/transport/grpc.mjs +7 -2
  93. package/dist/distribution/transport/tcp.mjs +7 -1
  94. package/dist/distribution.d.ts +7 -7
  95. package/dist/distribution.mjs +1199 -1092
  96. package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
  97. package/dist/embed-4H7RTBVL.mjs +221 -0
  98. package/dist/embeddings/openai.mjs +2 -2
  99. package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
  100. package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
  101. package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
  102. package/dist/index.browser.mjs +4 -16
  103. package/dist/index.d.ts +1495 -78
  104. package/dist/index.mjs +14 -50
  105. package/dist/index.react-native.mjs +11 -28
  106. package/dist/invalidation/filesystem.mjs +1 -2
  107. package/dist/languages/albanian.mjs +1 -1
  108. package/dist/languages/amharic.mjs +1 -1
  109. package/dist/languages/arabic.mjs +1 -1
  110. package/dist/languages/armenian.mjs +1 -1
  111. package/dist/languages/azerbaijani.mjs +1 -1
  112. package/dist/languages/bambara.mjs +1 -1
  113. package/dist/languages/basque.mjs +1 -1
  114. package/dist/languages/belarusian.mjs +1 -1
  115. package/dist/languages/bengali.mjs +1 -1
  116. package/dist/languages/bosnian.mjs +1 -1
  117. package/dist/languages/breton.mjs +1 -1
  118. package/dist/languages/bulgarian.mjs +1 -1
  119. package/dist/languages/burmese.mjs +1 -1
  120. package/dist/languages/catalan.mjs +1 -1
  121. package/dist/languages/chinese.mjs +1 -1
  122. package/dist/languages/croatian.mjs +1 -1
  123. package/dist/languages/czech.mjs +1 -1
  124. package/dist/languages/dagbani.mjs +1 -1
  125. package/dist/languages/danish.mjs +1 -1
  126. package/dist/languages/dutch.mjs +1 -1
  127. package/dist/languages/english.mjs +1 -1
  128. package/dist/languages/esperanto.mjs +1 -1
  129. package/dist/languages/estonian.mjs +1 -1
  130. package/dist/languages/ewe.mjs +1 -1
  131. package/dist/languages/faroese.mjs +1 -1
  132. package/dist/languages/fijian.mjs +1 -1
  133. package/dist/languages/finnish.mjs +1 -1
  134. package/dist/languages/french.mjs +1 -1
  135. package/dist/languages/ga.mjs +1 -1
  136. package/dist/languages/galician.mjs +1 -1
  137. package/dist/languages/georgian.mjs +1 -1
  138. package/dist/languages/german.mjs +1 -1
  139. package/dist/languages/greek.mjs +1 -1
  140. package/dist/languages/guarani.mjs +1 -1
  141. package/dist/languages/gujarati.mjs +1 -1
  142. package/dist/languages/haitian-creole.mjs +1 -1
  143. package/dist/languages/hausa.mjs +1 -1
  144. package/dist/languages/hawaiian.mjs +1 -1
  145. package/dist/languages/hebrew.mjs +1 -1
  146. package/dist/languages/hindi.mjs +1 -1
  147. package/dist/languages/hungarian.mjs +1 -1
  148. package/dist/languages/icelandic.mjs +1 -1
  149. package/dist/languages/igbo.mjs +1 -1
  150. package/dist/languages/indonesian.mjs +1 -1
  151. package/dist/languages/irish.mjs +1 -1
  152. package/dist/languages/italian.mjs +1 -1
  153. package/dist/languages/japanese.mjs +1 -1
  154. package/dist/languages/kannada.mjs +1 -1
  155. package/dist/languages/kazakh.mjs +1 -1
  156. package/dist/languages/khmer.mjs +1 -1
  157. package/dist/languages/kinyarwanda.mjs +1 -1
  158. package/dist/languages/kirundi.mjs +1 -1
  159. package/dist/languages/korean.mjs +1 -1
  160. package/dist/languages/kurmanji.mjs +1 -1
  161. package/dist/languages/kyrgyz.mjs +1 -1
  162. package/dist/languages/lao.mjs +1 -1
  163. package/dist/languages/latin.mjs +1 -1
  164. package/dist/languages/latvian.mjs +1 -1
  165. package/dist/languages/lingala.mjs +1 -1
  166. package/dist/languages/lithuanian.mjs +1 -1
  167. package/dist/languages/luxembourgish.mjs +1 -1
  168. package/dist/languages/macedonian.mjs +1 -1
  169. package/dist/languages/malagasy.mjs +1 -1
  170. package/dist/languages/malay.mjs +1 -1
  171. package/dist/languages/malayalam.mjs +1 -1
  172. package/dist/languages/maltese.mjs +1 -1
  173. package/dist/languages/maori.mjs +1 -1
  174. package/dist/languages/marathi.mjs +1 -1
  175. package/dist/languages/nepali.mjs +1 -1
  176. package/dist/languages/norwegian.mjs +1 -1
  177. package/dist/languages/oromo.mjs +1 -1
  178. package/dist/languages/persian.mjs +1 -1
  179. package/dist/languages/polish.mjs +1 -1
  180. package/dist/languages/portuguese.mjs +1 -1
  181. package/dist/languages/punjabi.mjs +1 -1
  182. package/dist/languages/romanian.mjs +1 -1
  183. package/dist/languages/russian.mjs +1 -1
  184. package/dist/languages/samoan.mjs +1 -1
  185. package/dist/languages/sanskrit.mjs +1 -1
  186. package/dist/languages/scottish-gaelic.mjs +1 -1
  187. package/dist/languages/serbian.mjs +1 -1
  188. package/dist/languages/shona.mjs +1 -1
  189. package/dist/languages/sinhala.mjs +1 -1
  190. package/dist/languages/slovak.mjs +1 -1
  191. package/dist/languages/slovenian.mjs +1 -1
  192. package/dist/languages/sorani.mjs +1 -1
  193. package/dist/languages/spanish.mjs +1 -1
  194. package/dist/languages/swahili.mjs +1 -1
  195. package/dist/languages/swedish.mjs +1 -1
  196. package/dist/languages/tagalog.mjs +1 -1
  197. package/dist/languages/tamil.mjs +1 -1
  198. package/dist/languages/tatar.mjs +1 -1
  199. package/dist/languages/telugu.mjs +1 -1
  200. package/dist/languages/thai.mjs +1 -1
  201. package/dist/languages/tibetan.mjs +1 -1
  202. package/dist/languages/tigrinya.mjs +1 -1
  203. package/dist/languages/tongan.mjs +1 -1
  204. package/dist/languages/turkish.mjs +1 -1
  205. package/dist/languages/twi.mjs +1 -1
  206. package/dist/languages/ukrainian.mjs +1 -1
  207. package/dist/languages/urdu.mjs +1 -1
  208. package/dist/languages/vietnamese.mjs +1 -1
  209. package/dist/languages/welsh.mjs +1 -1
  210. package/dist/languages/wolof.mjs +1 -1
  211. package/dist/languages/xhosa.mjs +1 -1
  212. package/dist/languages/yoruba.mjs +1 -1
  213. package/dist/languages/zulu.mjs +1 -1
  214. package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
  215. package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
  216. package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
  217. package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
  218. package/dist/optional-writes-DDFHYUOH.mjs +218 -0
  219. package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
  220. package/dist/partial-words.browser.mjs +1649 -0
  221. package/dist/partial-words.mjs +15 -14
  222. package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
  223. package/dist/pattern.mjs +6 -8
  224. package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
  225. package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
  226. package/dist/phrase-module-XK3T44VU.mjs +634 -0
  227. package/dist/query-actions-OMXJEC2J.mjs +118 -0
  228. package/dist/query-suggestions/merge-worker.mjs +18 -54
  229. package/dist/query-suggestions.browser.mjs +26 -227
  230. package/dist/query-suggestions.d.ts +2 -2
  231. package/dist/query-suggestions.mjs +16 -67
  232. package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
  233. package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
  234. package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
  235. package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
  236. package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
  237. package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
  238. package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
  239. package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
  240. package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
  241. package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
  242. package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
  243. package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
  244. package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
  245. package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
  246. package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
  247. package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
  248. package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
  249. package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
  250. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  251. package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
  252. package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
  253. package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
  254. package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
  255. package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
  256. package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
  257. package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
  258. package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
  259. package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
  260. package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
  261. package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
  262. package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
  263. package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
  264. package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
  265. package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
  266. package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
  267. package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
  268. package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
  269. package/dist/react-native/partial-words.mjs +954 -35
  270. package/dist/react-native/pattern.mjs +238 -111
  271. package/dist/react-native/query-suggestions.mjs +33 -239
  272. package/dist/react-native/sync.mjs +95 -52
  273. package/dist/react-native/synonyms.mjs +256 -112
  274. package/dist/react.d.ts +9 -9
  275. package/dist/react.mjs +6 -0
  276. package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
  277. package/dist/rerankers/http.mjs +2 -2
  278. package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
  279. package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
  280. package/dist/runtime-4GKUIAKV.mjs +595 -0
  281. package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
  282. package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
  283. package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
  284. package/dist/server.d.ts +33 -13
  285. package/dist/server.mjs +81 -100
  286. package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
  287. package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
  288. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  289. package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
  290. package/dist/sync/filesystem.d.ts +3 -3
  291. package/dist/sync/filesystem.mjs +27 -11
  292. package/dist/sync/indexeddb.browser.mjs +287 -0
  293. package/dist/sync/indexeddb.d.ts +3 -3
  294. package/dist/sync/indexeddb.mjs +7 -2
  295. package/dist/sync.browser.mjs +88 -72
  296. package/dist/sync.d.ts +4 -4
  297. package/dist/sync.mjs +89 -59
  298. package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
  299. package/dist/synonyms.mjs +15 -11
  300. package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
  301. package/dist/thread-SNILJXX3.mjs +30 -0
  302. package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
  303. package/dist/vector/search-worker.d.ts +7 -1
  304. package/dist/vector/search-worker.mjs +9 -13
  305. package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
  306. package/dist/workers/entry.mjs +60 -30
  307. package/openapi.json +66 -50
  308. package/package.json +233 -8
  309. package/dist/analysis-change-CKEMM2MC.mjs +0 -55
  310. package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
  311. package/dist/archive-history-E4R73RWD.mjs +0 -29
  312. package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
  313. package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
  314. package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
  315. package/dist/browser/chunk-25MTVDYD.mjs +0 -432
  316. package/dist/browser/chunk-3I27KXQD.mjs +0 -34
  317. package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
  318. package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
  319. package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
  320. package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
  321. package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
  322. package/dist/browser/chunk-AI5UI46D.mjs +0 -12
  323. package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
  324. package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
  325. package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
  326. package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
  327. package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
  328. package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
  329. package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
  330. package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
  331. package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
  332. package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
  333. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  334. package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
  335. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  336. package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
  337. package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
  338. package/dist/browser/chunk-MY2XKH22.mjs +0 -40
  339. package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
  340. package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
  341. package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
  342. package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
  343. package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
  344. package/dist/browser/chunk-TK63WFXK.mjs +0 -90
  345. package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
  346. package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
  347. package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
  348. package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
  349. package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
  350. package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
  351. package/dist/browser/chunk-V43N6TZO.mjs +0 -857
  352. package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
  353. package/dist/browser/chunk-WN464PTL.mjs +0 -375
  354. package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
  355. package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
  356. package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
  357. package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
  358. package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
  359. package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
  360. package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
  361. package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
  362. package/dist/browser/import-5ESVMFDJ.mjs +0 -20
  363. package/dist/browser/methods-KTQCSIN4.mjs +0 -30
  364. package/dist/browser/module-NDSQLPMQ.mjs +0 -25
  365. package/dist/browser/operations-OJ3BXS55.mjs +0 -5
  366. package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
  367. package/dist/browser/pool-3JLET5RO.mjs +0 -297
  368. package/dist/browser/release-HYXPVJVS.mjs +0 -32
  369. package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
  370. package/dist/browser/retention-TY3L3BFN.mjs +0 -211
  371. package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
  372. package/dist/browser/start-2UZTOBG5.mjs +0 -26
  373. package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
  374. package/dist/browser/write-ULXAN5B4.mjs +0 -196
  375. package/dist/chunk-2NE7VCYE.mjs +0 -212
  376. package/dist/chunk-2ROUFMQX.mjs +0 -543
  377. package/dist/chunk-37CD3OKS.mjs +0 -87
  378. package/dist/chunk-3MXV3VHR.mjs +0 -46
  379. package/dist/chunk-3OH67G7R.mjs +0 -214
  380. package/dist/chunk-4B7JIGQD.mjs +0 -106
  381. package/dist/chunk-5YNRKI2C.mjs +0 -81
  382. package/dist/chunk-6HTGOXIK.mjs +0 -875
  383. package/dist/chunk-6OEGXLBX.mjs +0 -57
  384. package/dist/chunk-6RC4K46J.mjs +0 -86
  385. package/dist/chunk-7GFE3PEY.mjs +0 -40
  386. package/dist/chunk-7GXJUJ24.mjs +0 -49
  387. package/dist/chunk-7V5IUP2T.mjs +0 -63
  388. package/dist/chunk-A4L7YTKX.mjs +0 -458
  389. package/dist/chunk-BEZH7VEH.mjs +0 -23
  390. package/dist/chunk-BNL5OPRV.mjs +0 -634
  391. package/dist/chunk-BTPGQC6Z.mjs +0 -27
  392. package/dist/chunk-C7XQYG6U.mjs +0 -50
  393. package/dist/chunk-CQZXZ75V.mjs +0 -562
  394. package/dist/chunk-CWS55TEX.mjs +0 -242
  395. package/dist/chunk-CZQ33DLW.mjs +0 -78
  396. package/dist/chunk-EEBCDP36.mjs +0 -32
  397. package/dist/chunk-FB6463IN.mjs +0 -1499
  398. package/dist/chunk-FCJ553HS.mjs +0 -655
  399. package/dist/chunk-FEZVDOWX.mjs +0 -6996
  400. package/dist/chunk-FOCV4CBZ.mjs +0 -87
  401. package/dist/chunk-FQBFZAWG.mjs +0 -27
  402. package/dist/chunk-G6HRSGOU.mjs +0 -74
  403. package/dist/chunk-GES7U2PM.mjs +0 -68
  404. package/dist/chunk-GH4WQ22V.mjs +0 -34
  405. package/dist/chunk-GPWMUPCA.mjs +0 -1126
  406. package/dist/chunk-GQPTDQCL.mjs +0 -54
  407. package/dist/chunk-HJSN7UYG.mjs +0 -10
  408. package/dist/chunk-I54T6CQP.mjs +0 -117
  409. package/dist/chunk-IPN7S4KX.mjs +0 -130
  410. package/dist/chunk-J4IJTTEX.mjs +0 -226
  411. package/dist/chunk-JSV6OQ4C.mjs +0 -74
  412. package/dist/chunk-K4ZIGL2O.mjs +0 -290
  413. package/dist/chunk-KAKQ74LS.mjs +0 -53
  414. package/dist/chunk-KCQYFO4K.mjs +0 -62
  415. package/dist/chunk-KHKIRS5D.mjs +0 -113
  416. package/dist/chunk-KOKM4PQN.mjs +0 -52
  417. package/dist/chunk-KP7TYHWT.mjs +0 -92
  418. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  419. package/dist/chunk-L5ORED7V.mjs +0 -25
  420. package/dist/chunk-LDNCLRZU.mjs +0 -218
  421. package/dist/chunk-LHKOSI2J.mjs +0 -58
  422. package/dist/chunk-LQG5G2LY.mjs +0 -39
  423. package/dist/chunk-MATMVH65.mjs +0 -9
  424. package/dist/chunk-MP7ABPTD.mjs +0 -189
  425. package/dist/chunk-MW6BR4QP.mjs +0 -316
  426. package/dist/chunk-MWD5VPN7.mjs +0 -21
  427. package/dist/chunk-ODYKO5TO.mjs +0 -101
  428. package/dist/chunk-OG7UGJ6F.mjs +0 -63
  429. package/dist/chunk-ORN5BLL7.mjs +0 -11
  430. package/dist/chunk-OYABFKTW.mjs +0 -1974
  431. package/dist/chunk-P5LANSH6.mjs +0 -121
  432. package/dist/chunk-PELHVTO7.mjs +0 -24
  433. package/dist/chunk-PIH5SJ4N.mjs +0 -7481
  434. package/dist/chunk-PSZI4O24.mjs +0 -253
  435. package/dist/chunk-PV2CG7EJ.mjs +0 -886
  436. package/dist/chunk-PWGYQXML.mjs +0 -470
  437. package/dist/chunk-Q2UL2TPC.mjs +0 -21
  438. package/dist/chunk-QHPYS427.mjs +0 -66
  439. package/dist/chunk-QJVOLKU6.mjs +0 -111
  440. package/dist/chunk-QLDABJZM.mjs +0 -20
  441. package/dist/chunk-QYRNLLLI.mjs +0 -99
  442. package/dist/chunk-RCX4WNLC.mjs +0 -151
  443. package/dist/chunk-RFBLQLLZ.mjs +0 -16
  444. package/dist/chunk-RIHIVFIQ.mjs +0 -240
  445. package/dist/chunk-RLOES5SX.mjs +0 -31
  446. package/dist/chunk-RWURUDSP.mjs +0 -2639
  447. package/dist/chunk-RZWWB2XR.mjs +0 -96
  448. package/dist/chunk-S5LN746F.mjs +0 -10
  449. package/dist/chunk-SOI6LTHM.mjs +0 -216
  450. package/dist/chunk-SQJWAIF4.mjs +0 -12
  451. package/dist/chunk-TDIK6NTV.mjs +0 -10
  452. package/dist/chunk-TQXHPFWT.mjs +0 -177
  453. package/dist/chunk-U55XJEIG.mjs +0 -18
  454. package/dist/chunk-UD5HIS4U.mjs +0 -382
  455. package/dist/chunk-UHICS3JI.mjs +0 -83
  456. package/dist/chunk-UJDJB3NE.mjs +0 -913
  457. package/dist/chunk-VD7J552I.mjs +0 -28
  458. package/dist/chunk-VSLSGASN.mjs +0 -30
  459. package/dist/chunk-VZ2ONHEE.mjs +0 -3715
  460. package/dist/chunk-W7OSSKH5.mjs +0 -1427
  461. package/dist/chunk-WZ7YV54N.mjs +0 -152
  462. package/dist/chunk-X5CV2E2B.mjs +0 -1
  463. package/dist/chunk-Y4WPNHMC.mjs +0 -50
  464. package/dist/chunk-YEKL4JBK.mjs +0 -96
  465. package/dist/chunk-ZXNQEW43.mjs +0 -18
  466. package/dist/chunk-ZYAY26KG.mjs +0 -29
  467. package/dist/direct-executor-V5DRIECI.mjs +0 -34
  468. package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
  469. package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
  470. package/dist/gzip-F4HMIEEQ.mjs +0 -1
  471. package/dist/import-GPKLTSKA.mjs +0 -56
  472. package/dist/methods-7IOINERF.mjs +0 -30
  473. package/dist/module-JYHFTVPS.mjs +0 -63
  474. package/dist/operations-G5EP6JPX.mjs +0 -5
  475. package/dist/phrase-module-XW5MKWP5.mjs +0 -330
  476. package/dist/pool-Y4A46T36.mjs +0 -298
  477. package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
  478. package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
  479. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
  480. package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
  481. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
  482. package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
  483. package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
  484. package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
  485. package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
  486. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
  487. package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
  488. package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
  489. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
  490. package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
  491. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
  492. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
  493. package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
  494. package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
  495. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
  496. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
  497. package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
  498. package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
  499. package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
  500. package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
  501. package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
  502. package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
  503. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
  504. package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
  505. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
  506. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
  507. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
  508. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
  509. package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
  510. package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
  511. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
  512. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
  513. package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
  514. package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
  515. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
  516. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
  517. package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
  518. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
  519. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
  520. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
  521. package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
  522. package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
  523. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
  524. package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
  525. package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
  526. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
  527. package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
  528. package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
  529. package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
  530. package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
  531. package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
  532. package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
  533. package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
  534. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
  535. package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
  536. package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
  537. package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
  538. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
  539. package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
  540. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
  541. package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
  542. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
  543. package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
  544. package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
  545. package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
  546. package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
  547. package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
  548. package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
  549. package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
  550. package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
  551. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
  552. package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
  553. package/dist/release-SMR4WJD4.mjs +0 -32
  554. package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
  555. package/dist/restore-queue-LZH7LMLR.mjs +0 -55
  556. package/dist/retention-3K7ARGAG.mjs +0 -66
  557. package/dist/runtime-COIUORCN.mjs +0 -177
  558. package/dist/snapshot-O7ICP7NJ.mjs +0 -783
  559. package/dist/start-S4PFP6RS.mjs +0 -66
  560. package/dist/sync-state-ZWHLAFLE.mjs +0 -15
  561. package/dist/thread-YLJ3ZPQ3.mjs +0 -50
  562. package/dist/write-2OBJGLFS.mjs +0 -233
  563. /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
  564. /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
  565. /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
@@ -1,4072 +0,0 @@
1
- import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-4B7JIGQD.mjs';
2
- import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-YEKL4JBK.mjs';
3
- import { closePartitionWriter, listPersistedIndexes, removeLeftoverRewriteFiles, drainIndexStateForUnload, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-PSZI4O24.mjs';
4
- import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, unpackEnvelopeBytes, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf } from './chunk-W7OSSKH5.mjs';
5
- import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-P5LANSH6.mjs';
6
- import './chunk-MATMVH65.mjs';
7
- import './chunk-VD7J552I.mjs';
8
- import './chunk-ZXNNR73R.mjs';
9
- import './chunk-WZ7YV54N.mjs';
10
- import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-CWS55TEX.mjs';
11
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-K4ZIGL2O.mjs';
12
- import './chunk-BUMREPLU.mjs';
13
- import './chunk-37CD3OKS.mjs';
14
- import { createDurableDirectory } from './chunk-HHBUCIUY.mjs';
15
- import { SEGMENT_HEADER_SIZE, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, buildEntry, captureWholePartition, checkpointLastSeqNo } from './chunk-GPWMUPCA.mjs';
16
- import './chunk-C7XQYG6U.mjs';
17
- import { readSegmentFile } from './chunk-BNL5OPRV.mjs';
18
- import './chunk-CQZXZ75V.mjs';
19
- import { adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-UHICS3JI.mjs';
20
- import './chunk-5FLRXF73.mjs';
21
- import './chunk-JDDXV6UF.mjs';
22
- import './chunk-KCQYFO4K.mjs';
23
- import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION, manifestWithinPartitionCount } from './chunk-JTYPJF3T.mjs';
24
- import { getFs } from './chunk-HLPRHOSO.mjs';
25
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
26
- import { sha256 } from './chunk-LNGE7SLY.mjs';
27
- import './chunk-ODYKO5TO.mjs';
28
- import './chunk-UD5HIS4U.mjs';
29
- import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
30
- import './chunk-GQPTDQCL.mjs';
31
- import { QUERY_HISTORY_FOLDER } from './chunk-5YNRKI2C.mjs';
32
- import { toError, packChecksummedEnvelope } from './chunk-FEZVDOWX.mjs';
33
- import './chunk-OYABFKTW.mjs';
34
- import './chunk-HJSN7UYG.mjs';
35
- import { detectRuntime, resolveWorkerEntry } from './chunk-LHKOSI2J.mjs';
36
- import './chunk-XIHUI4KB.mjs';
37
- import './chunk-3MXV3VHR.mjs';
38
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
39
- import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-VZ2ONHEE.mjs';
40
- import './chunk-PIH5SJ4N.mjs';
41
- import './chunk-GH4WQ22V.mjs';
42
- import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-3OH67G7R.mjs';
43
- import './chunk-IPN7S4KX.mjs';
44
- import { fnv1a } from './chunk-KAKQ74LS.mjs';
45
- import { crc32 } from './chunk-7GXJUJ24.mjs';
46
- import './chunk-Q2UL2TPC.mjs';
47
- import './chunk-CITFCZKF.mjs';
48
- import './chunk-RLOES5SX.mjs';
49
- import './chunk-PV2CG7EJ.mjs';
50
- import './chunk-FQBFZAWG.mjs';
51
- import { encodeDocument, requireValidFieldTypes } from './chunk-W5EB5XKI.mjs';
52
- import './chunk-SQJWAIF4.mjs';
53
- import './chunk-GES7U2PM.mjs';
54
- import './chunk-I54T6CQP.mjs';
55
- import './chunk-I5B7RXU7.mjs';
56
- import './chunk-Y4WPNHMC.mjs';
57
- import './chunk-RZWWB2XR.mjs';
58
- import './chunk-TGP5M5R7.mjs';
59
- import './chunk-G6HRSGOU.mjs';
60
- import './chunk-U2BKYEC7.mjs';
61
- import './chunk-6HTGOXIK.mjs';
62
- import { createRequestId } from './chunk-QHPYS427.mjs';
63
- import './chunk-QYRNLLLI.mjs';
64
- import './chunk-TDIK6NTV.mjs';
65
- import './chunk-FB6463IN.mjs';
66
- import './chunk-7GFE3PEY.mjs';
67
- import './chunk-ZXNQEW43.mjs';
68
- import './chunk-7V5IUP2T.mjs';
69
- import './chunk-L5ORED7V.mjs';
70
- import './chunk-X5CV2E2B.mjs';
71
- import './chunk-A4L7YTKX.mjs';
72
- import './chunk-U55XJEIG.mjs';
73
- import './chunk-KOKM4PQN.mjs';
74
- import './chunk-ORN5BLL7.mjs';
75
- import './chunk-BTPGQC6Z.mjs';
76
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
77
- import { MIN_M } from './chunk-ZXX5GZEX.mjs';
78
- import './chunk-RFBLQLLZ.mjs';
79
- import './chunk-KKQC526K.mjs';
80
- import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
81
- import { NarsilError, ErrorCodes } from './chunk-MP7ABPTD.mjs';
82
- import './chunk-J4IJTTEX.mjs';
83
- import { compareCodePoints } from './chunk-TBC3XUL2.mjs';
84
- import { decode, encode } from '@msgpack/msgpack';
85
-
86
- // src/persistence/durability/sync/layout.ts
87
- var SYNC_KEY_NUMBER_WIDTH = 16;
88
- function syncDirectoryPrefix(indexName) {
89
- return `${indexName}/sync/`;
90
- }
91
- function publishedSnapshotKey(indexName) {
92
- return `${syncDirectoryPrefix(indexName)}published`;
93
- }
94
- function historyRoot(indexName) {
95
- return `${indexName}/history/`;
96
- }
97
- function historyPrefix(indexName, partitionId) {
98
- return `${historyRoot(indexName)}${partitionId}/`;
99
- }
100
- function walSegmentPrefix(indexName, partitionId) {
101
- return `${indexName}/wal/${partitionId}/`;
102
- }
103
- function formatSyncKeyNumber(value) {
104
- return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
105
- }
106
- function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
107
- return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
108
- }
109
- function walSegmentKey(indexName, partitionId, startSeqNo) {
110
- return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
111
- }
112
- var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
113
- function parseHistoryKey(indexName, key) {
114
- const root = historyRoot(indexName);
115
- if (!key.startsWith(root)) return null;
116
- const match = HISTORY_TAIL.exec(key.slice(root.length));
117
- if (match === null) return null;
118
- const partitionId = Number.parseInt(match[1], 10);
119
- const startSeqNo = Number.parseInt(match[2], 10);
120
- const movedAtMs = Number.parseInt(match[3], 10);
121
- if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
122
- return null;
123
- }
124
- return { key, partitionId, startSeqNo, movedAtMs };
125
- }
126
- function parseWalSegmentStart(indexName, partitionId, key) {
127
- const prefix = walSegmentPrefix(indexName, partitionId);
128
- if (!key.startsWith(prefix)) return null;
129
- const tail = key.slice(prefix.length);
130
- if (!/^\d{16}$/.test(tail)) return null;
131
- const value = Number.parseInt(tail, 10);
132
- return Number.isSafeInteger(value) ? value : null;
133
- }
134
-
135
- // src/persistence/durability/sync/history.ts
136
- function compareHistorySegments(a, b) {
137
- return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
138
- }
139
- async function listHistorySegments(directory, indexName, partitionId) {
140
- const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
141
- const segments = [];
142
- for (const key of await directory.list(prefix)) {
143
- const segment = parseHistoryKey(indexName, key);
144
- if (segment !== null) segments.push(segment);
145
- }
146
- return segments.sort(compareHistorySegments);
147
- }
148
- async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
149
- await directory.move(
150
- walSegmentKey(indexName, partitionId, startSeqNo),
151
- historyKey(indexName, partitionId, startSeqNo, movedAtMs)
152
- );
153
- }
154
- async function removeHistory(directory, indexName, partitionId) {
155
- for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
156
- await directory.remove(segment.key);
157
- }
158
- }
159
-
160
- // src/persistence/durability/checkpoint.ts
161
- function walPrefix(indexName, partitionId) {
162
- return `${indexName}/wal/${partitionId}/`;
163
- }
164
- async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
165
- for (const { partitionId, lastSeqNo } of checkpoint) {
166
- const prefix = walPrefix(indexName, partitionId);
167
- const markerBytes = await directory.read(`${prefix}commit`);
168
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
169
- if (marker === null) {
170
- continue;
171
- }
172
- const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
173
- const segments = await directory.list(prefix);
174
- const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
175
- for (let i = 0; i < startSeqNos.length; i += 1) {
176
- const start = startSeqNos[i];
177
- if (start >= activeSegmentSeqNo) {
178
- break;
179
- }
180
- const nextStart = startSeqNos[i + 1];
181
- const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
182
- if (segmentCoversBeyondCheckpoint) {
183
- break;
184
- }
185
- if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
186
- else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
187
- }
188
- }
189
- }
190
- async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
191
- for (const [partitionId, partition] of [...partitions]) {
192
- if (partitionId < partitionCount) continue;
193
- try {
194
- await partition.walWriter.close();
195
- } catch (err) {
196
- onCloseError(err instanceof Error ? err : new Error(String(err)));
197
- }
198
- partitions.delete(partitionId);
199
- }
200
- const walRoot = `${indexName}/wal/`;
201
- for (const key of await directory.list(walRoot)) {
202
- const partitionId = walKeyPartitionId(key, walRoot);
203
- if (partitionId === null || partitionId < partitionCount) continue;
204
- await directory.remove(key);
205
- }
206
- }
207
- function walKeyPartitionId(key, walRoot) {
208
- const tail = key.slice(walRoot.length);
209
- const separator = tail.indexOf("/");
210
- if (separator <= 0) {
211
- return null;
212
- }
213
- const segment = tail.slice(0, separator);
214
- if (!/^\d+$/.test(segment)) {
215
- return null;
216
- }
217
- const value = Number.parseInt(segment, 10);
218
- return Number.isSafeInteger(value) ? value : null;
219
- }
220
- function parseSegmentStartSeqNo(key, prefix) {
221
- if (!key.startsWith(prefix)) {
222
- return null;
223
- }
224
- const tail = key.slice(prefix.length);
225
- if (!/^\d{16}$/.test(tail)) {
226
- return null;
227
- }
228
- const value = Number.parseInt(tail, 10);
229
- return Number.isSafeInteger(value) ? value : null;
230
- }
231
- function formatSegmentStart(startSeqNo) {
232
- return startSeqNo.toString().padStart(16, "0");
233
- }
234
-
235
- // src/persistence/durability/checkpoint-capture.ts
236
- function whileNoMutationApplies(partitions, capture2) {
237
- return new Promise((resolve, reject) => {
238
- let release = () => void 0;
239
- const released = new Promise((settle) => {
240
- release = settle;
241
- });
242
- function captureNow() {
243
- try {
244
- resolve(capture2());
245
- } catch (error) {
246
- reject(error);
247
- } finally {
248
- release();
249
- }
250
- }
251
- let waiting = partitions.length;
252
- if (waiting === 0) {
253
- captureNow();
254
- return;
255
- }
256
- for (const partition of partitions) {
257
- partition.appendChain = partition.appendChain.then(() => {
258
- waiting -= 1;
259
- if (waiting === 0) captureNow();
260
- return released;
261
- });
262
- }
263
- });
264
- }
265
- async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
266
- for (const [partitionId, partition] of [...indexState.partitions]) {
267
- await partition.appendChain;
268
- if (partition.failed !== null) continue;
269
- try {
270
- await partition.walWriter.commit();
271
- } catch (error) {
272
- failPartition2(partitionId, partition, error);
273
- }
274
- }
275
- }
276
- async function openNewLogSegments(indexState, failPartition2) {
277
- const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
278
- try {
279
- if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
280
- } catch (error) {
281
- failPartition2(partitionId, partition, error);
282
- return;
283
- }
284
- const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
285
- failPartition2(partitionId, partition, error);
286
- });
287
- partition.appendChain = rolled;
288
- await rolled;
289
- });
290
- await Promise.all(rolls);
291
- }
292
- function aTargetFailed(indexState, targets) {
293
- return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
294
- }
295
- function noWholePartitions() {
296
- return { contents: [], captured: [] };
297
- }
298
- function wholePartitionsWhereMostChanged(manager, rules) {
299
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
300
- return (targets) => {
301
- const whole = { contents: [], captured: [] };
302
- const changed = targets.filter(
303
- (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
304
- );
305
- const mostOfEachChanged = changed.every((target) => {
306
- const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
307
- const documents = manager.getPartition(target.partitionId).count();
308
- return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
309
- });
310
- const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
311
- if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
312
- for (const { partitionId } of changed) {
313
- const capture2 = captureRecords?.(partitionId);
314
- const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
315
- if (captured !== null) {
316
- whole.captured.push(captured);
317
- } else if (everyPartition || !aWorkerCanSerialise) {
318
- whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
319
- }
320
- }
321
- return whole;
322
- };
323
- }
324
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
325
- for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
326
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
327
- onCaptured?.();
328
- const documentCount = manager.countDocuments();
329
- const targets = [];
330
- let recordsLeftInLog = 0;
331
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
332
- const partition = indexState.partitions.get(partitionId);
333
- if (partition !== void 0 && partition.failed !== null) continue;
334
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
335
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
336
- appliedSeqNo,
337
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
338
- );
339
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
340
- const target = {
341
- partitionId,
342
- lastSeqNo,
343
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
344
- };
345
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
346
- targets.push(target);
347
- }
348
- const vectorPlans = /* @__PURE__ */ new Map();
349
- try {
350
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
351
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
352
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
353
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
354
- }
355
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
356
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
357
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
358
- } catch (error) {
359
- for (const plan of vectorPlans.values()) plan.release();
360
- throw error;
361
- }
362
- });
363
- }
364
- async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
365
- if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
366
- const priorVectors = priorManifest?.vectors ?? [];
367
- const priorCheckpoint = priorManifest?.checkpoint ?? [];
368
- const noPartitionChanged = capture2.targets.every(
369
- (target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
370
- );
371
- const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
372
- (fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
373
- );
374
- const everyPlanUnchanged = [...capture2.vectorPlans.values()].every((plan) => plan.unchanged);
375
- if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged && everyPlanUnchanged) {
376
- return { vectors: priorVectors, layouts: [], written: [] };
377
- }
378
- const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
379
- return { vectors: written.refs, layouts: written.layouts, written: written.written };
380
- }
381
-
382
- // src/persistence/durability/checkpoint-worker-entry.ts
383
- function resolveCheckpointWorkerEntry() {
384
- return resolveWorkerEntry(
385
- import.meta.url,
386
- /\/src\/persistence\/durability\/[^/]+$/,
387
- "persistence/durability/checkpoint-worker.mjs"
388
- );
389
- }
390
-
391
- // src/persistence/durability/checkpoint-worker-dispatch.ts
392
- var workerUsable = true;
393
- var pooledWorker = null;
394
- var workerBusy = false;
395
- var spawnedWorkerCount = 0;
396
- var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
397
- var idleTimer = null;
398
- async function spawnWorker() {
399
- const entryPoint = resolveCheckpointWorkerEntry();
400
- if (entryPoint === null) return null;
401
- try {
402
- const worker = await spawnNodeWorker(new URL(entryPoint));
403
- if (worker === null) {
404
- return null;
405
- }
406
- if (typeof worker.unref === "function") {
407
- worker.unref();
408
- }
409
- worker.on("error", () => discardWorker(worker));
410
- worker.on("exit", () => discardWorker(worker));
411
- spawnedWorkerCount += 1;
412
- return worker;
413
- } catch {
414
- return null;
415
- }
416
- }
417
- function discardWorker(worker) {
418
- if (pooledWorker === worker) {
419
- pooledWorker = null;
420
- }
421
- try {
422
- void worker.terminate();
423
- } catch {
424
- }
425
- }
426
- function unrefTimer(timer) {
427
- if (typeof timer.unref === "function") {
428
- timer.unref();
429
- }
430
- return timer;
431
- }
432
- function runWorker(worker, request) {
433
- return new Promise((resolve) => {
434
- let settled = false;
435
- let timeoutId = armSilenceTimeout();
436
- function armSilenceTimeout() {
437
- return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
438
- }
439
- const onMessage = (msg) => {
440
- const response = msg;
441
- if (response.type === "heartbeat") {
442
- clearTimeout(timeoutId);
443
- timeoutId = armSilenceTimeout();
444
- return;
445
- }
446
- if (response.type === "success") {
447
- settle({ written: response.segments, timedOut: false }, false);
448
- } else {
449
- settle({ written: null, timedOut: false }, true);
450
- }
451
- };
452
- const onError = () => {
453
- settle({ written: null, timedOut: false }, true);
454
- };
455
- const onExit = () => {
456
- settle({ written: null, timedOut: false }, true);
457
- };
458
- function settle(outcome, discard) {
459
- if (settled) {
460
- return;
461
- }
462
- settled = true;
463
- clearTimeout(timeoutId);
464
- worker.off("message", onMessage);
465
- worker.off("error", onError);
466
- worker.off("exit", onExit);
467
- if (discard) {
468
- discardWorker(worker);
469
- }
470
- resolve(outcome);
471
- }
472
- worker.on("message", onMessage);
473
- worker.on("error", onError);
474
- worker.on("exit", onExit);
475
- try {
476
- worker.postMessage(request);
477
- } catch {
478
- settle({ written: null, timedOut: false }, true);
479
- }
480
- });
481
- }
482
- function delay(ms) {
483
- return new Promise((resolve) => {
484
- unrefTimer(setTimeout(resolve, ms));
485
- });
486
- }
487
- function checkpointWorkerIsUsable() {
488
- const runtime = detectRuntime();
489
- return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
490
- }
491
- async function runCheckpointOnWorker(request) {
492
- const runtime = detectRuntime();
493
- if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
494
- return null;
495
- }
496
- if (workerBusy) {
497
- return null;
498
- }
499
- workerBusy = true;
500
- cancelIdleRetirement();
501
- try {
502
- if (pooledWorker === null) {
503
- pooledWorker = await spawnWorker();
504
- }
505
- const worker = pooledWorker;
506
- if (worker === null) {
507
- workerUsable = false;
508
- return null;
509
- }
510
- const outcome = await runWorker(worker, request);
511
- if (outcome.written !== null) {
512
- return outcome.written;
513
- }
514
- if (outcome.timedOut) {
515
- await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
516
- }
517
- return null;
518
- } finally {
519
- workerBusy = false;
520
- retireWorkerOnceIdle();
521
- }
522
- }
523
- function cancelIdleRetirement() {
524
- if (idleTimer !== null) {
525
- clearTimeout(idleTimer);
526
- idleTimer = null;
527
- }
528
- }
529
- function retireWorkerOnceIdle() {
530
- cancelIdleRetirement();
531
- idleTimer = unrefTimer(
532
- setTimeout(() => {
533
- idleTimer = null;
534
- if (!workerBusy) terminateCheckpointWorker();
535
- }, workerIdleMs)
536
- );
537
- }
538
- function terminateCheckpointWorker() {
539
- cancelIdleRetirement();
540
- const worker = pooledWorker;
541
- if (worker === null) {
542
- return;
543
- }
544
- pooledWorker = null;
545
- try {
546
- void worker.terminate();
547
- } catch {
548
- }
549
- }
550
-
551
- // src/persistence/durability/checkpoint-write.ts
552
- function workerCanWriteEverySegment(input) {
553
- if (!input.canOffload || input.wholePartitions.length > 0) return false;
554
- const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
555
- if (!analysesTextInProcessOnly) return true;
556
- return input.targets.every(
557
- (target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
558
- );
559
- }
560
- async function writeIndexCheckpointSegments(input) {
561
- const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
562
- const { keepsWriteRecords } = input;
563
- const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
564
- root: directory.root,
565
- metadata,
566
- targets,
567
- compactionThreshold,
568
- prior,
569
- capturedPartitions: [...capturedPartitions],
570
- keepsWriteRecords
571
- }) : null;
572
- if (offloaded !== null) {
573
- return offloaded;
574
- }
575
- return writeCheckpointSegments({
576
- store: directoryCheckpointStore(directory),
577
- directory,
578
- metadata,
579
- targets,
580
- compactionThreshold,
581
- prior,
582
- wholePartitions,
583
- capturedPartitions,
584
- keepsWriteRecords
585
- });
586
- }
587
-
588
- // src/persistence/durability/segment/vector-adoption.ts
589
- function recordWrittenVectors(vectorIndexes, written) {
590
- for (const field of written) {
591
- vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
592
- }
593
- }
594
- async function adoptVectorLayouts(store, vectorIndexes, layouts) {
595
- const pathOf = store.pathOf;
596
- if (pathOf === void 0) return;
597
- for (const layout of layouts) {
598
- const vecIndex = vectorIndexes.get(layout.fieldPath);
599
- if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
600
- const path = await pathOf.call(store, layout.key);
601
- await vecIndex.adoptDiskLayout({
602
- path,
603
- docIds: layout.docIds,
604
- dead: layout.dead,
605
- vectorsOffset: layout.vectorsOffset
606
- });
607
- }
608
- }
609
- async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
610
- const pathOf = store.pathOf;
611
- if (pathOf === void 0) return [...keys];
612
- const read = /* @__PURE__ */ new Set();
613
- for (const vectorIndex of vectorIndexes.values()) {
614
- for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
615
- }
616
- if (read.size === 0) return [...keys];
617
- const unread = [];
618
- for (const key of keys) {
619
- if (!read.has(await pathOf.call(store, key))) unread.push(key);
620
- }
621
- return unread;
622
- }
623
-
624
- // src/persistence/durability/write-stall.ts
625
- function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
626
- return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
627
- }
628
- async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
629
- while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
630
- await waitOutsideCountedWrite(
631
- () => new Promise((resume) => {
632
- indexState.stalledWrites.push(resume);
633
- })
634
- );
635
- }
636
- }
637
- function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
638
- const counted = indexState.mutationsSinceCheckpoint;
639
- const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
640
- indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
641
- indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
642
- releaseStalledWrites(indexState);
643
- }
644
- function releaseStalledWrites(indexState) {
645
- const waiting = indexState.stalledWrites.splice(0);
646
- for (const resume of waiting) resume();
647
- }
648
-
649
- // src/persistence/durability/checkpoint-run.ts
650
- async function bothOrTheFirstFailure(first, second) {
651
- const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
652
- if (firstSettled.status === "rejected") throw firstSettled.reason;
653
- if (secondSettled.status === "rejected") throw secondSettled.reason;
654
- return [firstSettled.value, secondSettled.value];
655
- }
656
- async function runDurableCheckpoint(input) {
657
- try {
658
- let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
659
- while (recordsLeftInLog > 0 && input.mayContinue()) {
660
- recordsLeftInLog = await writeOneBoundedCheckpoint(input);
661
- }
662
- } catch (err) {
663
- if (!isSystemError(err)) throw err;
664
- const error = new NarsilError(
665
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
666
- `The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
667
- {
668
- indexName: input.indexName,
669
- directory: input.directory.root,
670
- cause: err.message
671
- }
672
- );
673
- input.markFatal(error);
674
- throw error;
675
- }
676
- }
677
- function isSystemError(err) {
678
- return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
679
- }
680
- async function writeOneBoundedCheckpoint(input) {
681
- const manager = input.hooks.getManager(input.indexName);
682
- if (manager === void 0) {
683
- return 0;
684
- }
685
- const metadata = input.hooks.buildTermsMetadata(input.indexName);
686
- if (metadata === void 0) {
687
- return 0;
688
- }
689
- const fromMemory = input.fromMemory || input.hooks.mustRewriteTerms(input.indexName);
690
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
691
- const store = directoryCheckpointStore(input.directory);
692
- const writeRecords = input.hooks.writeRecords ?? null;
693
- const { indexName } = input;
694
- const storedManifest = await readCurrentManifest(store, input.indexName);
695
- const priorManifest = storedManifest === null ? null : manifestWithinPartitionCount(storedManifest, manager.partitionCount);
696
- const textOnDisk = metadata.textStorage === "disk";
697
- const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
698
- priorCheckpoint: priorManifest?.checkpoint ?? [],
699
- priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
700
- everyPartition: fromMemory,
701
- aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
702
- ...writeRecords === null ? {} : {
703
- captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
704
- }
705
- });
706
- let changesSinceCapture = null;
707
- const capture2 = await captureCheckpoint(
708
- input.indexState,
709
- manager,
710
- vectorIndexes,
711
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
712
- fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
713
- priorManifest?.vectors ?? [],
714
- writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
715
- textOnDisk ? () => {
716
- changesSinceCapture = manager.openCheckpointChangeLog();
717
- } : void 0,
718
- input.pinsRetiredVectors
719
- );
720
- try {
721
- await openNewLogSegments(input.indexState, input.failPartition);
722
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
723
- return changesSinceCapture;
724
- });
725
- } finally {
726
- for (const plan of capture2.vectorPlans.values()) plan.release();
727
- changesSinceCapture?.close();
728
- }
729
- }
730
- async function adoptWrittenSegments(input, manager, partitions, changes) {
731
- const { indexName, indexState, hooks } = input;
732
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-JYHFTVPS.mjs');
733
- const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
734
- if (adoption === null) return [];
735
- const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
736
- if (changes.partitionsReplaced) return false;
737
- const changed = changes.takeWrittenDocIds();
738
- swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
739
- hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
740
- });
741
- hooks.queueDiskAdoption?.({
742
- type: "adoptDiskSegments",
743
- indexName,
744
- partitions: actionPartitionsOf(adoption),
745
- changed: [...changed],
746
- requestId: createRequestId()
747
- });
748
- return true;
749
- });
750
- if (!adopted) {
751
- closeUnusedSegments(adoption);
752
- return [];
753
- }
754
- await hooks.awaitCopiesOfIndex?.(indexName);
755
- return adoption.droppedKeys;
756
- }
757
- async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
758
- const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
759
- const writeRecords = input.hooks.writeRecords ?? null;
760
- const { targets, documentCount } = capture2;
761
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
762
- if (aTargetFailed(input.indexState, targets)) return 0;
763
- const [liveVectors, segments] = await bothOrTheFirstFailure(
764
- writeCapturedVectors(store, input.indexName, capture2, priorManifest, fromMemory),
765
- writeIndexCheckpointSegments({
766
- directory: input.directory,
767
- metadata,
768
- targets,
769
- compactionThreshold: input.compactionThreshold,
770
- canOffload: input.canOffload,
771
- prior: priorManifest,
772
- wholePartitions: capture2.wholePartitions.contents,
773
- capturedPartitions: capture2.wholePartitions.captured,
774
- keepsWriteRecords: writeRecords !== null
775
- })
776
- );
777
- await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
778
- if (aTargetFailed(input.indexState, targets)) return 0;
779
- const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
780
- input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
781
- recordWrittenVectors(vectorIndexes, liveVectors.written);
782
- await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
783
- const changes = changesSinceCapture();
784
- const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
785
- if (!input.holdsCheckpointFiles()) {
786
- const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
787
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-JYHFTVPS.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
788
- const keptForSync = await input.keepsCheckpointKey();
789
- await removeCheckpointGarbage(
790
- store,
791
- removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
792
- );
793
- }
794
- const checkpointDocumentCount = documentCount;
795
- await input.queueMetadataWrite(input.indexName, async () => {
796
- const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
797
- if (checkpointMetadata === void 0) {
798
- return;
799
- }
800
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
801
- await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
802
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
803
- });
804
- const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
805
- await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
806
- if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
807
- await reclaimWalBeyondCount(
808
- input.directory,
809
- input.indexName,
810
- manager.partitionCount,
811
- input.indexState.partitions,
812
- input.markFatal
813
- );
814
- const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
815
- noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
816
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
817
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
818
- }
819
- function recordsBeyond(indexState, targets) {
820
- let records = 0;
821
- for (const [partitionId, partition] of indexState.partitions) {
822
- if (partition.failed !== null) continue;
823
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
824
- }
825
- return records;
826
- }
827
-
828
- // src/persistence/durability/checkpoint-turn.ts
829
- function createIndexLoads(vectorIndexesOf2) {
830
- const loads = /* @__PURE__ */ new Map();
831
- const startGraphRepairs = (indexName) => {
832
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
833
- vectorIndex.scheduleBuild();
834
- vectorIndex.startShuffledRebuildWhenDue();
835
- }
836
- };
837
- return {
838
- track(indexName, load, startsGraphRepairs = true) {
839
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
840
- loads.set(
841
- indexName,
842
- loaded.catch(() => void 0)
843
- );
844
- return loaded;
845
- },
846
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
847
- startGraphRepairs
848
- };
849
- }
850
- async function switchInTurn(turns, turn) {
851
- let swapped = false;
852
- await checkpointInTurn(turns, {
853
- joinsInFlight: false,
854
- mayStart: turn.mayStart,
855
- perform: async () => {
856
- swapped = turn.swap();
857
- if (swapped) await turn.checkpoint();
858
- },
859
- afterSettling: turn.afterSettling
860
- });
861
- return swapped;
862
- }
863
- async function checkpointInTurn(turns, turn) {
864
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
865
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
866
- if (!turn.mayStart()) return;
867
- const run = turn.perform().finally(() => {
868
- turns.checkpointInFlight = null;
869
- turn.afterSettling();
870
- });
871
- turns.checkpointInFlight = run;
872
- return run;
873
- }
874
-
875
- // src/persistence/durability/held-checkpoints.ts
876
- function failedPartitionIds(indexState) {
877
- const failed = [];
878
- for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
879
- return failed;
880
- }
881
- function createHeldCheckpoints(hooks) {
882
- const reported = /* @__PURE__ */ new WeakSet();
883
- function heldReport(indexName, indexState) {
884
- const partitionIds = failedPartitionIds(indexState);
885
- if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
886
- return {
887
- indexName,
888
- partitionIds,
889
- recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
890
- documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
891
- };
892
- }
893
- return {
894
- refuse(indexName, indexState) {
895
- const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
896
- if (partitionIds.length === 0) return;
897
- throw new NarsilError(
898
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
899
- `Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
900
- { indexName, partitionIds }
901
- );
902
- },
903
- holds(indexName, indexState) {
904
- const held = heldReport(indexName, indexState);
905
- if (held === null) return false;
906
- const unreported = held.partitionIds.some((partitionId) => {
907
- const partition = indexState.partitions.get(partitionId);
908
- return partition !== void 0 && !reported.has(partition);
909
- });
910
- if (unreported) {
911
- for (const partitionId of held.partitionIds) {
912
- const partition = indexState.partitions.get(partitionId);
913
- if (partition !== void 0) reported.add(partition);
914
- }
915
- hooks.onCheckpointHeld?.(held);
916
- }
917
- return true;
918
- },
919
- report(indexes) {
920
- const held = [];
921
- for (const [indexName, indexState] of indexes) {
922
- const entry = heldReport(indexName, indexState);
923
- if (entry !== null) held.push(entry);
924
- }
925
- return held;
926
- }
927
- };
928
- }
929
-
930
- // src/persistence/durability/restore-journal.ts
931
- var RESTORE_JOURNAL_PREFIX = ".swap-";
932
- var RESTORE_STAGING_PREFIX = ".restore-";
933
- var RETIRED_INDEX_PREFIX = ".retired-";
934
- var LOCK_FILE_KEY = ".narsil.lock";
935
- var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
936
- function digestOf(indexName) {
937
- let hex = "";
938
- for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
939
- return hex;
940
- }
941
- function journalKey(indexName) {
942
- return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
943
- }
944
- function randomSuffix() {
945
- return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
946
- }
947
- function newStagingPrefix() {
948
- return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
949
- }
950
- function newRetiredPrefix() {
951
- return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
952
- }
953
- function encodeJournal(indexName, stagingPrefix) {
954
- return new TextEncoder().encode(`${indexName}
955
- ${stagingPrefix}`);
956
- }
957
- function isStagingPrefix(prefix) {
958
- return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
959
- }
960
- function corruptJournal(key, reason) {
961
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
962
- }
963
- function readJournal(key, bytes) {
964
- const text = new TextDecoder().decode(bytes);
965
- const lineEnd = text.indexOf("\n");
966
- const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
967
- if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
968
- if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
969
- corruptJournal(key, "names an index that no index name can match");
970
- }
971
- const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
972
- if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
973
- corruptJournal(key, "names no staging prefix that a restore writes");
974
- }
975
- return { kind: "journal", indexName, stagingPrefix };
976
- }
977
- async function listedKeys(storage, indexName, stagingPrefix) {
978
- const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
979
- if (manifestBytes === null) {
980
- throw new NarsilError(
981
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
982
- `The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
983
- { indexName, stagingPrefix }
984
- );
985
- }
986
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
987
- if (decoded.kind !== "current") {
988
- throw new NarsilError(
989
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
990
- `The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
991
- { indexName }
992
- );
993
- }
994
- const root = `${indexName}/`;
995
- const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
996
- for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
997
- return keys;
998
- }
999
- function withoutTrailingSlash(prefix) {
1000
- return prefix.slice(0, -1);
1001
- }
1002
- async function finishJournal(storage, indexName, stagingPrefix) {
1003
- const root = `${indexName}/`;
1004
- const listed = await listedKeys(storage, indexName, stagingPrefix);
1005
- const staged = await storage.list(stagingPrefix);
1006
- let retiredPrefix = null;
1007
- if (storage.moveTree !== void 0 && staged.length > 0) {
1008
- retiredPrefix = newRetiredPrefix();
1009
- await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
1010
- await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
1011
- } else {
1012
- const stagedKeys = new Set(staged);
1013
- for (const key of listed) {
1014
- if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
1015
- }
1016
- }
1017
- for (const key of await storage.list(root)) {
1018
- if (!listed.has(key.slice(root.length))) await storage.remove(key);
1019
- }
1020
- await storage.remove(journalKey(indexName));
1021
- return retiredPrefix;
1022
- }
1023
- async function removePrefix(storage, prefix) {
1024
- if (storage.removeTree !== void 0) {
1025
- await storage.removeTree(prefix);
1026
- return;
1027
- }
1028
- for (const key of await storage.list(prefix)) await storage.remove(key);
1029
- }
1030
- async function commitJournal(storage, indexName, stagingPrefix) {
1031
- await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
1032
- }
1033
- function isTopLevelJournal(key) {
1034
- return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
1035
- }
1036
- async function removeStagingPrefixes(storage) {
1037
- const prefixes = /* @__PURE__ */ new Set();
1038
- for (const key of await storage.list(".")) {
1039
- const slash = key.indexOf("/");
1040
- if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
1041
- }
1042
- for (const prefix of prefixes) await removePrefix(storage, prefix);
1043
- }
1044
- async function recoverRestoreJournals(storage) {
1045
- for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
1046
- if (!isTopLevelJournal(key)) continue;
1047
- const bytes = await storage.read(key);
1048
- if (bytes === null) continue;
1049
- const reading = readJournal(key, bytes);
1050
- if (reading.kind === "leftover") {
1051
- await storage.remove(key);
1052
- continue;
1053
- }
1054
- await finishJournal(storage, reading.indexName, reading.stagingPrefix);
1055
- }
1056
- await removeStagingPrefixes(storage);
1057
- }
1058
-
1059
- // src/persistence/durability/restore-directory.ts
1060
- function missing(error) {
1061
- return error.code === "ENOENT";
1062
- }
1063
- function storageFailed(code, message, key, error) {
1064
- return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
1065
- }
1066
- function directoryRestoreStorage(directory) {
1067
- return {
1068
- directory,
1069
- read: (key) => directory.read(key),
1070
- write: (key, chunks) => directory.atomicWrite(key, chunks),
1071
- writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
1072
- remove: (key) => directory.remove(key),
1073
- list: (prefix) => directory.list(prefix),
1074
- moveKey: (from, to) => directory.move(from, to),
1075
- pathOf: (key) => directory.pathOf(key),
1076
- async moveTree(from, to) {
1077
- const source = await directory.pathOf(from);
1078
- const target = await directory.pathOf(to);
1079
- const fs = await getFs();
1080
- try {
1081
- await fs.rename(source, target);
1082
- } catch (error) {
1083
- if (missing(error)) return;
1084
- throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
1085
- }
1086
- await directory.syncDirectoryOf(to);
1087
- },
1088
- async removeTree(prefix) {
1089
- const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
1090
- const fs = await getFs();
1091
- try {
1092
- await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
1093
- } catch (error) {
1094
- throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
1095
- }
1096
- await directory.syncDirectoryOf(key);
1097
- }
1098
- };
1099
- }
1100
- function recoverDirectoryRestores(directory) {
1101
- return recoverRestoreJournals(directoryRestoreStorage(directory));
1102
- }
1103
-
1104
- // src/persistence/durability/restore-staging.ts
1105
- function toError2(error) {
1106
- return error instanceof Error ? error : new Error(String(error));
1107
- }
1108
- function pathUnder(path, root) {
1109
- if (!path.startsWith(root)) return false;
1110
- const next = path.charAt(root.length);
1111
- return next === "" || next === "/" || next === "\\";
1112
- }
1113
- function stagedDirectory(directory, staged, unstaged) {
1114
- const { readRange, byteLength } = directory;
1115
- return {
1116
- root: directory.root,
1117
- appendHandle: (key) => directory.appendHandle(staged(key)),
1118
- markerHandle: (key) => directory.markerHandle(staged(key)),
1119
- atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
1120
- syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
1121
- read: (key) => directory.read(staged(key)),
1122
- remove: (key) => directory.remove(staged(key)),
1123
- move: (from, to) => directory.move(staged(from), staged(to)),
1124
- list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
1125
- pathOf: (key) => directory.pathOf(staged(key)),
1126
- ...readRange === void 0 ? {} : {
1127
- readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
1128
- },
1129
- ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
1130
- };
1131
- }
1132
- function createRestoreStaging(storage, indexName, options = {}) {
1133
- const stagingPrefix = newStagingPrefix();
1134
- const indexRoot = `${indexName}/`;
1135
- let committed = false;
1136
- const staged = (key) => {
1137
- if (!key.startsWith(indexRoot)) {
1138
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
1139
- key
1140
- });
1141
- }
1142
- return `${stagingPrefix}${key.slice(indexRoot.length)}`;
1143
- };
1144
- const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
1145
- const store = {
1146
- read: (key) => storage.read(staged(key)),
1147
- write: (key, chunks) => storage.write(staged(key), chunks),
1148
- remove: (key) => storage.remove(staged(key)),
1149
- list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
1150
- };
1151
- const pathOf = storage.pathOf;
1152
- if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
1153
- if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
1154
- let retiredPrefix = null;
1155
- async function finishOnStorage() {
1156
- retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
1157
- }
1158
- return {
1159
- store,
1160
- async relocation() {
1161
- if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
1162
- const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
1163
- const to = await pathOf.call(storage, indexName);
1164
- return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
1165
- },
1166
- async commit() {
1167
- try {
1168
- await commitJournal(storage, indexName, stagingPrefix);
1169
- } catch (error) {
1170
- try {
1171
- await storage.remove(journalKey(indexName));
1172
- } catch (removal) {
1173
- committed = true;
1174
- options.onStorageLost?.(toError2(removal));
1175
- }
1176
- throw error;
1177
- }
1178
- committed = true;
1179
- },
1180
- async finish() {
1181
- try {
1182
- await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
1183
- } catch (error) {
1184
- options.onStorageLost?.(toError2(error));
1185
- }
1186
- },
1187
- async removeRetired() {
1188
- const prefix = retiredPrefix;
1189
- retiredPrefix = null;
1190
- if (prefix !== null) await removePrefix(storage, prefix);
1191
- },
1192
- async discard() {
1193
- if (committed) return;
1194
- await removePrefix(storage, stagingPrefix);
1195
- }
1196
- };
1197
- }
1198
- function adapterRestoreStorage(adapter) {
1199
- const store = adapterCheckpointStore(adapter);
1200
- return {
1201
- read: (key) => store.read(key),
1202
- write: (key, chunks) => store.write(key, chunks),
1203
- writeJournal: (key, bytes) => store.write(key, [bytes]),
1204
- remove: (key) => store.remove(key),
1205
- list: (prefix) => store.list(prefix),
1206
- async moveKey(from, to) {
1207
- const bytes = await store.read(from);
1208
- if (bytes === null) return;
1209
- await store.write(to, [bytes]);
1210
- await store.remove(from);
1211
- }
1212
- };
1213
- }
1214
-
1215
- // src/persistence/durability/archive-checkpoint.ts
1216
- function toError3(error) {
1217
- return error instanceof Error ? error : new Error(String(error));
1218
- }
1219
- function stageDirectoryRestore(deps, indexName) {
1220
- return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
1221
- async finish(run) {
1222
- try {
1223
- await run();
1224
- } catch (error) {
1225
- deps.markFatal(toError3(error));
1226
- throw error;
1227
- }
1228
- },
1229
- onStorageLost: deps.markFatal
1230
- });
1231
- }
1232
- function releaser(held, indexName) {
1233
- let released = false;
1234
- return () => {
1235
- if (released) return;
1236
- released = true;
1237
- const count = (held.get(indexName) ?? 1) - 1;
1238
- if (count <= 0) held.delete(indexName);
1239
- else held.set(indexName, count);
1240
- };
1241
- }
1242
- function createArchiveCheckpointAccess(deps) {
1243
- const store = directoryCheckpointStore(deps.directory);
1244
- return {
1245
- async openArchiveCheckpoint(indexName) {
1246
- deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
1247
- const release = releaser(deps.held, indexName);
1248
- try {
1249
- await deps.checkpoint(indexName, false);
1250
- await deps.checkpoint(indexName, true);
1251
- const manifestBytes = await deps.directory.read(manifestKey(indexName));
1252
- if (manifestBytes === null) {
1253
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
1254
- indexName
1255
- });
1256
- }
1257
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1258
- if (decoded.kind !== "current") {
1259
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
1260
- indexName
1261
- });
1262
- }
1263
- return {
1264
- manifestBytes,
1265
- manifest: decoded.manifest,
1266
- readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
1267
- readRange: async (ref, offset, length) => {
1268
- const readRange = deps.directory.readRange;
1269
- if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
1270
- const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
1271
- return whole.slice(offset, offset + length);
1272
- },
1273
- release,
1274
- directory: deps.directory
1275
- };
1276
- } catch (error) {
1277
- release();
1278
- throw error;
1279
- }
1280
- },
1281
- stageRestore(indexName) {
1282
- return stageDirectoryRestore(deps, indexName);
1283
- },
1284
- adoptRestoredCheckpoint(indexName, checkpoint) {
1285
- for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
1286
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
1287
- }
1288
- }
1289
- };
1290
- }
1291
-
1292
- // src/persistence/durability/query-history.ts
1293
- async function removeQueryHistoryOf(storage, indexName) {
1294
- const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
1295
- if (storage.kind === "directory") {
1296
- for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
1297
- return;
1298
- }
1299
- for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
1300
- }
1301
- function queryHistoryAccess(storage) {
1302
- return {
1303
- historyStorage: () => storage,
1304
- removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
1305
- };
1306
- }
1307
-
1308
- // src/persistence/durability/manager-unload.ts
1309
- function createIndexUnloading(deps) {
1310
- const { directory, hooks, indexes, metadataWrites } = deps;
1311
- return {
1312
- async removeIndex(indexName) {
1313
- hooks.writeRecords?.dropIndex(indexName);
1314
- const indexState = indexes.get(indexName);
1315
- if (indexState !== void 0) {
1316
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
1317
- indexes.delete(indexName);
1318
- metadataWrites.forget(indexName);
1319
- }
1320
- for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
1321
- for (const key of await directory.list(`${indexName}/`)) {
1322
- await directory.remove(key);
1323
- }
1324
- await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
1325
- deps.forgetSyncState(indexName);
1326
- },
1327
- async unloadIndex(indexName) {
1328
- const indexState = indexes.get(indexName);
1329
- if (indexState === void 0) {
1330
- return;
1331
- }
1332
- await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
1333
- indexes.delete(indexName);
1334
- metadataWrites.forget(indexName);
1335
- deps.forgetSyncState(indexName);
1336
- }
1337
- };
1338
- }
1339
-
1340
- // src/persistence/durability/sync/frame-cursor.ts
1341
- var SegmentFileMissing = class extends Error {
1342
- };
1343
- function corrupt(key, message, details) {
1344
- throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
1345
- }
1346
- function isMissingFile(error) {
1347
- return error?.code === "ENOENT";
1348
- }
1349
- var FrameCursor = class {
1350
- constructor(directory, key, limit) {
1351
- this.directory = directory;
1352
- this.key = key;
1353
- this.limit = limit;
1354
- }
1355
- buffer = new Uint8Array(0);
1356
- bufferStart = 0;
1357
- whole = null;
1358
- async load(offset, length) {
1359
- const readRange = this.directory.readRange;
1360
- if (readRange === void 0) {
1361
- if (this.whole === null) {
1362
- const bytes = await this.directory.read(this.key);
1363
- if (bytes === null) throw new SegmentFileMissing(this.key);
1364
- this.whole = bytes;
1365
- }
1366
- this.buffer = this.whole;
1367
- this.bufferStart = 0;
1368
- return;
1369
- }
1370
- const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
1371
- try {
1372
- this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
1373
- } catch (error) {
1374
- if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
1375
- throw error;
1376
- }
1377
- this.bufferStart = offset;
1378
- }
1379
- async bytesAt(offset, length) {
1380
- const end = offset + length;
1381
- if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
1382
- if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
1383
- return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
1384
- }
1385
- async checkHeader() {
1386
- if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
1387
- const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
1388
- if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
1389
- }
1390
- async frameLengthAt(offset) {
1391
- const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
1392
- const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
1393
- 0,
1394
- false
1395
- );
1396
- if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
1397
- corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
1398
- }
1399
- return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
1400
- }
1401
- async frameAt(offset, frameBytes) {
1402
- const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
1403
- const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
1404
- const payload = body.subarray(0, recordLength);
1405
- const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
1406
- if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
1407
- let seqNo;
1408
- try {
1409
- seqNo = decodeWalEntry(payload).seqNo;
1410
- } catch (error) {
1411
- corrupt(this.key, "A log record fails to decode", {
1412
- offset,
1413
- cause: error instanceof Error ? error.message : String(error)
1414
- });
1415
- }
1416
- return { seqNo, payload: payload.slice(), frameBytes };
1417
- }
1418
- };
1419
-
1420
- // src/persistence/durability/sync/log-reader.ts
1421
- async function walSources(directory, indexName, partitionId) {
1422
- const prefix = walSegmentPrefix(indexName, partitionId);
1423
- const markerBytes = await directory.read(`${prefix}commit`);
1424
- const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
1425
- if (marker === null) return { sources: [], head: 0 };
1426
- const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
1427
- const sources = [];
1428
- for (const key of await directory.list(prefix)) {
1429
- const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
1430
- if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
1431
- const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
1432
- if (readableBytes === null || readableBytes === void 0) continue;
1433
- sources.push({ key, startSeqNo, readableBytes, inHistory: false });
1434
- }
1435
- return { sources, head: highestDurableSeqNo };
1436
- }
1437
- async function openPartitionLog(directory, indexName, partitionId) {
1438
- const { sources, head } = await walSources(directory, indexName, partitionId);
1439
- const byStart = /* @__PURE__ */ new Map();
1440
- for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
1441
- const readableBytes = await directory.byteLength?.(segment.key);
1442
- if (readableBytes === null || readableBytes === void 0) continue;
1443
- byStart.set(segment.startSeqNo, {
1444
- key: segment.key,
1445
- startSeqNo: segment.startSeqNo,
1446
- readableBytes,
1447
- inHistory: true
1448
- });
1449
- }
1450
- for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
1451
- const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
1452
- return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
1453
- }
1454
- function sourceHolding(log, seqNo) {
1455
- let found = -1;
1456
- for (let i = 0; i < log.sources.length; i++) {
1457
- if (log.sources[i].startSeqNo > seqNo) break;
1458
- found = i;
1459
- }
1460
- return found;
1461
- }
1462
- async function offsetAfter(cursor, source, afterSeqNo) {
1463
- let offset = SEGMENT_HEADER_SIZE;
1464
- for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
1465
- offset += await cursor.frameLengthAt(offset);
1466
- }
1467
- return offset;
1468
- }
1469
- async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
1470
- const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1471
- await cursor.checkHeader();
1472
- let offset = await offsetAfter(cursor, source, afterSeqNo);
1473
- let checksNumbering = true;
1474
- while (offset < cursor.limit) {
1475
- const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
1476
- if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
1477
- checksNumbering = false;
1478
- if (frame.seqNo > afterSeqNo) {
1479
- offset = SEGMENT_HEADER_SIZE;
1480
- continue;
1481
- }
1482
- }
1483
- checksNumbering = false;
1484
- offset += frame.frameBytes;
1485
- if (frame.seqNo <= afterSeqNo) continue;
1486
- if (frame.seqNo > upToSeqNo) return true;
1487
- if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
1488
- read.frames.push(frame);
1489
- read.bytes += frame.payload.byteLength;
1490
- if (frame.seqNo === upToSeqNo) return true;
1491
- }
1492
- return false;
1493
- }
1494
- async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
1495
- const read = { frames: [], bytes: 0, gone: false };
1496
- if (afterSeqNo >= upToSeqNo) return read;
1497
- for (let attempt = 0; attempt < 2; attempt++) {
1498
- read.frames = [];
1499
- read.bytes = 0;
1500
- const log = await openPartitionLog(directory, indexName, partitionId);
1501
- let index = sourceHolding(log, afterSeqNo + 1);
1502
- if (index < 0) {
1503
- read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
1504
- return read;
1505
- }
1506
- try {
1507
- for (; index < log.sources.length; index++) {
1508
- const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
1509
- if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
1510
- }
1511
- return read;
1512
- } catch (error) {
1513
- if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
1514
- }
1515
- }
1516
- return read;
1517
- }
1518
- async function logBytesAfter(directory, log, afterSeqNo) {
1519
- const first = sourceHolding(log, afterSeqNo + 1);
1520
- let bytes = 0;
1521
- for (let i = Math.max(first, 0); i < log.sources.length; i++) {
1522
- const source = log.sources[i];
1523
- if (i === first) {
1524
- const cursor = new FrameCursor(directory, source.key, source.readableBytes);
1525
- try {
1526
- bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
1527
- } catch (error) {
1528
- if (!(error instanceof SegmentFileMissing)) throw error;
1529
- }
1530
- continue;
1531
- }
1532
- bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
1533
- }
1534
- return bytes;
1535
- }
1536
-
1537
- // src/persistence/durability/sync/pins.ts
1538
- function createFilePins(remove) {
1539
- const counts = /* @__PURE__ */ new Map();
1540
- const awaitingRemoval = /* @__PURE__ */ new Set();
1541
- function unpin(key) {
1542
- const count = (counts.get(key) ?? 1) - 1;
1543
- if (count > 0) {
1544
- counts.set(key, count);
1545
- return;
1546
- }
1547
- counts.delete(key);
1548
- if (!awaitingRemoval.delete(key)) return;
1549
- void remove(key).catch(() => void 0);
1550
- }
1551
- return {
1552
- pin(keys) {
1553
- const pinned = [...keys];
1554
- for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
1555
- let released = false;
1556
- return () => {
1557
- if (released) return;
1558
- released = true;
1559
- for (const key of pinned) unpin(key);
1560
- };
1561
- },
1562
- isPinned: (key) => counts.has(key),
1563
- removeWhenUnpinned(key) {
1564
- if (!counts.has(key)) return false;
1565
- awaitingRemoval.add(key);
1566
- return true;
1567
- }
1568
- };
1569
- }
1570
- var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
1571
- var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
1572
- function hexOf(bytes) {
1573
- let hex = "";
1574
- for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
1575
- return hex;
1576
- }
1577
- function isBytes(value) {
1578
- return value instanceof Uint8Array;
1579
- }
1580
- async function publishedFrom(indexName, payload, config, manifestBytes) {
1581
- const decoded = await decodeSegmentManifest(manifestBytes, indexName);
1582
- if (decoded.kind !== "current") {
1583
- throw new NarsilError(
1584
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1585
- `The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
1586
- { indexName }
1587
- );
1588
- }
1589
- return {
1590
- config,
1591
- manifestBytes,
1592
- manifest: decoded.manifest,
1593
- entityTag: hexOf(sha256(payload)),
1594
- keys: manifestReferencedKeys(decoded.manifest)
1595
- };
1596
- }
1597
- async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
1598
- const payload = encode({ config, manifest: manifestBytes });
1599
- const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
1600
- const published = await publishedFrom(indexName, payload, config, manifestBytes);
1601
- await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
1602
- return published;
1603
- }
1604
- async function readPublishedSnapshot(directory, indexName) {
1605
- const bytes = await directory.read(publishedSnapshotKey(indexName));
1606
- if (bytes === null) return null;
1607
- const { payload } = await openChecksummedEnvelope(bytes, {
1608
- envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
1609
- what: PUBLISHED_SNAPSHOT_WHAT,
1610
- allowsCompression: false
1611
- });
1612
- const raw = decode(payload);
1613
- if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
1614
- throw new NarsilError(
1615
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
1616
- `The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
1617
- { indexName }
1618
- );
1619
- }
1620
- return publishedFrom(indexName, payload, raw.config, raw.manifest);
1621
- }
1622
-
1623
- // src/persistence/durability/sync/retention.ts
1624
- async function firstLogStart(directory, indexName, partitionId) {
1625
- let first = Number.POSITIVE_INFINITY;
1626
- for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
1627
- const start = parseWalSegmentStart(indexName, partitionId, key);
1628
- if (start !== null && start < first) first = start;
1629
- }
1630
- return first;
1631
- }
1632
- async function sizedQueues(retention) {
1633
- const { directory, indexName } = retention;
1634
- const queues = /* @__PURE__ */ new Map();
1635
- for (const segment of await listHistorySegments(directory, indexName)) {
1636
- const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
1637
- if (bytes === void 0 || bytes === null) continue;
1638
- const queue = queues.get(segment.partitionId) ?? [];
1639
- queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
1640
- queues.set(segment.partitionId, queue);
1641
- }
1642
- for (const [partitionId, queue] of queues) {
1643
- const logStart = await firstLogStart(directory, indexName, partitionId);
1644
- for (let i = 0; i < queue.length; i++) {
1645
- const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
1646
- queue[i].lastSeqNo = nextStart - 1;
1647
- }
1648
- }
1649
- return queues;
1650
- }
1651
- function nextToDelete(queues, totalBytes, retention) {
1652
- const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
1653
- let oldest = null;
1654
- for (const queue of queues.values()) {
1655
- const head = queue[0];
1656
- if (head === void 0) continue;
1657
- if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
1658
- }
1659
- if (oldest === null) return null;
1660
- if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
1661
- return null;
1662
- }
1663
- async function enforceHistoryLimits(retention) {
1664
- const queues = await sizedQueues(retention);
1665
- let totalBytes = 0;
1666
- for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
1667
- let published = false;
1668
- let deleted = 0;
1669
- let segment = nextToDelete(queues, totalBytes, retention);
1670
- while (segment !== null) {
1671
- const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
1672
- if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
1673
- published = true;
1674
- await retention.publish();
1675
- }
1676
- await retention.directory.remove(segment.key);
1677
- queues.get(segment.partitionId)?.shift();
1678
- totalBytes -= segment.bytes;
1679
- deleted += 1;
1680
- segment = nextToDelete(queues, totalBytes, retention);
1681
- }
1682
- return deleted;
1683
- }
1684
-
1685
- // src/persistence/durability/sync/storage.ts
1686
- function createSyncStorage(deps) {
1687
- const { directory } = deps;
1688
- const store = directoryCheckpointStore(directory);
1689
- const cache = /* @__PURE__ */ new Map();
1690
- const publishing = /* @__PURE__ */ new Map();
1691
- const pins = createFilePins((key) => directory.remove(key));
1692
- async function published(indexName) {
1693
- const cached = cache.get(indexName);
1694
- if (cached !== void 0) return cached;
1695
- const read = await readPublishedSnapshot(directory, indexName);
1696
- cache.set(indexName, read);
1697
- return read;
1698
- }
1699
- async function removeUnlisted(indexName, previous, kept) {
1700
- const manifest = await readCurrentManifest(store, indexName);
1701
- const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
1702
- for (const key of previous) {
1703
- if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
1704
- await directory.remove(key);
1705
- }
1706
- }
1707
- async function publishNow(indexName, configurationOf) {
1708
- const archive = await deps.openArchiveCheckpoint(indexName);
1709
- try {
1710
- const config = await configurationOf(archive.manifest);
1711
- const previous = await published(indexName);
1712
- const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
1713
- cache.set(indexName, next);
1714
- if (previous !== null) {
1715
- await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
1716
- }
1717
- return next;
1718
- } finally {
1719
- archive.release();
1720
- }
1721
- }
1722
- return {
1723
- publish(indexName, configurationOf) {
1724
- const inFlight = publishing.get(indexName);
1725
- const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
1726
- const tracked = run.finally(() => {
1727
- if (publishing.get(indexName) === tracked) publishing.delete(indexName);
1728
- });
1729
- publishing.set(indexName, tracked);
1730
- return tracked;
1731
- },
1732
- published,
1733
- pin: (snapshot) => pins.pin(snapshot.keys),
1734
- async readFileRange(key, offset, length) {
1735
- const readRange = directory.readRange;
1736
- if (readRange !== void 0) return readRange.call(directory, key, offset, length);
1737
- const whole = await directory.read(key);
1738
- return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
1739
- },
1740
- async release(indexName) {
1741
- await deps.inCheckpointTurn(indexName, async () => {
1742
- const previous = await published(indexName);
1743
- await directory.remove(publishedSnapshotKey(indexName));
1744
- cache.set(indexName, null);
1745
- if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
1746
- await removeHistory(directory, indexName);
1747
- });
1748
- },
1749
- removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
1750
- async enforceHistoryLimits(indexName, limits, publish, nowMs) {
1751
- const current = await published(indexName);
1752
- return enforceHistoryLimits({
1753
- directory,
1754
- indexName,
1755
- limits,
1756
- nowMs,
1757
- publish,
1758
- publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
1759
- });
1760
- },
1761
- openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
1762
- readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
1763
- logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
1764
- currentManifest: (indexName) => readCurrentManifest(store, indexName),
1765
- async keepsCheckpointKey(indexName) {
1766
- const current = await published(indexName);
1767
- return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
1768
- },
1769
- async settleAtRecovery(indexName, active) {
1770
- cache.delete(indexName);
1771
- const publishedKey = publishedSnapshotKey(indexName);
1772
- for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
1773
- if (key !== publishedKey) await directory.remove(key);
1774
- }
1775
- if (!active) {
1776
- await directory.remove(publishedSnapshotKey(indexName));
1777
- await removeHistory(directory, indexName);
1778
- cache.set(indexName, null);
1779
- return /* @__PURE__ */ new Set();
1780
- }
1781
- try {
1782
- return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
1783
- } catch {
1784
- await directory.remove(publishedSnapshotKey(indexName));
1785
- cache.set(indexName, null);
1786
- return /* @__PURE__ */ new Set();
1787
- }
1788
- },
1789
- forget(indexName) {
1790
- cache.delete(indexName);
1791
- }
1792
- };
1793
- }
1794
-
1795
- // src/persistence/durability/manager-storage.ts
1796
- function createIndexStorageAccess(deps) {
1797
- const { directory, indexes } = deps;
1798
- const archiveAccess = createArchiveCheckpointAccess({
1799
- directory,
1800
- held: deps.heldCheckpointFiles,
1801
- checkpoint: deps.checkpoint,
1802
- registerPartition: deps.registerPartition,
1803
- markFatal: deps.markFatal
1804
- });
1805
- const syncStorage = createSyncStorage({
1806
- directory,
1807
- openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
1808
- inCheckpointTurn: (indexName, work) => {
1809
- const indexState = indexes.get(indexName);
1810
- if (indexState === void 0) return work();
1811
- return checkpointInTurn(indexState, {
1812
- joinsInFlight: false,
1813
- mayStart: () => true,
1814
- perform: work,
1815
- afterSettling: () => releaseStalledWrites(indexState)
1816
- });
1817
- }
1818
- });
1819
- const unloading = createIndexUnloading({
1820
- directory,
1821
- hooks: deps.hooks,
1822
- indexes,
1823
- metadataWrites: deps.metadataWrites,
1824
- closeWriterReportingFailure: deps.closeWriterReportingFailure,
1825
- forgetSyncState: syncStorage.forget
1826
- });
1827
- return { archiveAccess, syncStorage, unloading };
1828
- }
1829
-
1830
- // src/persistence/durability/metadata-write-queue.ts
1831
- function createMetadataWriteQueue() {
1832
- const writes = /* @__PURE__ */ new Map();
1833
- return {
1834
- queue(indexName, write) {
1835
- const previous = writes.get(indexName) ?? Promise.resolve();
1836
- const queued = previous.then(write);
1837
- writes.set(
1838
- indexName,
1839
- queued.catch(() => void 0)
1840
- );
1841
- return queued;
1842
- },
1843
- pending: (indexName) => writes.get(indexName),
1844
- forget(indexName) {
1845
- writes.delete(indexName);
1846
- }
1847
- };
1848
- }
1849
-
1850
- // src/persistence/durability/mutation-batch.ts
1851
- function toError4(err) {
1852
- return err instanceof Error ? err : new Error(String(err));
1853
- }
1854
- function requireAscendingSeqNo(record, seqNo, previousSeqNo) {
1855
- if (seqNo > previousSeqNo) return;
1856
- throw new NarsilError(
1857
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
1858
- `Log record ${seqNo} for partition ${record.partitionId} of index "${record.indexName}" does not follow record ${previousSeqNo}`,
1859
- { indexName: record.indexName, partitionId: record.partitionId, seqNo, previousSeqNo }
1860
- );
1861
- }
1862
- function failUnlessAlreadyFailed(outcomes, count, error) {
1863
- for (let i = 0; i < count; i++) {
1864
- const outcome = outcomes[i];
1865
- if (outcome === void 0 || outcome.ok) {
1866
- outcomes[i] = { ok: false, error };
1867
- }
1868
- }
1869
- }
1870
- function failPartition(partition, address, deps, err) {
1871
- const error = toError4(err);
1872
- if (partition.failed !== null) return partition.failed;
1873
- partition.failed = error;
1874
- if (!deps.failsPartitionAlone(address.indexName, address.partitionId, error)) deps.markFatal(error);
1875
- return error;
1876
- }
1877
- async function commitEveryPartition(indexes, deps) {
1878
- for (const [indexName, indexState] of indexes) {
1879
- for (const [partitionId, partition] of indexState.partitions) {
1880
- if (partition.failed !== null || deps.fatalError() !== null) continue;
1881
- try {
1882
- await partition.walWriter.commit();
1883
- } catch (err) {
1884
- failPartition(partition, { indexName, partitionId }, deps, err);
1885
- }
1886
- }
1887
- }
1888
- }
1889
- function partitionFailureFor(indexName, deps) {
1890
- return (partitionId, partition, error) => failPartition(partition, { indexName, partitionId }, deps, error);
1891
- }
1892
- async function recordBatchesByPartition(records, partitionOf, deps) {
1893
- const positionsByPartition = /* @__PURE__ */ new Map();
1894
- for (let i = 0; i < records.length; i++) {
1895
- const partition = partitionOf(records[i]);
1896
- const positions = positionsByPartition.get(partition);
1897
- if (positions === void 0) {
1898
- positionsByPartition.set(partition, [i]);
1899
- } else {
1900
- positions.push(i);
1901
- }
1902
- }
1903
- const outcomes = new Array(records.length);
1904
- await Promise.all(
1905
- [...positionsByPartition].map(async ([partition, positions]) => {
1906
- const partitionOutcomes = await recordPartitionBatch(
1907
- partition,
1908
- positions.map((position) => records[position]),
1909
- deps
1910
- );
1911
- for (let i = 0; i < positions.length; i++) {
1912
- outcomes[positions[i]] = partitionOutcomes[i];
1913
- }
1914
- })
1915
- );
1916
- return outcomes;
1917
- }
1918
- async function recordPartitionBatch(partition, records, deps) {
1919
- const outcomes = new Array(records.length);
1920
- let appendedCount = 0;
1921
- let commitsHere = false;
1922
- const appended = partition.appendChain.then(async () => {
1923
- const fatal = deps.fatalError();
1924
- if (fatal !== null) {
1925
- throw fatal;
1926
- }
1927
- if (partition.failed !== null) {
1928
- throw partition.failed;
1929
- }
1930
- const applied = [];
1931
- for (let i = 0; i < records.length; i++) {
1932
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
1933
- try {
1934
- await records[i].apply();
1935
- applied.push(i);
1936
- } catch (err) {
1937
- outcomes[i] = { ok: false, error: err };
1938
- }
1939
- }
1940
- if (applied.length === 0) {
1941
- return;
1942
- }
1943
- let lastSeqNo = partition.appliedSeqNo;
1944
- const entries = [];
1945
- try {
1946
- for (const i of applied) {
1947
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
1948
- let numbered;
1949
- try {
1950
- numbered = deps.numberEntry(records[i]);
1951
- } catch (err) {
1952
- outcomes[i] = { ok: false, error: err };
1953
- continue;
1954
- }
1955
- if (numbered === null) {
1956
- lastSeqNo = partition.seqOwner.next();
1957
- entries.push(deps.buildEntry(records[i], lastSeqNo));
1958
- } else {
1959
- requireAscendingSeqNo(records[i], numbered.seqNo, lastSeqNo);
1960
- partition.seqOwner.advanceTo(numbered.seqNo, numbered.primaryTerm);
1961
- lastSeqNo = numbered.seqNo;
1962
- entries.push(numbered);
1963
- }
1964
- outcomes[i] = { ok: true, seqNo: lastSeqNo };
1965
- if (!deps.defersCommit(records[i])) commitsHere = true;
1966
- }
1967
- if (entries.length > 0) await partition.walWriter.appendAll(entries);
1968
- } catch (err) {
1969
- throw failPartition(partition, records[0], deps, err);
1970
- }
1971
- partition.appliedSeqNo = lastSeqNo;
1972
- appendedCount = entries.length;
1973
- });
1974
- partition.appendChain = appended.catch(() => void 0);
1975
- try {
1976
- await appended;
1977
- if (deps.syncEachBatch && appendedCount > 0 && commitsHere) {
1978
- try {
1979
- await partition.walWriter.commit();
1980
- } catch (err) {
1981
- throw failPartition(partition, records[0], deps, err);
1982
- }
1983
- }
1984
- } catch (err) {
1985
- failUnlessAlreadyFailed(outcomes, records.length, toError4(err));
1986
- }
1987
- return outcomes;
1988
- }
1989
- async function commitAppended(partition, address, deps) {
1990
- await partition.appendChain;
1991
- const failed = partition.failed ?? deps.fatalError();
1992
- if (failed !== null) throw failed;
1993
- try {
1994
- await partition.walWriter.commit();
1995
- } catch (err) {
1996
- throw failPartition(partition, address, deps, err);
1997
- }
1998
- }
1999
-
2000
- // src/persistence/durability/mutation-entry.ts
2001
- function buildMutationEntry(record, seqNo) {
2002
- return buildEntry({
2003
- seqNo,
2004
- primaryTerm: SINGLE_NODE_PRIMARY_TERM,
2005
- operation: record.operation,
2006
- partitionId: record.partitionId,
2007
- indexName: record.indexName,
2008
- documentId: record.documentId,
2009
- document: record.document
2010
- });
2011
- }
2012
-
2013
- // src/persistence/durability/partition-failure.ts
2014
- function createPartitionFailureRouting() {
2015
- let onFailed = null;
2016
- return {
2017
- isolate(handler) {
2018
- onFailed = handler;
2019
- },
2020
- failsAlone(indexName, partitionId, error) {
2021
- if (onFailed === null) return false;
2022
- onFailed(indexName, partitionId, error);
2023
- return true;
2024
- }
2025
- };
2026
- }
2027
-
2028
- // src/persistence/durability/group-commit.ts
2029
- function createGroupCommitCoordinator(sync) {
2030
- let inFlight = null;
2031
- let next = null;
2032
- function makeBatch() {
2033
- let resolve = () => void 0;
2034
- let reject = () => void 0;
2035
- const promise = new Promise((res, rej) => {
2036
- resolve = res;
2037
- reject = rej;
2038
- });
2039
- return { promise, resolve, reject, waiters: 0 };
2040
- }
2041
- function drain() {
2042
- if (inFlight !== null || next === null) {
2043
- return;
2044
- }
2045
- const batch = next;
2046
- next = null;
2047
- inFlight = sync().then(() => {
2048
- batch.resolve();
2049
- }).catch((err) => {
2050
- batch.reject(err);
2051
- }).finally(() => {
2052
- inFlight = null;
2053
- drain();
2054
- });
2055
- }
2056
- return {
2057
- commit() {
2058
- if (next === null) {
2059
- next = makeBatch();
2060
- }
2061
- next.waiters += 1;
2062
- const promise = next.promise;
2063
- drain();
2064
- return promise;
2065
- },
2066
- get pendingCount() {
2067
- return next === null ? 0 : next.waiters;
2068
- }
2069
- };
2070
- }
2071
-
2072
- // src/persistence/durability/wal-writer.ts
2073
- var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
2074
- function segmentKey(indexName, partitionId, startSeqNo) {
2075
- const padded = startSeqNo.toString().padStart(16, "0");
2076
- return `${indexName}/wal/${partitionId}/${padded}`;
2077
- }
2078
- function parseSegmentStartSeqNo2(key) {
2079
- const tail = key.slice(key.lastIndexOf("/") + 1);
2080
- if (!SEGMENT_TAIL_PATTERN.test(tail)) {
2081
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Malformed WAL segment key: "${key}"`, { key });
2082
- }
2083
- const value = Number.parseInt(tail, 10);
2084
- if (!Number.isSafeInteger(value)) {
2085
- throw new NarsilError(
2086
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2087
- `WAL segment start sequence number is out of range: "${key}"`,
2088
- {
2089
- key
2090
- }
2091
- );
2092
- }
2093
- return value;
2094
- }
2095
- function createWalWriter(directory, config) {
2096
- const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
2097
- const commitKey = `${config.indexName}/wal/${config.partitionId}/commit`;
2098
- const preparedKey = nextSegmentKey(config.indexName, config.partitionId);
2099
- let nextSegmentPrepared = false;
2100
- let handle = null;
2101
- let activeKey = null;
2102
- let activeStartSeqNo = 0;
2103
- let activeBytes = 0;
2104
- let highestAppendedSeqNo = 0;
2105
- let highestDurableSeqNo = 0;
2106
- let markerWriter = null;
2107
- let durabilityLock = Promise.resolve();
2108
- const coordinator = createGroupCommitCoordinator(() => withDurabilityLock(flushActiveSegment));
2109
- function withDurabilityLock(fn) {
2110
- const result = durabilityLock.then(fn);
2111
- durabilityLock = result.then(
2112
- () => void 0,
2113
- () => void 0
2114
- );
2115
- return result;
2116
- }
2117
- async function getMarkerWriter() {
2118
- if (markerWriter === null) {
2119
- const writer = await createMarkerWriter(directory, config.indexName, config.partitionId);
2120
- markerWriter = writer;
2121
- if (writer.existingHighestDurableSeqNo > highestDurableSeqNo) {
2122
- highestDurableSeqNo = writer.existingHighestDurableSeqNo;
2123
- }
2124
- if (writer.created) {
2125
- await directory.syncDirectoryOf(commitKey);
2126
- }
2127
- }
2128
- return markerWriter;
2129
- }
2130
- async function flushActiveSegment() {
2131
- if (handle === null) {
2132
- return;
2133
- }
2134
- const writer = await getMarkerWriter();
2135
- const durableByteLength = activeBytes;
2136
- const activeSegmentSeqNo = activeStartSeqNo;
2137
- const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
2138
- await handle.sync();
2139
- await writer.commit({ activeSegmentSeqNo, durableByteLength, highestDurableSeqNo: durableSeqNo });
2140
- if (durableSeqNo > highestDurableSeqNo) {
2141
- highestDurableSeqNo = durableSeqNo;
2142
- }
2143
- }
2144
- async function openSegmentByKey(key) {
2145
- const nextHandle = await directory.appendHandle(key);
2146
- const existingSize = await nextHandle.size();
2147
- let createdNew = false;
2148
- if (existingSize === 0) {
2149
- await nextHandle.append(writeSegmentHeader());
2150
- activeBytes = SEGMENT_HEADER_SIZE;
2151
- createdNew = true;
2152
- } else {
2153
- activeBytes = existingSize;
2154
- }
2155
- handle = nextHandle;
2156
- activeKey = key;
2157
- activeStartSeqNo = parseSegmentStartSeqNo2(key);
2158
- if (createdNew) {
2159
- await directory.syncDirectoryOf(key);
2160
- }
2161
- return createdNew;
2162
- }
2163
- async function findActiveSegmentKey() {
2164
- const prefix = `${config.indexName}/wal/${config.partitionId}/`;
2165
- const keys = (await directory.list(prefix)).filter((k) => SEGMENT_TAIL_PATTERN.test(k.slice(prefix.length))).sort(compareCodePoints);
2166
- return keys.length > 0 ? keys[keys.length - 1] : null;
2167
- }
2168
- async function ensureSegment(seqNo) {
2169
- if (handle !== null) {
2170
- return;
2171
- }
2172
- const existing = await findActiveSegmentKey();
2173
- const createdNew = await openSegmentByKey(existing ?? segmentKey(config.indexName, config.partitionId, seqNo));
2174
- if (createdNew) {
2175
- await withDurabilityLock(flushActiveSegment);
2176
- }
2177
- }
2178
- async function rollUnderLock(startSeqNo) {
2179
- if (handle !== null) {
2180
- await flushActiveSegment();
2181
- await handle.close();
2182
- handle = null;
2183
- }
2184
- await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
2185
- await flushActiveSegment();
2186
- }
2187
- async function rollToNewSegment(startSeqNo) {
2188
- await withDurabilityLock(() => rollUnderLock(startSeqNo));
2189
- }
2190
- function activeSegmentHoldsRecords() {
2191
- return handle !== null && activeBytes > SEGMENT_HEADER_SIZE && highestAppendedSeqNo >= activeStartSeqNo;
2192
- }
2193
- async function prepareNextSegment() {
2194
- const prepared = await directory.appendHandle(preparedKey);
2195
- try {
2196
- if (await prepared.size() > 0) await prepared.truncate(0);
2197
- await prepared.append(writeSegmentHeader());
2198
- await prepared.sync();
2199
- nextSegmentPrepared = true;
2200
- } finally {
2201
- await prepared.close();
2202
- }
2203
- }
2204
- async function swapToPreparedSegment(sealed) {
2205
- const writer = await getMarkerWriter();
2206
- const startSeqNo = highestAppendedSeqNo + 1;
2207
- const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
2208
- const key = segmentKey(config.indexName, config.partitionId, startSeqNo);
2209
- await Promise.all([sealed.sync(), directory.move(preparedKey, key)]);
2210
- handle = await directory.appendHandle(key);
2211
- activeKey = key;
2212
- activeStartSeqNo = startSeqNo;
2213
- activeBytes = SEGMENT_HEADER_SIZE;
2214
- await writer.commit({
2215
- activeSegmentSeqNo: startSeqNo,
2216
- durableByteLength: SEGMENT_HEADER_SIZE,
2217
- highestDurableSeqNo: durableSeqNo
2218
- });
2219
- highestDurableSeqNo = durableSeqNo;
2220
- await sealed.close();
2221
- }
2222
- async function maybeRoll(nextSeqNo, appendedBytes) {
2223
- if (handle === null || activeBytes <= SEGMENT_HEADER_SIZE) {
2224
- return;
2225
- }
2226
- if (activeBytes >= segmentMaxBytes || activeBytes + appendedBytes > segmentMaxBytes) {
2227
- await rollToNewSegment(nextSeqNo);
2228
- }
2229
- }
2230
- function joinFrames(frames) {
2231
- if (frames.length === 1) {
2232
- return frames[0];
2233
- }
2234
- let totalBytes = 0;
2235
- for (const frame of frames) {
2236
- totalBytes += frame.length;
2237
- }
2238
- const joined = new Uint8Array(totalBytes);
2239
- let offset = 0;
2240
- for (const frame of frames) {
2241
- joined.set(frame, offset);
2242
- offset += frame.length;
2243
- }
2244
- return joined;
2245
- }
2246
- function framesThatFit(frames, start) {
2247
- const room = segmentMaxBytes - activeBytes;
2248
- let end = start + 1;
2249
- let bytes = frames[start].length;
2250
- while (end < frames.length && bytes + frames[end].length <= room) {
2251
- bytes += frames[end].length;
2252
- end += 1;
2253
- }
2254
- return end;
2255
- }
2256
- async function appendGroup(frames, entries) {
2257
- const activeHandle = handle;
2258
- if (activeHandle === null) {
2259
- throw new NarsilError(
2260
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2261
- `WAL segment for "${config.indexName}" partition ${config.partitionId} is not open`,
2262
- { indexName: config.indexName, partitionId: config.partitionId }
2263
- );
2264
- }
2265
- const joined = joinFrames([...frames]);
2266
- if (activeBytes + joined.length > MAX_FILE_BYTES) {
2267
- throw new NarsilError(
2268
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
2269
- `A log record of ${joined.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
2270
- { indexName: config.indexName, partitionId: config.partitionId, bytes: joined.length }
2271
- );
2272
- }
2273
- await activeHandle.append(joined);
2274
- activeBytes += joined.length;
2275
- const lastSeqNo = entries[entries.length - 1].seqNo;
2276
- if (lastSeqNo > highestAppendedSeqNo) {
2277
- highestAppendedSeqNo = lastSeqNo;
2278
- }
2279
- }
2280
- async function appendFrames(entries) {
2281
- if (entries.length === 0) {
2282
- return;
2283
- }
2284
- const frames = [];
2285
- for (const entry of entries) {
2286
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
2287
- frames.push(frameRecord(entry));
2288
- }
2289
- await ensureSegment(entries[0].seqNo);
2290
- let start = 0;
2291
- while (start < frames.length) {
2292
- await maybeRoll(entries[start].seqNo, frames[start].length);
2293
- const end = framesThatFit(frames, start);
2294
- await appendGroup(frames.slice(start, end), entries.slice(start, end));
2295
- start = end;
2296
- }
2297
- }
2298
- return {
2299
- async appendAll(entries) {
2300
- await appendFrames(entries);
2301
- },
2302
- async appendDurable(entry) {
2303
- await appendFrames([entry]);
2304
- await coordinator.commit();
2305
- },
2306
- async commit() {
2307
- await coordinator.commit();
2308
- },
2309
- rollToNewSegment,
2310
- async prepareCheckpointRoll() {
2311
- if (activeSegmentHoldsRecords()) await prepareNextSegment();
2312
- },
2313
- async rollForCheckpoint() {
2314
- await withDurabilityLock(async () => {
2315
- const sealed = handle;
2316
- const prepared = nextSegmentPrepared;
2317
- nextSegmentPrepared = false;
2318
- if (sealed === null || !activeSegmentHoldsRecords()) return;
2319
- if (prepared) await swapToPreparedSegment(sealed);
2320
- else await rollUnderLock(highestAppendedSeqNo + 1);
2321
- });
2322
- },
2323
- async close() {
2324
- await withDurabilityLock(async () => {
2325
- if (handle !== null) {
2326
- await flushActiveSegment();
2327
- await handle.close();
2328
- handle = null;
2329
- }
2330
- if (markerWriter !== null) {
2331
- await markerWriter.close();
2332
- markerWriter = null;
2333
- }
2334
- activeKey = null;
2335
- });
2336
- },
2337
- async abandon() {
2338
- await withDurabilityLock(async () => {
2339
- const closing = [handle?.close(), markerWriter?.close()];
2340
- handle = null;
2341
- markerWriter = null;
2342
- activeKey = null;
2343
- await Promise.allSettled(closing);
2344
- });
2345
- },
2346
- get activeSegmentKey() {
2347
- return activeKey;
2348
- }
2349
- };
2350
- }
2351
-
2352
- // src/persistence/durability/partition-registry.ts
2353
- function createPartitionRegistry(directory, segmentMaxBytes) {
2354
- const indexes = /* @__PURE__ */ new Map();
2355
- function indexState(indexName) {
2356
- let state = indexes.get(indexName);
2357
- if (state === void 0) {
2358
- state = {
2359
- partitions: /* @__PURE__ */ new Map(),
2360
- mutationsSinceCheckpoint: 0,
2361
- documentBytesSinceCheckpoint: 0,
2362
- checkpointInFlight: null,
2363
- unloading: false,
2364
- stalledWrites: []
2365
- };
2366
- indexes.set(indexName, state);
2367
- }
2368
- return state;
2369
- }
2370
- function freshPartition(indexName, partitionId, position) {
2371
- return {
2372
- walWriter: createWalWriter(directory, { indexName, partitionId, segmentMaxBytes }),
2373
- seqOwner: createSeqOwner(position.seqNo, position.primaryTerm),
2374
- appendChain: Promise.resolve(),
2375
- appliedSeqNo: position.seqNo,
2376
- failed: null
2377
- };
2378
- }
2379
- return {
2380
- directory,
2381
- indexes,
2382
- indexState,
2383
- freshPartition,
2384
- partition(indexName, partitionId, startSeqNo, primaryTerm = SINGLE_NODE_PRIMARY_TERM) {
2385
- const state = indexState(indexName);
2386
- let partition = state.partitions.get(partitionId);
2387
- if (partition === void 0) {
2388
- partition = freshPartition(indexName, partitionId, { seqNo: startSeqNo, primaryTerm });
2389
- state.partitions.set(partitionId, partition);
2390
- }
2391
- return partition;
2392
- }
2393
- };
2394
- }
2395
- var SNAPSHOT_BUNDLE_VERSION = 2;
2396
- function invalid(message) {
2397
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
2398
- }
2399
- function vectorIndexesOf(raw) {
2400
- if (raw === void 0 || raw === null) return {};
2401
- if (typeof raw !== "object" || Array.isArray(raw)) invalid("vectorIndexes must be a map");
2402
- const result = {};
2403
- for (const [fieldPath, parts] of Object.entries(raw)) {
2404
- result[fieldPath] = decodeVectorIndexParts(parts);
2405
- }
2406
- return result;
2407
- }
2408
- function checkpointOf(raw) {
2409
- if (!Array.isArray(raw)) return [];
2410
- const result = [];
2411
- for (const entry of raw) {
2412
- if (entry === null || typeof entry !== "object") continue;
2413
- const { partitionId, lastSeqNo, primaryTerm } = entry;
2414
- if (typeof partitionId !== "number" || typeof lastSeqNo !== "number" || typeof primaryTerm !== "number") continue;
2415
- result.push({ partitionId, lastSeqNo, primaryTerm });
2416
- }
2417
- return result;
2418
- }
2419
- async function decodeSnapshotBundle(data) {
2420
- const { payloadBytes } = await unpackEnvelopeBytes(data);
2421
- const raw = decode(payloadBytes);
2422
- if (raw.version !== SNAPSHOT_BUNDLE_VERSION) {
2423
- throw new NarsilError(
2424
- ErrorCodes.ENVELOPE_VERSION_MISMATCH,
2425
- `The snapshot bundle holds layout ${String(raw.version)}, which this release cannot open. Export the index with the release that wrote it and import the export`,
2426
- { version: raw.version }
2427
- );
2428
- }
2429
- if (raw.schema === null || typeof raw.schema !== "object" || Array.isArray(raw.schema)) invalid("missing schema");
2430
- requireValidFieldTypes(raw.schema);
2431
- if (typeof raw.language !== "string") invalid("missing language");
2432
- if (!Array.isArray(raw.partitions) || !raw.partitions.every((part) => part instanceof Uint8Array)) {
2433
- invalid("missing partitions");
2434
- }
2435
- return {
2436
- schema: raw.schema,
2437
- language: raw.language,
2438
- partitions: raw.partitions,
2439
- vectorIndexes: vectorIndexesOf(raw.vectorIndexes),
2440
- checkpoint: checkpointOf(raw.checkpoint)
2441
- };
2442
- }
2443
- function countSnapshotBundleDocuments(bundle) {
2444
- let total = 0;
2445
- for (const partitionBytes of bundle.partitions)
2446
- total += Object.keys(deserializePayloadV2(partitionBytes).documents).length;
2447
- return total;
2448
- }
2449
- async function loadSnapshotBundle(bytes, deps) {
2450
- const bundle = await decodeSnapshotBundle(bytes);
2451
- while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
2452
- if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
2453
- for (let i = 0; i < bundle.partitions.length; i += 1) {
2454
- deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
2455
- }
2456
- for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
2457
- const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
2458
- deps.vectorIndexes.get(fieldPath)?.deserialize(rebuildsGraph ? parts.map((part) => ({ ...part, graphs: [] })) : parts);
2459
- }
2460
- return bundle.checkpoint;
2461
- }
2462
-
2463
- // src/persistence/durability/earlier-layout/manifest.ts
2464
- function isKey(value) {
2465
- return typeof value === "string" && value.length > 0 && !value.includes("\0");
2466
- }
2467
- function checkpointOf2(raw) {
2468
- if (!Array.isArray(raw)) return [];
2469
- const result = [];
2470
- for (const entry of raw) {
2471
- if (!isRecord(entry) || !isUint32(entry.partitionId)) continue;
2472
- if (!isSequenceNumber(entry.lastSeqNo) || !isSequenceNumber(entry.primaryTerm)) continue;
2473
- result.push({ partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm });
2474
- }
2475
- return result;
2476
- }
2477
- function segmentsOf(raw, partitionId) {
2478
- if (raw === void 0) return [];
2479
- if (!Array.isArray(raw) || raw.length > MAX_SEGMENTS_PER_PARTITION) {
2480
- invalidManifest("has malformed segment references", { partitionId });
2481
- }
2482
- const seen = /* @__PURE__ */ new Set();
2483
- return raw.map((segment) => {
2484
- if (!isRecord(segment) || !isId(segment.id) || !isUint32(segment.docCount) || !isUint32(segment.tombstoneCount) || !isKey(segment.key) || seen.has(segment.id)) {
2485
- invalidManifest("has an invalid segment reference", { partitionId });
2486
- }
2487
- seen.add(segment.id);
2488
- return { id: segment.id, key: segment.key, docCount: segment.docCount, tombstoneCount: segment.tombstoneCount };
2489
- });
2490
- }
2491
- function partitionsOf(raw) {
2492
- if (!Array.isArray(raw)) invalidManifest("is missing partition entries");
2493
- return raw.map((entry) => {
2494
- if (!isRecord(entry) || !isUint32(entry.partitionId)) invalidManifest("has a partition without a valid id");
2495
- const segments = segmentsOf(entry.segments, entry.partitionId);
2496
- const nextSegmentId = entry.nextSegmentId;
2497
- if (!isId(nextSegmentId) || segments.some((segment) => segment.id >= nextSegmentId)) {
2498
- invalidManifest("has an invalid next segment id", { partitionId: entry.partitionId });
2499
- }
2500
- return { partitionId: entry.partitionId, nextSegmentId, segments };
2501
- });
2502
- }
2503
- function vectorFileOf(raw, fieldPath) {
2504
- if (!isRecord(raw) || !isId(raw.id) || !isKey(raw.key)) invalidManifest("has an invalid vector file reference");
2505
- const count = raw.count;
2506
- if (!isUint32(count) || count > VECTOR_FILE_MAX_VECTORS) invalidManifest(`gives "${fieldPath}" an invalid count`);
2507
- const dead = raw.dead ?? null;
2508
- if (dead !== null && (!(dead instanceof Uint8Array) || dead.byteLength !== deadBytesFor(count))) {
2509
- invalidManifest(`marks the dead vectors of "${fieldPath}" with the wrong number of bytes`);
2510
- }
2511
- return { id: raw.id, key: raw.key, count, dead };
2512
- }
2513
- function vectorsOf(raw) {
2514
- if (raw === void 0 || raw === null) return [];
2515
- if (!Array.isArray(raw)) invalidManifest("has malformed vector references");
2516
- return raw.map((field) => {
2517
- if (!isRecord(field) || typeof field.fieldPath !== "string") invalidManifest("has an invalid vector reference");
2518
- const fieldPath = field.fieldPath;
2519
- if (!isId(field.nextFileId) || !Array.isArray(field.files) || !isId(field.graphGeneration)) {
2520
- invalidManifest(`gives "${fieldPath}" an invalid vector reference`);
2521
- }
2522
- const graphKey = field.graphKey ?? null;
2523
- if (graphKey !== null && (!isKey(graphKey) || field.graphGeneration === 0)) {
2524
- invalidManifest(`gives "${fieldPath}" an invalid graph key`);
2525
- }
2526
- return {
2527
- fieldPath,
2528
- nextFileId: field.nextFileId,
2529
- files: field.files.map((file) => vectorFileOf(file, fieldPath)),
2530
- graphGeneration: field.graphGeneration,
2531
- graphKey
2532
- };
2533
- });
2534
- }
2535
- function decodeEarlierManifest(raw) {
2536
- const schema = raw.schema;
2537
- if (!isRecord(schema)) invalidManifest("is missing its schema");
2538
- requireValidFieldTypes(schema);
2539
- if (typeof raw.language !== "string") invalidManifest("is missing its language");
2540
- return {
2541
- schema,
2542
- language: raw.language,
2543
- checkpoint: checkpointOf2(raw.checkpoint),
2544
- partitions: partitionsOf(raw.partitions),
2545
- vectors: vectorsOf(raw.vectors)
2546
- };
2547
- }
2548
- function earlierReferencedKeys(manifest) {
2549
- const keys = /* @__PURE__ */ new Set();
2550
- for (const partition of manifest.partitions) {
2551
- for (const segment of partition.segments) keys.add(segment.key);
2552
- }
2553
- for (const vector of manifest.vectors) {
2554
- for (const file of vector.files) keys.add(file.key);
2555
- if (vector.graphKey !== null) keys.add(vector.graphKey);
2556
- }
2557
- return keys;
2558
- }
2559
-
2560
- // src/persistence/durability/earlier-layout/merge.ts
2561
- function mergeTimeOrderedSegments(ordered, fallback) {
2562
- const winner = resolveWinners(ordered);
2563
- const first = ordered[0]?.partition;
2564
- const merged = {
2565
- indexName: first?.indexName ?? fallback.indexName,
2566
- partitionId: first?.partitionId ?? fallback.partitionId,
2567
- totalPartitions: first?.totalPartitions ?? fallback.totalPartitions,
2568
- language: first?.language ?? fallback.language,
2569
- schema: {},
2570
- docCount: 0,
2571
- avgDocLength: 0,
2572
- documents: /* @__PURE__ */ Object.create(null),
2573
- invertedIndex: /* @__PURE__ */ Object.create(null),
2574
- fieldIndexes: {
2575
- numeric: /* @__PURE__ */ Object.create(null),
2576
- boolean: /* @__PURE__ */ Object.create(null),
2577
- enum: /* @__PURE__ */ Object.create(null),
2578
- geopoint: /* @__PURE__ */ Object.create(null)
2579
- },
2580
- statistics: {
2581
- totalDocuments: 0,
2582
- totalFieldLengths: /* @__PURE__ */ Object.create(null),
2583
- averageFieldLengths: /* @__PURE__ */ Object.create(null),
2584
- docFrequencies: /* @__PURE__ */ Object.create(null)
2585
- }
2586
- };
2587
- for (let segIndex = 0; segIndex < ordered.length; segIndex += 1) {
2588
- const part = ordered[segIndex].partition;
2589
- addSegmentSchema(merged.schema, part.schema);
2590
- collectDocuments(merged, part, winner, segIndex);
2591
- collectInvertedIndex(merged, part, winner, segIndex);
2592
- collectFieldIndexes(merged, part, winner, segIndex);
2593
- collectSurfaceForms(merged, part);
2594
- }
2595
- recomputeStatistics(merged);
2596
- validateMergedPartition(merged);
2597
- return merged;
2598
- }
2599
- function countLiveDocuments(ordered) {
2600
- return resolveWinners(ordered).size;
2601
- }
2602
- function resolveWinners(ordered) {
2603
- const winner = /* @__PURE__ */ new Map();
2604
- for (let segIndex = 0; segIndex < ordered.length; segIndex += 1) {
2605
- const segment = ordered[segIndex];
2606
- for (const docId of segment.tombstones) {
2607
- winner.delete(docId);
2608
- }
2609
- for (const docId of Object.keys(segment.partition.documents)) {
2610
- winner.set(docId, segIndex);
2611
- }
2612
- }
2613
- return winner;
2614
- }
2615
- function owns(winner, docId, segIndex) {
2616
- return winner.get(docId) === segIndex;
2617
- }
2618
- function addSegmentSchema(agreed, schema) {
2619
- for (const [field, type] of Object.entries(schema)) {
2620
- const existing = agreed[field];
2621
- if (existing !== void 0 && existing !== type) {
2622
- throw new NarsilError(
2623
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2624
- `Segment schemas disagree on field "${field}": "${existing}" vs "${type}"`,
2625
- { field, existing, found: type }
2626
- );
2627
- }
2628
- agreed[field] = type;
2629
- }
2630
- }
2631
- function collectDocuments(merged, part, winner, segIndex) {
2632
- for (const [docId, doc] of Object.entries(part.documents)) {
2633
- if (owns(winner, docId, segIndex)) {
2634
- merged.documents[docId] = doc;
2635
- }
2636
- }
2637
- }
2638
- function collectInvertedIndex(merged, part, winner, segIndex) {
2639
- for (const [token, list] of Object.entries(part.invertedIndex)) {
2640
- let target = merged.invertedIndex[token];
2641
- for (const posting of list.postings) {
2642
- if (!owns(winner, posting.docId, segIndex)) continue;
2643
- if (target === void 0) {
2644
- target = { docFrequency: 0, postings: [] };
2645
- merged.invertedIndex[token] = target;
2646
- }
2647
- target.postings.push(posting);
2648
- }
2649
- }
2650
- }
2651
- function collectEntries(target, source, winner, segIndex) {
2652
- for (const [path, entries] of Object.entries(source)) {
2653
- for (const entry of entries) {
2654
- if (!owns(winner, entry.docId, segIndex)) continue;
2655
- let bucket = target[path];
2656
- if (bucket === void 0) {
2657
- bucket = [];
2658
- target[path] = bucket;
2659
- }
2660
- bucket.push(entry);
2661
- }
2662
- }
2663
- }
2664
- function collectFieldIndexes(merged, part, winner, segIndex) {
2665
- collectEntries(merged.fieldIndexes.numeric, part.fieldIndexes.numeric, winner, segIndex);
2666
- collectEntries(merged.fieldIndexes.geopoint, part.fieldIndexes.geopoint, winner, segIndex);
2667
- for (const [path, idx] of Object.entries(part.fieldIndexes.boolean)) {
2668
- for (const [docIds, flag] of [
2669
- [idx.trueDocs, true],
2670
- [idx.falseDocs, false]
2671
- ]) {
2672
- for (const docId of docIds) {
2673
- if (!owns(winner, docId, segIndex)) continue;
2674
- let target = merged.fieldIndexes.boolean[path];
2675
- if (target === void 0) {
2676
- target = { trueDocs: [], falseDocs: [] };
2677
- merged.fieldIndexes.boolean[path] = target;
2678
- }
2679
- (flag ? target.trueDocs : target.falseDocs).push(docId);
2680
- }
2681
- }
2682
- }
2683
- for (const [path, values] of Object.entries(part.fieldIndexes.enum)) {
2684
- for (const [value, docIds] of Object.entries(values)) {
2685
- for (const docId of docIds) {
2686
- if (!owns(winner, docId, segIndex)) continue;
2687
- let target = merged.fieldIndexes.enum[path];
2688
- if (target === void 0) {
2689
- target = /* @__PURE__ */ Object.create(null);
2690
- merged.fieldIndexes.enum[path] = target;
2691
- }
2692
- let bucket = target[value];
2693
- if (bucket === void 0) {
2694
- bucket = [];
2695
- target[value] = bucket;
2696
- }
2697
- bucket.push(docId);
2698
- }
2699
- }
2700
- }
2701
- }
2702
- function collectSurfaceForms(merged, part) {
2703
- if (!part.surfaceForms) return;
2704
- let target = merged.surfaceForms;
2705
- if (!target) {
2706
- target = /* @__PURE__ */ Object.create(null);
2707
- merged.surfaceForms = target;
2708
- }
2709
- for (const [surface, value] of Object.entries(part.surfaceForms)) {
2710
- const count = typeof value === "number" ? value : value[0];
2711
- const token = typeof value === "number" ? surface : value[1];
2712
- const existing = target[surface];
2713
- if (existing === void 0) {
2714
- target[surface] = value;
2715
- continue;
2716
- }
2717
- const existingCount = typeof existing === "number" ? existing : existing[0];
2718
- const total = existingCount + count;
2719
- target[surface] = token === surface ? total : [total, token];
2720
- }
2721
- }
2722
- function recomputeStatistics(merged) {
2723
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
2724
- let totalDocuments = 0;
2725
- for (const doc of Object.values(merged.documents)) {
2726
- totalDocuments += 1;
2727
- for (const [field, length] of Object.entries(doc.fieldLengths)) {
2728
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
2729
- }
2730
- }
2731
- const docFrequencies = /* @__PURE__ */ Object.create(null);
2732
- for (const [token, list] of Object.entries(merged.invertedIndex)) {
2733
- const distinct = /* @__PURE__ */ new Set();
2734
- for (const posting of list.postings) distinct.add(posting.docId);
2735
- list.docFrequency = distinct.size;
2736
- docFrequencies[token] = distinct.size;
2737
- }
2738
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
2739
- for (const [field, length] of Object.entries(totalFieldLengths)) {
2740
- averageFieldLengths[field] = totalDocuments > 0 ? length / totalDocuments : 0;
2741
- }
2742
- merged.statistics.totalDocuments = totalDocuments;
2743
- merged.statistics.totalFieldLengths = totalFieldLengths;
2744
- merged.statistics.averageFieldLengths = averageFieldLengths;
2745
- merged.statistics.docFrequencies = docFrequencies;
2746
- merged.docCount = totalDocuments;
2747
- merged.avgDocLength = Object.values(averageFieldLengths).reduce((sum, value) => sum + value, 0);
2748
- }
2749
- function validateMergedPartition(merged) {
2750
- for (const [token, list] of Object.entries(merged.invertedIndex)) {
2751
- for (const posting of list.postings) {
2752
- if (!(posting.docId in merged.documents)) {
2753
- throw new NarsilError(
2754
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2755
- `Merged segment posting references document "${posting.docId}" that is absent from the partition`,
2756
- { token, docId: posting.docId }
2757
- );
2758
- }
2759
- }
2760
- }
2761
- }
2762
- function normalizeTombstones(raw) {
2763
- if (raw === void 0) return [];
2764
- if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
2765
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment file has a malformed tombstone list");
2766
- }
2767
- return raw;
2768
- }
2769
- async function openContainer(store, key) {
2770
- const bytes = await store.read(key);
2771
- if (bytes === null) {
2772
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Segment file "${key}" is missing`, { key });
2773
- }
2774
- const { payloadBytes } = await unpackEnvelopeBytes(bytes);
2775
- const raw = decode(payloadBytes);
2776
- if (!(raw.payload instanceof Uint8Array)) {
2777
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment file is missing its partition payload", { key });
2778
- }
2779
- return { payload: raw.payload, tombstones: normalizeTombstones(raw.tombstones) };
2780
- }
2781
- async function readEarlierSegmentContents(store, key) {
2782
- const { payload, tombstones } = await openContainer(store, key);
2783
- return { partition: deserializePayloadV2(payload), tombstones };
2784
- }
2785
- async function readEarlierSegmentDocuments(store, key) {
2786
- const { payload, tombstones } = await openContainer(store, key);
2787
- const stored = documentsOfPayloadV2(payload);
2788
- const documents = [];
2789
- for (const [docId, document] of Object.entries(stored.documents)) {
2790
- documents.push({ docId, document: document.fields });
2791
- }
2792
- return { schema: stored.schema, documents, tombstones };
2793
- }
2794
-
2795
- // src/persistence/durability/checkpoint-count.ts
2796
- var decoder = new TextDecoder();
2797
- async function countPagedPartition(store, partition) {
2798
- const { directory } = store;
2799
- if (directory === void 0 || partition.segments.length === 0) return null;
2800
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-JYHFTVPS.mjs');
2801
- for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2802
- const scans = [];
2803
- try {
2804
- for (const ref of partition.segments) scans.push(await openPagedSegmentScan(directory, ref));
2805
- let live = 0;
2806
- for (let at = 0; at < scans.length; at++) {
2807
- for (let ordinal = 0; ordinal < scans[at].count; ordinal++) {
2808
- if (countsAsLatestHolder(scans, at, scans[at].idAt(ordinal))) live += 1;
2809
- }
2810
- }
2811
- return live;
2812
- } finally {
2813
- for (const scan of scans) scan.close();
2814
- }
2815
- }
2816
- async function countCurrentLayout(store, manifest) {
2817
- let total = 0;
2818
- for (const partition of manifest.partitions) {
2819
- const paged = await countPagedPartition(store, partition);
2820
- if (paged !== null) {
2821
- total += paged;
2822
- continue;
2823
- }
2824
- const live = /* @__PURE__ */ new Set();
2825
- for (const ref of partition.segments) {
2826
- const contents = await readSegmentFile(store, ref);
2827
- for (const docId of contents.tombstones) live.delete(docId);
2828
- const ids = contents.arrays.idTable;
2829
- for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
2830
- live.add(decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
2831
- }
2832
- }
2833
- total += live.size;
2834
- }
2835
- return total;
2836
- }
2837
- async function countCheckpointDocuments(store, indexName) {
2838
- const decoded = await readSegmentManifest(store, indexName);
2839
- if (decoded?.kind === "current") return countCurrentLayout(store, decoded.manifest);
2840
- if (decoded?.kind === "earlier") {
2841
- let total = 0;
2842
- for (const partition of decodeEarlierManifest(decoded.payload).partitions) {
2843
- const ordered = [];
2844
- for (const segment of partition.segments) ordered.push(await readEarlierSegmentContents(store, segment.key));
2845
- total += countLiveDocuments(ordered);
2846
- }
2847
- return total;
2848
- }
2849
- const bundle = await store.read(snapshotBundleKey(indexName));
2850
- return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
2851
- }
2852
- function removeWhatTheSegmentReplaces(manager, segment) {
2853
- manager.beginBatchRemove();
2854
- try {
2855
- for (const docId of segment.tombstones) {
2856
- if (manager.has(docId)) manager.remove(docId);
2857
- }
2858
- for (const { docId } of segment.documents) {
2859
- if (manager.has(docId)) manager.remove(docId);
2860
- }
2861
- } finally {
2862
- manager.endBatchRemove();
2863
- }
2864
- }
2865
- function attachDocuments(manager, partitionId, segment) {
2866
- if (segment.documents.length === 0) return;
2867
- const payload = buildSegmentPayload(
2868
- segment.documents,
2869
- manager.config.schema,
2870
- manager.language,
2871
- resolvePartitionInsertOptions(manager.config, manager.analysis),
2872
- manager.config.trackPositions ?? true
2873
- );
2874
- const documents = segment.documents.map((entry) => entry.document);
2875
- const snapshot = freezeSegmentShared(payload, documents);
2876
- if (snapshot === null) {
2877
- manager.mergeSegment(partitionId, payload, documents);
2878
- return;
2879
- }
2880
- manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
2881
- }
2882
- async function loadPartition(store, indexName, partition, deps) {
2883
- if (deps.storedTermsAreCurrent) {
2884
- const agreedSchema = {};
2885
- for (const ref of partition.segments) {
2886
- const segment = await readEarlierSegmentDocuments(store, ref.key);
2887
- addSegmentSchema(agreedSchema, segment.schema);
2888
- removeWhatTheSegmentReplaces(deps.manager, segment);
2889
- attachDocuments(deps.manager, partition.partitionId, segment);
2890
- }
2891
- return;
2892
- }
2893
- const ordered = [];
2894
- for (const ref of partition.segments) ordered.push(await readEarlierSegmentContents(store, ref.key));
2895
- const merged = mergeTimeOrderedSegments(ordered, {
2896
- indexName,
2897
- partitionId: partition.partitionId,
2898
- totalPartitions: deps.manager.partitionCount,
2899
- language: deps.manager.language.name
2900
- });
2901
- deps.manager.deserializePartition(partition.partitionId, merged);
2902
- }
2903
- async function readGraph(store, ref) {
2904
- if (ref.graphKey === null) return null;
2905
- const bytes = await store.read(ref.graphKey);
2906
- if (bytes === null) {
2907
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The graph file "${ref.graphKey}" is missing`, {
2908
- key: ref.graphKey
2909
- });
2910
- }
2911
- const { payload } = await openChecksummedEnvelope(bytes, {
2912
- envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
2913
- what: `graph file "${ref.graphKey}"`,
2914
- allowsCompression: true
2915
- });
2916
- const graph = decodeVectorGraphPayload(decode(payload));
2917
- if (graph.graphs.some((numbered) => numbered.m < MIN_M)) return null;
2918
- return graphPartsOfEarlierPayload(graph);
2919
- }
2920
- async function loadVectorField(store, ref, vectorIndex, fileRefs) {
2921
- let liveVectors = 0;
2922
- for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
2923
- const restore = vectorIndex.restoreCheckpoint({
2924
- liveVectors,
2925
- holdsGraph: ref.graphKey !== null,
2926
- graph: await readGraph(store, ref)
2927
- });
2928
- for (const file of ref.files) {
2929
- const bytes = await store.read(file.key);
2930
- if (bytes === null) {
2931
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The vector file "${file.key}" is missing`, {
2932
- key: file.key
2933
- });
2934
- }
2935
- const { header, payload: payloadBytes } = await openChecksummedEnvelope(bytes, {
2936
- envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
2937
- what: `vector file "${file.key}"`,
2938
- allowsCompression: false
2939
- });
2940
- const payload = decodeVectorFilePayload(decode(payloadBytes));
2941
- if (payload.docIds.length !== file.count) {
2942
- throw new NarsilError(
2943
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2944
- `The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
2945
- { key: file.key }
2946
- );
2947
- }
2948
- fileRefs.set(file.key, { key: file.key, bytes: bytes.byteLength, checksum: header.checksum });
2949
- restore.addFile({
2950
- id: file.id,
2951
- key: file.key,
2952
- payload,
2953
- dead: file.dead,
2954
- location: store.pathOf === void 0 ? null : {
2955
- path: await store.pathOf(file.key),
2956
- vectorsOffset: HEADER_SIZE + header.payloadLength - payload.vectors.byteLength
2957
- }
2958
- });
2959
- }
2960
- restore.finish();
2961
- }
2962
- async function loadEarlierLayout(store, indexName, manifest, deps) {
2963
- let highestPartitionId = manifest.partitions.length - 1;
2964
- for (const partition of manifest.partitions) {
2965
- if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
2966
- }
2967
- growPartitionsTo(deps.manager, highestPartitionId);
2968
- for (const partition of manifest.partitions) {
2969
- if (partition.partitionId >= deps.manager.partitionCount) {
2970
- throw new NarsilError(
2971
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
2972
- `Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
2973
- { indexName, partitionId: partition.partitionId }
2974
- );
2975
- }
2976
- await loadPartition(store, indexName, partition, deps);
2977
- }
2978
- const vectorFileRefs = /* @__PURE__ */ new Map();
2979
- for (const vector of manifest.vectors) {
2980
- const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
2981
- if (vectorIndex === void 0) continue;
2982
- await loadVectorField(store, vector, vectorIndex, vectorFileRefs);
2983
- }
2984
- return { checkpoint: manifest.checkpoint, vectorFileRefs };
2985
- }
2986
-
2987
- // src/persistence/durability/earlier-layout/upgrade.ts
2988
- async function loadEarlier(deps, earlier) {
2989
- const loadDeps = {
2990
- manager: deps.manager,
2991
- vectorIndexes: deps.vectorIndexes,
2992
- storedTermsAreCurrent: deps.storedTermsAreCurrent
2993
- };
2994
- if (earlier.kind === "bundle") {
2995
- await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
2996
- return { checkpoint: await loadSnapshotBundle(earlier.bytes, loadDeps), fileRefs: /* @__PURE__ */ new Map() };
2997
- }
2998
- await reclaimUnlistedKeys(deps.store, deps.indexName, earlierReferencedKeys(earlier.manifest));
2999
- const loaded = await loadEarlierLayout(deps.store, deps.indexName, earlier.manifest, loadDeps);
3000
- return { checkpoint: loaded.checkpoint, fileRefs: loaded.vectorFileRefs };
3001
- }
3002
- function priorVectorsOf(earlier, fileRefs) {
3003
- if (earlier.kind === "bundle") return [];
3004
- return earlier.manifest.vectors.map((field) => ({
3005
- fieldPath: field.fieldPath,
3006
- nextFileId: field.nextFileId,
3007
- graphGeneration: field.graphGeneration,
3008
- graphParts: [],
3009
- files: field.files.flatMap((file) => {
3010
- const ref = fileRefs.get(file.key);
3011
- return ref === void 0 ? [] : [{ ...file, bytes: ref.bytes, checksum: ref.checksum }];
3012
- })
3013
- }));
3014
- }
3015
- function priorPartitionsOf(earlier, partitionCount) {
3016
- if (earlier.kind === "bundle") return [];
3017
- return earlier.manifest.partitions.filter((partition) => partition.partitionId < partitionCount).map((partition) => ({ partitionId: partition.partitionId, nextSegmentId: partition.nextSegmentId, segments: [] }));
3018
- }
3019
- async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
3020
- const priorVectors = priorVectorsOf(earlier, fileRefs);
3021
- const plans = /* @__PURE__ */ new Map();
3022
- for (const [fieldPath, vectorIndex] of deps.vectorIndexes) {
3023
- await vectorIndex.completeGraph();
3024
- const listed = priorVectors.find((ref) => ref.fieldPath === fieldPath);
3025
- plans.set(
3026
- fieldPath,
3027
- vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), true)
3028
- );
3029
- }
3030
- const written = await writeCheckpointSegments({
3031
- store: deps.store,
3032
- metadata: deps.metadata,
3033
- targets,
3034
- compactionThreshold: deps.compactionThreshold,
3035
- prior: { checkpoint: [], partitions: priorPartitionsOf(earlier, deps.manager.partitionCount) },
3036
- wholePartitions: targets.map((target) => partitionContentsOf(deps.manager, target.partitionId))
3037
- });
3038
- const vectors = await writeLiveVectors({
3039
- store: deps.store,
3040
- indexName: deps.indexName,
3041
- plans,
3042
- priorVectors,
3043
- priorFileRefs: fileRefs
3044
- });
3045
- const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
3046
- recordWrittenVectors(deps.vectorIndexes, vectors.written);
3047
- await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
3048
- await removeCheckpointGarbage(
3049
- deps.store,
3050
- await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
3051
- );
3052
- }
3053
- async function upgradeEarlierLayout(deps, earlier) {
3054
- deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
3055
- try {
3056
- const loaded = await loadEarlier(deps, earlier);
3057
- const targets = await deps.replay(loaded.checkpoint);
3058
- await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
3059
- deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
3060
- return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
3061
- } catch (error) {
3062
- const failure = error instanceof Error ? error : new Error(String(error));
3063
- deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
3064
- throw failure;
3065
- }
3066
- }
3067
-
3068
- // src/persistence/durability/checkpoint-recovery.ts
3069
- async function earlierLayoutOf(deps, decoded) {
3070
- if (decoded?.kind === "earlier") return { kind: "manifest", manifest: decodeEarlierManifest(decoded.payload) };
3071
- if (decoded !== null) return null;
3072
- const bundle = await deps.store.read(snapshotBundleKey(deps.indexName));
3073
- return bundle === null ? null : { kind: "bundle", bytes: bundle };
3074
- }
3075
- async function recoverCheckpoint(deps) {
3076
- const decoded = await readSegmentManifest(deps.store, deps.indexName);
3077
- const earlier = await earlierLayoutOf(deps, decoded);
3078
- if (earlier !== null) {
3079
- const upgraded = await upgradeEarlierLayout(deps, earlier);
3080
- return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
3081
- }
3082
- if (decoded === null || decoded.kind !== "current") {
3083
- if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
3084
- return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
3085
- }
3086
- await deps.store.remove(snapshotBundleKey(deps.indexName));
3087
- if (!deps.keepsUnlistedFiles) {
3088
- const referenced = manifestReferencedKeys(decoded.manifest);
3089
- for (const key of deps.keptForSync) referenced.add(key);
3090
- await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
3091
- }
3092
- const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
3093
- manager: deps.manager,
3094
- vectorIndexes: deps.vectorIndexes,
3095
- writeRecords: deps.writeRecords ?? null
3096
- });
3097
- return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
3098
- }
3099
-
3100
- // src/persistence/durability/stored-analysis.ts
3101
- function generationOfStoredTerms(store, indexName) {
3102
- return readSegmentManifest(store, indexName).then((decoded) => {
3103
- if (decoded === null) return null;
3104
- return decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
3105
- });
3106
- }
3107
- function promotePendingAnalysis(metadata) {
3108
- const pending = metadata.pendingAnalysis;
3109
- if (pending === void 0) return;
3110
- metadata.language = pending.language;
3111
- metadata.analysisGeneration = pending.generation;
3112
- metadata.analysisRevision = void 0;
3113
- metadata.tokenizer = pending.tokenizer;
3114
- metadata.stopWords = pending.stopWords;
3115
- metadata.stopWordList = pending.stopWordList;
3116
- metadata.pendingAnalysis = void 0;
3117
- }
3118
- async function settleStoredAnalysis(store, metadata, metadataOnly) {
3119
- if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
3120
- const stored = await generationOfStoredTerms(store, metadata.indexName);
3121
- const current = metadata.analysisGeneration ?? 0;
3122
- if (stored === null || stored === current) return false;
3123
- if (stored === metadata.pendingAnalysis?.generation) {
3124
- promotePendingAnalysis(metadata);
3125
- return true;
3126
- }
3127
- throw new NarsilError(
3128
- ErrorCodes.CONFIG_INVALID,
3129
- `The saved terms of index "${metadata.indexName}" come from analysis generation ${stored}, which matches no settings that its metadata records`,
3130
- {
3131
- indexName: metadata.indexName,
3132
- storedGeneration: stored,
3133
- currentGeneration: current,
3134
- pendingGeneration: metadata.pendingAnalysis?.generation
3135
- }
3136
- );
3137
- }
3138
-
3139
- // src/persistence/durability/recover-index.ts
3140
- async function writeCountedMetadata(deps, indexName, documentCount) {
3141
- await deps.queueMetadataWrite(indexName, async () => {
3142
- const metadata = deps.hooks.buildMetadata(indexName, documentCount);
3143
- if (metadata === void 0) return;
3144
- const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3145
- await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
3146
- }).catch(() => void 0);
3147
- }
3148
- async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3149
- const { directory, hooks } = deps;
3150
- const store = directoryCheckpointStore(directory);
3151
- const metadata = await loadMetadata(directory, indexName);
3152
- if (metadata === null) {
3153
- return;
3154
- }
3155
- let derivedCount;
3156
- if (metadataOnly && metadata.documentCount === void 0) {
3157
- try {
3158
- derivedCount = await countCheckpointDocuments(store, indexName);
3159
- metadata.documentCount = derivedCount;
3160
- } catch {
3161
- derivedCount = void 0;
3162
- }
3163
- }
3164
- const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
3165
- await hooks.createIndexFromMetadata(metadata, !metadataOnly);
3166
- if (metadataOnly) {
3167
- if (derivedCount !== void 0 || promoted) {
3168
- await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
3169
- }
3170
- return;
3171
- }
3172
- const manager = hooks.getManager(indexName);
3173
- const current = hooks.buildMetadata(indexName);
3174
- if (manager === void 0 || current === void 0) {
3175
- return;
3176
- }
3177
- const writeRecords = hooks.writeRecords ?? null;
3178
- writeRecords?.dropIndex(indexName);
3179
- const replayDeps = {
3180
- manager,
3181
- vectorFieldPaths: hooks.getVectorFieldPaths(indexName),
3182
- vectorIndexes: hooks.getVectorIndexes(indexName),
3183
- writeRecords,
3184
- defersGraphBuilds: true
3185
- };
3186
- const replay = async (checkpoint) => {
3187
- const replayed = [];
3188
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
3189
- const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
3190
- writeRecords?.loadTermHistory(
3191
- indexName,
3192
- partitionId,
3193
- checkpoint.find((entry) => entry.partitionId === partitionId)
3194
- );
3195
- const { highestSeqNo, highestPrimaryTerm } = await replayWal(
3196
- directory,
3197
- indexName,
3198
- partitionId,
3199
- fromSeqNo,
3200
- replayDeps
3201
- );
3202
- const primaryTerm = Math.max(
3203
- checkpointPrimaryTerm(checkpoint, partitionId),
3204
- highestPrimaryTerm,
3205
- SINGLE_NODE_PRIMARY_TERM
3206
- );
3207
- replayed.push({ partitionId, lastSeqNo: highestSeqNo, primaryTerm });
3208
- }
3209
- return replayed;
3210
- };
3211
- const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
3212
- const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
3213
- const recovered = await recoverCheckpoint({
3214
- store,
3215
- indexName,
3216
- metadata: current,
3217
- manager,
3218
- vectorIndexes: replayDeps.vectorIndexes,
3219
- storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3220
- compactionThreshold: deps.compactionThreshold,
3221
- keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
3222
- keptForSync,
3223
- replay,
3224
- onLayoutUpgrade: hooks.onLayoutUpgrade,
3225
- writeRecords
3226
- });
3227
- for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
3228
- deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
3229
- }
3230
- if (recovered.upgradedDocumentCount !== null || promoted) {
3231
- await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
3232
- }
3233
- }
3234
-
3235
- // src/persistence/durability/manager.ts
3236
- function createDurabilityManager(config, hooks, directoryOverride) {
3237
- const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
3238
- const canOffloadCheckpoint = directoryOverride === void 0;
3239
- const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
3240
- const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
3241
- const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
3242
- const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
3243
- const mode = config.mode ?? "sync";
3244
- const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
3245
- const registry = createPartitionRegistry(directory, segmentMaxBytes);
3246
- const indexes = registry.indexes;
3247
- const getOrCreateIndexState = registry.indexState;
3248
- const getOrCreatePartition = registry.partition;
3249
- let entryNumbering = null;
3250
- let commitDeferral = null;
3251
- const partitionFailures = createPartitionFailureRouting();
3252
- const heldCheckpoints = createHeldCheckpoints(hooks);
3253
- const metadataWrites = createMetadataWriteQueue();
3254
- const queueMetadataWrite = metadataWrites.queue;
3255
- let checkpointTimer = null;
3256
- let asyncFlushTimer = null;
3257
- let shuttingDown = false;
3258
- let fatalError = null;
3259
- const heldCheckpointFiles = /* @__PURE__ */ new Map();
3260
- const loads = createIndexLoads(hooks.getVectorIndexes);
3261
- const directoryWatch = createDirectoryWatch(directory, () => shuttingDown || fatalError !== null, markFatal);
3262
- function stopTimers() {
3263
- directoryWatch.stop();
3264
- if (checkpointTimer !== null) clearInterval(checkpointTimer);
3265
- if (asyncFlushTimer !== null) clearInterval(asyncFlushTimer);
3266
- checkpointTimer = null;
3267
- asyncFlushTimer = null;
3268
- }
3269
- function markFatal(error) {
3270
- if (fatalError !== null) return;
3271
- fatalError = error;
3272
- stopTimers();
3273
- hooks.onFatalError(error);
3274
- }
3275
- async function closeWriterReportingFailure(closePromise) {
3276
- try {
3277
- await closePromise;
3278
- } catch (err) {
3279
- markFatal(toError(err));
3280
- }
3281
- }
3282
- function startCheckpointTimer() {
3283
- if (checkpointTimer !== null || shuttingDown || fatalError !== null || checkpointIntervalMs <= 0) {
3284
- return;
3285
- }
3286
- checkpointTimer = unrefInterval(() => void runScheduledCheckpoints(), checkpointIntervalMs);
3287
- }
3288
- function startAsyncFlushTimer() {
3289
- if (mode !== "async" || asyncFlushTimer !== null || shuttingDown || fatalError !== null || flushIntervalMs <= 0) {
3290
- return;
3291
- }
3292
- asyncFlushTimer = unrefInterval(() => void flushAllPartitions(), flushIntervalMs);
3293
- }
3294
- function flushAllPartitions() {
3295
- return commitEveryPartition(indexes, partitionBatchDeps);
3296
- }
3297
- async function runScheduledCheckpoints() {
3298
- if (fatalError !== null) {
3299
- return;
3300
- }
3301
- for (const indexName of [...indexes.keys()]) {
3302
- try {
3303
- await checkpointIndex(indexName);
3304
- } catch (err) {
3305
- markFatal(toError(err));
3306
- }
3307
- }
3308
- }
3309
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
3310
- const indexState = indexes.get(indexName);
3311
- if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
3312
- return checkpointInTurn(indexState, {
3313
- joinsInFlight: !afterInFlight && !pinsRetiredVectors,
3314
- mayStart: () => !indexState.unloading && !shuttingDown,
3315
- perform: () => performCheckpoint(indexName, indexState, fromMemory, pinsRetiredVectors),
3316
- afterSettling: () => releaseStalledWrites(indexState)
3317
- });
3318
- }
3319
- async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
3320
- heldCheckpoints.refuse(indexName, indexes.get(indexName));
3321
- return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors);
3322
- }
3323
- async function performCheckpoint(indexName, indexState, fromMemory = false, pinsRetiredVectors = false) {
3324
- await directoryWatch.verify();
3325
- if (fatalError !== null) throw fatalError;
3326
- await runDurableCheckpoint({
3327
- directory,
3328
- hooks,
3329
- indexName,
3330
- indexState,
3331
- compactionThreshold,
3332
- canOffload: canOffloadCheckpoint,
3333
- fromMemory,
3334
- pinsRetiredVectors,
3335
- mutationThreshold: checkpointMutationThreshold,
3336
- mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3337
- holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
3338
- keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
3339
- queueMetadataWrite,
3340
- markFatal,
3341
- failPartition: partitionFailureFor(indexName, partitionBatchDeps)
3342
- });
3343
- }
3344
- const partitionBatchDeps = {
3345
- syncEachBatch: mode === "sync",
3346
- fatalError: () => fatalError,
3347
- markFatal,
3348
- failsPartitionAlone: partitionFailures.failsAlone,
3349
- buildEntry: buildMutationEntry,
3350
- numberEntry: (record) => record.replicated ?? (entryNumbering === null ? null : entryNumbering(record)),
3351
- defersCommit: (record) => commitDeferral?.(record) === true
3352
- };
3353
- async function recordMutations(records) {
3354
- if (fatalError !== null) {
3355
- const error = fatalError;
3356
- return records.map(() => ({ ok: false, error }));
3357
- }
3358
- const outcomes = await recordBatchesByPartition(
3359
- records,
3360
- (record) => getOrCreatePartition(record.indexName, record.partitionId, 0),
3361
- partitionBatchDeps
3362
- );
3363
- const recordedByIndex = /* @__PURE__ */ new Map();
3364
- for (let i = 0; i < records.length; i++) {
3365
- if (!outcomes[i].ok) continue;
3366
- const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
3367
- recorded.records += 1;
3368
- recorded.documentBytes += records[i].document?.byteLength ?? 0;
3369
- recordedByIndex.set(records[i].indexName, recorded);
3370
- }
3371
- for (const [indexName, recorded] of recordedByIndex) {
3372
- const indexState = getOrCreateIndexState(indexName);
3373
- indexState.mutationsSinceCheckpoint += recorded.records;
3374
- indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
3375
- if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
3376
- void checkpointIndex(indexName).catch((err) => {
3377
- markFatal(toError(err));
3378
- });
3379
- }
3380
- await stallWhileCheckpointsFallBehind(indexState);
3381
- }
3382
- if (recordedByIndex.size === 0) return outcomes;
3383
- if (mode === "sync") await directoryWatch.verify();
3384
- const lost = mode === "sync" ? fatalError : null;
3385
- if (lost !== null) return outcomes.map((outcome) => outcome.ok ? { ok: false, error: lost } : outcome);
3386
- await directoryWatch.verifyOnce();
3387
- directoryWatch.start();
3388
- startCheckpointTimer();
3389
- startAsyncFlushTimer();
3390
- return outcomes;
3391
- }
3392
- const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
3393
- directory,
3394
- hooks,
3395
- indexes,
3396
- metadataWrites,
3397
- heldCheckpointFiles,
3398
- checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
3399
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
3400
- markFatal,
3401
- closeWriterReportingFailure
3402
- });
3403
- const recoveryDeps = {
3404
- directory,
3405
- hooks,
3406
- compactionThreshold,
3407
- registerPartition: (name, partitionId, startSeqNo, primaryTerm) => {
3408
- getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
3409
- },
3410
- holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
3411
- queueMetadataWrite,
3412
- syncStorage
3413
- };
3414
- function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3415
- return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
3416
- }
3417
- return {
3418
- isActive: () => true,
3419
- async recover(metadataOnly = false) {
3420
- await directoryWatch.claim();
3421
- await recoverDirectoryRestores(directory);
3422
- const names = await listPersistedIndexes(directory);
3423
- await removeLeftoverRewriteFiles(directory, names);
3424
- for (const indexName of names) {
3425
- await recoverIndex(indexName, metadataOnly, false);
3426
- }
3427
- await directoryWatch.verify();
3428
- directoryWatch.start();
3429
- startCheckpointTimer();
3430
- for (const indexName of names) loads.startGraphRepairs(indexName);
3431
- },
3432
- recoverIndex(indexName) {
3433
- return recoverIndex(indexName);
3434
- },
3435
- highestPersistedSeqNo(indexName, partitionId) {
3436
- return indexes.get(indexName)?.partitions.get(partitionId)?.appliedSeqNo ?? 0;
3437
- },
3438
- partitionRegistry: registry,
3439
- numberEntriesWith(numbering, defersCommit) {
3440
- entryNumbering = numbering;
3441
- commitDeferral = defersCommit ?? null;
3442
- },
3443
- isolatePartitionFailures: partitionFailures.isolate,
3444
- keepWriteRecordsWith(keeper) {
3445
- hooks.writeRecords = keeper;
3446
- },
3447
- rewriteStorage: () => ({ directory, writeRecords: hooks.writeRecords ?? null }),
3448
- heldCheckpoints: () => heldCheckpoints.report(indexes),
3449
- async commitPartition(indexName, partitionId) {
3450
- const partition = indexes.get(indexName)?.partitions.get(partitionId);
3451
- if (mode !== "sync" || partition === void 0) return;
3452
- await commitAppended(partition, { indexName, partitionId }, partitionBatchDeps);
3453
- await directoryWatch.verify();
3454
- if (fatalError !== null) throw fatalError;
3455
- },
3456
- async recordMutation(record) {
3457
- const [outcome] = await recordMutations([record]);
3458
- if (!outcome.ok) {
3459
- throw outcome.error;
3460
- }
3461
- return outcome.seqNo;
3462
- },
3463
- recordMutations,
3464
- persistMetadata(indexName) {
3465
- return queueMetadataWrite(indexName, async () => {
3466
- const metadata = hooks.buildMetadata(indexName);
3467
- if (metadata === void 0) {
3468
- return;
3469
- }
3470
- const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3471
- await directory.atomicWrite(`${indexName}/meta`, bytes);
3472
- getOrCreateIndexState(indexName);
3473
- });
3474
- },
3475
- checkpoint(indexName) {
3476
- return requestedCheckpoint(indexName);
3477
- },
3478
- checkpointFromMemory(indexName) {
3479
- return requestedCheckpoint(indexName, true);
3480
- },
3481
- async checkpointAfterSwitch(indexName, swap) {
3482
- const indexState = getOrCreateIndexState(indexName);
3483
- heldCheckpoints.refuse(indexName, indexState);
3484
- return switchInTurn(indexState, {
3485
- mayStart: () => !indexState.unloading && !shuttingDown,
3486
- swap,
3487
- checkpoint: () => performCheckpoint(indexName, indexState, true),
3488
- afterSettling: () => releaseStalledWrites(indexState)
3489
- });
3490
- },
3491
- async checkpointAfterInFlight(indexName) {
3492
- await loads.settled(indexName);
3493
- await checkpointIndex(indexName, false, true).catch((err) => markFatal(toError(err)));
3494
- },
3495
- async checkpointAll() {
3496
- for (const indexName of [...indexes.keys()]) {
3497
- await checkpointIndex(indexName);
3498
- }
3499
- },
3500
- ...unloading,
3501
- syncStorage: () => syncStorage,
3502
- ...queryHistoryAccess({ kind: "directory", directory }),
3503
- ...archiveAccess,
3504
- async shutdown() {
3505
- shuttingDown = true;
3506
- stopTimers();
3507
- for (const indexState of indexes.values()) {
3508
- if (indexState.checkpointInFlight !== null) {
3509
- await indexState.checkpointInFlight.catch(() => void 0);
3510
- }
3511
- for (const partition of indexState.partitions.values()) {
3512
- await closeWriterReportingFailure(closePartitionWriter(partition));
3513
- }
3514
- }
3515
- indexes.clear();
3516
- terminateCheckpointWorker();
3517
- await directoryWatch.release();
3518
- }
3519
- };
3520
- }
3521
-
3522
- // src/persistence/durability/snapshot-only/checkpoint.ts
3523
- function capture(input, prior) {
3524
- const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
3525
- const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
3526
- const targets = [];
3527
- const contents = [];
3528
- for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
3529
- const lastSeqNo = input.appliedSeqNoByPartition.get(partitionId) ?? 0;
3530
- const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
3531
- if (unchanged) continue;
3532
- targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
3533
- contents.push(partitionContentsOf(input.manager, partitionId));
3534
- }
3535
- const plans = /* @__PURE__ */ new Map();
3536
- for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
3537
- const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
3538
- plans.set(fieldPath, vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key)));
3539
- }
3540
- return { targets, plans, documentCount: input.manager.countDocuments(), contents };
3541
- }
3542
- async function writeCaptured(input, store, prior, captured) {
3543
- const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, input.manager.partitionCount);
3544
- const written = await writeCheckpointSegments({
3545
- store,
3546
- metadata: input.metadata,
3547
- targets: captured.targets,
3548
- compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3549
- prior: priorWithinCount,
3550
- wholePartitions: captured.contents
3551
- });
3552
- const vectors = await writeLiveVectors({
3553
- store,
3554
- indexName: input.indexName,
3555
- plans: captured.plans,
3556
- priorVectors: prior?.vectors ?? []
3557
- });
3558
- const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
3559
- recordWrittenVectors(input.vectorIndexes, vectors.written);
3560
- await removeCheckpointGarbage(
3561
- store,
3562
- await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
3563
- );
3564
- }
3565
- async function writeSnapshotOnlyCheckpoint(input) {
3566
- const store = adapterCheckpointStore(input.adapter);
3567
- for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
3568
- const prior = await readCurrentManifest(store, input.indexName);
3569
- const captured = capture(input, prior);
3570
- await writeCaptured(input, store, prior, captured);
3571
- return captured.documentCount;
3572
- }
3573
-
3574
- // src/persistence/durability/snapshot-only/index-state.ts
3575
- function createSnapshotIndexState() {
3576
- return {
3577
- mutationsSinceCheckpoint: 0,
3578
- checkpointInFlight: null,
3579
- applyChain: Promise.resolve(),
3580
- unloading: false,
3581
- appliedSeqNoByPartition: /* @__PURE__ */ new Map()
3582
- };
3583
- }
3584
-
3585
- // src/persistence/durability/snapshot-only/restores.ts
3586
- function toError5(error) {
3587
- return error instanceof Error ? error : new Error(String(error));
3588
- }
3589
- function createPendingRestores(onFatalError) {
3590
- const pending = /* @__PURE__ */ new Map();
3591
- return {
3592
- finishLater(indexName, run) {
3593
- let running = null;
3594
- const finish = () => {
3595
- running ??= run().then(
3596
- () => {
3597
- if (pending.get(indexName) === finish) pending.delete(indexName);
3598
- },
3599
- (error) => {
3600
- running = null;
3601
- throw error;
3602
- }
3603
- );
3604
- return running;
3605
- };
3606
- pending.set(indexName, finish);
3607
- void finish().catch((error) => onFatalError(toError5(error)));
3608
- return Promise.resolve();
3609
- },
3610
- async settle(indexName) {
3611
- await pending.get(indexName)?.();
3612
- },
3613
- forget(indexName) {
3614
- pending.delete(indexName);
3615
- },
3616
- async settleAll() {
3617
- for (const finish of [...pending.values()]) await finish().catch(() => void 0);
3618
- }
3619
- };
3620
- }
3621
-
3622
- // src/persistence/durability/snapshot-only/index.ts
3623
- function metadataKey(indexName) {
3624
- return `${indexName}/meta`;
3625
- }
3626
- function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
3627
- const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
3628
- const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
3629
- const indexes = /* @__PURE__ */ new Map();
3630
- const metadataWrites = /* @__PURE__ */ new Map();
3631
- const loads = createIndexLoads(hooks.getVectorIndexes);
3632
- const restoreStorage = adapterRestoreStorage(adapter);
3633
- const restores = createPendingRestores(hooks.onFatalError);
3634
- let checkpointTimer = null;
3635
- let shuttingDown = false;
3636
- function settleRestore(indexName) {
3637
- return restores.settle(indexName);
3638
- }
3639
- function getOrCreateIndexState(indexName) {
3640
- let state = indexes.get(indexName);
3641
- if (state === void 0) {
3642
- state = createSnapshotIndexState();
3643
- indexes.set(indexName, state);
3644
- }
3645
- return state;
3646
- }
3647
- function startCheckpointTimer() {
3648
- if (checkpointTimer !== null || shuttingDown || checkpointIntervalMs <= 0) {
3649
- return;
3650
- }
3651
- checkpointTimer = setInterval(() => {
3652
- void runScheduledCheckpoints();
3653
- }, checkpointIntervalMs);
3654
- if (typeof checkpointTimer.unref === "function") {
3655
- checkpointTimer.unref();
3656
- }
3657
- }
3658
- async function runScheduledCheckpoints() {
3659
- for (const indexName of [...indexes.keys()]) {
3660
- try {
3661
- await checkpointIndex(indexName);
3662
- } catch (err) {
3663
- hooks.onFatalError(err instanceof Error ? err : new Error(String(err)));
3664
- }
3665
- }
3666
- }
3667
- async function performCheckpoint(indexName, indexState, fromMemory) {
3668
- await settleRestore(indexName);
3669
- const manager = hooks.getManager(indexName);
3670
- const metadata = hooks.buildTermsMetadata(indexName);
3671
- if (manager === void 0 || metadata === void 0) {
3672
- return;
3673
- }
3674
- const documentCount = await writeSnapshotOnlyCheckpoint({
3675
- adapter,
3676
- indexName,
3677
- metadata,
3678
- manager,
3679
- vectorIndexes: hooks.getVectorIndexes(indexName),
3680
- appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
3681
- rewritesEveryPartition: fromMemory || hooks.mustRewriteTerms(indexName)
3682
- });
3683
- hooks.onManifestCommitted(indexName, metadata.analysisGeneration ?? 0);
3684
- await queueMetadataWrite(indexName, async () => {
3685
- const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
3686
- if (checkpointMetadata === void 0) return;
3687
- const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
3688
- await adapter.save(metadataKey(indexName), metadataBytes);
3689
- hooks.recordCheckpoint?.(indexName, documentCount, manager.partitionCount);
3690
- });
3691
- indexState.mutationsSinceCheckpoint = 0;
3692
- if (publisher !== void 0) {
3693
- const partitionIds = [];
3694
- for (let i = 0; i < manager.partitionCount; i += 1) {
3695
- partitionIds.push(i);
3696
- }
3697
- await publisher.publishPartitions(indexName, partitionIds);
3698
- }
3699
- }
3700
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
3701
- const indexState = indexes.get(indexName);
3702
- if (indexState === void 0 || indexState.unloading) return;
3703
- return checkpointInTurn(indexState, {
3704
- joinsInFlight: !afterInFlight,
3705
- mayStart: () => !indexState.unloading && !shuttingDown,
3706
- perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
3707
- afterSettling: () => void 0
3708
- });
3709
- }
3710
- function queueMetadataWrite(indexName, write) {
3711
- const previous = metadataWrites.get(indexName) ?? Promise.resolve();
3712
- const queued = previous.then(() => settleRestore(indexName)).then(write);
3713
- metadataWrites.set(
3714
- indexName,
3715
- queued.catch(() => void 0)
3716
- );
3717
- return queued;
3718
- }
3719
- async function saveMetadata(indexName, documentCount) {
3720
- const metadata = hooks.buildMetadata(indexName, documentCount);
3721
- if (metadata === void 0) return;
3722
- await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(metadata, { checksum: true }));
3723
- }
3724
- function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
3725
- return loads.track(indexName, loadIndex(indexName, metadataOnly), startsGraphRepairs);
3726
- }
3727
- async function loadIndex(indexName, metadataOnly) {
3728
- await settleRestore(indexName);
3729
- const metaBytes = await adapter.load(metadataKey(indexName));
3730
- if (metaBytes === null) {
3731
- return;
3732
- }
3733
- const { metadata } = await readMetadataEnvelope(metaBytes);
3734
- const promoted = await settleStoredAnalysis(adapterCheckpointStore(adapter), metadata, metadataOnly);
3735
- let derivedCount;
3736
- if (metadataOnly && metadata.documentCount === void 0) {
3737
- try {
3738
- derivedCount = await countStoredBundleDocuments(indexName);
3739
- metadata.documentCount = derivedCount;
3740
- } catch {
3741
- derivedCount = void 0;
3742
- }
3743
- }
3744
- await hooks.createIndexFromMetadata(metadata, !metadataOnly);
3745
- if (metadataOnly) {
3746
- if (derivedCount !== void 0 || promoted) {
3747
- const counted = derivedCount ?? metadata.documentCount;
3748
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, counted)).catch(() => void 0);
3749
- }
3750
- return;
3751
- }
3752
- const manager = hooks.getManager(indexName);
3753
- const current = hooks.buildMetadata(indexName);
3754
- if (manager === void 0 || current === void 0) {
3755
- return;
3756
- }
3757
- const recovered = await recoverCheckpoint({
3758
- store: adapterCheckpointStore(adapter),
3759
- indexName,
3760
- metadata: current,
3761
- manager,
3762
- vectorIndexes: hooks.getVectorIndexes(indexName),
3763
- storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
3764
- compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
3765
- keepsUnlistedFiles: false,
3766
- keptForSync: /* @__PURE__ */ new Set(),
3767
- replay: async (checkpoint) => checkpoint,
3768
- onLayoutUpgrade: hooks.onLayoutUpgrade
3769
- });
3770
- const indexState = getOrCreateIndexState(indexName);
3771
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
3772
- const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
3773
- if (persistedSeqNo > 0) {
3774
- indexState.appliedSeqNoByPartition.set(partitionId, persistedSeqNo);
3775
- }
3776
- }
3777
- if (recovered.upgradedDocumentCount !== null || promoted) {
3778
- const upgradedCount = recovered.upgradedDocumentCount ?? manager.countDocuments();
3779
- await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
3780
- }
3781
- }
3782
- function countStoredBundleDocuments(indexName) {
3783
- return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
3784
- }
3785
- async function recordMutation(record) {
3786
- const indexState = getOrCreateIndexState(record.indexName);
3787
- const buffered = indexState.applyChain.then(async () => {
3788
- await record.apply();
3789
- const appliedSeqNo2 = (indexState.appliedSeqNoByPartition.get(record.partitionId) ?? 0) + 1;
3790
- indexState.appliedSeqNoByPartition.set(record.partitionId, appliedSeqNo2);
3791
- return appliedSeqNo2;
3792
- });
3793
- indexState.applyChain = buffered.then(
3794
- () => void 0,
3795
- () => void 0
3796
- );
3797
- const appliedSeqNo = await buffered;
3798
- indexState.mutationsSinceCheckpoint += 1;
3799
- startCheckpointTimer();
3800
- if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
3801
- void checkpointIndex(record.indexName).catch((err) => {
3802
- hooks.onFatalError(err instanceof Error ? err : new Error(String(err)));
3803
- });
3804
- }
3805
- return appliedSeqNo;
3806
- }
3807
- async function drainIndex(indexName) {
3808
- const indexState = indexes.get(indexName);
3809
- if (indexState !== void 0) {
3810
- indexState.unloading = true;
3811
- try {
3812
- await indexState.applyChain;
3813
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
3814
- await metadataWrites.get(indexName);
3815
- } catch (error) {
3816
- indexState.unloading = false;
3817
- throw error;
3818
- }
3819
- }
3820
- indexes.delete(indexName);
3821
- metadataWrites.delete(indexName);
3822
- }
3823
- return {
3824
- isActive() {
3825
- return true;
3826
- },
3827
- async recover(metadataOnly = false) {
3828
- await recoverRestoreJournals(restoreStorage);
3829
- const names = (await adapter.list("")).filter((key) => key.endsWith("/meta")).map((key) => key.slice(0, -"/meta".length));
3830
- for (const indexName of names) await recoverIndex(indexName, metadataOnly, false);
3831
- startCheckpointTimer();
3832
- for (const indexName of names) loads.startGraphRepairs(indexName);
3833
- },
3834
- recoverIndex(indexName) {
3835
- return recoverIndex(indexName);
3836
- },
3837
- highestPersistedSeqNo(indexName, partitionId) {
3838
- return indexes.get(indexName)?.appliedSeqNoByPartition.get(partitionId) ?? 0;
3839
- },
3840
- recordMutation,
3841
- async recordMutations(records) {
3842
- const outcomes = [];
3843
- for (const record of records) {
3844
- try {
3845
- outcomes.push({ ok: true, seqNo: await recordMutation(record) });
3846
- } catch (error) {
3847
- outcomes.push({ ok: false, error });
3848
- }
3849
- }
3850
- return outcomes;
3851
- },
3852
- persistMetadata(indexName) {
3853
- return queueMetadataWrite(indexName, async () => {
3854
- const metadata = hooks.buildMetadata(indexName);
3855
- if (metadata === void 0) {
3856
- return;
3857
- }
3858
- const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3859
- await adapter.save(metadataKey(indexName), bytes);
3860
- getOrCreateIndexState(indexName);
3861
- });
3862
- },
3863
- checkpoint(indexName) {
3864
- return checkpointIndex(indexName);
3865
- },
3866
- checkpointFromMemory(indexName) {
3867
- return checkpointIndex(indexName, true);
3868
- },
3869
- async checkpointAfterSwitch(indexName, swap) {
3870
- const indexState = getOrCreateIndexState(indexName);
3871
- return switchInTurn(indexState, {
3872
- mayStart: () => !indexState.unloading && !shuttingDown,
3873
- swap,
3874
- checkpoint: () => performCheckpoint(indexName, indexState, true),
3875
- afterSettling: () => void 0
3876
- });
3877
- },
3878
- async checkpointAfterInFlight(indexName) {
3879
- await loads.settled(indexName);
3880
- await checkpointIndex(indexName, false, true).catch((err) => {
3881
- hooks.onFatalError(err instanceof Error ? err : new Error(String(err)));
3882
- });
3883
- },
3884
- async checkpointAll() {
3885
- for (const indexName of [...indexes.keys()]) {
3886
- await checkpointIndex(indexName);
3887
- }
3888
- },
3889
- stageRestore(indexName) {
3890
- return createRestoreStaging(restoreStorage, indexName, {
3891
- finish: (run) => restores.finishLater(indexName, run),
3892
- onStorageLost: hooks.onFatalError
3893
- });
3894
- },
3895
- adoptRestoredCheckpoint(indexName, checkpoint) {
3896
- const indexState = getOrCreateIndexState(indexName);
3897
- indexState.appliedSeqNoByPartition.clear();
3898
- for (const { partitionId, lastSeqNo } of checkpoint) {
3899
- if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
3900
- }
3901
- },
3902
- ...queryHistoryAccess({ kind: "adapter", adapter }),
3903
- async removeIndex(indexName) {
3904
- await settleRestore(indexName).catch(() => void 0);
3905
- restores.forget(indexName);
3906
- await restoreStorage.remove(journalKey(indexName));
3907
- await drainIndex(indexName);
3908
- for (const key of await adapter.list(`${indexName}/`)) {
3909
- await adapter.delete(key);
3910
- }
3911
- await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
3912
- },
3913
- async unloadIndex(indexName) {
3914
- await settleRestore(indexName);
3915
- await drainIndex(indexName);
3916
- },
3917
- async reloadIndex(indexName) {
3918
- const indexState = getOrCreateIndexState(indexName);
3919
- const reload = indexState.applyChain.then(() => recoverIndex(indexName));
3920
- indexState.applyChain = reload.catch(() => void 0);
3921
- await reload;
3922
- },
3923
- async shutdown() {
3924
- shuttingDown = true;
3925
- if (checkpointTimer !== null) {
3926
- clearInterval(checkpointTimer);
3927
- checkpointTimer = null;
3928
- }
3929
- await restores.settleAll();
3930
- for (const indexState of indexes.values()) {
3931
- if (indexState.checkpointInFlight !== null) {
3932
- await indexState.checkpointInFlight.catch(() => void 0);
3933
- }
3934
- }
3935
- indexes.clear();
3936
- }
3937
- };
3938
- }
3939
-
3940
- // src/engine/durability-integration.ts
3941
- function buildMetadata(indexName, hooks, documentCount) {
3942
- const config = hooks.getIndexConfig(indexName);
3943
- if (config === void 0) {
3944
- return void 0;
3945
- }
3946
- return metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount, documentCount);
3947
- }
3948
- function createDurabilityIntegration(tier, hooks) {
3949
- const managerHooks = {
3950
- getManager: hooks.getManager,
3951
- getVectorFieldPaths: hooks.getVectorFieldPaths,
3952
- getVectorIndexes: hooks.getVectorIndexes,
3953
- buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
3954
- buildTermsMetadata(indexName) {
3955
- const config = hooks.getTermsConfig(indexName);
3956
- return config === void 0 ? void 0 : metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount);
3957
- },
3958
- mustRewriteTerms: hooks.mustRewriteTerms,
3959
- onManifestCommitted: hooks.onManifestCommitted,
3960
- ...hooks.queueDiskAdoption === void 0 ? {} : { queueDiskAdoption: hooks.queueDiskAdoption },
3961
- ...hooks.awaitCopiesOfIndex === void 0 ? {} : { awaitCopiesOfIndex: hooks.awaitCopiesOfIndex },
3962
- createIndexFromMetadata: hooks.createIndexFromMetadata,
3963
- recordCheckpoint: hooks.recordCheckpoint,
3964
- onLayoutUpgrade: hooks.onLayoutUpgrade,
3965
- onCheckpointHeld: hooks.onCheckpointHeld,
3966
- onFatalError: hooks.onFatalError,
3967
- ...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
3968
- ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
3969
- };
3970
- const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
3971
- function partitionFor(indexName, docId) {
3972
- const partitionManager = hooks.getManager(indexName);
3973
- const partitionCount = partitionManager?.partitionCount ?? 1;
3974
- return fnv1a(docId) % Math.max(1, partitionCount);
3975
- }
3976
- async function recordMutation(indexName, docId, operation, document, apply) {
3977
- const partitionId = partitionFor(indexName, docId);
3978
- const seqNo = await manager.recordMutation({
3979
- indexName,
3980
- partitionId,
3981
- operation,
3982
- documentId: docId,
3983
- document,
3984
- apply
3985
- });
3986
- return { indexName, partitionId, seqNo };
3987
- }
3988
- return {
3989
- manager,
3990
- recordInsertOrUpdate(indexName, docId, document, apply) {
3991
- return recordMutation(indexName, docId, "INDEX", encodeDocument(document), apply);
3992
- },
3993
- async recordMutationBatch(indexName, mutations) {
3994
- const records = [];
3995
- const outcomes = new Array(mutations.length);
3996
- const recordPositions = [];
3997
- for (let i = 0; i < mutations.length; i++) {
3998
- if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
3999
- const mutation = mutations[i];
4000
- try {
4001
- records.push({
4002
- indexName,
4003
- partitionId: partitionFor(indexName, mutation.docId),
4004
- operation: mutation.document === null ? "DELETE" : "INDEX",
4005
- documentId: mutation.docId,
4006
- document: mutation.document === null ? null : encodeDocument(mutation.document),
4007
- apply: mutation.apply
4008
- });
4009
- recordPositions.push(i);
4010
- } catch (error) {
4011
- outcomes[i] = { ok: false, error };
4012
- }
4013
- }
4014
- const recorded = await manager.recordMutations(records);
4015
- for (let i = 0; i < recorded.length; i++) {
4016
- const outcome = recorded[i];
4017
- outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
4018
- }
4019
- return outcomes;
4020
- },
4021
- recordRemove(indexName, docId, apply) {
4022
- return recordMutation(indexName, docId, "DELETE", null, apply);
4023
- }
4024
- };
4025
- }
4026
-
4027
- // src/engine/durability-wiring.ts
4028
- function createDurabilityFromTier(tier, wiring) {
4029
- return createDurabilityIntegration(tier, {
4030
- checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
4031
- getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
4032
- getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
4033
- getVectorIndexes: (indexName) => (wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName)?.getVectorIndexes() : void 0) ?? /* @__PURE__ */ new Map(),
4034
- getIndexConfig: (indexName) => {
4035
- const entry = wiring.indexRegistry.get(indexName);
4036
- return entry === void 0 ? void 0 : durableMetadataConfigOf(entry);
4037
- },
4038
- getTermsConfig: (indexName) => {
4039
- const entry = wiring.indexRegistry.get(indexName);
4040
- return entry === void 0 ? void 0 : metadataConfigOf(entry);
4041
- },
4042
- mustRewriteTerms: (indexName) => (wiring.indexRegistry.get(indexName)?.durableAnalysis ?? null) !== null,
4043
- onManifestCommitted(indexName, analysisGeneration) {
4044
- const entry = wiring.indexRegistry.get(indexName);
4045
- if (entry !== void 0 && entry.analysisGeneration === analysisGeneration) entry.durableAnalysis = null;
4046
- },
4047
- queueDiskAdoption: wiring.queueDiskAdoption,
4048
- awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
4049
- createIndexFromMetadata: wiring.createIndexFromMetadata,
4050
- recordCheckpoint: wiring.recordCheckpoint,
4051
- onLayoutUpgrade: wiring.emitLayoutUpgrade,
4052
- onCheckpointHeld: wiring.emitCheckpointHeld,
4053
- onFatalError: wiring.emitFatalError,
4054
- keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4055
- onHistoryMoved(indexName) {
4056
- if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4057
- import('./retention-3K7ARGAG.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4058
- }
4059
- });
4060
- }
4061
- async function reloadIndexAndCopies(orchestrator, durability, indexName) {
4062
- const manager = durability?.manager;
4063
- if (manager?.reloadIndex === void 0) return;
4064
- const wasScaledOut = orchestrator.desyncIndex(indexName);
4065
- try {
4066
- await manager.reloadIndex(indexName);
4067
- } finally {
4068
- await orchestrator.resyncIndex(indexName, wasScaledOut);
4069
- }
4070
- }
4071
-
4072
- export { createDurabilityFromTier, reloadIndexAndCopies };