@delali/narsil 0.3.1-next.156 → 0.3.1-next.160

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (564) hide show
  1. package/dist/adapters/filesystem.mjs +3 -3
  2. package/dist/adapters/gcs.mjs +3 -3
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +3 -3
  5. package/dist/browser/{chunk-3V7DNNZL.mjs → chunk-4RNSFVWI.mjs} +20997 -19899
  6. package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
  7. package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
  8. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  9. package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
  10. package/dist/browser/{durability-config-TIYKW6XX.mjs → durability-config-SGB5ZP5K.mjs} +8 -3
  11. package/dist/browser/embed-RF2MPGAO.mjs +222 -0
  12. package/dist/{geo-index-MRH5JMIR.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
  13. package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
  14. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  15. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  16. package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
  17. package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
  18. package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
  19. package/dist/{react-native/chunks/restore-hooks-IBGIY3DR.mjs → browser/restore-queue-S4BWOGMU.mjs} +98 -36
  20. package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
  21. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  22. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  23. package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
  24. package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
  25. package/dist/browser/{chunk-SSTPBZZX.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
  26. package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
  27. package/dist/{capabilities-Da4ywVVl.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
  28. package/dist/chunk-2JRERUNZ.mjs +217 -0
  29. package/dist/chunk-2RIS76RJ.mjs +212 -0
  30. package/dist/chunk-3B775F3C.mjs +252 -0
  31. package/dist/{chunk-K5QFDYST.mjs → chunk-3D6RZGJH.mjs} +3 -3
  32. package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
  33. package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
  34. package/dist/chunk-6EZW2GQL.mjs +978 -0
  35. package/dist/chunk-75UNINFA.mjs +271 -0
  36. package/dist/chunk-77AUVDQ5.mjs +7 -0
  37. package/dist/{browser/chunk-23XRXIBW.mjs → chunk-7MWYORQC.mjs} +10 -3
  38. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  39. package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
  40. package/dist/{chunk-XAXKV65G.mjs → chunk-BXM7XM4G.mjs} +7 -3
  41. package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
  42. package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
  43. package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
  44. package/dist/{chunk-I2KPOLOZ.mjs → chunk-FE3MRIBN.mjs} +104 -13
  45. package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
  46. package/dist/{chunk-ZKP7G7TV.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
  47. package/dist/{chunk-2UV4QBHW.mjs → chunk-GF4K7IUK.mjs} +317 -120
  48. package/dist/chunk-GXCDWLQI.mjs +193 -0
  49. package/dist/{chunk-25KDNIZZ.mjs → chunk-H64YDF5M.mjs} +3 -3
  50. package/dist/{browser/chunk-CRE3G7F3.mjs → chunk-HOOR4CP5.mjs} +63 -7
  51. package/dist/chunk-IB4BYLE3.mjs +127 -0
  52. package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
  53. package/dist/chunk-IH3G6SJO.mjs +40 -0
  54. package/dist/chunk-JV74RKCY.mjs +3041 -0
  55. package/dist/{chunk-FRBFVQFN.mjs → chunk-K67I3RF4.mjs} +261 -101
  56. package/dist/chunk-KLVMK3R5.mjs +47 -0
  57. package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
  58. package/dist/chunk-LF4GB4PO.mjs +281 -0
  59. package/dist/chunk-NC4QLCZC.mjs +59 -0
  60. package/dist/chunk-OOO54OVC.mjs +627 -0
  61. package/dist/chunk-OUWZKNO3.mjs +207 -0
  62. package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
  63. package/dist/{chunk-6I6AYAXS.mjs → chunk-PTTMGRG5.mjs} +9 -10
  64. package/dist/{browser/chunk-SRMZXJSL.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
  65. package/dist/chunk-QBVTYY4J.mjs +54 -0
  66. package/dist/{react-native/chunks/chunk-GXX3DQXP.mjs → chunk-R4CE4IVU.mjs} +352 -165
  67. package/dist/chunk-S2POFTLN.mjs +10 -0
  68. package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
  69. package/dist/{chunk-FAKFUDZZ.mjs → chunk-SHGN3QIF.mjs} +80 -13
  70. package/dist/chunk-SXIRM4MX.mjs +3549 -0
  71. package/dist/{react-native/chunks/chunk-EJ33LCHV.mjs → chunk-TGPKS2UM.mjs} +47 -1
  72. package/dist/chunk-TINW4BSH.mjs +31301 -0
  73. package/dist/{cold-module-6MO2XJF2.mjs → chunk-TSLV5P2J.mjs} +397 -79
  74. package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
  75. package/dist/chunk-U6B7DUQW.mjs +788 -0
  76. package/dist/{chunk-FYOVBEMM.mjs → chunk-V3SGPXWQ.mjs} +8 -9
  77. package/dist/chunk-W34VJXY7.mjs +4723 -0
  78. package/dist/chunk-YETKPX5M.mjs +5 -0
  79. package/dist/chunk-ZQ6QY42L.mjs +59 -0
  80. package/dist/{client-DRv7UVeY.d.ts → client-CTp8RRiK.d.ts} +16 -4
  81. package/dist/client.d.ts +8 -8
  82. package/dist/client.mjs +70 -18
  83. package/dist/{config-DMwffLKn.d.ts → config-PCNzWjHL.d.ts} +6 -3
  84. package/dist/{config-CoSig5Ox.d.ts → config-_WT5grSL.d.ts} +3 -3
  85. package/dist/core-leaf-KXFGP74A.mjs +7 -0
  86. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  87. package/dist/direct-executor-MKTZBR26.mjs +16 -0
  88. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  89. package/dist/distribution/coordinator/etcd.mjs +7 -4
  90. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  91. package/dist/distribution/coordinator/in-memory.mjs +1 -2
  92. package/dist/distribution/transport/grpc.mjs +7 -2
  93. package/dist/distribution/transport/tcp.mjs +7 -1
  94. package/dist/distribution.d.ts +7 -7
  95. package/dist/distribution.mjs +1192 -1077
  96. package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
  97. package/dist/embed-4H7RTBVL.mjs +221 -0
  98. package/dist/embeddings/openai.mjs +2 -2
  99. package/dist/{engine-CFxPYqEk.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
  100. package/dist/{browser/geo-index-HHWGMNIS.mjs → geo-index-2ZMMWWSK.mjs} +190 -92
  101. package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
  102. package/dist/index.browser.mjs +4 -15
  103. package/dist/index.d.ts +1495 -78
  104. package/dist/index.mjs +14 -50
  105. package/dist/index.react-native.mjs +11 -28
  106. package/dist/invalidation/filesystem.mjs +1 -2
  107. package/dist/languages/albanian.mjs +1 -1
  108. package/dist/languages/amharic.mjs +1 -1
  109. package/dist/languages/arabic.mjs +1 -1
  110. package/dist/languages/armenian.mjs +1 -1
  111. package/dist/languages/azerbaijani.mjs +1 -1
  112. package/dist/languages/bambara.mjs +1 -1
  113. package/dist/languages/basque.mjs +1 -1
  114. package/dist/languages/belarusian.mjs +1 -1
  115. package/dist/languages/bengali.mjs +1 -1
  116. package/dist/languages/bosnian.mjs +1 -1
  117. package/dist/languages/breton.mjs +1 -1
  118. package/dist/languages/bulgarian.mjs +1 -1
  119. package/dist/languages/burmese.mjs +1 -1
  120. package/dist/languages/catalan.mjs +1 -1
  121. package/dist/languages/chinese.mjs +1 -1
  122. package/dist/languages/croatian.mjs +1 -1
  123. package/dist/languages/czech.mjs +1 -1
  124. package/dist/languages/dagbani.mjs +1 -1
  125. package/dist/languages/danish.mjs +1 -1
  126. package/dist/languages/dutch.mjs +1 -1
  127. package/dist/languages/english.mjs +1 -1
  128. package/dist/languages/esperanto.mjs +1 -1
  129. package/dist/languages/estonian.mjs +1 -1
  130. package/dist/languages/ewe.mjs +1 -1
  131. package/dist/languages/faroese.mjs +1 -1
  132. package/dist/languages/fijian.mjs +1 -1
  133. package/dist/languages/finnish.mjs +1 -1
  134. package/dist/languages/french.mjs +1 -1
  135. package/dist/languages/ga.mjs +1 -1
  136. package/dist/languages/galician.mjs +1 -1
  137. package/dist/languages/georgian.mjs +1 -1
  138. package/dist/languages/german.mjs +1 -1
  139. package/dist/languages/greek.mjs +1 -1
  140. package/dist/languages/guarani.mjs +1 -1
  141. package/dist/languages/gujarati.mjs +1 -1
  142. package/dist/languages/haitian-creole.mjs +1 -1
  143. package/dist/languages/hausa.mjs +1 -1
  144. package/dist/languages/hawaiian.mjs +1 -1
  145. package/dist/languages/hebrew.mjs +1 -1
  146. package/dist/languages/hindi.mjs +1 -1
  147. package/dist/languages/hungarian.mjs +1 -1
  148. package/dist/languages/icelandic.mjs +1 -1
  149. package/dist/languages/igbo.mjs +1 -1
  150. package/dist/languages/indonesian.mjs +1 -1
  151. package/dist/languages/irish.mjs +1 -1
  152. package/dist/languages/italian.mjs +1 -1
  153. package/dist/languages/japanese.mjs +1 -1
  154. package/dist/languages/kannada.mjs +1 -1
  155. package/dist/languages/kazakh.mjs +1 -1
  156. package/dist/languages/khmer.mjs +1 -1
  157. package/dist/languages/kinyarwanda.mjs +1 -1
  158. package/dist/languages/kirundi.mjs +1 -1
  159. package/dist/languages/korean.mjs +1 -1
  160. package/dist/languages/kurmanji.mjs +1 -1
  161. package/dist/languages/kyrgyz.mjs +1 -1
  162. package/dist/languages/lao.mjs +1 -1
  163. package/dist/languages/latin.mjs +1 -1
  164. package/dist/languages/latvian.mjs +1 -1
  165. package/dist/languages/lingala.mjs +1 -1
  166. package/dist/languages/lithuanian.mjs +1 -1
  167. package/dist/languages/luxembourgish.mjs +1 -1
  168. package/dist/languages/macedonian.mjs +1 -1
  169. package/dist/languages/malagasy.mjs +1 -1
  170. package/dist/languages/malay.mjs +1 -1
  171. package/dist/languages/malayalam.mjs +1 -1
  172. package/dist/languages/maltese.mjs +1 -1
  173. package/dist/languages/maori.mjs +1 -1
  174. package/dist/languages/marathi.mjs +1 -1
  175. package/dist/languages/nepali.mjs +1 -1
  176. package/dist/languages/norwegian.mjs +1 -1
  177. package/dist/languages/oromo.mjs +1 -1
  178. package/dist/languages/persian.mjs +1 -1
  179. package/dist/languages/polish.mjs +1 -1
  180. package/dist/languages/portuguese.mjs +1 -1
  181. package/dist/languages/punjabi.mjs +1 -1
  182. package/dist/languages/romanian.mjs +1 -1
  183. package/dist/languages/russian.mjs +1 -1
  184. package/dist/languages/samoan.mjs +1 -1
  185. package/dist/languages/sanskrit.mjs +1 -1
  186. package/dist/languages/scottish-gaelic.mjs +1 -1
  187. package/dist/languages/serbian.mjs +1 -1
  188. package/dist/languages/shona.mjs +1 -1
  189. package/dist/languages/sinhala.mjs +1 -1
  190. package/dist/languages/slovak.mjs +1 -1
  191. package/dist/languages/slovenian.mjs +1 -1
  192. package/dist/languages/sorani.mjs +1 -1
  193. package/dist/languages/spanish.mjs +1 -1
  194. package/dist/languages/swahili.mjs +1 -1
  195. package/dist/languages/swedish.mjs +1 -1
  196. package/dist/languages/tagalog.mjs +1 -1
  197. package/dist/languages/tamil.mjs +1 -1
  198. package/dist/languages/tatar.mjs +1 -1
  199. package/dist/languages/telugu.mjs +1 -1
  200. package/dist/languages/thai.mjs +1 -1
  201. package/dist/languages/tibetan.mjs +1 -1
  202. package/dist/languages/tigrinya.mjs +1 -1
  203. package/dist/languages/tongan.mjs +1 -1
  204. package/dist/languages/turkish.mjs +1 -1
  205. package/dist/languages/twi.mjs +1 -1
  206. package/dist/languages/ukrainian.mjs +1 -1
  207. package/dist/languages/urdu.mjs +1 -1
  208. package/dist/languages/vietnamese.mjs +1 -1
  209. package/dist/languages/welsh.mjs +1 -1
  210. package/dist/languages/wolof.mjs +1 -1
  211. package/dist/languages/xhosa.mjs +1 -1
  212. package/dist/languages/yoruba.mjs +1 -1
  213. package/dist/languages/zulu.mjs +1 -1
  214. package/dist/{long-query-thread-YA4IAYSY.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
  215. package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
  216. package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
  217. package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
  218. package/dist/optional-writes-DDFHYUOH.mjs +218 -0
  219. package/dist/{paged-partition-archive-PXW5XAYU.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
  220. package/dist/partial-words.browser.mjs +1649 -0
  221. package/dist/partial-words.mjs +15 -14
  222. package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
  223. package/dist/pattern.mjs +6 -8
  224. package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
  225. package/dist/persistence/durability/checkpoint-worker.mjs +27 -58
  226. package/dist/phrase-module-XK3T44VU.mjs +634 -0
  227. package/dist/query-actions-OMXJEC2J.mjs +118 -0
  228. package/dist/query-suggestions/merge-worker.mjs +18 -54
  229. package/dist/query-suggestions.browser.mjs +26 -226
  230. package/dist/query-suggestions.d.ts +2 -2
  231. package/dist/query-suggestions.mjs +16 -67
  232. package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
  233. package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
  234. package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
  235. package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
  236. package/dist/react-native/chunks/chunk-FTQ7JNMU.mjs +898 -0
  237. package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
  238. package/dist/{chunk-TBC3XUL2.mjs → react-native/chunks/chunk-HRDMIQ63.mjs} +53 -1
  239. package/dist/react-native/chunks/{chunk-KV46CT4D.mjs → chunk-IY6U6HRA.mjs} +63 -7
  240. package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
  241. package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
  242. package/dist/react-native/chunks/chunk-OK3WUA6N.mjs +54 -0
  243. package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
  244. package/dist/react-native/chunks/{chunk-DZ7H7D2C.mjs → chunk-TQ26BKBH.mjs} +6 -0
  245. package/dist/react-native/chunks/{chunk-6VBXNFLE.mjs → chunk-VUJCSQG5.mjs} +3 -1
  246. package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
  247. package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
  248. package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
  249. package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
  250. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  251. package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
  252. package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
  253. package/dist/react-native/chunks/{geo-index-PX3BIHAH.mjs → geo-index-QEYKEVPH.mjs} +190 -93
  254. package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
  255. package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
  256. package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
  257. package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
  258. package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
  259. package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
  260. package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
  261. package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
  262. package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
  263. package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
  264. package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
  265. package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
  266. package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
  267. package/dist/react-native/chunks/{chunk-UQ4OAXKU.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
  268. package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
  269. package/dist/react-native/partial-words.mjs +954 -35
  270. package/dist/react-native/pattern.mjs +238 -111
  271. package/dist/react-native/query-suggestions.mjs +33 -239
  272. package/dist/react-native/sync.mjs +127 -50
  273. package/dist/react-native/synonyms.mjs +256 -112
  274. package/dist/react.d.ts +9 -9
  275. package/dist/react.mjs +6 -0
  276. package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
  277. package/dist/rerankers/http.mjs +2 -2
  278. package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
  279. package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
  280. package/dist/runtime-4GKUIAKV.mjs +595 -0
  281. package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
  282. package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
  283. package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
  284. package/dist/server.d.ts +35 -13
  285. package/dist/server.mjs +90 -101
  286. package/dist/{browser/durability-wiring-A7BVG2DW.mjs → storage-leaf-HMIIOUYN.mjs} +2348 -902
  287. package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
  288. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  289. package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
  290. package/dist/sync/filesystem.d.ts +3 -3
  291. package/dist/sync/filesystem.mjs +27 -11
  292. package/dist/sync/indexeddb.browser.mjs +287 -0
  293. package/dist/sync/indexeddb.d.ts +3 -3
  294. package/dist/sync/indexeddb.mjs +7 -2
  295. package/dist/sync.browser.mjs +120 -70
  296. package/dist/sync.d.ts +4 -4
  297. package/dist/sync.mjs +121 -57
  298. package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
  299. package/dist/synonyms.mjs +15 -11
  300. package/dist/{tasks-qrJojgST.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
  301. package/dist/thread-SNILJXX3.mjs +30 -0
  302. package/dist/{types-BZCGjfLn.d.ts → types-CTg0KaMy.d.ts} +2 -2
  303. package/dist/vector/search-worker.d.ts +7 -1
  304. package/dist/vector/search-worker.mjs +9 -13
  305. package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
  306. package/dist/workers/entry.mjs +60 -30
  307. package/openapi.json +137 -50
  308. package/package.json +233 -8
  309. package/dist/analysis-change-5BQOQ4BF.mjs +0 -55
  310. package/dist/analysis-change-entry-TXX6JSMX.mjs +0 -1
  311. package/dist/archive-history-E4R73RWD.mjs +0 -29
  312. package/dist/browser/analysis-change-435CUEPZ.mjs +0 -933
  313. package/dist/browser/analysis-change-entry-HZXKPPGV.mjs +0 -1
  314. package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
  315. package/dist/browser/chunk-2CFI5VVV.mjs +0 -241
  316. package/dist/browser/chunk-2TI4SSUP.mjs +0 -315
  317. package/dist/browser/chunk-3I27KXQD.mjs +0 -34
  318. package/dist/browser/chunk-3SSRJMFN.mjs +0 -100
  319. package/dist/browser/chunk-4NGKC4H4.mjs +0 -2635
  320. package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
  321. package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
  322. package/dist/browser/chunk-6LWDPLTB.mjs +0 -177
  323. package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
  324. package/dist/browser/chunk-7U3WVJUP.mjs +0 -150
  325. package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
  326. package/dist/browser/chunk-BHFQATSC.mjs +0 -432
  327. package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
  328. package/dist/browser/chunk-EA63E3D2.mjs +0 -857
  329. package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
  330. package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
  331. package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
  332. package/dist/browser/chunk-GPPOG6PT.mjs +0 -9
  333. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  334. package/dist/browser/chunk-IZXVW2ZJ.mjs +0 -217
  335. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  336. package/dist/browser/chunk-K4Q6KX3R.mjs +0 -2528
  337. package/dist/browser/chunk-KZWSX6BH.mjs +0 -645
  338. package/dist/browser/chunk-MGI4BCXC.mjs +0 -63
  339. package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
  340. package/dist/browser/chunk-MOG4OVDK.mjs +0 -51
  341. package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
  342. package/dist/browser/chunk-NQAME3RK.mjs +0 -21
  343. package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
  344. package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
  345. package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
  346. package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
  347. package/dist/browser/chunk-STOWQFT3.mjs +0 -1829
  348. package/dist/browser/chunk-TK63WFXK.mjs +0 -90
  349. package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
  350. package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
  351. package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
  352. package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
  353. package/dist/browser/chunk-WI4MR75E.mjs +0 -86
  354. package/dist/browser/chunk-WN464PTL.mjs +0 -375
  355. package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
  356. package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
  357. package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
  358. package/dist/browser/cold-module-2EHCIV4K.mjs +0 -759
  359. package/dist/browser/engine-hooks-E4UD67JD.mjs +0 -17
  360. package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
  361. package/dist/browser/import-DXYB6756.mjs +0 -19
  362. package/dist/browser/methods-KTQCSIN4.mjs +0 -30
  363. package/dist/browser/module-NIYMEA5R.mjs +0 -23
  364. package/dist/browser/operations-OJ3BXS55.mjs +0 -5
  365. package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
  366. package/dist/browser/pool-3JLET5RO.mjs +0 -297
  367. package/dist/browser/release-76MXRPGV.mjs +0 -33
  368. package/dist/browser/restore-hooks-YPN4ZFCY.mjs +0 -114
  369. package/dist/browser/restore-queue-G3RQGL3Y.mjs +0 -55
  370. package/dist/browser/retention-ZWVMNC3Q.mjs +0 -210
  371. package/dist/browser/snapshot-VEQNIE6C.mjs +0 -963
  372. package/dist/browser/start-BM442C6R.mjs +0 -25
  373. package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
  374. package/dist/browser/write-2EVMZWDE.mjs +0 -194
  375. package/dist/chunk-2NE7VCYE.mjs +0 -212
  376. package/dist/chunk-37CD3OKS.mjs +0 -87
  377. package/dist/chunk-3HYC4JRF.mjs +0 -50
  378. package/dist/chunk-3MXV3VHR.mjs +0 -46
  379. package/dist/chunk-3OH67G7R.mjs +0 -214
  380. package/dist/chunk-46KY2PJD.mjs +0 -905
  381. package/dist/chunk-4KKXNJM6.mjs +0 -242
  382. package/dist/chunk-5YNRKI2C.mjs +0 -81
  383. package/dist/chunk-6HTGOXIK.mjs +0 -875
  384. package/dist/chunk-6OEGXLBX.mjs +0 -57
  385. package/dist/chunk-6RC4K46J.mjs +0 -86
  386. package/dist/chunk-7GFE3PEY.mjs +0 -40
  387. package/dist/chunk-7GXJUJ24.mjs +0 -49
  388. package/dist/chunk-7LXHHCXZ.mjs +0 -1397
  389. package/dist/chunk-7V5IUP2T.mjs +0 -63
  390. package/dist/chunk-A4L7YTKX.mjs +0 -458
  391. package/dist/chunk-BEZH7VEH.mjs +0 -23
  392. package/dist/chunk-BTPGQC6Z.mjs +0 -27
  393. package/dist/chunk-BU4SAIC4.mjs +0 -196
  394. package/dist/chunk-BWZ3P4MU.mjs +0 -20
  395. package/dist/chunk-CSLGFPFQ.mjs +0 -218
  396. package/dist/chunk-CZQ33DLW.mjs +0 -78
  397. package/dist/chunk-DP5C7YFQ.mjs +0 -543
  398. package/dist/chunk-DWEPMFJE.mjs +0 -240
  399. package/dist/chunk-EEBCDP36.mjs +0 -32
  400. package/dist/chunk-EFBDHH5P.mjs +0 -7081
  401. package/dist/chunk-FB6463IN.mjs +0 -1499
  402. package/dist/chunk-FCJ553HS.mjs +0 -655
  403. package/dist/chunk-FNGAM5B7.mjs +0 -2639
  404. package/dist/chunk-FOCV4CBZ.mjs +0 -87
  405. package/dist/chunk-FQBFZAWG.mjs +0 -27
  406. package/dist/chunk-G6HRSGOU.mjs +0 -74
  407. package/dist/chunk-GES7U2PM.mjs +0 -68
  408. package/dist/chunk-GH4WQ22V.mjs +0 -34
  409. package/dist/chunk-GQPTDQCL.mjs +0 -54
  410. package/dist/chunk-HFPMMRQT.mjs +0 -206
  411. package/dist/chunk-HJSN7UYG.mjs +0 -10
  412. package/dist/chunk-I53G4JAU.mjs +0 -1974
  413. package/dist/chunk-I54T6CQP.mjs +0 -117
  414. package/dist/chunk-IPN7S4KX.mjs +0 -130
  415. package/dist/chunk-J4IJTTEX.mjs +0 -226
  416. package/dist/chunk-JDJBQLRO.mjs +0 -1126
  417. package/dist/chunk-JWRUUCWD.mjs +0 -151
  418. package/dist/chunk-K4ZIGL2O.mjs +0 -290
  419. package/dist/chunk-KAKQ74LS.mjs +0 -53
  420. package/dist/chunk-KF4EXSM4.mjs +0 -21
  421. package/dist/chunk-KHKIRS5D.mjs +0 -113
  422. package/dist/chunk-KOKM4PQN.mjs +0 -52
  423. package/dist/chunk-KP7TYHWT.mjs +0 -92
  424. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  425. package/dist/chunk-L5ORED7V.mjs +0 -25
  426. package/dist/chunk-L6V6BST3.mjs +0 -1435
  427. package/dist/chunk-LHKOSI2J.mjs +0 -58
  428. package/dist/chunk-LQG5G2LY.mjs +0 -39
  429. package/dist/chunk-MATMVH65.mjs +0 -9
  430. package/dist/chunk-MP7ABPTD.mjs +0 -189
  431. package/dist/chunk-MQOJIO5B.mjs +0 -316
  432. package/dist/chunk-NF4DT3IH.mjs +0 -85
  433. package/dist/chunk-ORN5BLL7.mjs +0 -11
  434. package/dist/chunk-P5ZI2P2W.mjs +0 -7481
  435. package/dist/chunk-PELHVTO7.mjs +0 -24
  436. package/dist/chunk-PWC4VB3Q.mjs +0 -106
  437. package/dist/chunk-Q2UL2TPC.mjs +0 -21
  438. package/dist/chunk-QHPYS427.mjs +0 -66
  439. package/dist/chunk-QJVOLKU6.mjs +0 -111
  440. package/dist/chunk-QOW2JPUI.mjs +0 -634
  441. package/dist/chunk-QYRNLLLI.mjs +0 -99
  442. package/dist/chunk-RDXO6JST.mjs +0 -63
  443. package/dist/chunk-RFBLQLLZ.mjs +0 -16
  444. package/dist/chunk-RLOES5SX.mjs +0 -31
  445. package/dist/chunk-RZWWB2XR.mjs +0 -96
  446. package/dist/chunk-S5LN746F.mjs +0 -10
  447. package/dist/chunk-SA5LOGO3.mjs +0 -470
  448. package/dist/chunk-SQJWAIF4.mjs +0 -12
  449. package/dist/chunk-TDIK6NTV.mjs +0 -10
  450. package/dist/chunk-TGS5UXBE.mjs +0 -216
  451. package/dist/chunk-TQXHPFWT.mjs +0 -177
  452. package/dist/chunk-TZ3355VS.mjs +0 -121
  453. package/dist/chunk-TZTRKM33.mjs +0 -74
  454. package/dist/chunk-U55XJEIG.mjs +0 -18
  455. package/dist/chunk-UHICS3JI.mjs +0 -83
  456. package/dist/chunk-VD7J552I.mjs +0 -28
  457. package/dist/chunk-VTTRFF2C.mjs +0 -3711
  458. package/dist/chunk-WX4LAO6M.mjs +0 -656
  459. package/dist/chunk-WZ7YV54N.mjs +0 -152
  460. package/dist/chunk-X5CV2E2B.mjs +0 -1
  461. package/dist/chunk-XXHN7J46.mjs +0 -101
  462. package/dist/chunk-Y4WPNHMC.mjs +0 -50
  463. package/dist/chunk-YEKL4JBK.mjs +0 -96
  464. package/dist/chunk-YTWZVGCB.mjs +0 -382
  465. package/dist/chunk-Z2KAFCRT.mjs +0 -30
  466. package/dist/chunk-ZXNQEW43.mjs +0 -18
  467. package/dist/chunk-ZYAY26KG.mjs +0 -29
  468. package/dist/direct-executor-FTMYY3TI.mjs +0 -34
  469. package/dist/durability-wiring-VIF5MOSY.mjs +0 -4064
  470. package/dist/engine-hooks-EGUGNZYE.mjs +0 -54
  471. package/dist/gzip-F4HMIEEQ.mjs +0 -1
  472. package/dist/import-XKDPJQDP.mjs +0 -56
  473. package/dist/methods-7IOINERF.mjs +0 -30
  474. package/dist/module-FQ4JWNQJ.mjs +0 -62
  475. package/dist/operations-G5EP6JPX.mjs +0 -5
  476. package/dist/phrase-module-XW5MKWP5.mjs +0 -330
  477. package/dist/pool-Y4A46T36.mjs +0 -298
  478. package/dist/react-native/chunks/analysis-change-LT5QBQ3H.mjs +0 -926
  479. package/dist/react-native/chunks/analysis-change-entry-SCVWK6OU.mjs +0 -1
  480. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
  481. package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
  482. package/dist/react-native/chunks/chunk-3JWDECAZ.mjs +0 -858
  483. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
  484. package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
  485. package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
  486. package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
  487. package/dist/react-native/chunks/chunk-4JYO3JD2.mjs +0 -242
  488. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
  489. package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +0 -85
  490. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
  491. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
  492. package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
  493. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
  494. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
  495. package/dist/react-native/chunks/chunk-DNXOMPAQ.mjs +0 -191
  496. package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
  497. package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
  498. package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
  499. package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
  500. package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
  501. package/dist/react-native/chunks/chunk-G43IPIDG.mjs +0 -151
  502. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
  503. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
  504. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
  505. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
  506. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
  507. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
  508. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
  509. package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
  510. package/dist/react-native/chunks/chunk-MIP4SOR6.mjs +0 -22803
  511. package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
  512. package/dist/react-native/chunks/chunk-MUWD2H2S.mjs +0 -21
  513. package/dist/react-native/chunks/chunk-NB3LP33V.mjs +0 -315
  514. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
  515. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
  516. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
  517. package/dist/react-native/chunks/chunk-PDRWWPKV.mjs +0 -2532
  518. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
  519. package/dist/react-native/chunks/chunk-POPDAGH6.mjs +0 -2636
  520. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
  521. package/dist/react-native/chunks/chunk-QUKQNI3H.mjs +0 -647
  522. package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
  523. package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
  524. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
  525. package/dist/react-native/chunks/chunk-TP6KCKER.mjs +0 -63
  526. package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
  527. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
  528. package/dist/react-native/chunks/chunk-VWUDIOT6.mjs +0 -218
  529. package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
  530. package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
  531. package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
  532. package/dist/react-native/chunks/chunk-ZLWVW734.mjs +0 -100
  533. package/dist/react-native/chunks/chunk-ZXZFN7FI.mjs +0 -432
  534. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
  535. package/dist/react-native/chunks/cold-module-KVZYUGRY.mjs +0 -772
  536. package/dist/react-native/chunks/durability-wiring-34OYO32E.mjs +0 -4942
  537. package/dist/react-native/chunks/engine-hooks-M7JV46NH.mjs +0 -30
  538. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
  539. package/dist/react-native/chunks/import-H6AQREWI.mjs +0 -32
  540. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
  541. package/dist/react-native/chunks/module-LBZPGOWH.mjs +0 -36
  542. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
  543. package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
  544. package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
  545. package/dist/react-native/chunks/release-D7ZTIANM.mjs +0 -33
  546. package/dist/react-native/chunks/restore-queue-ZZX54AH7.mjs +0 -55
  547. package/dist/react-native/chunks/retention-EW4TTDFB.mjs +0 -223
  548. package/dist/react-native/chunks/snapshot-REYVIN3X.mjs +0 -975
  549. package/dist/react-native/chunks/start-BQK6KN3R.mjs +0 -38
  550. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
  551. package/dist/react-native/chunks/write-FQAYCZF5.mjs +0 -207
  552. package/dist/release-BH3DKW3T.mjs +0 -33
  553. package/dist/restore-hooks-CDERDEF4.mjs +0 -153
  554. package/dist/restore-queue-7NNSTROA.mjs +0 -55
  555. package/dist/retention-QSVBBNRM.mjs +0 -66
  556. package/dist/runtime-COIUORCN.mjs +0 -177
  557. package/dist/snapshot-RH4NSKIZ.mjs +0 -782
  558. package/dist/start-MCKCOGIW.mjs +0 -66
  559. package/dist/sync-state-ZWHLAFLE.mjs +0 -15
  560. package/dist/thread-O4SBJJJS.mjs +0 -50
  561. package/dist/write-3FK7NG3Y.mjs +0 -232
  562. /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
  563. /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
  564. /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
@@ -1,546 +1,201 @@
1
- import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-K4Q6KX3R.mjs';
2
- import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-G64Q6OJF.mjs';
3
- import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-2CFI5VVV.mjs';
4
- import './chunk-YJHGOJAC.mjs';
5
- import { sha256 } from './chunk-M3MQE2X7.mjs';
6
- import './chunk-3SSRJMFN.mjs';
7
- import './chunk-BHFQATSC.mjs';
8
- import { QUERY_HISTORY_FOLDER } from './chunk-7IGKGWSQ.mjs';
9
- import './chunk-3I27KXQD.mjs';
10
- import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
11
- import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-EA63E3D2.mjs';
12
- import { applyDeleteEntry, applyIndexEntry } from './chunk-KZWSX6BH.mjs';
13
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-ZI2V4VD7.mjs';
14
- import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-23XRXIBW.mjs';
15
- import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-3V7DNNZL.mjs';
16
- import './chunk-ML7THVLJ.mjs';
17
- import { createRequestId } from './chunk-VI7BQR4U.mjs';
18
- import './chunk-JPTQZWJQ.mjs';
19
- import { encodeDocument, openChecksummedEnvelope, crc32, requireValidFieldTypes, HEADER_SIZE } from './chunk-STOWQFT3.mjs';
20
- import { fnv1a } from './chunk-RA6LSBCM.mjs';
21
- import './chunk-MOG4OVDK.mjs';
22
- import './chunk-TK63WFXK.mjs';
23
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WN464PTL.mjs';
24
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime, resolveWorkerEntry } from './chunk-XAWNZ4G2.mjs';
25
- import './chunk-TY7L6OXO.mjs';
26
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
27
- import { compareCodePoints, MIN_M } from './chunk-NXQG7DRA.mjs';
28
- import { MAX_FILE_BYTES } from './chunk-GPPOG6PT.mjs';
29
- import './chunk-WXLBZJMT.mjs';
30
- import { NarsilError, ErrorCodes } from './chunk-SRMZXJSL.mjs';
31
- import './chunk-JNOAKDUM.mjs';
32
- import { decode, encode } from '@msgpack/msgpack';
1
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES, scheduleSyncRetention } from './chunk-FE3MRIBN.mjs';
2
+ export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-FE3MRIBN.mjs';
3
+ import { openPagedSegmentScan, countsAsLatestHolder, buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, chunksOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, bytesThatSegmentsHoldInMemory, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, createDetachedIndex, checkPagedSegmentFully, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readMetadataEnvelope, snapshotCheckpointFor, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, readSegmentManifest, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, growPartitionsTo, createMarkerWriter, metadataConfigOf, durableMetadataConfigOf } from './chunk-SXIRM4MX.mjs';
4
+ export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedSegmentFully, countsAsLatestHolder, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, openPagedSegmentScan, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-SXIRM4MX.mjs';
5
+ import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-TSLV5P2J.mjs';
6
+ export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-TSLV5P2J.mjs';
7
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-2JRERUNZ.mjs';
8
+ export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-2JRERUNZ.mjs';
9
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-4CAMH7FO.mjs';
10
+ import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-OOO54OVC.mjs';
11
+ export { readSnapshotArchive } from './chunk-OOO54OVC.mjs';
12
+ import { createDurableDirectory } from './chunk-TW543F33.mjs';
13
+ import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-W34VJXY7.mjs';
14
+ export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-W34VJXY7.mjs';
15
+ import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-U6B7DUQW.mjs';
16
+ import './chunk-5FLRXF73.mjs';
17
+ import './chunk-JDDXV6UF.mjs';
18
+ import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7MWYORQC.mjs';
19
+ import { getFs } from './chunk-HLPRHOSO.mjs';
20
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
21
+ import { encode } from './chunk-77AUVDQ5.mjs';
22
+ export { bindStorageHost } from './chunk-77AUVDQ5.mjs';
23
+ import { sha256 } from './chunk-LNGE7SLY.mjs';
24
+ import { setNestedValue } from './chunk-QBVTYY4J.mjs';
25
+ import { rewriteSpillOf } from './chunk-IB4BYLE3.mjs';
26
+ import { deserializePartition } from './chunk-2RIS76RJ.mjs';
27
+ import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LAWIJRMJ.mjs';
28
+ export { createHNSWIndex, osqBitsOf } from './chunk-LAWIJRMJ.mjs';
29
+ import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
30
+ import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-6EZW2GQL.mjs';
31
+ import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createRequestId, createPartitionRouter, buildSegmentPayloads } from './chunk-TINW4BSH.mjs';
32
+ import { encodeDocument } from './chunk-KLVMK3R5.mjs';
33
+ import './chunk-AB27FWZM.mjs';
34
+ import { decode } from './chunk-GXCDWLQI.mjs';
35
+ import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
36
+ import './chunk-7T4LAYT4.mjs';
37
+ import './chunk-JV74RKCY.mjs';
38
+ import { flattenSchema, parseVectorDimension, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, MAX_DOCUMENT_NESTING_DEPTH, requireValidFieldTypes } from './chunk-R4CE4IVU.mjs';
39
+ import './chunk-U2BKYEC7.mjs';
40
+ import { detectRuntime, spawnNodeWorker, resolveWorkerEntry } from './chunk-OUWZKNO3.mjs';
41
+ import { STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3B775F3C.mjs';
42
+ import './chunk-E2F6LZR3.mjs';
43
+ import { MIN_M } from './chunk-ZXX5GZEX.mjs';
44
+ import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
45
+ import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
46
+ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
33
47
 
34
- // src/persistence/durability/node-fs.browser.ts
35
- function filesystemUnavailable() {
36
- throw new NarsilError(
37
- ErrorCodes.CONFIG_INVALID,
38
- "Filesystem durability requires a Node.js runtime; use the memory or IndexedDB persistence adapters in the browser"
39
- );
40
- }
41
- async function getFs() {
42
- return filesystemUnavailable();
43
- }
44
- async function getPath() {
45
- return filesystemUnavailable();
46
- }
47
-
48
- // src/persistence/durability/lock-holder.browser.ts
49
- async function holderIsLive(_holder, _self, _lockPath) {
50
- return true;
51
- }
52
- function holdDirectoryOpen(_directory) {
53
- return () => void 0;
54
- }
55
- async function removeStaleLock(_lockPath, _asidePath, _isStale) {
56
- }
57
-
58
- // src/persistence/durability/directory-lock.ts
59
- var LOCK_FILE_NAME = ".narsil.lock";
60
- var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
61
- var lockedDirectories = /* @__PURE__ */ new Set();
62
- function errnoOf(err) {
63
- return err?.code;
64
- }
65
- function integerOrNull(text) {
66
- const value = Number.parseInt(text ?? "", 10);
67
- return Number.isSafeInteger(value) ? value : null;
68
- }
69
- function parseHolder(text) {
70
- const [pidText, threadText, startText, nonceText] = text.split("\n");
71
- const pid = integerOrNull(pidText);
72
- if (pid === null || pid <= 0) return null;
73
- const nonce = nonceText?.trim() ?? "";
74
- return {
75
- pid,
76
- threadId: integerOrNull(threadText),
77
- processStartMs: integerOrNull(startText),
78
- nonce: nonce.length > 0 ? nonce : null
79
- };
80
- }
81
- function formatHolder(holder) {
82
- return `${holder.pid}
83
- ${holder.threadId ?? ""}
84
- ${holder.processStartMs ?? ""}
85
- ${holder.nonce ?? ""}
86
- `;
87
- }
88
- async function readHolder(fs, lockPath) {
89
- try {
90
- return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
91
- } catch (err) {
92
- if (errnoOf(err) === "ENOENT") return "absent";
93
- throw err;
94
- }
95
- }
96
- async function settledHolder(fs, lockPath) {
97
- const first = await readHolder(fs, lockPath);
98
- if (first !== "unreadable") return first;
99
- await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
100
- return readHolder(fs, lockPath);
101
- }
102
- function heldElsewhere(root, holder, lockPath) {
103
- return new NarsilError(
104
- ErrorCodes.CONFIG_INVALID,
105
- `The durability directory "${root}" is in use by ${holder}, and only one engine may write a write-ahead log directory. Point this engine at another directory, or shut the other engine down first. Where no engine uses the directory, delete "${lockPath}"`,
106
- { directory: root }
107
- );
108
- }
109
- function holderLabel(holder, self) {
110
- if (holder.pid !== self.pid) return `process ${holder.pid}`;
111
- return holder.threadId === self.threadId ? "another copy of Narsil in this thread" : "another thread of this process";
112
- }
113
- async function createLockFileExclusively(fs, lockPath, content) {
114
- try {
115
- await fs.writeFile(lockPath, content, { flag: "wx" });
116
- return true;
117
- } catch (err) {
118
- if (errnoOf(err) === "EEXIST") return false;
119
- throw err;
120
- }
121
- }
122
- async function createLockFile(fs, lockPath, draftPath, content) {
123
- await fs.writeFile(draftPath, content, { flag: "w" });
124
- try {
125
- await fs.link(draftPath, lockPath);
126
- return true;
127
- } catch (err) {
128
- const code = errnoOf(err);
129
- if (code === "EEXIST") return false;
130
- if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
131
- return createLockFileExclusively(fs, lockPath, content);
48
+ // src/persistence/paged/adopt.ts
49
+ function yieldToEventLoop() {
50
+ return new Promise((resolve) => setTimeout(resolve, 0));
51
+ }
52
+ function applyNewerTombstones(prepared) {
53
+ for (let at = 0; at < prepared.length; at++) {
54
+ if (!prepared[at].isNew) continue;
55
+ for (let newer = at + 1; newer < prepared.length; newer++) {
56
+ const access = prepared[newer].segment.paged;
57
+ if (access === null) continue;
58
+ for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
59
+ }
60
+ }
61
+ }
62
+ async function applyNewerTombstonesInTurns(prepared) {
63
+ let applied = 0;
64
+ for (let at = 0; at < prepared.length; at++) {
65
+ if (!prepared[at].isNew) continue;
66
+ for (let newer = at + 1; newer < prepared.length; newer++) {
67
+ const access = prepared[newer].segment.paged;
68
+ if (access === null) continue;
69
+ for (const docId of access.snapshot.tombstones) {
70
+ prepared[at].segment.tombstoneDocument(docId);
71
+ applied += 1;
72
+ if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
73
+ }
132
74
  }
133
- throw err;
134
- } finally {
135
- await fs.unlink(draftPath).catch(() => void 0);
136
75
  }
137
76
  }
138
- async function acquire(fs, root, lockPath, self) {
139
- const draftPath = `${lockPath}.${self.pid}.${self.threadId ?? 0}`;
140
- const content = formatHolder(self);
141
- for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
142
- if (await createLockFile(fs, lockPath, draftPath, content)) return;
143
- const holder = await settledHolder(fs, lockPath);
144
- if (typeof holder === "object" && await holderIsLive()) {
145
- throw heldElsewhere(root, holderLabel(holder, self), lockPath);
77
+ async function prepareDiskAdoption(directory, manager, partitions) {
78
+ for (const entry of partitions) {
79
+ for (const ref of entry.segments) {
80
+ if (!await isPagedSegmentFile(directory, ref.key)) return null;
146
81
  }
147
- await removeStaleLock();
148
- }
149
- throw heldElsewhere(root, "another process", lockPath);
150
- }
151
- async function describeSelf() {
152
- return {
153
- pid: process.pid,
154
- threadId: await nodeThreadId(),
155
- processStartMs: Math.round(Date.now() - process.uptime() * 1e3),
156
- nonce: globalThis.crypto.randomUUID()
157
- };
158
- }
159
- async function lockDirectory(root) {
160
- const fs = await getFs();
161
- const pathMod = await getPath();
162
- const resolvedRoot = pathMod.resolve(root);
163
- const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
164
- const self = await describeSelf();
165
- async function holds() {
166
- const holder = await readHolder(fs, lockPath);
167
- return typeof holder === "object" && holder.nonce === self.nonce;
168
- }
169
- async function removeOwnLockFile() {
170
- if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
171
- }
172
- let directoryKey = null;
173
- let closeDirectory = () => void 0;
174
- try {
175
- await fs.mkdir(resolvedRoot, { recursive: true });
176
- const stat = await fs.stat(resolvedRoot, { bigint: true });
177
- const key = `${stat.dev}:${stat.ino}`;
178
- if (lockedDirectories.has(key)) throw heldElsewhere(root, "another engine in this process", lockPath);
179
- lockedDirectories.add(key);
180
- directoryKey = key;
181
- await acquire(fs, root, lockPath, self);
182
- closeDirectory = holdDirectoryOpen(resolvedRoot);
183
- } catch (err) {
184
- if (directoryKey !== null) lockedDirectories.delete(directoryKey);
185
- await removeOwnLockFile();
186
- if (err instanceof NarsilError) throw err;
187
- throw new NarsilError(
188
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
189
- `The engine cannot lock the durability directory "${root}"`,
190
- { directory: root, cause: err instanceof Error ? err.message : String(err) }
191
- );
192
82
  }
193
- const heldKey = directoryKey;
194
- let released = false;
195
- return {
196
- holds,
197
- async release() {
198
- if (released) return;
199
- released = true;
200
- lockedDirectories.delete(heldKey);
201
- await removeOwnLockFile();
202
- closeDirectory();
203
- }
204
- };
205
- }
206
-
207
- // src/persistence/durability/list-recursive.ts
208
- async function listRecursive(dir, base, pathMod, fs) {
209
- const results = [];
210
- let entries;
211
- try {
212
- entries = await fs.readdir(dir, { withFileTypes: true });
213
- } catch (err) {
214
- if (err.code === "ENOENT") {
215
- return results;
83
+ const prepared = [];
84
+ const droppedKeys = [];
85
+ for (const entry of partitions) {
86
+ if (entry.partitionId >= manager.partitionCount) continue;
87
+ const partition = manager.getPartition(entry.partitionId);
88
+ const held = /* @__PURE__ */ new Map();
89
+ if (isCompositePartition(partition)) {
90
+ for (const segment of partition.frozenSegments()) {
91
+ if (segment.paged !== null) held.set(segment.paged.key, segment);
92
+ }
216
93
  }
217
- throw err;
218
- }
219
- for (const entry of entries) {
220
- const full = pathMod.join(dir, entry.name);
221
- if (entry.isDirectory()) {
222
- results.push(...await listRecursive(full, base, pathMod, fs));
223
- } else {
224
- results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
94
+ const list = [];
95
+ for (const ref of entry.segments) {
96
+ const existing = held.get(ref.key);
97
+ if (existing !== void 0) {
98
+ held.delete(ref.key);
99
+ list.push({ segment: existing, isNew: false });
100
+ continue;
101
+ }
102
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
103
+ list.push({ segment: createPagedFrozenSegment(snapshot), isNew: true });
225
104
  }
105
+ droppedKeys.push(...held.keys());
106
+ await applyNewerTombstonesInTurns(list);
107
+ prepared.push({ partitionId: entry.partitionId, prepared: list });
226
108
  }
227
- return results;
228
- }
229
-
230
- // src/persistence/durability/durable-filesystem.ts
231
- function validateRelativeKey(key) {
232
- if (key.length === 0) {
233
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
234
- }
235
- if (key.includes("\0")) {
236
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
237
- }
238
- }
239
- function wrapFsyncError(err, key) {
240
- throw new NarsilError(
241
- ErrorCodes.PERSISTENCE_FSYNC_FAILED,
242
- `fsync failed for "${key}"; the write is not acknowledged and the node must recover from the durable log`,
243
- { key, cause: err instanceof Error ? err.message : String(err) }
244
- );
109
+ return { partitions: prepared, droppedKeys };
245
110
  }
246
- async function openReadWriteCreating(filePath, fs) {
247
- try {
248
- return await fs.open(filePath, "r+");
249
- } catch (err) {
250
- if (err.code !== "ENOENT") {
251
- throw err;
252
- }
253
- }
254
- try {
255
- return await fs.open(filePath, "wx+");
256
- } catch (err) {
257
- if (err.code !== "EEXIST") {
258
- throw err;
111
+ function keysThatSegmentsStillRead(manager) {
112
+ const keys = /* @__PURE__ */ new Set();
113
+ for (const partition of manager.getAllPartitions()) {
114
+ if (!isCompositePartition(partition)) continue;
115
+ for (const segment of partition.frozenSegments()) {
116
+ if (segment.paged !== null) keys.add(segment.paged.key);
259
117
  }
260
- return fs.open(filePath, "r+");
261
118
  }
119
+ return keys;
262
120
  }
263
- async function writeFully(handle, buffer, key) {
264
- let offset = 0;
265
- while (offset < buffer.length) {
266
- let bytesWritten;
267
- try {
268
- ;
269
- ({ bytesWritten } = await handle.write(buffer, offset, buffer.length - offset));
270
- } catch (err) {
271
- throw new NarsilError(
272
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
273
- `Write to "${key}" failed; the write is not acknowledged`,
274
- {
275
- key,
276
- written: offset,
277
- total: buffer.length,
278
- cause: err instanceof Error ? err.message : String(err)
279
- }
280
- );
281
- }
282
- if (bytesWritten <= 0) {
283
- throw new NarsilError(
284
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
285
- `Write to "${key}" made no progress; the file would be truncated and the write is not acknowledged`,
286
- { key, written: offset, total: buffer.length }
287
- );
121
+ function closeUnusedSegments(adoption) {
122
+ for (const { prepared } of adoption.partitions) {
123
+ for (const { segment, isNew } of prepared) {
124
+ if (isNew) segment.paged?.close();
288
125
  }
289
- offset += bytesWritten;
290
126
  }
291
127
  }
292
- async function fsyncDirectory(dirPath, fs) {
293
- let handle = null;
128
+ function actionPartitionsOf(adoption) {
129
+ return adoption.partitions.map(({ partitionId, prepared }) => ({
130
+ partitionId,
131
+ segments: prepared.map(({ segment, isNew }) => {
132
+ const access = segment.paged;
133
+ if (access !== null && isNew) return { snapshot: access.snapshot };
134
+ return { segmentId: segment.segmentId };
135
+ })
136
+ }));
137
+ }
138
+ function swapPreparedPartitions(manager, partitions, changed, onSwapped) {
139
+ for (const { partitionId, prepared } of partitions) {
140
+ const kept = [];
141
+ for (const { segment } of prepared) {
142
+ if (segment.arrays.documentCount > 0) kept.push(segment);
143
+ else segment.paged?.close();
144
+ }
145
+ adoptDiskSegments(manager, partitionId, kept, changed);
146
+ onSwapped?.(partitionId, kept);
147
+ }
148
+ }
149
+ function adoptDiskSegments(manager, partitionId, segments, changed) {
150
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
151
+ const before = composite.frozenSegments();
152
+ const kept = [];
153
+ for (const docId of changed) {
154
+ if (docPartitionMap.get(docId) !== partitionId) continue;
155
+ const document = composite.getRef(docId);
156
+ if (document !== void 0) kept.push([docId, document]);
157
+ }
158
+ const dropped = before.filter((segment) => !segments.includes(segment));
159
+ for (const segment of segments) {
160
+ if (before.includes(segment)) continue;
161
+ for (const docId of changed) segment.tombstoneDocument(docId);
162
+ }
163
+ composite.replaceFrozenSegments(segments);
164
+ composite.live.clear();
165
+ for (const [docId, pid] of docPartitionMap) {
166
+ if (pid === partitionId) docPartitionMap.delete(docId);
167
+ }
168
+ composite.live.beginBatch();
294
169
  try {
295
- handle = await fs.open(dirPath, "r");
296
- await handle.sync();
297
- } catch (err) {
298
- const code = err.code;
299
- if (code === "EISDIR" || code === "EPERM" || code === "EINVAL" || code === "ENOTSUP") {
300
- return;
170
+ for (const [docId, document] of kept) {
171
+ composite.live.insert(docId, document, manager.config.schema, manager.language, insertOptions);
172
+ docPartitionMap.set(docId, partitionId);
301
173
  }
302
- wrapFsyncError(err, dirPath);
303
174
  } finally {
304
- if (handle !== null) {
305
- await handle.close().catch(() => void 0);
306
- }
307
- }
308
- }
309
- function createDurableDirectory(root) {
310
- if (root.trim().length === 0) {
311
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Durable directory requires a non-empty root path");
312
- }
313
- async function resolve(key) {
314
- validateRelativeKey(key);
315
- const pathMod = await getPath();
316
- const resolvedBase = pathMod.resolve(root);
317
- const resolvedPath = pathMod.resolve(root, key);
318
- if (resolvedPath !== resolvedBase && !resolvedPath.startsWith(resolvedBase + pathMod.sep)) {
319
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: path traversal detected", { key });
320
- }
321
- return { path: resolvedPath, resolvedBase, pathMod };
322
- }
323
- async function ensureDir(filePath, pathMod, fs) {
324
- await fs.mkdir(pathMod.dirname(filePath), { recursive: true });
325
- }
326
- return {
327
- get root() {
328
- return root;
329
- },
330
- lock() {
331
- return lockDirectory(root);
332
- },
333
- async identity() {
334
- const fs = await getFs();
335
- const pathMod = await getPath();
336
- try {
337
- const stat = await fs.stat(pathMod.resolve(root), { bigint: true });
338
- return `${stat.dev}:${stat.ino}`;
339
- } catch (err) {
340
- if (err.code === "ENOENT") return null;
341
- throw err;
342
- }
343
- },
344
- async appendHandle(key) {
345
- const { path: filePath, pathMod } = await resolve(key);
346
- const fs = await getFs();
347
- await ensureDir(filePath, pathMod, fs);
348
- const handle = await fs.open(filePath, "a");
349
- return {
350
- async append(bytes) {
351
- await writeFully(handle, bytes, key);
352
- },
353
- async sync() {
354
- try {
355
- await handle.sync();
356
- } catch (err) {
357
- wrapFsyncError(err, key);
358
- }
359
- },
360
- async truncate(byteLength) {
361
- await handle.truncate(byteLength);
362
- try {
363
- await handle.sync();
364
- } catch (err) {
365
- wrapFsyncError(err, key);
366
- }
367
- },
368
- async size() {
369
- const stat = await handle.stat();
370
- return stat.size;
371
- },
372
- async close() {
373
- await handle.close();
374
- }
375
- };
376
- },
377
- async markerHandle(key) {
378
- const { path: filePath, pathMod } = await resolve(key);
379
- const fs = await getFs();
380
- await ensureDir(filePath, pathMod, fs);
381
- const handle = await openReadWriteCreating(filePath, fs);
382
- return {
383
- async read() {
384
- const stat = await handle.stat();
385
- if (stat.size === 0) {
386
- return new Uint8Array(0);
387
- }
388
- const bytes = new Uint8Array(stat.size);
389
- await handle.read(bytes, 0, stat.size, 0);
390
- return bytes;
391
- },
392
- async writeSlot(offset, slot) {
393
- try {
394
- await handle.write(slot, 0, slot.length, offset);
395
- } catch (err) {
396
- throw new NarsilError(
397
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
398
- `Write to "${key}" failed; the write is not acknowledged`,
399
- { key, cause: err instanceof Error ? err.message : String(err) }
400
- );
401
- }
402
- try {
403
- await handle.sync();
404
- } catch (err) {
405
- wrapFsyncError(err, key);
406
- }
407
- },
408
- async close() {
409
- await handle.close();
410
- }
411
- };
412
- },
413
- async syncDirectoryOf(key) {
414
- const { path: filePath, pathMod } = await resolve(key);
415
- const fs = await getFs();
416
- await fsyncDirectory(pathMod.dirname(filePath), fs);
417
- },
418
- async atomicWrite(key, data) {
419
- const { path: filePath, pathMod } = await resolve(key);
420
- const fs = await getFs();
421
- await ensureDir(filePath, pathMod, fs);
422
- const dir = pathMod.dirname(filePath);
423
- const tempName = `${pathMod.basename(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
424
- const tempPath = pathMod.join(dir, tempName);
425
- const buffers = Array.isArray(data) ? data : [data];
426
- let handle = null;
427
- try {
428
- handle = await fs.open(tempPath, "w");
429
- for (const buffer of buffers) {
430
- await writeFully(handle, buffer, tempPath);
431
- }
432
- try {
433
- await handle.sync();
434
- } catch (err) {
435
- wrapFsyncError(err, tempPath);
436
- }
437
- await handle.close();
438
- handle = null;
439
- await fs.rename(tempPath, filePath);
440
- await fsyncDirectory(dir, fs);
441
- } catch (err) {
442
- if (handle !== null) {
443
- await handle.close().catch(() => void 0);
444
- }
445
- await fs.unlink(tempPath).catch(() => void 0);
446
- throw err;
447
- }
448
- },
449
- async read(key) {
450
- const { path: filePath } = await resolve(key);
451
- const fs = await getFs();
452
- try {
453
- const data = await fs.readFile(filePath);
454
- return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
455
- } catch (err) {
456
- if (err.code === "ENOENT") {
457
- return null;
458
- }
459
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read "${key}"`, {
460
- key,
461
- cause: err instanceof Error ? err.message : String(err)
462
- });
463
- }
464
- },
465
- async readRange(key, offset, length) {
466
- const { path: filePath } = await resolve(key);
467
- const fs = await getFs();
468
- let handle = null;
469
- try {
470
- handle = await fs.open(filePath, "r");
471
- const bytes = new Uint8Array(length);
472
- let filled = 0;
473
- while (filled < length) {
474
- const { bytesRead } = await handle.read(bytes, filled, length - filled, offset + filled);
475
- if (bytesRead === 0) break;
476
- filled += bytesRead;
477
- }
478
- if (filled < length) {
479
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
480
- key,
481
- offset,
482
- length
483
- });
484
- }
485
- return bytes;
486
- } finally {
487
- await handle?.close();
488
- }
489
- },
490
- async byteLength(key) {
491
- const { path: filePath } = await resolve(key);
492
- const fs = await getFs();
493
- try {
494
- return (await fs.stat(filePath)).size;
495
- } catch (err) {
496
- if (err.code === "ENOENT") {
497
- return null;
498
- }
499
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read the length of "${key}"`, {
500
- key,
501
- cause: err instanceof Error ? err.message : String(err)
502
- });
503
- }
504
- },
505
- async remove(key) {
506
- const { path: filePath, pathMod } = await resolve(key);
507
- const fs = await getFs();
508
- try {
509
- await fs.unlink(filePath);
510
- } catch (err) {
511
- if (err.code === "ENOENT") {
512
- return;
513
- }
514
- throw new NarsilError(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, {
515
- key,
516
- cause: err instanceof Error ? err.message : String(err)
517
- });
175
+ composite.live.endBatch();
176
+ }
177
+ for (const segment of dropped) segment.paged?.close();
178
+ }
179
+ function adoptSegmentsOnWorker(manager, action) {
180
+ const partitions = action.partitions.map(({ partitionId, segments }) => {
181
+ const partition = manager.getPartition(partitionId);
182
+ const held = isCompositePartition(partition) ? partition.frozenSegments() : [];
183
+ const prepared = segments.map((item) => {
184
+ if ("snapshot" in item) return { segment: createPagedFrozenSegment(item.snapshot), isNew: true };
185
+ const existing = held.find((segment) => segment.segmentId === item.segmentId);
186
+ if (existing === void 0) {
187
+ throw new NarsilError(
188
+ ErrorCodes.PARTITION_CORRUPTED,
189
+ `Partition ${partitionId} of "${action.indexName}" holds no segment "${item.segmentId}" to keep`,
190
+ { indexName: action.indexName, partitionId, segmentId: item.segmentId }
191
+ );
518
192
  }
519
- await fsyncDirectory(pathMod.dirname(filePath), fs);
520
- },
521
- async move(from, to) {
522
- const source = await resolve(from);
523
- const target = await resolve(to);
524
- const fs = await getFs();
525
- await ensureDir(target.path, target.pathMod, fs);
526
- await fs.rename(source.path, target.path);
527
- const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
528
- for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
529
- },
530
- async list(prefix) {
531
- const fs = await getFs();
532
- const pathMod = await getPath();
533
- const resolvedBase = pathMod.resolve(root);
534
- const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
535
- const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
536
- const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
537
- return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
538
- },
539
- async pathOf(key) {
540
- const { path: filePath } = await resolve(key);
541
- return filePath;
542
- }
543
- };
193
+ return { segment: existing, isNew: false };
194
+ });
195
+ applyNewerTombstones(prepared);
196
+ return { partitionId, prepared };
197
+ });
198
+ swapPreparedPartitions(manager, partitions, new Set(action.changed));
544
199
  }
545
200
 
546
201
  // src/persistence/durability/sync/layout.ts
@@ -654,6 +309,7 @@ async function reclaimWalBeyondCount(directory, indexName, partitionCount, parti
654
309
  await partition.walWriter.close();
655
310
  } catch (err) {
656
311
  onCloseError(err instanceof Error ? err : new Error(String(err)));
312
+ throw err;
657
313
  }
658
314
  partitions.delete(partitionId);
659
315
  }
@@ -692,136 +348,6 @@ function formatSegmentStart(startSeqNo) {
692
348
  return startSeqNo.toString().padStart(16, "0");
693
349
  }
694
350
 
695
- // src/persistence/durability/marker-writer.ts
696
- function markerKey(indexName, partitionId) {
697
- return `${indexName}/wal/${partitionId}/commit`;
698
- }
699
- async function createMarkerWriter(directory, indexName, partitionId) {
700
- const key = markerKey(indexName, partitionId);
701
- const handle = await directory.markerHandle(key);
702
- const existing = await handle.read();
703
- const current = readCommitMarker(existing);
704
- const createdFresh = existing.length === 0;
705
- let lastSlot = current?.slotIndex ?? null;
706
- let writeSeq = current?.state.writeSeq ?? 0;
707
- return {
708
- created: createdFresh,
709
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
710
- async commit(state) {
711
- const slotIndex = nextMarkerSlot(lastSlot);
712
- writeSeq += 1;
713
- const slot = encodeMarkerSlot({ ...state, writeSeq });
714
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
715
- lastSlot = slotIndex;
716
- },
717
- async close() {
718
- await handle.close();
719
- }
720
- };
721
- }
722
-
723
- // src/persistence/durability/recovery.ts
724
- async function removeLeftoverRewriteFiles(directory, names) {
725
- for (const indexName of names) {
726
- for (const place of ["rebuilt", "received"]) {
727
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
728
- }
729
- }
730
- }
731
- async function listPersistedIndexes(directory) {
732
- const metaKeys = await directory.list("");
733
- const names = [];
734
- for (const key of metaKeys) {
735
- if (key.endsWith("/meta")) {
736
- names.push(key.slice(0, -"/meta".length));
737
- }
738
- }
739
- return names;
740
- }
741
- async function loadMetadata(directory, indexName) {
742
- const bytes = await directory.read(`${indexName}/meta`);
743
- if (bytes === null) {
744
- return null;
745
- }
746
- const { metadata } = await readMetadataEnvelope(bytes);
747
- return metadata;
748
- }
749
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
750
- await directory.remove(nextSegmentKey(indexName, partitionId));
751
- const read = await readWalSegments(directory, indexName, partitionId);
752
- if (read === null) {
753
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
754
- }
755
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
756
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
757
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
758
- }
759
- let highestSeqNo = fromSeqNoExclusive;
760
- let highestPrimaryTerm = 0;
761
- const replay = async (entries) => {
762
- for (const entry of entries) {
763
- if (entry.seqNo <= highestSeqNo) {
764
- continue;
765
- }
766
- if (turnIsSpent()) await yieldTurn();
767
- applyEntry(entry, deps);
768
- highestSeqNo = entry.seqNo;
769
- if (entry.primaryTerm > highestPrimaryTerm) {
770
- highestPrimaryTerm = entry.primaryTerm;
771
- }
772
- }
773
- };
774
- await replay(durableEntries);
775
- if (activeTail !== null) {
776
- await replay(activeTail.entries);
777
- }
778
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
779
- if (activeTail !== null) {
780
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
781
- }
782
- return { highestSeqNo, highestPrimaryTerm };
783
- }
784
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
785
- const lastReplayed = tail.entries.at(-1);
786
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
787
- const handle = await directory.appendHandle(tail.key);
788
- try {
789
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
790
- else await handle.sync();
791
- } finally {
792
- await handle.close();
793
- }
794
- if (lastReplayed === void 0) return;
795
- const writer = await createMarkerWriter(directory, indexName, partitionId);
796
- try {
797
- await writer.commit({
798
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
799
- durableByteLength: tail.cleanEnd,
800
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
801
- });
802
- } finally {
803
- await writer.close();
804
- }
805
- }
806
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
807
- for (const { key, startSeqNo } of segments) {
808
- if (startSeqNo > activeSegmentSeqNo) {
809
- await directory.remove(key);
810
- }
811
- }
812
- }
813
- function applyEntry(entry, deps) {
814
- deps.writeRecords?.replayed(entry);
815
- if (entry.operation === "DELETE") {
816
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
817
- return;
818
- }
819
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
820
- }
821
- function snapshotCheckpointFor(checkpoint, partitionId) {
822
- return checkpointLastSeqNo(checkpoint, partitionId);
823
- }
824
-
825
351
  // src/persistence/durability/checkpoint-capture.ts
826
352
  function whileNoMutationApplies(partitions, capture2) {
827
353
  return new Promise((resolve, reject) => {
@@ -829,9 +355,9 @@ function whileNoMutationApplies(partitions, capture2) {
829
355
  const released = new Promise((settle) => {
830
356
  release = settle;
831
357
  });
832
- function captureNow() {
358
+ async function captureNow() {
833
359
  try {
834
- resolve(capture2());
360
+ resolve(await capture2());
835
361
  } catch (error) {
836
362
  reject(error);
837
363
  } finally {
@@ -840,13 +366,13 @@ function whileNoMutationApplies(partitions, capture2) {
840
366
  }
841
367
  let waiting = partitions.length;
842
368
  if (waiting === 0) {
843
- captureNow();
369
+ void captureNow();
844
370
  return;
845
371
  }
846
372
  for (const partition of partitions) {
847
373
  partition.appendChain = partition.appendChain.then(() => {
848
374
  waiting -= 1;
849
- if (waiting === 0) captureNow();
375
+ if (waiting === 0) void captureNow();
850
376
  return released;
851
377
  });
852
378
  }
@@ -886,9 +412,11 @@ function noWholePartitions() {
886
412
  return { contents: [], captured: [] };
887
413
  }
888
414
  function wholePartitionsWhereMostChanged(manager, rules) {
889
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
415
+ const { priorCheckpoint, priorPartitionIds, aWorkerCanSerialise, captureRecords } = rules;
890
416
  return (targets) => {
891
417
  const whole = { contents: [], captured: [] };
418
+ const everyPartition = rules.everyPartition || requiresLayoutCheckpoint(manager) || priorPartitionIds.length > 0 && priorPartitionIds.length !== manager.partitionCount;
419
+ if (rules.usesLogForUnchangedLayout && !everyPartition) return whole;
892
420
  const changed = targets.filter(
893
421
  (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
894
422
  );
@@ -911,45 +439,57 @@ function wholePartitionsWhereMostChanged(manager, rules) {
911
439
  return whole;
912
440
  };
913
441
  }
914
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
442
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
915
443
  for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
916
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
917
- onCaptured?.();
918
- const documentCount = manager.countDocuments();
919
- const targets = [];
920
- let recordsLeftInLog = 0;
921
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
922
- const partition = indexState.partitions.get(partitionId);
923
- if (partition !== void 0 && partition.failed !== null) continue;
924
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
925
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
926
- appliedSeqNo,
927
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
928
- );
929
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
930
- const target = {
931
- partitionId,
932
- lastSeqNo,
933
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
934
- };
935
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
936
- targets.push(target);
937
- }
938
- const vectorPlans = /* @__PURE__ */ new Map();
939
- try {
940
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
941
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
942
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
943
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
444
+ return whileNoMutationApplies(
445
+ [...indexState.partitions.values()],
446
+ () => captureAppliedPrefix(manager, (cutoffs) => {
447
+ onCaptured?.();
448
+ const partitionCount = manager.partitionCount;
449
+ const previous = priorCheckpointThatLimitsTheRecords;
450
+ const changedLayout = requiresLayoutCheckpoint(manager) || previous !== void 0 && previous.length > 0 && previous.length !== partitionCount;
451
+ const bounded = changedLayout ? void 0 : previous;
452
+ const documentCount = manager.countDocuments();
453
+ const targets = [];
454
+ let recordsLeftInLog = 0;
455
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
456
+ const partition = indexState.partitions.get(partitionId);
457
+ if (partition !== void 0 && partition.failed !== null) continue;
458
+ const appliedSeqNo = Math.min(partition?.appliedSeqNo ?? 0, cutoffs.get(partitionId) ?? Infinity);
459
+ const lastSeqNo = bounded === void 0 ? appliedSeqNo : Math.min(appliedSeqNo, snapshotCheckpointFor(bounded, partitionId) + LOG_RECORDS_PER_CHECKPOINT);
460
+ recordsLeftInLog += appliedSeqNo - lastSeqNo;
461
+ const target = {
462
+ partitionId,
463
+ lastSeqNo,
464
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
465
+ };
466
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
467
+ targets.push(target);
944
468
  }
945
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
946
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
947
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
948
- } catch (error) {
949
- for (const plan of vectorPlans.values()) plan.release();
950
- throw error;
951
- }
952
- });
469
+ const vectorPlans = /* @__PURE__ */ new Map();
470
+ try {
471
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
472
+ const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
473
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
474
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, true));
475
+ }
476
+ const memoryMatchesTheTargets = recordsLeftInLog === 0;
477
+ const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
478
+ return {
479
+ partitionCount,
480
+ targets,
481
+ documentCount,
482
+ vectorPlans,
483
+ wholePartitions,
484
+ recordsLeftInLog,
485
+ unreplayedCutoffs: cutoffs
486
+ };
487
+ } catch (error) {
488
+ for (const plan of vectorPlans.values()) plan.release();
489
+ throw error;
490
+ }
491
+ })
492
+ );
953
493
  }
954
494
  async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
955
495
  if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
@@ -969,8 +509,22 @@ async function writeCapturedVectors(store, indexName, capture2, priorManifest, r
969
509
  return { vectors: written.refs, layouts: written.layouts, written: written.written };
970
510
  }
971
511
 
972
- // src/persistence/durability/checkpoint-worker-entry.ts
973
- function resolveCheckpointWorkerEntry() {
512
+ // src/persistence/durability/checkpoint-prior.ts
513
+ function priorForLayout(manifest, partitionCount) {
514
+ const checkpoints = new Map(manifest?.checkpoint.map((entry) => [entry.partitionId, entry]));
515
+ const partitions = new Map(manifest?.partitions.map((entry) => [entry.partitionId, entry]));
516
+ const prior = { checkpoint: [], partitions: [] };
517
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
518
+ prior.checkpoint.push(
519
+ checkpoints.get(partitionId) ?? { partitionId, lastSeqNo: 0, primaryTerm: SINGLE_NODE_PRIMARY_TERM }
520
+ );
521
+ prior.partitions.push(partitions.get(partitionId) ?? { partitionId, nextSegmentId: 0, segments: [] });
522
+ }
523
+ return prior;
524
+ }
525
+
526
+ // src/persistence/durability/checkpoint-worker-entry.ts
527
+ function resolveCheckpointWorkerEntry() {
974
528
  return resolveWorkerEntry(
975
529
  import.meta.url,
976
530
  /\/src\/persistence\/durability\/[^/]+$/,
@@ -1281,13 +835,15 @@ async function writeOneBoundedCheckpoint(input) {
1281
835
  const store = directoryCheckpointStore(input.directory);
1282
836
  const writeRecords = input.hooks.writeRecords ?? null;
1283
837
  const { indexName } = input;
1284
- const priorManifest = await readCurrentManifest(store, input.indexName);
838
+ const storedManifest = await readCurrentManifest(store, input.indexName);
839
+ const priorManifest = storedManifest === null ? null : manifestWithinPartitionCount(storedManifest, manager.partitionCount);
1285
840
  const textOnDisk = metadata.textStorage === "disk";
1286
841
  const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1287
842
  priorCheckpoint: priorManifest?.checkpoint ?? [],
1288
843
  priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1289
844
  everyPartition: fromMemory,
1290
845
  aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
846
+ usesLogForUnchangedLayout: textOnDisk,
1291
847
  ...writeRecords === null ? {} : {
1292
848
  captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1293
849
  }
@@ -1297,20 +853,28 @@ async function writeOneBoundedCheckpoint(input) {
1297
853
  input.indexState,
1298
854
  manager,
1299
855
  vectorIndexes,
1300
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
856
+ wholePartitionsOf,
1301
857
  fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1302
858
  priorManifest?.vectors ?? [],
1303
859
  writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1304
860
  textOnDisk ? () => {
1305
861
  changesSinceCapture = manager.openCheckpointChangeLog();
1306
- } : void 0,
1307
- input.pinsRetiredVectors
862
+ } : void 0
1308
863
  );
1309
864
  try {
1310
865
  await openNewLogSegments(input.indexState, input.failPartition);
1311
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1312
- return changesSinceCapture;
1313
- });
866
+ return await finishBoundedCheckpoint(
867
+ input,
868
+ manager,
869
+ store,
870
+ { ...metadata, partitionCount: capture2.partitionCount },
871
+ priorManifest,
872
+ capture2,
873
+ fromMemory,
874
+ () => {
875
+ return changesSinceCapture;
876
+ }
877
+ );
1314
878
  } finally {
1315
879
  for (const plan of capture2.vectorPlans.values()) plan.release();
1316
880
  changesSinceCapture?.close();
@@ -1318,24 +882,26 @@ async function writeOneBoundedCheckpoint(input) {
1318
882
  }
1319
883
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1320
884
  const { indexName, indexState, hooks } = input;
1321
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-NIYMEA5R.mjs');
1322
885
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1323
886
  if (adoption === null) return [];
1324
- const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1325
- if (changes.partitionsReplaced) return false;
1326
- const changed = changes.takeWrittenDocIds();
1327
- swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1328
- hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1329
- });
1330
- hooks.queueDiskAdoption?.({
1331
- type: "adoptDiskSegments",
1332
- indexName,
1333
- partitions: actionPartitionsOf(adoption),
1334
- changed: [...changed],
1335
- requestId: createRequestId()
1336
- });
1337
- return true;
1338
- });
887
+ const adopted = await whileNoMutationApplies(
888
+ [...indexState.partitions.values()],
889
+ () => manager.publication.write(() => {
890
+ if (changes.partitionsReplaced) return false;
891
+ const changed = changes.takeWrittenDocIds();
892
+ swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
893
+ hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
894
+ });
895
+ hooks.queueDiskAdoption?.({
896
+ type: "adoptDiskSegments",
897
+ indexName,
898
+ partitions: actionPartitionsOf(adoption),
899
+ changed: [...changed],
900
+ requestId: createRequestId()
901
+ });
902
+ return true;
903
+ })
904
+ );
1339
905
  if (!adopted) {
1340
906
  closeUnusedSegments(adoption);
1341
907
  return [];
@@ -1357,7 +923,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1357
923
  targets,
1358
924
  compactionThreshold: input.compactionThreshold,
1359
925
  canOffload: input.canOffload,
1360
- prior: priorManifest,
926
+ prior: priorForLayout(priorManifest, capture2.partitionCount),
1361
927
  wholePartitions: capture2.wholePartitions.contents,
1362
928
  capturedPartitions: capture2.wholePartitions.captured,
1363
929
  keepsWriteRecords: writeRecords !== null
@@ -1373,7 +939,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1373
939
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1374
940
  if (!input.holdsCheckpointFiles()) {
1375
941
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1376
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-NIYMEA5R.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
942
+ const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1377
943
  const keptForSync = await input.keepsCheckpointKey();
1378
944
  await removeCheckpointGarbage(
1379
945
  store,
@@ -1386,9 +952,12 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1386
952
  if (checkpointMetadata === void 0) {
1387
953
  return;
1388
954
  }
1389
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
955
+ const bytes = await writeMetadataEnvelope(
956
+ { ...checkpointMetadata, partitionCount: capture2.partitionCount },
957
+ { checksum: true }
958
+ );
1390
959
  await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1391
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
960
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
1392
961
  });
1393
962
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1394
963
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
@@ -1396,69 +965,36 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1396
965
  await reclaimWalBeyondCount(
1397
966
  input.directory,
1398
967
  input.indexName,
1399
- manager.partitionCount,
968
+ capture2.partitionCount,
1400
969
  input.indexState.partitions,
1401
970
  input.markFatal
1402
971
  );
972
+ await input.directory.remove(transitionKey(input.indexName));
1403
973
  const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1404
974
  noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1405
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1406
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
975
+ const eligibleBeyond = recordsBeyond(input.indexState, targets, capture2.unreplayedCutoffs);
976
+ const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
977
+ return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
1407
978
  }
1408
- function recordsBeyond(indexState, targets) {
979
+ function recordsBeyond(indexState, targets, cutoffs) {
1409
980
  let records = 0;
1410
981
  for (const [partitionId, partition] of indexState.partitions) {
1411
982
  if (partition.failed !== null) continue;
1412
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
983
+ records += Math.max(
984
+ 0,
985
+ Math.min(partition.appliedSeqNo, cutoffs?.get(partitionId) ?? Infinity) - snapshotCheckpointFor(targets, partitionId)
986
+ );
1413
987
  }
1414
988
  return records;
1415
989
  }
1416
990
 
1417
- // src/persistence/durability/checkpoint-turn.ts
1418
- function createIndexLoads(vectorIndexesOf2) {
1419
- const loads = /* @__PURE__ */ new Map();
1420
- const startGraphRepairs = (indexName) => {
1421
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1422
- vectorIndex.scheduleBuild();
1423
- vectorIndex.startShuffledRebuildWhenDue();
1424
- }
1425
- };
1426
- return {
1427
- track(indexName, load, startsGraphRepairs = true) {
1428
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1429
- loads.set(
1430
- indexName,
1431
- loaded.catch(() => void 0)
1432
- );
1433
- return loaded;
1434
- },
1435
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1436
- startGraphRepairs
1437
- };
991
+ // src/persistence/durability/deferred-mutations.ts
992
+ var deferred = /* @__PURE__ */ new WeakSet();
993
+ function deferMutationPublication(record) {
994
+ deferred.add(record);
1438
995
  }
1439
- async function switchInTurn(turns, turn) {
1440
- let swapped = false;
1441
- await checkpointInTurn(turns, {
1442
- joinsInFlight: false,
1443
- mayStart: turn.mayStart,
1444
- perform: async () => {
1445
- swapped = turn.swap();
1446
- if (swapped) await turn.checkpoint();
1447
- },
1448
- afterSettling: turn.afterSettling
1449
- });
1450
- return swapped;
1451
- }
1452
- async function checkpointInTurn(turns, turn) {
1453
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1454
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1455
- if (!turn.mayStart()) return;
1456
- const run = turn.perform().finally(() => {
1457
- turns.checkpointInFlight = null;
1458
- turn.afterSettling();
1459
- });
1460
- turns.checkpointInFlight = run;
1461
- return run;
996
+ function hasDeferredPublication(record) {
997
+ return deferred.has(record);
1462
998
  }
1463
999
 
1464
1000
  // src/persistence/durability/held-checkpoints.ts
@@ -1516,24 +1052,6 @@ function createHeldCheckpoints(hooks) {
1516
1052
  };
1517
1053
  }
1518
1054
 
1519
- // src/persistence/durability/manager-state.ts
1520
- function closePartitionWriter(partition) {
1521
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1522
- }
1523
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1524
- indexState.unloading = true;
1525
- try {
1526
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1527
- await pendingMetadataWrite;
1528
- for (const partition of indexState.partitions.values()) {
1529
- await closeWalWriter(closePartitionWriter(partition));
1530
- }
1531
- } catch (error) {
1532
- indexState.unloading = false;
1533
- throw error;
1534
- }
1535
- }
1536
-
1537
1055
  // src/persistence/durability/restore-journal.ts
1538
1056
  var RESTORE_JOURNAL_PREFIX = ".swap-";
1539
1057
  var RESTORE_STAGING_PREFIX = ".restore-";
@@ -2174,6 +1692,8 @@ function createFilePins(remove) {
2174
1692
  }
2175
1693
  };
2176
1694
  }
1695
+
1696
+ // src/persistence/durability/sync/published.ts
2177
1697
  var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2178
1698
  var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2179
1699
  function hexOf(bytes) {
@@ -2842,13 +2362,13 @@ function createWalWriter(directory, config) {
2842
2362
  for (const frame of frames) {
2843
2363
  totalBytes += frame.length;
2844
2364
  }
2845
- const joined = new Uint8Array(totalBytes);
2365
+ const joined2 = new Uint8Array(totalBytes);
2846
2366
  let offset = 0;
2847
2367
  for (const frame of frames) {
2848
- joined.set(frame, offset);
2368
+ joined2.set(frame, offset);
2849
2369
  offset += frame.length;
2850
2370
  }
2851
- return joined;
2371
+ return joined2;
2852
2372
  }
2853
2373
  function framesThatFit(frames, start) {
2854
2374
  const room = segmentMaxBytes - activeBytes;
@@ -2869,16 +2389,16 @@ function createWalWriter(directory, config) {
2869
2389
  { indexName: config.indexName, partitionId: config.partitionId }
2870
2390
  );
2871
2391
  }
2872
- const joined = joinFrames([...frames]);
2873
- if (activeBytes + joined.length > MAX_FILE_BYTES) {
2392
+ const joined2 = joinFrames([...frames]);
2393
+ if (activeBytes + joined2.length > MAX_FILE_BYTES) {
2874
2394
  throw new NarsilError(
2875
2395
  ErrorCodes.PERSISTENCE_SAVE_FAILED,
2876
- `A log record of ${joined.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
2877
- { indexName: config.indexName, partitionId: config.partitionId, bytes: joined.length }
2396
+ `A log record of ${joined2.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
2397
+ { indexName: config.indexName, partitionId: config.partitionId, bytes: joined2.length }
2878
2398
  );
2879
2399
  }
2880
- await activeHandle.append(joined);
2881
- activeBytes += joined.length;
2400
+ await activeHandle.append(joined2);
2401
+ activeBytes += joined2.length;
2882
2402
  const lastSeqNo = entries[entries.length - 1].seqNo;
2883
2403
  if (lastSeqNo > highestAppendedSeqNo) {
2884
2404
  highestAppendedSeqNo = lastSeqNo;
@@ -2999,6 +2519,109 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
2999
2519
  }
3000
2520
  };
3001
2521
  }
2522
+
2523
+ // src/serialization/payload-v2.ts
2524
+ function validateHnswMetric(value) {
2525
+ if (value === void 0 || value === null) return void 0;
2526
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
2527
+ return value;
2528
+ }
2529
+ return void 0;
2530
+ }
2531
+ function storedDocumentsOf(raw) {
2532
+ requireValidFieldTypes(raw.schema ?? {});
2533
+ const documents = {};
2534
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
2535
+ documents[docId] = {
2536
+ fields: doc.fields,
2537
+ fieldLengths: doc.field_lengths ?? {}
2538
+ };
2539
+ }
2540
+ dropStoredVectorValues(documents, raw.schema ?? {});
2541
+ return documents;
2542
+ }
2543
+ function documentsOfPayloadV2(data) {
2544
+ const raw = decode(data);
2545
+ return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
2546
+ }
2547
+ function deserializePayloadV2(data) {
2548
+ const raw = decode(data);
2549
+ const documents = storedDocumentsOf(raw);
2550
+ const fieldNames = raw.inverted_index?.field_names ?? [];
2551
+ const invertedIndex = {};
2552
+ for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
2553
+ const count = list.ids.length;
2554
+ const postings = new Array(count);
2555
+ for (let i = 0; i < count; i++) {
2556
+ postings[i] = {
2557
+ docId: list.ids[i],
2558
+ termFrequency: list.tf[i],
2559
+ field: fieldNames[list.fi[i]] ?? "",
2560
+ positions: list.pos ? list.pos[i] : []
2561
+ };
2562
+ }
2563
+ invertedIndex[token] = {
2564
+ docFrequency: list.df,
2565
+ postings
2566
+ };
2567
+ }
2568
+ const numeric = {};
2569
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
2570
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
2571
+ }
2572
+ const boolean = {};
2573
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
2574
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
2575
+ }
2576
+ const geopoint = {};
2577
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
2578
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
2579
+ }
2580
+ const vectorData = {};
2581
+ for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
2582
+ vectorData[field] = {
2583
+ dimension: data2.dimension,
2584
+ vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
2585
+ hnswGraph: data2.hnsw_graph ? {
2586
+ entryPoint: data2.hnsw_graph.entry_point,
2587
+ maxLayer: data2.hnsw_graph.max_layer,
2588
+ m: data2.hnsw_graph.m,
2589
+ efConstruction: data2.hnsw_graph.ef_construction,
2590
+ metric: validateHnswMetric(data2.hnsw_graph.metric),
2591
+ nodes: data2.hnsw_graph.nodes
2592
+ } : null,
2593
+ codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
2594
+ };
2595
+ }
2596
+ return {
2597
+ indexName: raw.index_name,
2598
+ partitionId: raw.partition_id,
2599
+ totalPartitions: raw.total_partitions,
2600
+ language: raw.language ?? "english",
2601
+ schema: raw.schema ?? {},
2602
+ docCount: raw.doc_count ?? 0,
2603
+ avgDocLength: raw.avg_doc_length ?? 0,
2604
+ documents,
2605
+ invertedIndex,
2606
+ fieldIndexes: {
2607
+ numeric,
2608
+ boolean,
2609
+ enum: raw.field_indexes?.enum ?? {},
2610
+ geopoint,
2611
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
2612
+ },
2613
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
2614
+ vectorData,
2615
+ statistics: {
2616
+ totalDocuments: raw.statistics?.total_documents ?? 0,
2617
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
2618
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
2619
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
2620
+ }
2621
+ };
2622
+ }
2623
+
2624
+ // src/persistence/durability/earlier-layout/bundle.ts
3002
2625
  var SNAPSHOT_BUNDLE_VERSION = 2;
3003
2626
  function invalid(message) {
3004
2627
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
@@ -3058,7 +2681,7 @@ async function loadSnapshotBundle(bytes, deps) {
3058
2681
  while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
3059
2682
  if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
3060
2683
  for (let i = 0; i < bundle.partitions.length; i += 1) {
3061
- deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
2684
+ deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
3062
2685
  }
3063
2686
  for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
3064
2687
  const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
@@ -3366,6 +2989,8 @@ function validateMergedPartition(merged) {
3366
2989
  }
3367
2990
  }
3368
2991
  }
2992
+
2993
+ // src/persistence/durability/earlier-layout/segment.ts
3369
2994
  function normalizeTombstones(raw) {
3370
2995
  if (raw === void 0) return [];
3371
2996
  if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
@@ -3404,7 +3029,6 @@ var decoder = new TextDecoder();
3404
3029
  async function countPagedPartition(store, partition) {
3405
3030
  const { directory } = store;
3406
3031
  if (directory === void 0 || partition.segments.length === 0) return null;
3407
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-NIYMEA5R.mjs');
3408
3032
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
3409
3033
  const scans = [];
3410
3034
  try {
@@ -3456,6 +3080,8 @@ async function countCheckpointDocuments(store, indexName) {
3456
3080
  const bundle = await store.read(snapshotBundleKey(indexName));
3457
3081
  return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
3458
3082
  }
3083
+
3084
+ // src/persistence/durability/earlier-layout/load.ts
3459
3085
  function removeWhatTheSegmentReplaces(manager, segment) {
3460
3086
  manager.beginBatchRemove();
3461
3087
  try {
@@ -3471,20 +3097,21 @@ function removeWhatTheSegmentReplaces(manager, segment) {
3471
3097
  }
3472
3098
  function attachDocuments(manager, partitionId, segment) {
3473
3099
  if (segment.documents.length === 0) return;
3474
- const payload = buildSegmentPayload(
3100
+ const outputs = buildSegmentPayloads(
3475
3101
  segment.documents,
3476
3102
  manager.config.schema,
3477
3103
  manager.language,
3478
3104
  resolvePartitionInsertOptions(manager.config, manager.analysis),
3479
3105
  manager.config.trackPositions ?? true
3480
3106
  );
3481
- const documents = segment.documents.map((entry) => entry.document);
3482
- const snapshot = freezeSegmentShared(payload, documents);
3483
- if (snapshot === null) {
3484
- manager.mergeSegment(partitionId, payload, documents);
3485
- return;
3107
+ const prepared = outputs.map(({ payload, documents: members }) => {
3108
+ const documents = members.map((entry) => entry.document);
3109
+ return { payload, documents, snapshot: freezeSegmentShared(payload, documents) };
3110
+ });
3111
+ for (const { payload, documents, snapshot } of prepared) {
3112
+ if (snapshot === null) manager.mergeSegment(partitionId, payload, documents);
3113
+ else manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
3486
3114
  }
3487
- manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
3488
3115
  }
3489
3116
  async function loadPartition(store, indexName, partition, deps) {
3490
3117
  if (deps.storedTermsAreCurrent) {
@@ -3505,7 +3132,7 @@ async function loadPartition(store, indexName, partition, deps) {
3505
3132
  totalPartitions: deps.manager.partitionCount,
3506
3133
  language: deps.manager.language.name
3507
3134
  });
3508
- deps.manager.deserializePartition(partition.partitionId, merged);
3135
+ deps.manager.deserializePartition(partition.partitionId, merged, deserializePartition);
3509
3136
  }
3510
3137
  async function readGraph(store, ref) {
3511
3138
  if (ref.graphKey === null) return null;
@@ -3704,13 +3331,68 @@ async function recoverCheckpoint(deps) {
3704
3331
  return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
3705
3332
  }
3706
3333
 
3707
- // src/persistence/durability/stored-analysis.ts
3708
- function generationOfStoredTerms(store, indexName) {
3709
- return readSegmentManifest(store, indexName).then((decoded) => {
3710
- if (decoded === null) return null;
3711
- return decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
3712
- });
3334
+ // src/persistence/durability/layout-transition/recovery.ts
3335
+ async function settleStoredLayout(store, metadata) {
3336
+ const transition = await readTransition(store, metadata.indexName);
3337
+ const decoded = await readSegmentManifest(store, metadata.indexName);
3338
+ const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
3339
+ if (transition !== null && partitionCount !== transition.oldPartitionCount && partitionCount !== transition.newPartitionCount) {
3340
+ invalidTransition("matches neither the manifest nor the metadata");
3341
+ }
3342
+ if (transition !== null && decoded === null && partitionCount !== transition.oldPartitionCount) {
3343
+ invalidTransition("has no old checkpoint or matching metadata");
3344
+ }
3345
+ const changedMetadata = partitionCount !== metadata.partitionCount;
3346
+ metadata.partitionCount = partitionCount;
3347
+ return {
3348
+ transition,
3349
+ changedMetadata,
3350
+ replaysOldLayout: transition !== null && partitionCount === transition.oldPartitionCount
3351
+ };
3352
+ }
3353
+ async function checkRetiredHead(directory, indexName, head) {
3354
+ const wal = await readWalSegments(directory, indexName, head.partitionId);
3355
+ if (wal === null) return;
3356
+ if (wal.highestReadFromWal > head.lastSeqNo || wal.marker.state.highestDurableSeqNo > head.lastSeqNo || wal.activeTail?.entries.some((entry) => entry.seqNo > head.lastSeqNo)) {
3357
+ invalidTransition("has records above a retired partition head");
3358
+ }
3359
+ }
3360
+ async function replayAcrossLayout(directory, indexName, transition, checkpoint, deps) {
3361
+ const terms = [];
3362
+ for (const head of transition.oldHeads) {
3363
+ const from = checkpointLastSeqNo(checkpoint, head.partitionId);
3364
+ if (from > head.lastSeqNo) invalidTransition("has an old head below its checkpoint");
3365
+ const replayed2 = await replayWalUpTo(directory, indexName, head.partitionId, from, head.lastSeqNo, deps);
3366
+ if (replayed2.highestSeqNo < head.lastSeqNo) invalidTransition("is missing an admitted old record");
3367
+ terms[head.partitionId] = Math.max(
3368
+ head.primaryTerm,
3369
+ replayed2.highestPrimaryTerm,
3370
+ checkpointPrimaryTerm(checkpoint, head.partitionId)
3371
+ );
3372
+ if (head.partitionId >= transition.newPartitionCount) await checkRetiredHead(directory, indexName, head);
3373
+ }
3374
+ const spill = await rewriteSpillOf({ directory, writeRecords: deps.writeRecords ?? null }, deps.manager);
3375
+ await createRebalancer().rebalance(
3376
+ deps.manager,
3377
+ transition.newPartitionCount,
3378
+ createPartitionRouter(),
3379
+ void 0,
3380
+ spill
3381
+ );
3382
+ const replayed = [];
3383
+ for (let partitionId = 0; partitionId < transition.newPartitionCount; partitionId++) {
3384
+ const head = transition.oldHeads[partitionId];
3385
+ const next = await replayWal(directory, indexName, partitionId, head?.lastSeqNo ?? 0, deps);
3386
+ replayed.push({
3387
+ partitionId,
3388
+ lastSeqNo: next.highestSeqNo,
3389
+ primaryTerm: Math.max(terms[partitionId] ?? SINGLE_NODE_PRIMARY_TERM, next.highestPrimaryTerm)
3390
+ });
3391
+ }
3392
+ return { checkpoint: replayed, removeUnread: async () => spill?.removeUnread(deps.manager) };
3713
3393
  }
3394
+
3395
+ // src/persistence/durability/stored-analysis.ts
3714
3396
  function promotePendingAnalysis(metadata) {
3715
3397
  const pending = metadata.pendingAnalysis;
3716
3398
  if (pending === void 0) return;
@@ -3724,9 +3406,13 @@ function promotePendingAnalysis(metadata) {
3724
3406
  }
3725
3407
  async function settleStoredAnalysis(store, metadata, metadataOnly) {
3726
3408
  if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
3727
- const stored = await generationOfStoredTerms(store, metadata.indexName);
3409
+ const decoded = await readSegmentManifest(store, metadata.indexName);
3410
+ const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
3411
+ const changedLayout = partitionCount !== metadata.partitionCount;
3412
+ metadata.partitionCount = partitionCount;
3413
+ const stored = decoded === null ? null : decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
3728
3414
  const current = metadata.analysisGeneration ?? 0;
3729
- if (stored === null || stored === current) return false;
3415
+ if (stored === null || stored === current) return changedLayout;
3730
3416
  if (stored === metadata.pendingAnalysis?.generation) {
3731
3417
  promotePendingAnalysis(metadata);
3732
3418
  return true;
@@ -3750,15 +3436,16 @@ async function writeCountedMetadata(deps, indexName, documentCount) {
3750
3436
  if (metadata === void 0) return;
3751
3437
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3752
3438
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
3753
- }).catch(() => void 0);
3439
+ });
3754
3440
  }
3755
3441
  async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3756
3442
  const { directory, hooks } = deps;
3757
3443
  const store = directoryCheckpointStore(directory);
3758
3444
  const metadata = await loadMetadata(directory, indexName);
3759
3445
  if (metadata === null) {
3760
- return;
3446
+ return null;
3761
3447
  }
3448
+ const layout = metadataOnly ? { transition: null, changedMetadata: false, replaysOldLayout: false } : await settleStoredLayout(store, metadata);
3762
3449
  let derivedCount;
3763
3450
  if (metadataOnly && metadata.documentCount === void 0) {
3764
3451
  try {
@@ -3771,15 +3458,15 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3771
3458
  const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
3772
3459
  await hooks.createIndexFromMetadata(metadata, !metadataOnly);
3773
3460
  if (metadataOnly) {
3774
- if (derivedCount !== void 0 || promoted) {
3461
+ if (derivedCount !== void 0 || promoted || layout.changedMetadata) {
3775
3462
  await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
3776
3463
  }
3777
- return;
3464
+ return null;
3778
3465
  }
3779
3466
  const manager = hooks.getManager(indexName);
3780
3467
  const current = hooks.buildMetadata(indexName);
3781
3468
  if (manager === void 0 || current === void 0) {
3782
- return;
3469
+ return null;
3783
3470
  }
3784
3471
  const writeRecords = hooks.writeRecords ?? null;
3785
3472
  writeRecords?.dropIndex(indexName);
@@ -3790,7 +3477,13 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3790
3477
  writeRecords,
3791
3478
  defersGraphBuilds: true
3792
3479
  };
3480
+ let removeUnread = async () => void 0;
3793
3481
  const replay = async (checkpoint) => {
3482
+ if (layout.replaysOldLayout && layout.transition !== null) {
3483
+ const replayed2 = await replayAcrossLayout(directory, indexName, layout.transition, checkpoint, replayDeps);
3484
+ removeUnread = replayed2.removeUnread;
3485
+ return replayed2.checkpoint;
3486
+ }
3794
3487
  const replayed = [];
3795
3488
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
3796
3489
  const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
@@ -3834,9 +3527,19 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3834
3527
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
3835
3528
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
3836
3529
  }
3837
- if (recovered.upgradedDocumentCount !== null || promoted) {
3530
+ if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
3838
3531
  await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
3839
3532
  }
3533
+ return layout.transition === null ? null : removeUnread;
3534
+ }
3535
+ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkpoint) {
3536
+ const removeUnread = await recoverPersistedIndex(deps, indexName, metadataOnly);
3537
+ if (removeUnread === null) return;
3538
+ try {
3539
+ await checkpoint();
3540
+ } finally {
3541
+ await removeUnread();
3542
+ }
3840
3543
  }
3841
3544
 
3842
3545
  // src/persistence/durability/manager.ts
@@ -3913,21 +3616,25 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3913
3616
  }
3914
3617
  }
3915
3618
  }
3916
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
3619
+ async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false, requested = false) {
3917
3620
  const indexState = indexes.get(indexName);
3918
3621
  if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
3919
- return checkpointInTurn(indexState, {
3920
- joinsInFlight: !afterInFlight && !pinsRetiredVectors,
3921
- mayStart: () => !indexState.unloading && !shuttingDown,
3922
- perform: () => performCheckpoint(indexName, indexState, fromMemory, pinsRetiredVectors),
3923
- afterSettling: () => releaseStalledWrites(indexState)
3924
- });
3622
+ return holdScheduledCheckpoint(
3623
+ checkpointInTurn(indexState, {
3624
+ joinsInFlight: !afterInFlight && !pinsRetiredVectors,
3625
+ mayStart: () => !indexState.unloading && !shuttingDown,
3626
+ perform: () => performCheckpoint(indexName, indexState, fromMemory),
3627
+ afterSettling: () => releaseStalledWrites(indexState)
3628
+ }),
3629
+ requested
3630
+ );
3925
3631
  }
3926
3632
  async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
3927
3633
  heldCheckpoints.refuse(indexName, indexes.get(indexName));
3928
- return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors);
3634
+ await requireCheckpointLayout(hooks.getManager(indexName));
3635
+ return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors, true);
3929
3636
  }
3930
- async function performCheckpoint(indexName, indexState, fromMemory = false, pinsRetiredVectors = false) {
3637
+ async function performCheckpoint(indexName, indexState, fromMemory = false) {
3931
3638
  await directoryWatch.verify();
3932
3639
  if (fatalError !== null) throw fatalError;
3933
3640
  await runDurableCheckpoint({
@@ -3938,7 +3645,6 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3938
3645
  compactionThreshold,
3939
3646
  canOffload: canOffloadCheckpoint,
3940
3647
  fromMemory,
3941
- pinsRetiredVectors,
3942
3648
  mutationThreshold: checkpointMutationThreshold,
3943
3649
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3944
3650
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
@@ -3970,9 +3676,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3970
3676
  const recordedByIndex = /* @__PURE__ */ new Map();
3971
3677
  for (let i = 0; i < records.length; i++) {
3972
3678
  if (!outcomes[i].ok) continue;
3973
- const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
3679
+ const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0, deferred: false };
3974
3680
  recorded.records += 1;
3975
3681
  recorded.documentBytes += records[i].document?.byteLength ?? 0;
3682
+ recorded.deferred ||= hasDeferredPublication(records[i]);
3976
3683
  recordedByIndex.set(records[i].indexName, recorded);
3977
3684
  }
3978
3685
  for (const [indexName, recorded] of recordedByIndex) {
@@ -3984,7 +3691,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3984
3691
  markFatal(toError(err));
3985
3692
  });
3986
3693
  }
3987
- await stallWhileCheckpointsFallBehind(indexState);
3694
+ if (!recorded.deferred) await stallWhileCheckpointsFallBehind(indexState);
3988
3695
  }
3989
3696
  if (recordedByIndex.size === 0) return outcomes;
3990
3697
  if (mode === "sync") await directoryWatch.verify();
@@ -4019,7 +3726,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4019
3726
  syncStorage
4020
3727
  };
4021
3728
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
4022
- return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
3729
+ const recovered = recoverWithLayoutCheckpoint(
3730
+ recoveryDeps,
3731
+ indexName,
3732
+ metadataOnly,
3733
+ () => performCheckpoint(indexName, getOrCreateIndexState(indexName), true)
3734
+ );
3735
+ return loads.track(indexName, recovered, startsGraphRepairs);
4023
3736
  }
4024
3737
  return {
4025
3738
  isActive: () => true,
@@ -4101,7 +3814,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4101
3814
  },
4102
3815
  async checkpointAll() {
4103
3816
  for (const indexName of [...indexes.keys()]) {
4104
- await checkpointIndex(indexName);
3817
+ await requestedCheckpoint(indexName);
4105
3818
  }
4106
3819
  },
4107
3820
  ...unloading,
@@ -4127,31 +3840,47 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4127
3840
  }
4128
3841
 
4129
3842
  // src/persistence/durability/snapshot-only/checkpoint.ts
4130
- function capture(input, prior) {
3843
+ function capture(input, prior, cutoffs) {
4131
3844
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
4132
3845
  const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
4133
3846
  const targets = [];
4134
3847
  const contents = [];
3848
+ const positions = /* @__PURE__ */ new Map();
4135
3849
  for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
4136
- const lastSeqNo = input.appliedSeqNoByPartition.get(partitionId) ?? 0;
3850
+ const lastSeqNo = Math.min(
3851
+ input.appliedSeqNoByPartition.get(partitionId) ?? 0,
3852
+ cutoffs.get(partitionId) ?? Infinity
3853
+ );
3854
+ positions.set(partitionId, lastSeqNo);
4137
3855
  const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
4138
3856
  if (unchanged) continue;
4139
3857
  targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
4140
3858
  contents.push(partitionContentsOf(input.manager, partitionId));
4141
3859
  }
4142
3860
  const plans = /* @__PURE__ */ new Map();
4143
- for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
4144
- const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
4145
- plans.set(fieldPath, vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key)));
3861
+ try {
3862
+ for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
3863
+ const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
3864
+ plans.set(
3865
+ fieldPath,
3866
+ vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
3867
+ );
3868
+ }
3869
+ return {
3870
+ targets,
3871
+ plans,
3872
+ documentCount: input.manager.countDocuments(),
3873
+ contents,
3874
+ positions,
3875
+ partitionCount: input.manager.partitionCount
3876
+ };
3877
+ } catch (error) {
3878
+ for (const plan of plans.values()) plan.release();
3879
+ throw error;
4146
3880
  }
4147
- return { targets, plans, documentCount: input.manager.countDocuments(), contents };
4148
3881
  }
4149
3882
  async function writeCaptured(input, store, prior, captured) {
4150
- const partitionCount = input.manager.partitionCount;
4151
- const priorWithinCount = prior === null ? null : {
4152
- checkpoint: prior.checkpoint.filter((entry) => entry.partitionId < partitionCount),
4153
- partitions: prior.partitions.filter((partition) => partition.partitionId < partitionCount)
4154
- };
3883
+ const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, captured.partitionCount);
4155
3884
  const written = await writeCheckpointSegments({
4156
3885
  store,
4157
3886
  metadata: input.metadata,
@@ -4177,9 +3906,17 @@ async function writeSnapshotOnlyCheckpoint(input) {
4177
3906
  const store = adapterCheckpointStore(input.adapter);
4178
3907
  for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
4179
3908
  const prior = await readCurrentManifest(store, input.indexName);
4180
- const captured = capture(input, prior);
4181
- await writeCaptured(input, store, prior, captured);
4182
- return captured.documentCount;
3909
+ const captured = await captureAppliedPrefix(input.manager, (cutoffs) => capture(input, prior, cutoffs));
3910
+ try {
3911
+ await writeCaptured(input, store, prior, captured);
3912
+ return {
3913
+ documentCount: captured.documentCount,
3914
+ positions: captured.positions,
3915
+ partitionCount: captured.partitionCount
3916
+ };
3917
+ } finally {
3918
+ for (const plan of captured.plans.values()) plan.release();
3919
+ }
4183
3920
  }
4184
3921
 
4185
3922
  // src/persistence/durability/snapshot-only/index-state.ts
@@ -4192,6 +3929,20 @@ function createSnapshotIndexState() {
4192
3929
  appliedSeqNoByPartition: /* @__PURE__ */ new Map()
4193
3930
  };
4194
3931
  }
3932
+ function metadataKey(indexName) {
3933
+ return `${indexName}/meta`;
3934
+ }
3935
+ function noteSnapshotCommitted(state, positions, partitionCount) {
3936
+ let recordsLeft = 0;
3937
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
3938
+ if (partitionId >= partitionCount) {
3939
+ state.appliedSeqNoByPartition.delete(partitionId);
3940
+ continue;
3941
+ }
3942
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
3943
+ }
3944
+ state.mutationsSinceCheckpoint = recordsLeft;
3945
+ }
4195
3946
 
4196
3947
  // src/persistence/durability/snapshot-only/restores.ts
4197
3948
  function toError5(error) {
@@ -4231,9 +3982,6 @@ function createPendingRestores(onFatalError) {
4231
3982
  }
4232
3983
 
4233
3984
  // src/persistence/durability/snapshot-only/index.ts
4234
- function metadataKey(indexName) {
4235
- return `${indexName}/meta`;
4236
- }
4237
3985
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4238
3986
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
4239
3987
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
@@ -4279,10 +4027,8 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4279
4027
  await settleRestore(indexName);
4280
4028
  const manager = hooks.getManager(indexName);
4281
4029
  const metadata = hooks.buildTermsMetadata(indexName);
4282
- if (manager === void 0 || metadata === void 0) {
4283
- return;
4284
- }
4285
- const documentCount = await writeSnapshotOnlyCheckpoint({
4030
+ if (manager === void 0 || metadata === void 0) return;
4031
+ const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
4286
4032
  adapter,
4287
4033
  indexName,
4288
4034
  metadata,
@@ -4295,11 +4041,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4295
4041
  await queueMetadataWrite(indexName, async () => {
4296
4042
  const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
4297
4043
  if (checkpointMetadata === void 0) return;
4044
+ checkpointMetadata.partitionCount = partitionCount;
4298
4045
  const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
4299
4046
  await adapter.save(metadataKey(indexName), metadataBytes);
4300
- hooks.recordCheckpoint?.(indexName, documentCount, manager.partitionCount);
4047
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
4301
4048
  });
4302
- indexState.mutationsSinceCheckpoint = 0;
4049
+ noteSnapshotCommitted(indexState, positions, partitionCount);
4303
4050
  if (publisher !== void 0) {
4304
4051
  const partitionIds = [];
4305
4052
  for (let i = 0; i < manager.partitionCount; i += 1) {
@@ -4308,15 +4055,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4308
4055
  await publisher.publishPartitions(indexName, partitionIds);
4309
4056
  }
4310
4057
  }
4311
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
4058
+ async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, requested = false) {
4312
4059
  const indexState = indexes.get(indexName);
4313
4060
  if (indexState === void 0 || indexState.unloading) return;
4314
- return checkpointInTurn(indexState, {
4315
- joinsInFlight: !afterInFlight,
4316
- mayStart: () => !indexState.unloading && !shuttingDown,
4317
- perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
4318
- afterSettling: () => void 0
4319
- });
4061
+ if (requested) await requireCheckpointLayout(hooks.getManager(indexName));
4062
+ return holdScheduledCheckpoint(
4063
+ checkpointInTurn(indexState, {
4064
+ joinsInFlight: !afterInFlight,
4065
+ mayStart: () => !indexState.unloading && !shuttingDown,
4066
+ perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
4067
+ afterSettling: () => void 0
4068
+ }),
4069
+ requested
4070
+ );
4320
4071
  }
4321
4072
  function queueMetadataWrite(indexName, write) {
4322
4073
  const previous = metadataWrites.get(indexName) ?? Promise.resolve();
@@ -4346,7 +4097,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4346
4097
  let derivedCount;
4347
4098
  if (metadataOnly && metadata.documentCount === void 0) {
4348
4099
  try {
4349
- derivedCount = await countStoredBundleDocuments(indexName);
4100
+ derivedCount = await countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
4350
4101
  metadata.documentCount = derivedCount;
4351
4102
  } catch {
4352
4103
  derivedCount = void 0;
@@ -4390,9 +4141,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4390
4141
  await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
4391
4142
  }
4392
4143
  }
4393
- function countStoredBundleDocuments(indexName) {
4394
- return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
4395
- }
4396
4144
  async function recordMutation(record) {
4397
4145
  const indexState = getOrCreateIndexState(record.indexName);
4398
4146
  const buffered = indexState.applyChain.then(async () => {
@@ -4472,10 +4220,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4472
4220
  });
4473
4221
  },
4474
4222
  checkpoint(indexName) {
4475
- return checkpointIndex(indexName);
4223
+ return checkpointIndex(indexName, false, false, true);
4476
4224
  },
4477
4225
  checkpointFromMemory(indexName) {
4478
- return checkpointIndex(indexName, true);
4226
+ return checkpointIndex(indexName, true, true, true);
4479
4227
  },
4480
4228
  async checkpointAfterSwitch(indexName, swap) {
4481
4229
  const indexState = getOrCreateIndexState(indexName);
@@ -4494,7 +4242,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4494
4242
  },
4495
4243
  async checkpointAll() {
4496
4244
  for (const indexName of [...indexes.keys()]) {
4497
- await checkpointIndex(indexName);
4245
+ await checkpointIndex(indexName, false, false, true);
4498
4246
  }
4499
4247
  },
4500
4248
  stageRestore(indexName) {
@@ -4554,7 +4302,13 @@ function buildMetadata(indexName, hooks, documentCount) {
4554
4302
  if (config === void 0) {
4555
4303
  return void 0;
4556
4304
  }
4557
- return metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount, documentCount);
4305
+ const manager = hooks.getManager(indexName);
4306
+ return metadataFromConfig(
4307
+ indexName,
4308
+ config,
4309
+ manager === void 0 ? void 0 : metadataPartitionCount(manager),
4310
+ documentCount
4311
+ );
4558
4312
  }
4559
4313
  function createDurabilityIntegration(tier, hooks) {
4560
4314
  const managerHooks = {
@@ -4579,22 +4333,30 @@ function createDurabilityIntegration(tier, hooks) {
4579
4333
  ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4580
4334
  };
4581
4335
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4582
- function partitionFor(indexName, docId) {
4583
- const partitionManager = hooks.getManager(indexName);
4584
- const partitionCount = partitionManager?.partitionCount ?? 1;
4585
- return fnv1a(docId) % Math.max(1, partitionCount);
4586
- }
4587
4336
  async function recordMutation(indexName, docId, operation, document, apply) {
4588
- const partitionId = partitionFor(indexName, docId);
4589
- const seqNo = await manager.recordMutation({
4590
- indexName,
4591
- partitionId,
4592
- operation,
4593
- documentId: docId,
4594
- document,
4595
- apply
4337
+ return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
4338
+ const partitionId = fnv1a(docId) % partitionCount;
4339
+ let queued;
4340
+ try {
4341
+ const record = {
4342
+ indexName,
4343
+ partitionId,
4344
+ operation,
4345
+ documentId: docId,
4346
+ document,
4347
+ async apply() {
4348
+ queued = await apply() ?? void 0;
4349
+ if (queued !== void 0) deferMutationPublication(record);
4350
+ }
4351
+ };
4352
+ const seqNo = await manager.recordMutation(record);
4353
+ queued?.commit(partitionId, partitionCount, seqNo);
4354
+ return { indexName, partitionId, seqNo };
4355
+ } catch (error) {
4356
+ queued?.fail(error);
4357
+ throw error;
4358
+ }
4596
4359
  });
4597
- return { indexName, partitionId, seqNo };
4598
4360
  }
4599
4361
  return {
4600
4362
  manager,
@@ -4605,29 +4367,48 @@ function createDurabilityIntegration(tier, hooks) {
4605
4367
  const records = [];
4606
4368
  const outcomes = new Array(mutations.length);
4607
4369
  const recordPositions = [];
4370
+ const queued = [];
4608
4371
  for (let i = 0; i < mutations.length; i++) {
4609
4372
  if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
4610
4373
  const mutation = mutations[i];
4611
4374
  try {
4612
- records.push({
4375
+ const record = {
4613
4376
  indexName,
4614
- partitionId: partitionFor(indexName, mutation.docId),
4377
+ partitionId: 0,
4615
4378
  operation: mutation.document === null ? "DELETE" : "INDEX",
4616
4379
  documentId: mutation.docId,
4617
4380
  document: mutation.document === null ? null : encodeDocument(mutation.document),
4618
- apply: mutation.apply
4619
- });
4381
+ async apply() {
4382
+ queued[i] = await mutation.apply() ?? void 0;
4383
+ if (queued[i] !== void 0) deferMutationPublication(record);
4384
+ }
4385
+ };
4386
+ records.push(record);
4620
4387
  recordPositions.push(i);
4621
4388
  } catch (error) {
4622
4389
  outcomes[i] = { ok: false, error };
4623
4390
  }
4624
4391
  }
4625
- const recorded = await manager.recordMutations(records);
4626
- for (let i = 0; i < recorded.length; i++) {
4627
- const outcome = recorded[i];
4628
- outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
4629
- }
4630
- return outcomes;
4392
+ return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
4393
+ let recorded;
4394
+ try {
4395
+ for (const record of records) {
4396
+ record.partitionId = fnv1a(record.documentId) % partitionCount;
4397
+ }
4398
+ recorded = await manager.recordMutations(records);
4399
+ } catch (error) {
4400
+ for (const entry of queued) entry?.fail(error);
4401
+ throw error;
4402
+ }
4403
+ for (let i = 0; i < recorded.length; i++) {
4404
+ const outcome = recorded[i];
4405
+ const entry = queued[recordPositions[i]];
4406
+ if (outcome.ok) entry?.commit(records[i].partitionId, partitionCount, outcome.seqNo);
4407
+ else entry?.fail(outcome.error);
4408
+ outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
4409
+ }
4410
+ return outcomes;
4411
+ });
4631
4412
  },
4632
4413
  recordRemove(indexName, docId, apply) {
4633
4414
  return recordMutation(indexName, docId, "DELETE", null, apply);
@@ -4635,6 +4416,172 @@ function createDurabilityIntegration(tier, hooks) {
4635
4416
  };
4636
4417
  }
4637
4418
 
4419
+ // src/engine/invalidation.ts
4420
+ function createInvalidationFromConfig(config, tierKind, deps, indexRegistry) {
4421
+ if (config?.invalidation === void 0) {
4422
+ return null;
4423
+ }
4424
+ if (tierKind !== "snapshot") {
4425
+ throw new NarsilError(
4426
+ ErrorCodes.CONFIG_INVALID,
4427
+ "Invalidation requires a shared persistence adapter. The write-ahead log tier owns its directory exclusively, so configure a non-filesystem persistence adapter to share data between instances"
4428
+ );
4429
+ }
4430
+ return createInvalidationIntegration({
4431
+ adapter: config.invalidation,
4432
+ instanceId: generateId(),
4433
+ ...deps,
4434
+ listBroadcastIndexNames() {
4435
+ const indexes = [];
4436
+ for (const [name, entry] of indexRegistry) {
4437
+ if (entry.config.defaultScoring === "broadcast") indexes.push(name);
4438
+ }
4439
+ return indexes;
4440
+ }
4441
+ });
4442
+ }
4443
+ function createInvalidationIntegration(deps) {
4444
+ const { adapter, instanceId, reloadIndex, getManager, listBroadcastIndexNames, onError } = deps;
4445
+ const foreignStats = /* @__PURE__ */ new Map();
4446
+ const reloadChains = /* @__PURE__ */ new Map();
4447
+ let broadcastTimer = null;
4448
+ let stopped = false;
4449
+ function toError6(err) {
4450
+ return err instanceof Error ? err : new Error(String(err));
4451
+ }
4452
+ function pruneStaleForeignStats(entries) {
4453
+ const cutoff = Date.now() - FOREIGN_STATS_TTL_MS;
4454
+ for (const [sourceInstanceId, entry] of entries) {
4455
+ if (entry.receivedAt < cutoff) {
4456
+ entries.delete(sourceInstanceId);
4457
+ }
4458
+ }
4459
+ }
4460
+ function scheduleReload(indexName) {
4461
+ const chain = reloadChains.get(indexName) ?? Promise.resolve();
4462
+ const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError6(err)));
4463
+ reloadChains.set(indexName, next);
4464
+ }
4465
+ function storeForeignStats(indexName, sourceInstanceId, stats) {
4466
+ let entries = foreignStats.get(indexName);
4467
+ if (entries === void 0) {
4468
+ entries = /* @__PURE__ */ new Map();
4469
+ foreignStats.set(indexName, entries);
4470
+ }
4471
+ entries.set(sourceInstanceId, { stats, receivedAt: Date.now() });
4472
+ pruneStaleForeignStats(entries);
4473
+ }
4474
+ function handleEvent(event) {
4475
+ try {
4476
+ if (event.type === "partition") {
4477
+ if (event.sourceInstanceId === instanceId) {
4478
+ return;
4479
+ }
4480
+ scheduleReload(event.indexName);
4481
+ return;
4482
+ }
4483
+ if (event.instanceId === instanceId) {
4484
+ return;
4485
+ }
4486
+ storeForeignStats(event.indexName, event.instanceId, event.stats);
4487
+ } catch (err) {
4488
+ onError(toError6(err));
4489
+ }
4490
+ }
4491
+ async function publishOwnStatistics() {
4492
+ for (const indexName of listBroadcastIndexNames()) {
4493
+ const manager = getManager(indexName);
4494
+ if (manager === void 0) {
4495
+ continue;
4496
+ }
4497
+ const aggregate = manager.statisticsMessage();
4498
+ try {
4499
+ await adapter.publish({
4500
+ type: "statistics",
4501
+ indexName,
4502
+ instanceId,
4503
+ stats: {
4504
+ totalDocs: aggregate.totalDocuments,
4505
+ docFrequencies: aggregate.docFrequencies,
4506
+ totalFieldLengths: aggregate.totalFieldLengths
4507
+ }
4508
+ });
4509
+ } catch (err) {
4510
+ onError(toError6(err));
4511
+ }
4512
+ }
4513
+ }
4514
+ return {
4515
+ instanceId,
4516
+ async publishPartitions(indexName, partitions) {
4517
+ if (stopped || partitions.length === 0) {
4518
+ return;
4519
+ }
4520
+ try {
4521
+ await adapter.publish({
4522
+ type: "partition",
4523
+ indexName,
4524
+ partitions,
4525
+ timestamp: Date.now(),
4526
+ sourceInstanceId: instanceId
4527
+ });
4528
+ } catch (err) {
4529
+ onError(toError6(err));
4530
+ }
4531
+ },
4532
+ broadcastStats(indexName, tokens) {
4533
+ const manager = getManager(indexName);
4534
+ if (manager === void 0) {
4535
+ return void 0;
4536
+ }
4537
+ const own = manager.getAggregateStats(tokens);
4538
+ const entries = foreignStats.get(indexName);
4539
+ if (entries === void 0 || entries.size === 0) {
4540
+ return mergePartitionStats([own]);
4541
+ }
4542
+ pruneStaleForeignStats(entries);
4543
+ const combined = [own];
4544
+ for (const [, entry] of entries) {
4545
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
4546
+ for (const token of tokens) {
4547
+ if (Object.hasOwn(entry.stats.docFrequencies, token))
4548
+ docFrequencies[token] = entry.stats.docFrequencies[token];
4549
+ }
4550
+ combined.push({
4551
+ totalDocuments: entry.stats.totalDocs,
4552
+ docFrequencies,
4553
+ totalFieldLengths: entry.stats.totalFieldLengths
4554
+ });
4555
+ }
4556
+ return mergePartitionStats(combined);
4557
+ },
4558
+ async start() {
4559
+ await adapter.subscribe(handleEvent);
4560
+ if (broadcastTimer !== null || stopped) {
4561
+ return;
4562
+ }
4563
+ broadcastTimer = setInterval(() => {
4564
+ void publishOwnStatistics();
4565
+ }, STATISTICS_BROADCAST_INTERVAL_MS);
4566
+ if (typeof broadcastTimer.unref === "function") {
4567
+ broadcastTimer.unref();
4568
+ }
4569
+ },
4570
+ async shutdown() {
4571
+ stopped = true;
4572
+ if (broadcastTimer !== null) {
4573
+ clearInterval(broadcastTimer);
4574
+ broadcastTimer = null;
4575
+ }
4576
+ try {
4577
+ await adapter.shutdown();
4578
+ } catch (err) {
4579
+ onError(toError6(err));
4580
+ }
4581
+ }
4582
+ };
4583
+ }
4584
+
4638
4585
  // src/engine/durability-wiring.ts
4639
4586
  function createDurabilityFromTier(tier, wiring) {
4640
4587
  return createDurabilityIntegration(tier, {
@@ -4665,9 +4612,1508 @@ function createDurabilityFromTier(tier, wiring) {
4665
4612
  keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4666
4613
  onHistoryMoved(indexName) {
4667
4614
  if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4668
- import('./retention-ZWVMNC3Q.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4615
+ try {
4616
+ scheduleSyncRetention(wiring.engineCore(), indexName);
4617
+ } catch (error) {
4618
+ wiring.emitFatalError(error instanceof Error ? error : new Error(String(error)));
4619
+ }
4620
+ }
4621
+ });
4622
+ }
4623
+ async function reloadIndexAndCopies(orchestrator, durability, indexName) {
4624
+ const manager = durability?.manager;
4625
+ if (manager?.reloadIndex === void 0) return;
4626
+ const wasScaledOut = orchestrator.desyncIndex(indexName);
4627
+ try {
4628
+ await manager.reloadIndex(indexName);
4629
+ } finally {
4630
+ await orchestrator.resyncIndex(indexName, wasScaledOut);
4631
+ }
4632
+ }
4633
+
4634
+ // src/engine/export/lines.ts
4635
+ var NEWLINE = 10;
4636
+ function joined(parts, total) {
4637
+ if (parts.length === 1) return parts[0];
4638
+ const bytes = new Uint8Array(total);
4639
+ let offset = 0;
4640
+ for (const part of parts) {
4641
+ bytes.set(part, offset);
4642
+ offset += part.byteLength;
4643
+ }
4644
+ return bytes;
4645
+ }
4646
+ async function* exportLines(source, maxLineBytes) {
4647
+ const decoder3 = new TextDecoder("utf-8", { fatal: true });
4648
+ let parts = [];
4649
+ let pending = 0;
4650
+ let overflowing = false;
4651
+ let line = 0;
4652
+ const finishLine = () => {
4653
+ line += 1;
4654
+ if (overflowing) {
4655
+ overflowing = false;
4656
+ parts = [];
4657
+ pending = 0;
4658
+ return { line, refused: "too-long" };
4659
+ }
4660
+ const bytes = joined(parts, pending);
4661
+ parts = [];
4662
+ pending = 0;
4663
+ try {
4664
+ return { line, text: decoder3.decode(bytes) };
4665
+ } catch {
4666
+ return { line, refused: "not-utf8" };
4667
+ }
4668
+ };
4669
+ for await (const chunk of chunksOf(source)) {
4670
+ let start = 0;
4671
+ for (; ; ) {
4672
+ const newline = chunk.indexOf(NEWLINE, start);
4673
+ const end = newline < 0 ? chunk.byteLength : newline;
4674
+ const piece = chunk.subarray(start, end);
4675
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
4676
+ if (piece.byteLength > 0) parts.push(piece.slice());
4677
+ pending += piece.byteLength;
4678
+ } else {
4679
+ overflowing = true;
4680
+ parts = [];
4681
+ }
4682
+ if (newline < 0) break;
4683
+ yield finishLine();
4684
+ start = newline + 1;
4669
4685
  }
4686
+ }
4687
+ if (pending > 0 || overflowing) yield finishLine();
4688
+ }
4689
+
4690
+ // src/serialization/base64.ts
4691
+ var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4692
+ var PAD = 61;
4693
+ var BYTES_PER_GROUP = 3;
4694
+ var CHARACTERS_PER_GROUP = 4;
4695
+ var ASCII_LIMIT = 128;
4696
+ var ENCODE = new Uint8Array(ALPHABET.length);
4697
+ var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
4698
+ for (let at = 0; at < ALPHABET.length; at++) {
4699
+ ENCODE[at] = ALPHABET.charCodeAt(at);
4700
+ DECODE[ALPHABET.charCodeAt(at)] = at;
4701
+ }
4702
+ var asciiDecoder = new TextDecoder();
4703
+ function encodeBase64(bytes) {
4704
+ const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
4705
+ let write = 0;
4706
+ let read = 0;
4707
+ for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
4708
+ const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
4709
+ out[write++] = ENCODE[group >>> 18];
4710
+ out[write++] = ENCODE[group >>> 12 & 63];
4711
+ out[write++] = ENCODE[group >>> 6 & 63];
4712
+ out[write++] = ENCODE[group & 63];
4713
+ }
4714
+ const left = bytes.byteLength - read;
4715
+ if (left > 0) {
4716
+ const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
4717
+ out[write++] = ENCODE[group >>> 18];
4718
+ out[write++] = ENCODE[group >>> 12 & 63];
4719
+ out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
4720
+ out[write++] = PAD;
4721
+ }
4722
+ return asciiDecoder.decode(out);
4723
+ }
4724
+ function sextetAt(text, at) {
4725
+ const code = text.charCodeAt(at);
4726
+ return code < ASCII_LIMIT ? DECODE[code] : -1;
4727
+ }
4728
+ function decodeExactBase64(text) {
4729
+ if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
4730
+ const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
4731
+ const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
4732
+ const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
4733
+ let write = 0;
4734
+ for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
4735
+ const a2 = sextetAt(text, at);
4736
+ const b2 = sextetAt(text, at + 1);
4737
+ const c2 = sextetAt(text, at + 2);
4738
+ const d = sextetAt(text, at + 3);
4739
+ if ((a2 | b2 | c2 | d) < 0) return null;
4740
+ const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
4741
+ out[write++] = group >>> 16;
4742
+ out[write++] = group >>> 8 & 255;
4743
+ out[write++] = group & 255;
4744
+ }
4745
+ if (padding === 0) return out;
4746
+ const a = sextetAt(text, fullEnd);
4747
+ const b = sextetAt(text, fullEnd + 1);
4748
+ const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
4749
+ if ((a | b | c) < 0) return null;
4750
+ if (padding === 2 && (b & 15) !== 0) return null;
4751
+ if (padding === 1 && (c & 3) !== 0) return null;
4752
+ out[write++] = a << 2 | b >>> 4;
4753
+ if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
4754
+ return out;
4755
+ }
4756
+
4757
+ // src/engine/export/tags.ts
4758
+ var TAG_PREFIX = "$";
4759
+ var DATE_TAG = "$date";
4760
+ var BYTES_TAG = "$bytes";
4761
+ var NUMBER_TAG = "$number";
4762
+ var OBJECT_TAG = "$object";
4763
+ var MILLISECONDS_PER_SECOND = 1e3;
4764
+ var NANOSECONDS_PER_MILLISECOND = 1e6;
4765
+ var MAX_NANOSECONDS = 999999999;
4766
+ var MAX_DATE_MILLISECONDS = 864e13;
4767
+ var ESCAPED_TAG_PREFIX = "\\u0024";
4768
+ var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
4769
+ ["NaN", Number.NaN],
4770
+ ["Infinity", Number.POSITIVE_INFINITY],
4771
+ ["-Infinity", Number.NEGATIVE_INFINITY]
4772
+ ]);
4773
+ function refuse(path, reason) {
4774
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
4775
+ field: path
4670
4776
  });
4671
4777
  }
4778
+ function isRecord2(value) {
4779
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4780
+ }
4781
+ function startsWithTagPrefix(key) {
4782
+ return key.startsWith(TAG_PREFIX);
4783
+ }
4784
+ function soleTagKey(record) {
4785
+ let sole = null;
4786
+ for (const key of Object.keys(record)) {
4787
+ if (sole !== null) return null;
4788
+ sole = key;
4789
+ }
4790
+ return sole !== null && startsWithTagPrefix(sole) ? sole : null;
4791
+ }
4792
+ function dateOf(value, path) {
4793
+ if (!Array.isArray(value) || value.length !== 2)
4794
+ refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
4795
+ const seconds = value[0];
4796
+ const nanoseconds = value[1];
4797
+ if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
4798
+ refuse(
4799
+ path,
4800
+ `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
4801
+ );
4802
+ }
4803
+ if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
4804
+ refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
4805
+ }
4806
+ const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
4807
+ if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
4808
+ refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
4809
+ return new Date(milliseconds);
4810
+ }
4811
+ function bytesOf(value, path) {
4812
+ const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
4813
+ if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
4814
+ return bytes;
4815
+ }
4816
+ function numberOf(value, path) {
4817
+ const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
4818
+ if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
4819
+ return number;
4820
+ }
4821
+ function requireDepth(path, depth) {
4822
+ if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
4823
+ }
4824
+ function untagMap(record, path, depth) {
4825
+ requireDepth(path, depth);
4826
+ for (const key of Object.keys(record)) {
4827
+ record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
4828
+ }
4829
+ return record;
4830
+ }
4831
+ function untagList(list, path, depth) {
4832
+ requireDepth(path, depth);
4833
+ for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
4834
+ return list;
4835
+ }
4836
+ function untagTag(tag, value, path, depth) {
4837
+ switch (tag) {
4838
+ case DATE_TAG:
4839
+ return dateOf(value, path);
4840
+ case BYTES_TAG:
4841
+ return bytesOf(value, path);
4842
+ case NUMBER_TAG:
4843
+ return numberOf(value, path);
4844
+ case OBJECT_TAG:
4845
+ if (!isRecord2(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
4846
+ return untagMap(value, path, depth);
4847
+ default:
4848
+ return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
4849
+ }
4850
+ }
4851
+ function untag(value, path, depth) {
4852
+ if (typeof value !== "object" || value === null) return value;
4853
+ if (Array.isArray(value)) return untagList(value, path, depth);
4854
+ const record = value;
4855
+ const tag = soleTagKey(record);
4856
+ return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
4857
+ }
4858
+ function untagDocumentLine(text, parsed) {
4859
+ if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
4860
+ return untagMap(parsed, "", 0);
4861
+ }
4862
+
4863
+ // src/engine/export/import.ts
4864
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
4865
+ var DOCUMENT_EXPORT_VERSION = 1;
4866
+ function invalidHeader(message) {
4867
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
4868
+ }
4869
+ function isRecord3(value) {
4870
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4871
+ }
4872
+ function configOfHeader(line) {
4873
+ if (line === void 0) invalidHeader("is missing");
4874
+ if (!("text" in line)) {
4875
+ invalidHeader(
4876
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
4877
+ );
4878
+ }
4879
+ let raw;
4880
+ try {
4881
+ raw = JSON.parse(line.text);
4882
+ } catch {
4883
+ invalidHeader("is not JSON");
4884
+ }
4885
+ if (!isRecord3(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
4886
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
4887
+ }
4888
+ const configuration = requireConfigurationShape(raw.index, "document export header");
4889
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
4890
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
4891
+ return {
4892
+ schema: configuration.schema,
4893
+ language: configuration.language,
4894
+ ...fields,
4895
+ ...embedding === void 0 ? {} : {
4896
+ embedding: {
4897
+ fields: embedding.fields,
4898
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
4899
+ }
4900
+ }
4901
+ };
4902
+ }
4903
+ async function prepareIndex(engine, indexName, config) {
4904
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
4905
+ await engine.createIndex(indexName, config);
4906
+ return;
4907
+ }
4908
+ const existing = flattenSchema(engine.getStats(indexName).schema);
4909
+ const exported = flattenSchema(config.schema);
4910
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
4911
+ for (const field of fields) {
4912
+ if (existing[field] === exported[field]) continue;
4913
+ throw new NarsilError(
4914
+ ErrorCodes.CONFIG_INVALID,
4915
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
4916
+ { indexName, field }
4917
+ );
4918
+ }
4919
+ }
4920
+ function highestReportedAt(refused) {
4921
+ let highest = 0;
4922
+ for (let at = 1; at < refused.length; at++) {
4923
+ if (refused[at].line > refused[highest].line) highest = at;
4924
+ }
4925
+ return highest;
4926
+ }
4927
+ function refuse2(state, line, code, message) {
4928
+ state.result.refusedCount += 1;
4929
+ const refused = state.result.refused;
4930
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
4931
+ refused.push({ line, code, message });
4932
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
4933
+ return;
4934
+ }
4935
+ if (line >= refused[state.highestReported].line) return;
4936
+ refused[state.highestReported] = { line, code, message };
4937
+ state.highestReported = highestReportedAt(refused);
4938
+ }
4939
+ async function flush(state) {
4940
+ if (state.batch.length === 0) return;
4941
+ const batch = state.batch;
4942
+ state.batch = [];
4943
+ state.batchIds = /* @__PURE__ */ new Set();
4944
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
4945
+ const inserts = [];
4946
+ const updates = [];
4947
+ for (const entry of batch) {
4948
+ if (await state.engine.has(state.indexName, entry.docId))
4949
+ updates.push({ docId: entry.docId, document: entry.document });
4950
+ else inserts.push(entry.document);
4951
+ }
4952
+ for (const outcome of [
4953
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
4954
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
4955
+ ]) {
4956
+ if (outcome === null) continue;
4957
+ state.result.imported += outcome.succeeded.length;
4958
+ for (const failure of outcome.failed) {
4959
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
4960
+ refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
4961
+ }
4962
+ }
4963
+ }
4964
+ async function takeLine(state, entry) {
4965
+ if (!("text" in entry)) {
4966
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
4967
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
4968
+ return;
4969
+ }
4970
+ if (entry.text.length === 0) return;
4971
+ let document;
4972
+ try {
4973
+ document = JSON.parse(entry.text);
4974
+ } catch {
4975
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
4976
+ return;
4977
+ }
4978
+ if (!isRecord3(document) || typeof document.id !== "string" || document.id.length === 0) {
4979
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
4980
+ return;
4981
+ }
4982
+ try {
4983
+ untagDocumentLine(entry.text, document);
4984
+ } catch (error) {
4985
+ if (!(error instanceof NarsilError)) throw error;
4986
+ refuse2(state, entry.line, error.code, error.message);
4987
+ return;
4988
+ }
4989
+ if (state.batchIds.has(document.id)) await flush(state);
4990
+ state.batch.push({ line: entry.line, docId: document.id, document });
4991
+ state.batchIds.add(document.id);
4992
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
4993
+ }
4994
+ async function readDocumentExport(engine, indexName, source) {
4995
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
4996
+ const header = await lines.next();
4997
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
4998
+ const state = {
4999
+ engine,
5000
+ indexName,
5001
+ batch: [],
5002
+ batchIds: /* @__PURE__ */ new Set(),
5003
+ result: { imported: 0, refusedCount: 0, refused: [] },
5004
+ highestReported: 0
5005
+ };
5006
+ for await (const entry of lines) {
5007
+ await takeLine(state, entry);
5008
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
5009
+ }
5010
+ await flush(state);
5011
+ state.result.refused.sort((a, b) => a.line - b.line);
5012
+ return state.result;
5013
+ }
5014
+
5015
+ // src/engine/export/json-line.ts
5016
+ function refuse3(path, kind, docId) {
5017
+ throw new NarsilError(
5018
+ ErrorCodes.DOC_VALIDATION_FAILED,
5019
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
5020
+ { docId, field: path }
5021
+ );
5022
+ }
5023
+ function encodeNumber(value, path, target) {
5024
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
5025
+ if (!target.tagged) refuse3(path, String(value), target.docId);
5026
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
5027
+ }
5028
+ function encodeDate(value, path, target) {
5029
+ const milliseconds = value.getTime();
5030
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse3(path, "a date", target.docId);
5031
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
5032
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
5033
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
5034
+ }
5035
+ function encodeBytes(value, path, target) {
5036
+ if (!target.tagged) refuse3(path, "bytes", target.docId);
5037
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
5038
+ }
5039
+ function encodeList(value, path, target) {
5040
+ target.parts.push("[");
5041
+ for (let at = 0; at < value.length; at++) {
5042
+ if (at > 0) target.parts.push(",");
5043
+ encodeValue(value[at], `${path}[${at}]`, target);
5044
+ }
5045
+ target.parts.push("]");
5046
+ }
5047
+ function encodeMap(value, path, target) {
5048
+ const proto = Object.getPrototypeOf(value);
5049
+ if (proto !== Object.prototype && proto !== null) refuse3(path, "a value other than a plain object", target.docId);
5050
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
5051
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
5052
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
5053
+ for (let at = 0; at < entries.length; at++) {
5054
+ const [key, entry] = entries[at];
5055
+ if (at > 0) target.parts.push(",");
5056
+ target.parts.push(JSON.stringify(key), ":");
5057
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
5058
+ }
5059
+ target.parts.push(wrapped ? "}}" : "}");
5060
+ }
5061
+ function encodeValue(value, path, target) {
5062
+ if (value === null) {
5063
+ target.parts.push("null");
5064
+ return;
5065
+ }
5066
+ switch (typeof value) {
5067
+ case "string":
5068
+ target.parts.push(JSON.stringify(value));
5069
+ return;
5070
+ case "boolean":
5071
+ target.parts.push(value ? "true" : "false");
5072
+ return;
5073
+ case "number":
5074
+ target.parts.push(encodeNumber(value, path, target));
5075
+ return;
5076
+ case "object":
5077
+ break;
5078
+ default:
5079
+ refuse3(path, `a ${typeof value}`, target.docId);
5080
+ }
5081
+ if (Array.isArray(value)) encodeList(value, path, target);
5082
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
5083
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
5084
+ else encodeMap(value, path, target);
5085
+ }
5086
+ function encodeExactJson(value, docId) {
5087
+ const target = { docId, tagged: false, parts: [] };
5088
+ encodeValue(value, "", target);
5089
+ return target.parts.join("");
5090
+ }
5091
+ function encodeDocumentJson(document, docId) {
5092
+ const target = { docId, tagged: true, parts: [] };
5093
+ encodeValue(document, "", target);
5094
+ return target.parts.join("");
5095
+ }
5096
+
5097
+ // src/engine/export/paged-lines.ts
5098
+ async function coldFieldOf(checkpoint, field) {
5099
+ const { directory } = checkpoint;
5100
+ if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
5101
+ if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
5102
+ const fieldType = checkpoint.manifest.schema[field.fieldPath];
5103
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
5104
+ if (dimension === null) return null;
5105
+ const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
5106
+ return openColdVectorField({
5107
+ fieldPath: field.fieldPath,
5108
+ dimension,
5109
+ metric: "cosine",
5110
+ codeBits: null,
5111
+ files: await Promise.all(field.files.map(withPath)),
5112
+ graphParts: await Promise.all(field.graphParts.map(withPath)),
5113
+ resolvePartition: () => void 0
5114
+ });
5115
+ }
5116
+ async function openFileVectorReads(checkpoint, wholeFileReads) {
5117
+ const fields = /* @__PURE__ */ new Map();
5118
+ for (const field of checkpoint.manifest.vectors) {
5119
+ const opened = await coldFieldOf(checkpoint, field);
5120
+ if (opened !== null) fields.set(field.fieldPath, opened);
5121
+ }
5122
+ const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
5123
+ return {
5124
+ async vectorOf(fieldPath, docId) {
5125
+ const field = fields.get(fieldPath);
5126
+ if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
5127
+ const number = field.numberOf(docId);
5128
+ return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
5129
+ },
5130
+ close() {
5131
+ for (const field of fields.values()) field.close();
5132
+ fallback?.close();
5133
+ }
5134
+ };
5135
+ }
5136
+ async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
5137
+ const { directory } = checkpoint;
5138
+ if (directory === void 0) return;
5139
+ const scans = [];
5140
+ try {
5141
+ for (const ref of partition.segments) {
5142
+ const paged = await isPagedSegmentFile(directory, ref.key);
5143
+ scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
5144
+ }
5145
+ for (let at = 0; at < scans.length; at++) {
5146
+ const scan = scans[at];
5147
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) {
5148
+ const docId = scan.idAt(ordinal);
5149
+ if (!countsAsLatestHolder(scans, at, docId)) continue;
5150
+ const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
5151
+ document.id = docId;
5152
+ for (const field of checkpoint.manifest.vectors) {
5153
+ const vector = await vectors.vectorOf(field.fieldPath, docId);
5154
+ if (vector !== null) setNestedValue(document, field.fieldPath, vector);
5155
+ }
5156
+ yield `${encodeDocumentJson(document, docId)}
5157
+ `;
5158
+ }
5159
+ }
5160
+ } finally {
5161
+ for (const scan of scans) scan.close();
5162
+ }
5163
+ }
5164
+
5165
+ // src/engine/export/write.ts
5166
+ var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
5167
+ var FLOAT32_BYTES = 4;
5168
+ var ENVELOPE_VERSION_OFFSET = 4;
5169
+ async function locateVectors(checkpoint) {
5170
+ const located = /* @__PURE__ */ new Map();
5171
+ for (const field of checkpoint.manifest.vectors) {
5172
+ const byDocId = /* @__PURE__ */ new Map();
5173
+ for (const ref of field.files) {
5174
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
5175
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
5176
+ what: `vector file "${ref.key}"`,
5177
+ allowsCompression: false
5178
+ });
5179
+ const decoded = decodeVectorFilePayload(decode(payload));
5180
+ for (let position = 0; position < decoded.docIds.length; position++) {
5181
+ if (isDead(ref.dead, position)) continue;
5182
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
5183
+ }
5184
+ }
5185
+ located.set(field.fieldPath, byDocId);
5186
+ }
5187
+ return located;
5188
+ }
5189
+ async function readVector(checkpoint, location) {
5190
+ const width = location.dimension * FLOAT32_BYTES;
5191
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
5192
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
5193
+ }
5194
+ async function readSegment(checkpoint, ref) {
5195
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
5196
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
5197
+ what: `segment "${ref.key}"`,
5198
+ allowsCompression: true
5199
+ });
5200
+ return decodeKeywordSegment(payload, false);
5201
+ }
5202
+ var decoder2 = new TextDecoder();
5203
+ function idAt(contents, ordinal) {
5204
+ const ids = contents.arrays.idTable;
5205
+ return decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
5206
+ }
5207
+ async function winnersOf(checkpoint, partition) {
5208
+ const winners = /* @__PURE__ */ new Map();
5209
+ for (let at = 0; at < partition.segments.length; at++) {
5210
+ const contents = await readSegment(checkpoint, partition.segments[at]);
5211
+ for (const docId of contents.tombstones) winners.delete(docId);
5212
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
5213
+ }
5214
+ return winners;
5215
+ }
5216
+ async function wholeSegmentScan(checkpoint, ref) {
5217
+ const contents = await readSegment(checkpoint, ref);
5218
+ const table = contents.arrays.documentTable;
5219
+ const ids = /* @__PURE__ */ new Set();
5220
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
5221
+ return {
5222
+ count: contents.arrays.documentCount,
5223
+ schema: contents.schema,
5224
+ tombstones: new Set(contents.tombstones),
5225
+ idAt: (ordinal) => idAt(contents, ordinal),
5226
+ holds: (docId) => ids.has(docId),
5227
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
5228
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
5229
+ table.discardedFields
5230
+ ),
5231
+ close: () => void 0
5232
+ };
5233
+ }
5234
+ async function wholeFileVectorReads(checkpoint) {
5235
+ const located = await locateVectors(checkpoint);
5236
+ return {
5237
+ async vectorOf(fieldPath, docId) {
5238
+ const location = located.get(fieldPath)?.get(docId);
5239
+ return location === void 0 ? null : readVector(checkpoint, location);
5240
+ },
5241
+ close: () => located.clear()
5242
+ };
5243
+ }
5244
+ async function* pagedDocumentLines(checkpoint) {
5245
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
5246
+ try {
5247
+ for (const partition of checkpoint.manifest.partitions) {
5248
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
5249
+ }
5250
+ } finally {
5251
+ vectors.close();
5252
+ }
5253
+ }
5254
+ async function anyPartitionIsPaged(checkpoint) {
5255
+ for (const partition of checkpoint.manifest.partitions) {
5256
+ for (const ref of partition.segments) {
5257
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
5258
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
5259
+ }
5260
+ }
5261
+ return false;
5262
+ }
5263
+ async function* documentLines(checkpoint) {
5264
+ if (await anyPartitionIsPaged(checkpoint)) {
5265
+ yield* pagedDocumentLines(checkpoint);
5266
+ return;
5267
+ }
5268
+ const vectors = await locateVectors(checkpoint);
5269
+ for (const partition of checkpoint.manifest.partitions) {
5270
+ const winners = await winnersOf(checkpoint, partition);
5271
+ for (let at = 0; at < partition.segments.length; at++) {
5272
+ const contents = await readSegment(checkpoint, partition.segments[at]);
5273
+ const table = contents.arrays.documentTable;
5274
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
5275
+ const docId = idAt(contents, ordinal);
5276
+ if (winners.get(docId) !== at) continue;
5277
+ const stored = decodeStoredDocument(
5278
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
5279
+ table.discardedFields
5280
+ );
5281
+ const document = { id: docId, ...stored };
5282
+ document.id = docId;
5283
+ for (const [fieldPath, located] of vectors) {
5284
+ const location = located.get(docId);
5285
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
5286
+ }
5287
+ yield `${encodeDocumentJson(document, docId)}
5288
+ `;
5289
+ }
5290
+ }
5291
+ }
5292
+ }
5293
+ async function* exportLines2(header, checkpoint) {
5294
+ yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
5295
+ `;
5296
+ yield* documentLines(checkpoint);
5297
+ }
5298
+ function documentExportStream(header, checkpoint) {
5299
+ const lines = exportLines2(header, checkpoint);
5300
+ const encoder = new TextEncoder();
5301
+ let released = false;
5302
+ const release = () => {
5303
+ if (released) return;
5304
+ released = true;
5305
+ checkpoint.release();
5306
+ };
5307
+ return new ReadableStream({
5308
+ async pull(controller) {
5309
+ try {
5310
+ let text = "";
5311
+ while (text.length < EXPORT_CHUNK_BYTES) {
5312
+ const step = await lines.next();
5313
+ if (step.done) {
5314
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
5315
+ release();
5316
+ controller.close();
5317
+ return;
5318
+ }
5319
+ text += step.value;
5320
+ }
5321
+ controller.enqueue(encoder.encode(text));
5322
+ } catch (error) {
5323
+ release();
5324
+ controller.error(error);
5325
+ }
5326
+ },
5327
+ async cancel() {
5328
+ release();
5329
+ await lines.return(void 0);
5330
+ }
5331
+ });
5332
+ }
5333
+
5334
+ // src/query-suggestions/archive-history.ts
5335
+ function inDayOrder(left, right) {
5336
+ if (left.day !== right.day) return left.day < right.day ? -1 : 1;
5337
+ return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
5338
+ }
5339
+ async function* historyArchiveEntries(core, indexName) {
5340
+ const runtime = attachedRuntimeOf(core);
5341
+ const files = runtime?.host.files ?? historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
5342
+ await runtime?.saveDay(indexName);
5343
+ const names = [];
5344
+ for (const key of await listHistoryFolder(files, indexName)) {
5345
+ const name = historyFileOf(indexName, key);
5346
+ if (name !== null) names.push(name);
5347
+ }
5348
+ names.sort(inDayOrder);
5349
+ let position = 0;
5350
+ for (const name of names) {
5351
+ const bytes = await files.read(name.key);
5352
+ if (bytes === null || bytes === "too-long") continue;
5353
+ yield { name: `${QUERY_HISTORY_ARCHIVE_PREFIX}${position}`, bytes };
5354
+ position += 1;
5355
+ }
5356
+ }
5357
+
5358
+ // src/persistence/archive/validate.ts
5359
+ function requireDimension(key, found, expected) {
5360
+ if (expected === found) return;
5361
+ throw new NarsilError(
5362
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5363
+ `The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
5364
+ { key, found, expected }
5365
+ );
5366
+ }
5367
+ function requireSchemaOfManifest(key, schema, manifest) {
5368
+ for (const [field, type] of Object.entries(schema)) {
5369
+ const listed = manifest.schema[field];
5370
+ if (listed === void 0 || listed === type) continue;
5371
+ throw new NarsilError(
5372
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5373
+ `The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
5374
+ { key, field }
5375
+ );
5376
+ }
5377
+ }
5378
+ async function validateStagedCheckpoint(store, manifest) {
5379
+ for (const partition of manifest.partitions) {
5380
+ for (const ref of partition.segments) {
5381
+ const { directory } = store;
5382
+ if (directory !== void 0) {
5383
+ if (await isPagedSegmentFile(directory, ref.key)) {
5384
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
5385
+ continue;
5386
+ }
5387
+ }
5388
+ const contents = await readSegmentFile(store, ref);
5389
+ requireSchemaOfManifest(ref.key, contents.schema, manifest);
5390
+ decodeEveryStoredDocument(contents.arrays.documentTable);
5391
+ }
5392
+ }
5393
+ for (const field of manifest.vectors) {
5394
+ const fieldType = manifest.schema[field.fieldPath];
5395
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
5396
+ for (const file of field.files) {
5397
+ const { payload } = await readVectorFile(store, file, field.fieldPath);
5398
+ requireDimension(file.key, payload.dimension, dimension);
5399
+ }
5400
+ await readGraphParts(store, field);
5401
+ }
5402
+ }
5403
+
5404
+ // src/engine/snapshot/restore-setup.ts
5405
+ function prepareRestoredIndex(configuration, deps, reject) {
5406
+ const configFields = restoredConfigFields(configuration, reject);
5407
+ const embedding = restoredEmbedding(configuration.embedding, reject);
5408
+ const languageName = configuration.language ?? "";
5409
+ const language = getLanguage(languageName);
5410
+ const schema = validateSchema(configuration.schema);
5411
+ validatePatternValueLimit(configFields.patternValueLimit);
5412
+ requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
5413
+ validateVectorPromotion(configFields.vectorPromotion);
5414
+ validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
5415
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
5416
+ if (configFields.required !== void 0 && configFields.required.length > 0) {
5417
+ validateRequiredFieldsInSchema(configFields.required, schema);
5418
+ }
5419
+ const config = {
5420
+ schema,
5421
+ language: languageName,
5422
+ ...configFields,
5423
+ ...embedding !== void 0 ? {
5424
+ embedding: {
5425
+ fields: embedding.fields,
5426
+ ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
5427
+ }
5428
+ } : {}
5429
+ };
5430
+ resolveIndexAnalysis(config);
5431
+ const adapterName = embedding?.adapter ?? null;
5432
+ let embeddingAdapter = null;
5433
+ if (embedding !== void 0) {
5434
+ const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
5435
+ if (candidate) {
5436
+ validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
5437
+ embeddingAdapter = candidate;
5438
+ }
5439
+ }
5440
+ const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
5441
+ const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
5442
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
5443
+ return {
5444
+ config,
5445
+ language,
5446
+ embeddingAdapter,
5447
+ adapterName,
5448
+ storedRevision,
5449
+ analysisGeneration,
5450
+ storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
5451
+ synonyms,
5452
+ querySuggestions: storedStateOf(configuration)
5453
+ };
5454
+ }
5455
+ function restoredSynonyms(raw, config, language, reject) {
5456
+ if (raw === void 0 || raw === null) return null;
5457
+ return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
5458
+ }
5459
+ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
5460
+ const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema);
5461
+ if (loadingSchemaCode !== null) await loadingSchemaCode;
5462
+ const detached = createDetachedIndex(
5463
+ indexName,
5464
+ prepared.config,
5465
+ prepared.language,
5466
+ prepared.synonyms?.lookup ?? null,
5467
+ deps.executor
5468
+ );
5469
+ while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
5470
+ return detached;
5471
+ }
5472
+ function restoredRegistryEntry(prepared, manager, deps) {
5473
+ return {
5474
+ config: prepared.config,
5475
+ language: prepared.language,
5476
+ embeddingAdapter: prepared.embeddingAdapter,
5477
+ embeddingAdapterName: prepared.adapterName,
5478
+ vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
5479
+ indexUuid: null,
5480
+ heldPartitions: null,
5481
+ documentCount: manager.countDocuments(),
5482
+ partitionCount: manager.partitionCount,
5483
+ analysisRevision: prepared.storedRevision,
5484
+ arrayPositionGap: prepared.storedArrayPositionGap,
5485
+ synonyms: prepared.synonyms?.encoded ?? null,
5486
+ analysisGeneration: prepared.analysisGeneration,
5487
+ pendingAnalysis: null,
5488
+ durableAnalysis: null,
5489
+ sync: null,
5490
+ querySuggestions: prepared.querySuggestions
5491
+ };
5492
+ }
5493
+
5494
+ // src/engine/snapshot/replace.ts
5495
+ function releaseTimedOut(indexName, timeoutMs) {
5496
+ return new NarsilError(
5497
+ ErrorCodes.WORKER_TIMEOUT,
5498
+ `The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
5499
+ { indexName, timeoutMs }
5500
+ );
5501
+ }
5502
+ async function withinLimit(indexName, release, work, deps) {
5503
+ release.settled = work.catch(() => {
5504
+ release.reloads = true;
5505
+ });
5506
+ let timer;
5507
+ const limit = new Promise((_, reject) => {
5508
+ timer = setTimeout(() => {
5509
+ release.timedOut = true;
5510
+ reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
5511
+ }, deps.releaseTimeoutMs);
5512
+ });
5513
+ try {
5514
+ await Promise.race([work, limit]);
5515
+ } finally {
5516
+ clearTimeout(timer);
5517
+ }
5518
+ }
5519
+ function vectorIndexesOf2(indexName, deps) {
5520
+ const manager = deps.executor.getManager(indexName);
5521
+ return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
5522
+ }
5523
+ async function holdVectorCopies(vectorIndexes, release) {
5524
+ for (const vectorIndex of vectorIndexes) {
5525
+ if (release.held.includes(vectorIndex)) continue;
5526
+ release.held.push(vectorIndex);
5527
+ await vectorIndex.holdWorkerCopies();
5528
+ }
5529
+ }
5530
+ async function releaseCopies(indexName, release, deps) {
5531
+ release.reloads = await deps.copies.releaseIndexCopies(indexName);
5532
+ await holdVectorCopies(vectorIndexesOf2(indexName, deps), release);
5533
+ }
5534
+ async function endRelease(indexName, release, deps) {
5535
+ await release.settled;
5536
+ for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
5537
+ await deps.copies.endCopyRelease(indexName, release.reloads);
5538
+ }
5539
+ function persistedCheckpoint(indexName, partitionCount, deps) {
5540
+ const durable = deps.durability?.manager;
5541
+ if (durable === void 0) return [];
5542
+ return Array.from({ length: partitionCount }, (_, partitionId) => ({
5543
+ partitionId,
5544
+ lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
5545
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
5546
+ }));
5547
+ }
5548
+ async function commitToStorage(indexName, restored, previousPartitions, deps) {
5549
+ const durable = deps.durability?.manager;
5550
+ const staging = restored.staging;
5551
+ if (durable === void 0 || staging === null) return;
5552
+ const previous = persistedCheckpoint(indexName, previousPartitions, deps);
5553
+ if (previousPartitions > 0) await durable.unloadIndex(indexName);
5554
+ const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
5555
+ try {
5556
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
5557
+ await staging.commit();
5558
+ } catch (error) {
5559
+ if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
5560
+ throw error;
5561
+ }
5562
+ await staging.finish();
5563
+ const relocate = await staging.relocation().catch(() => null);
5564
+ if (relocate === null) return;
5565
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
5566
+ const { manager } = restored.detached;
5567
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
5568
+ const partition = manager.getPartition(partitionId);
5569
+ if (!isCompositePartition(partition)) continue;
5570
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
5571
+ }
5572
+ }
5573
+ function activate(indexName, restored, entry, deps) {
5574
+ deps.executor.installDetachedIndex(restored.detached);
5575
+ deps.indexRegistry.set(indexName, entry);
5576
+ restored.detached.attach();
5577
+ deps.clearAnalysisStale(indexName);
5578
+ const { prepared } = restored;
5579
+ if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
5580
+ deps.markAnalysisStale({
5581
+ indexName,
5582
+ language: prepared.language.name,
5583
+ storedRevision: prepared.storedRevision,
5584
+ currentRevision: prepared.language.revision
5585
+ });
5586
+ }
5587
+ if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
5588
+ deps.forgetWatermark(indexName);
5589
+ }
5590
+ async function replaceExisting(indexName, restored, entry, deps) {
5591
+ const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
5592
+ try {
5593
+ await deps.abandonAnalysisChange(indexName);
5594
+ await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
5595
+ await deps.indexState.replace(indexName, async (wasOpen) => {
5596
+ await deps.abandonAnalysisRebuild(indexName);
5597
+ const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
5598
+ const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
5599
+ await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
5600
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
5601
+ await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
5602
+ if (previous === void 0) activate(indexName, restored, entry, deps);
5603
+ else await previous.publication.write(() => activate(indexName, restored, entry, deps));
5604
+ await deps.querySuggestions?.afterActivate(entry);
5605
+ });
5606
+ } finally {
5607
+ const ending = endRelease(indexName, release, deps).catch(() => void 0);
5608
+ if (!release.timedOut) await ending;
5609
+ }
5610
+ }
5611
+ async function createFresh(indexName, restored, entry, deps) {
5612
+ deps.copies.desyncIndex(indexName);
5613
+ deps.executor.installDetachedIndex(restored.detached);
5614
+ deps.indexRegistry.set(indexName, entry);
5615
+ try {
5616
+ await deps.indexState.registerReplacing(indexName, async () => {
5617
+ await commitToStorage(indexName, restored, 0, deps);
5618
+ activate(indexName, restored, entry, deps);
5619
+ await deps.querySuggestions?.afterActivate(entry);
5620
+ });
5621
+ } catch (error) {
5622
+ if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
5623
+ throw error;
5624
+ } finally {
5625
+ await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
5626
+ }
5627
+ }
5628
+ async function stageMetadata(indexName, staging, entry) {
5629
+ const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
5630
+ await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
5631
+ }
5632
+ async function installRestoredIndex(indexName, restored, deps, history = null) {
5633
+ const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
5634
+ const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
5635
+ if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
5636
+ let replaced;
5637
+ try {
5638
+ if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
5639
+ await deps.querySuggestions?.beforeInstall(entry, deps.indexRegistry.get(indexName));
5640
+ if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
5641
+ replaced = deps.indexRegistry.get(indexName);
5642
+ if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
5643
+ else await createFresh(indexName, restored, entry, deps);
5644
+ } catch (error) {
5645
+ if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
5646
+ else deps.executor.discardDetachedIndex(restored.detached);
5647
+ throw error;
5648
+ }
5649
+ await restored.staging?.removeRetired().catch(() => void 0);
5650
+ const durable = deps.durability?.manager;
5651
+ if (restored.staging === null && durable !== void 0) {
5652
+ await durable.persistMetadata(indexName);
5653
+ await durable.checkpoint(indexName);
5654
+ }
5655
+ if (previousSync !== null) {
5656
+ const storage = durable?.syncStorage?.();
5657
+ if (storage !== void 0)
5658
+ await oneSyncChangeAtATime(deps.indexRegistry, indexName, () => storage.release(indexName));
5659
+ emitSyncChange(deps.indexRegistry, { kind: "epoch", indexName });
5660
+ }
5661
+ await deps.querySuggestions?.afterInstall(entry, history);
5662
+ if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
5663
+ await deps.announceIndexCreated(indexName, restored.prepared.config);
5664
+ for (const vectorIndex of restored.detached.manager.getVectorIndexes().values()) {
5665
+ vectorIndex.scheduleBuild();
5666
+ vectorIndex.startShuffledRebuildWhenDue();
5667
+ }
5668
+ }
5669
+
5670
+ // src/engine/snapshot/archive-restore.ts
5671
+ function rejectConfiguration(message) {
5672
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
5673
+ }
5674
+ function renamed(manifest, sourceName, indexName) {
5675
+ const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
5676
+ return {
5677
+ ...manifest,
5678
+ partitions: manifest.partitions.map((partition) => ({
5679
+ ...partition,
5680
+ segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
5681
+ })),
5682
+ vectors: manifest.vectors.map((field) => ({
5683
+ ...field,
5684
+ files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
5685
+ graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
5686
+ }))
5687
+ };
5688
+ }
5689
+ function requireSameSchema(prepared, manifest) {
5690
+ const configured = flattenSchema(prepared.config.schema);
5691
+ const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
5692
+ for (const field of keys) {
5693
+ if (configured[field] === manifest.schema[field]) continue;
5694
+ throw new NarsilError(
5695
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5696
+ `The archive's configuration gives field "${field}" another type than its manifest does`,
5697
+ { field }
5698
+ );
5699
+ }
5700
+ }
5701
+ function partitionCountOf(manifest) {
5702
+ let count = 1;
5703
+ for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
5704
+ return count;
5705
+ }
5706
+ function missingPart(what) {
5707
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
5708
+ }
5709
+ function keepsTextOnDisk(prepared, staging) {
5710
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
5711
+ }
5712
+ async function stageArchive(indexName, reader, staging, deps, staged) {
5713
+ await readSnapshotArchive(reader, {
5714
+ history: async (chunks) => {
5715
+ if (staged.history === void 0) {
5716
+ staged.history = await deps.querySuggestions?.historySink(staged.prepared?.querySuggestions ?? null) ?? null;
5717
+ }
5718
+ await staged.history?.add(chunks);
5719
+ },
5720
+ config: async (bytes) => {
5721
+ staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
5722
+ },
5723
+ manifest: async (manifest2, sourceName) => {
5724
+ if (staged.prepared === null) missingPart("configuration");
5725
+ requireSameSchema(staged.prepared, manifest2);
5726
+ const replacesIndex = deps.indexRegistry.has(indexName);
5727
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
5728
+ requireRestoreMemory(
5729
+ indexName,
5730
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
5731
+ replacesIndex
5732
+ );
5733
+ staged.manifest = renamed(manifest2, sourceName, indexName);
5734
+ },
5735
+ file: async (entryName, _ref, chunks) => {
5736
+ if (staged.manifest === null) missingPart("manifest");
5737
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
5738
+ }
5739
+ });
5740
+ const { prepared, manifest } = staged;
5741
+ if (prepared === null) missingPart("configuration");
5742
+ if (manifest === null) missingPart("manifest");
5743
+ const { directory } = staging;
5744
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
5745
+ const replacesIndex = deps.indexRegistry.has(indexName);
5746
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
5747
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
5748
+ }
5749
+ await validateStagedCheckpoint(staging, manifest);
5750
+ return { prepared, manifest };
5751
+ }
5752
+ async function loadStagedIndex(indexName, staged, staging, store, deps) {
5753
+ const { prepared, manifest } = staged;
5754
+ await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
5755
+ const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
5756
+ try {
5757
+ await loadSegmentedCheckpoint(
5758
+ store,
5759
+ indexName,
5760
+ manifest,
5761
+ { manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
5762
+ true
5763
+ );
5764
+ } catch (error) {
5765
+ deps.executor.discardDetachedIndex(detached);
5766
+ throw error;
5767
+ }
5768
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint };
5769
+ }
5770
+ async function restoreArchive(indexName, reader, deps) {
5771
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
5772
+ const store = staging?.store ?? memoryCheckpointStore();
5773
+ const archive = { prepared: null, manifest: null, history: void 0 };
5774
+ try {
5775
+ const staged = await stageArchive(indexName, reader, store, deps, archive);
5776
+ const restored = await loadStagedIndex(indexName, staged, staging, store, deps);
5777
+ await installRestoredIndex(indexName, restored, deps, archive.history ?? null);
5778
+ } catch (error) {
5779
+ await archive.history?.abandon().catch(() => void 0);
5780
+ throw error;
5781
+ } finally {
5782
+ await staging?.discard().catch(() => void 0);
5783
+ await reader.close();
5784
+ }
5785
+ }
5786
+
5787
+ // src/engine/snapshot/earlier-restore.ts
5788
+ function invalidSnapshot(message, details) {
5789
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
5790
+ }
5791
+ async function decodeEarlierSnapshot(data) {
5792
+ const payloadBytes = await unpackIndexSnapshotEnvelope(data);
5793
+ let decoded;
5794
+ try {
5795
+ decoded = decode(payloadBytes);
5796
+ } catch (err) {
5797
+ invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
5798
+ bytes: data.length
5799
+ });
5800
+ }
5801
+ if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
5802
+ invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
5803
+ }
5804
+ const envelope = decoded;
5805
+ if (envelope.version !== 1 && envelope.version !== 2) {
5806
+ invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
5807
+ version: envelope.version
5808
+ });
5809
+ }
5810
+ if (!envelope.schema || typeof envelope.schema !== "object")
5811
+ invalidSnapshot("Invalid snapshot: missing or invalid schema");
5812
+ if (!envelope.language || typeof envelope.language !== "string") {
5813
+ invalidSnapshot("Invalid snapshot: missing or invalid language");
5814
+ }
5815
+ if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
5816
+ return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
5817
+ }
5818
+ function vectorsOfSnapshot(envelope) {
5819
+ const byDocument = /* @__PURE__ */ new Map();
5820
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
5821
+ for (const part of decodeVectorIndexParts(rawParts)) {
5822
+ const values = bytesToVectors(part.vectors);
5823
+ for (let i = 0; i < part.docIds.length; i++) {
5824
+ let fields = byDocument.get(part.docIds[i]);
5825
+ if (fields === void 0) {
5826
+ fields = /* @__PURE__ */ new Map();
5827
+ byDocument.set(part.docIds[i], fields);
5828
+ }
5829
+ fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
5830
+ }
5831
+ }
5832
+ }
5833
+ return byDocument;
5834
+ }
5835
+ function footprintOf(envelope, prepared, replacesIndex) {
5836
+ let partitionBytes = 0;
5837
+ for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
5838
+ const vectorCounts = /* @__PURE__ */ new Map();
5839
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
5840
+ let count = 0;
5841
+ for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
5842
+ vectorCounts.set(fieldPath, count);
5843
+ }
5844
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
5845
+ }
5846
+ function documentsOfPartition(envelope, payload) {
5847
+ return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
5848
+ }
5849
+ function insertRestoredDocument(manager, docId, fields, vectors) {
5850
+ manager.insert(docId, fields, { skipClone: true });
5851
+ if (vectors === void 0) return;
5852
+ try {
5853
+ insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
5854
+ } catch (err) {
5855
+ manager.remove(docId);
5856
+ throw err;
5857
+ }
5858
+ }
5859
+ function insertSnapshotDocuments(envelope, manager) {
5860
+ const vectors = vectorsOfSnapshot(envelope);
5861
+ for (const payload of envelope.partitions ?? []) {
5862
+ for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
5863
+ insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
5864
+ }
5865
+ }
5866
+ for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
5867
+ }
5868
+ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
5869
+ const entry = restoredRegistryEntry(prepared, built.manager, deps);
5870
+ const archive = await buildArchiveFromMemory({
5871
+ indexName,
5872
+ metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
5873
+ manager: built.manager,
5874
+ vectorIndexes: built.manager.getVectorIndexes(),
5875
+ appliedSeqNo: () => 0,
5876
+ partitionId: null
5877
+ });
5878
+ try {
5879
+ for (const ref of manifestFileRefs(archive.manifest))
5880
+ await staging.store.write(ref.key, [await archive.readFile(ref)]);
5881
+ } finally {
5882
+ archive.release();
5883
+ }
5884
+ await validateStagedCheckpoint(staging.store, archive.manifest);
5885
+ return archive.manifest;
5886
+ }
5887
+ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
5888
+ const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
5889
+ let manifest;
5890
+ try {
5891
+ insertSnapshotDocuments(envelope, built.manager);
5892
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
5893
+ manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
5894
+ } catch (err) {
5895
+ deps.executor.discardDetachedIndex(built);
5896
+ throw err;
5897
+ }
5898
+ deps.executor.discardDetachedIndex(built);
5899
+ return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
5900
+ }
5901
+ async function restoreEarlierSnapshot(indexName, data, deps) {
5902
+ const envelope = await decodeEarlierSnapshot(data);
5903
+ const prepared = prepareRestoredIndex(envelope, deps);
5904
+ const replacesIndex = deps.indexRegistry.has(indexName);
5905
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
5906
+ const analysedNow = {
5907
+ ...prepared,
5908
+ storedRevision: prepared.language.revision,
5909
+ storedArrayPositionGap: ARRAY_POSITION_GAP
5910
+ };
5911
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
5912
+ try {
5913
+ await installRestoredIndex(
5914
+ indexName,
5915
+ await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
5916
+ deps
5917
+ );
5918
+ } finally {
5919
+ await staging?.discard().catch(() => void 0);
5920
+ }
5921
+ }
5922
+
5923
+ // src/engine/snapshot/index.ts
5924
+ var NRSL_MAGIC_BYTES = 4;
5925
+ async function checkpointForArchive(deps) {
5926
+ const durable = deps.durability?.manager;
5927
+ if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
5928
+ return buildArchiveFromMemory({
5929
+ indexName: deps.indexName,
5930
+ metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
5931
+ manager: deps.manager,
5932
+ vectorIndexes: deps.manager.getVectorIndexes(),
5933
+ appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
5934
+ partitionId: null
5935
+ });
5936
+ }
5937
+ async function* queryHistoryEntries(core, indexName) {
5938
+ yield* historyArchiveEntries(core, indexName);
5939
+ }
5940
+ async function createSnapshot(deps, coreWithAddedHistory) {
5941
+ const trailingEntries = coreWithAddedHistory === null ? void 0 : () => queryHistoryEntries(coreWithAddedHistory, deps.indexName);
5942
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
5943
+ const checkpoint = await checkpointForArchive(deps);
5944
+ try {
5945
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
5946
+ const config = await encodeIndexConfiguration(deps.entry);
5947
+ return archiveStream({
5948
+ ...checkpoint,
5949
+ config,
5950
+ sourceIndexName: deps.indexName,
5951
+ ...trailingEntries !== void 0 ? { trailingEntries } : {}
5952
+ });
5953
+ }
5954
+ } catch (error) {
5955
+ checkpoint.release();
5956
+ throw error;
5957
+ }
5958
+ checkpoint.release();
5959
+ }
5960
+ throw new NarsilError(
5961
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
5962
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
5963
+ { indexName: deps.indexName }
5964
+ );
5965
+ }
5966
+ function requireSnapshotSource(data) {
5967
+ if (data instanceof Uint8Array) return data;
5968
+ if (data !== null && typeof data === "object") {
5969
+ if (typeof data.getReader === "function") return data;
5970
+ if (typeof data[Symbol.asyncIterator] === "function") {
5971
+ return data;
5972
+ }
5973
+ }
5974
+ throw new NarsilError(
5975
+ ErrorCodes.DOC_VALIDATION_FAILED,
5976
+ "Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
5977
+ );
5978
+ }
5979
+ async function readEarlierSnapshotFile(reader) {
5980
+ const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
5981
+ if (length > MAX_FILE_BYTES) {
5982
+ throw new NarsilError(
5983
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5984
+ `The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
5985
+ { bytes: length }
5986
+ );
5987
+ }
5988
+ const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
5989
+ if (bytes.byteLength < length) {
5990
+ throw new NarsilError(
5991
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5992
+ "The snapshot ends before the payload its header declares"
5993
+ );
5994
+ }
5995
+ const trailing = await reader.readExactly(1);
5996
+ if (trailing !== null && trailing.byteLength > 0) {
5997
+ throw new NarsilError(
5998
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5999
+ "The snapshot holds bytes after the payload its header declares"
6000
+ );
6001
+ }
6002
+ return bytes;
6003
+ }
6004
+ async function restoreFromSnapshot(indexName, data, deps) {
6005
+ const reader = createByteReader(requireSnapshotSource(data));
6006
+ const head = await reader.peek(NRSL_MAGIC_BYTES);
6007
+ if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
6008
+ let bytes;
6009
+ try {
6010
+ bytes = await readEarlierSnapshotFile(reader);
6011
+ } finally {
6012
+ await reader.close();
6013
+ }
6014
+ await restoreEarlierSnapshot(indexName, bytes, deps);
6015
+ return;
6016
+ }
6017
+ if (head.byteLength === 0) {
6018
+ await reader.close();
6019
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
6020
+ }
6021
+ await restoreArchive(indexName, reader, deps);
6022
+ }
6023
+
6024
+ // src/engine/sync/status.ts
6025
+ function syncStatusOf(sync) {
6026
+ if (sync === null) return { enabled: false, active: false, epoch: null };
6027
+ return {
6028
+ enabled: true,
6029
+ ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
6030
+ ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {},
6031
+ active: sync.active,
6032
+ epoch: sync.epoch
6033
+ };
6034
+ }
6035
+
6036
+ // src/engine/sync/methods.ts
6037
+ async function updateSyncOf(core, indexName, setting) {
6038
+ core.guardShutdown();
6039
+ core.requireIndex(indexName);
6040
+ await updateIndexSync(core, indexName, setting);
6041
+ }
6042
+ function syncOf(core, indexName) {
6043
+ core.guardShutdown();
6044
+ return syncStatusOf(core.requireIndex(indexName).sync);
6045
+ }
6046
+
6047
+ // src/persistence/paged/rewrite-spill.ts
6048
+ function openRewriteSpill(storage, manager, target) {
6049
+ const config = target?.config ?? manager.config;
6050
+ return createRewriteSpill(storage, manager.indexName, {
6051
+ schema: flattenSchema(config.schema),
6052
+ trackPositions: config.trackPositions ?? true,
6053
+ partialWordAnalysis: target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis,
6054
+ textAnalysis: {
6055
+ language: target?.language ?? manager.language,
6056
+ options: resolvePartitionInsertOptions(config, target?.analysis ?? manager.analysis)
6057
+ }
6058
+ });
6059
+ }
6060
+ function createRewriteSpill(storage, indexName, analysis) {
6061
+ const { directory, writeRecords } = storage;
6062
+ const prefix = `${indexName}/rebuilt/${generateId()}/`;
6063
+ const opened = [];
6064
+ let nextFile = 0;
6065
+ async function removeFiles() {
6066
+ for (const key of await directory.list(prefix)) await directory.remove(key);
6067
+ }
6068
+ function freezeLive(partition) {
6069
+ if (!isCompositePartition(partition)) return null;
6070
+ return partition.freezeLiveTail((payload, documents) => {
6071
+ const snapshot = freezeSegmentShared(payload, documents, void 0, analysis.partialWordAnalysis);
6072
+ return snapshot === null ? createFrozenSegment(payload, documents) : createSharedFrozenSegment(snapshot);
6073
+ });
6074
+ }
6075
+ return {
6076
+ newPartition(partitionId) {
6077
+ const live = createPartitionIndex(partitionId, analysis.trackPositions);
6078
+ return createCompositePartition(partitionId, analysis.trackPositions, live, analysis.textAnalysis);
6079
+ },
6080
+ async spillIfFull(partition, partitionId) {
6081
+ if (!isCompositePartition(partition)) return;
6082
+ if (partition.live.estimateMemoryBytes() < REWRITE_SPILL_BYTES) return;
6083
+ const frozen = freezeLive(partition);
6084
+ if (frozen === null) return;
6085
+ const key = `${prefix}${nextFile}`;
6086
+ nextFile += 1;
6087
+ const written = await writePagedSegment({
6088
+ directory,
6089
+ key,
6090
+ schema: analysis.schema,
6091
+ tombstones: [],
6092
+ selections: [{ segment: frozen, keep: (ordinal) => !frozen.isTombstoned(ordinal) }],
6093
+ partialWordAnalysis: analysis.partialWordAnalysis,
6094
+ textAnalysis: analysis.textAnalysis,
6095
+ ...writeRecords === null ? {} : { writeRecords: writeRecords.captureSegments(indexName, partitionId, [frozen]) }
6096
+ });
6097
+ const listing = await pagedListingOf(directory, { key, bytes: written.bytes, checksum: written.checksum });
6098
+ const segment = createPagedFrozenSegment(openPagedSnapshot(listing));
6099
+ opened.push(segment);
6100
+ partition.swapFrozenSegments([frozen.segmentId], segment);
6101
+ },
6102
+ async abandon() {
6103
+ for (const segment of opened.splice(0)) segment.paged?.close();
6104
+ await removeFiles();
6105
+ },
6106
+ async removeUnread(manager) {
6107
+ for (const partition of manager.getAllPartitions()) {
6108
+ if (!isCompositePartition(partition)) continue;
6109
+ for (const segment of partition.frozenSegments()) {
6110
+ if (segment.paged?.key.startsWith(prefix)) return;
6111
+ }
6112
+ }
6113
+ for (const segment of opened.splice(0)) segment.paged?.close();
6114
+ await removeFiles();
6115
+ }
6116
+ };
6117
+ }
4672
6118
 
4673
- export { createDurabilityFromTier };
6119
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, actionPartitionsOf, adoptSegmentsOnWorker, checkpointForArchive, closeUnusedSegments, createDurabilityFromTier, createInvalidationFromConfig, createSnapshot, documentExportStream, keysThatSegmentsStillRead, openFileVectorReads, openRewriteSpill, pagedPartitionLines, prepareDiskAdoption, readDocumentExport, reloadIndexAndCopies, restoreFromSnapshot, swapPreparedPartitions, syncOf, updateSyncOf };