@delali/narsil 0.3.1-next.157 → 0.3.1-next.160

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (565) hide show
  1. package/dist/adapters/filesystem.mjs +3 -3
  2. package/dist/adapters/gcs.mjs +3 -3
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/adapters/s3.mjs +3 -3
  5. package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
  6. package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
  7. package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
  8. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  9. package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
  10. package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
  11. package/dist/browser/embed-RF2MPGAO.mjs +222 -0
  12. package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
  13. package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
  14. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  15. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  16. package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
  17. package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
  18. package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
  19. package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
  20. package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
  21. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  22. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  23. package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
  24. package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
  25. package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
  26. package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
  27. package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
  28. package/dist/chunk-2JRERUNZ.mjs +217 -0
  29. package/dist/chunk-2RIS76RJ.mjs +212 -0
  30. package/dist/chunk-3B775F3C.mjs +252 -0
  31. package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
  32. package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
  33. package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
  34. package/dist/chunk-6EZW2GQL.mjs +978 -0
  35. package/dist/chunk-75UNINFA.mjs +271 -0
  36. package/dist/chunk-77AUVDQ5.mjs +7 -0
  37. package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
  38. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  39. package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
  40. package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
  41. package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
  42. package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
  43. package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
  44. package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
  45. package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
  46. package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
  47. package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
  48. package/dist/chunk-GXCDWLQI.mjs +193 -0
  49. package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
  50. package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
  51. package/dist/chunk-IB4BYLE3.mjs +127 -0
  52. package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
  53. package/dist/chunk-IH3G6SJO.mjs +40 -0
  54. package/dist/chunk-JV74RKCY.mjs +3041 -0
  55. package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
  56. package/dist/chunk-KLVMK3R5.mjs +47 -0
  57. package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
  58. package/dist/chunk-LF4GB4PO.mjs +281 -0
  59. package/dist/chunk-NC4QLCZC.mjs +59 -0
  60. package/dist/chunk-OOO54OVC.mjs +627 -0
  61. package/dist/chunk-OUWZKNO3.mjs +207 -0
  62. package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
  63. package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
  64. package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
  65. package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
  66. package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
  67. package/dist/chunk-S2POFTLN.mjs +10 -0
  68. package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
  69. package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
  70. package/dist/chunk-SXIRM4MX.mjs +3549 -0
  71. package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
  72. package/dist/chunk-TINW4BSH.mjs +31301 -0
  73. package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
  74. package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
  75. package/dist/chunk-U6B7DUQW.mjs +788 -0
  76. package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
  77. package/dist/chunk-W34VJXY7.mjs +4723 -0
  78. package/dist/chunk-YETKPX5M.mjs +5 -0
  79. package/dist/chunk-ZQ6QY42L.mjs +59 -0
  80. package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
  81. package/dist/client.d.ts +8 -8
  82. package/dist/client.mjs +66 -18
  83. package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
  84. package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
  85. package/dist/core-leaf-KXFGP74A.mjs +7 -0
  86. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  87. package/dist/direct-executor-MKTZBR26.mjs +16 -0
  88. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  89. package/dist/distribution/coordinator/etcd.mjs +7 -4
  90. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  91. package/dist/distribution/coordinator/in-memory.mjs +1 -2
  92. package/dist/distribution/transport/grpc.mjs +7 -2
  93. package/dist/distribution/transport/tcp.mjs +7 -1
  94. package/dist/distribution.d.ts +7 -7
  95. package/dist/distribution.mjs +1199 -1092
  96. package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
  97. package/dist/embed-4H7RTBVL.mjs +221 -0
  98. package/dist/embeddings/openai.mjs +2 -2
  99. package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
  100. package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
  101. package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
  102. package/dist/index.browser.mjs +4 -16
  103. package/dist/index.d.ts +1495 -78
  104. package/dist/index.mjs +14 -50
  105. package/dist/index.react-native.mjs +11 -28
  106. package/dist/invalidation/filesystem.mjs +1 -2
  107. package/dist/languages/albanian.mjs +1 -1
  108. package/dist/languages/amharic.mjs +1 -1
  109. package/dist/languages/arabic.mjs +1 -1
  110. package/dist/languages/armenian.mjs +1 -1
  111. package/dist/languages/azerbaijani.mjs +1 -1
  112. package/dist/languages/bambara.mjs +1 -1
  113. package/dist/languages/basque.mjs +1 -1
  114. package/dist/languages/belarusian.mjs +1 -1
  115. package/dist/languages/bengali.mjs +1 -1
  116. package/dist/languages/bosnian.mjs +1 -1
  117. package/dist/languages/breton.mjs +1 -1
  118. package/dist/languages/bulgarian.mjs +1 -1
  119. package/dist/languages/burmese.mjs +1 -1
  120. package/dist/languages/catalan.mjs +1 -1
  121. package/dist/languages/chinese.mjs +1 -1
  122. package/dist/languages/croatian.mjs +1 -1
  123. package/dist/languages/czech.mjs +1 -1
  124. package/dist/languages/dagbani.mjs +1 -1
  125. package/dist/languages/danish.mjs +1 -1
  126. package/dist/languages/dutch.mjs +1 -1
  127. package/dist/languages/english.mjs +1 -1
  128. package/dist/languages/esperanto.mjs +1 -1
  129. package/dist/languages/estonian.mjs +1 -1
  130. package/dist/languages/ewe.mjs +1 -1
  131. package/dist/languages/faroese.mjs +1 -1
  132. package/dist/languages/fijian.mjs +1 -1
  133. package/dist/languages/finnish.mjs +1 -1
  134. package/dist/languages/french.mjs +1 -1
  135. package/dist/languages/ga.mjs +1 -1
  136. package/dist/languages/galician.mjs +1 -1
  137. package/dist/languages/georgian.mjs +1 -1
  138. package/dist/languages/german.mjs +1 -1
  139. package/dist/languages/greek.mjs +1 -1
  140. package/dist/languages/guarani.mjs +1 -1
  141. package/dist/languages/gujarati.mjs +1 -1
  142. package/dist/languages/haitian-creole.mjs +1 -1
  143. package/dist/languages/hausa.mjs +1 -1
  144. package/dist/languages/hawaiian.mjs +1 -1
  145. package/dist/languages/hebrew.mjs +1 -1
  146. package/dist/languages/hindi.mjs +1 -1
  147. package/dist/languages/hungarian.mjs +1 -1
  148. package/dist/languages/icelandic.mjs +1 -1
  149. package/dist/languages/igbo.mjs +1 -1
  150. package/dist/languages/indonesian.mjs +1 -1
  151. package/dist/languages/irish.mjs +1 -1
  152. package/dist/languages/italian.mjs +1 -1
  153. package/dist/languages/japanese.mjs +1 -1
  154. package/dist/languages/kannada.mjs +1 -1
  155. package/dist/languages/kazakh.mjs +1 -1
  156. package/dist/languages/khmer.mjs +1 -1
  157. package/dist/languages/kinyarwanda.mjs +1 -1
  158. package/dist/languages/kirundi.mjs +1 -1
  159. package/dist/languages/korean.mjs +1 -1
  160. package/dist/languages/kurmanji.mjs +1 -1
  161. package/dist/languages/kyrgyz.mjs +1 -1
  162. package/dist/languages/lao.mjs +1 -1
  163. package/dist/languages/latin.mjs +1 -1
  164. package/dist/languages/latvian.mjs +1 -1
  165. package/dist/languages/lingala.mjs +1 -1
  166. package/dist/languages/lithuanian.mjs +1 -1
  167. package/dist/languages/luxembourgish.mjs +1 -1
  168. package/dist/languages/macedonian.mjs +1 -1
  169. package/dist/languages/malagasy.mjs +1 -1
  170. package/dist/languages/malay.mjs +1 -1
  171. package/dist/languages/malayalam.mjs +1 -1
  172. package/dist/languages/maltese.mjs +1 -1
  173. package/dist/languages/maori.mjs +1 -1
  174. package/dist/languages/marathi.mjs +1 -1
  175. package/dist/languages/nepali.mjs +1 -1
  176. package/dist/languages/norwegian.mjs +1 -1
  177. package/dist/languages/oromo.mjs +1 -1
  178. package/dist/languages/persian.mjs +1 -1
  179. package/dist/languages/polish.mjs +1 -1
  180. package/dist/languages/portuguese.mjs +1 -1
  181. package/dist/languages/punjabi.mjs +1 -1
  182. package/dist/languages/romanian.mjs +1 -1
  183. package/dist/languages/russian.mjs +1 -1
  184. package/dist/languages/samoan.mjs +1 -1
  185. package/dist/languages/sanskrit.mjs +1 -1
  186. package/dist/languages/scottish-gaelic.mjs +1 -1
  187. package/dist/languages/serbian.mjs +1 -1
  188. package/dist/languages/shona.mjs +1 -1
  189. package/dist/languages/sinhala.mjs +1 -1
  190. package/dist/languages/slovak.mjs +1 -1
  191. package/dist/languages/slovenian.mjs +1 -1
  192. package/dist/languages/sorani.mjs +1 -1
  193. package/dist/languages/spanish.mjs +1 -1
  194. package/dist/languages/swahili.mjs +1 -1
  195. package/dist/languages/swedish.mjs +1 -1
  196. package/dist/languages/tagalog.mjs +1 -1
  197. package/dist/languages/tamil.mjs +1 -1
  198. package/dist/languages/tatar.mjs +1 -1
  199. package/dist/languages/telugu.mjs +1 -1
  200. package/dist/languages/thai.mjs +1 -1
  201. package/dist/languages/tibetan.mjs +1 -1
  202. package/dist/languages/tigrinya.mjs +1 -1
  203. package/dist/languages/tongan.mjs +1 -1
  204. package/dist/languages/turkish.mjs +1 -1
  205. package/dist/languages/twi.mjs +1 -1
  206. package/dist/languages/ukrainian.mjs +1 -1
  207. package/dist/languages/urdu.mjs +1 -1
  208. package/dist/languages/vietnamese.mjs +1 -1
  209. package/dist/languages/welsh.mjs +1 -1
  210. package/dist/languages/wolof.mjs +1 -1
  211. package/dist/languages/xhosa.mjs +1 -1
  212. package/dist/languages/yoruba.mjs +1 -1
  213. package/dist/languages/zulu.mjs +1 -1
  214. package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
  215. package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
  216. package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
  217. package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
  218. package/dist/optional-writes-DDFHYUOH.mjs +218 -0
  219. package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
  220. package/dist/partial-words.browser.mjs +1649 -0
  221. package/dist/partial-words.mjs +15 -14
  222. package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
  223. package/dist/pattern.mjs +6 -8
  224. package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
  225. package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
  226. package/dist/phrase-module-XK3T44VU.mjs +634 -0
  227. package/dist/query-actions-OMXJEC2J.mjs +118 -0
  228. package/dist/query-suggestions/merge-worker.mjs +18 -54
  229. package/dist/query-suggestions.browser.mjs +26 -227
  230. package/dist/query-suggestions.d.ts +2 -2
  231. package/dist/query-suggestions.mjs +16 -67
  232. package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
  233. package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
  234. package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
  235. package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
  236. package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
  237. package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
  238. package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
  239. package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
  240. package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
  241. package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
  242. package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
  243. package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
  244. package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
  245. package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
  246. package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
  247. package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
  248. package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
  249. package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
  250. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  251. package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
  252. package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
  253. package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
  254. package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
  255. package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
  256. package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
  257. package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
  258. package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
  259. package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
  260. package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
  261. package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
  262. package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
  263. package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
  264. package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
  265. package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
  266. package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
  267. package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
  268. package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
  269. package/dist/react-native/partial-words.mjs +954 -35
  270. package/dist/react-native/pattern.mjs +238 -111
  271. package/dist/react-native/query-suggestions.mjs +33 -239
  272. package/dist/react-native/sync.mjs +95 -52
  273. package/dist/react-native/synonyms.mjs +256 -112
  274. package/dist/react.d.ts +9 -9
  275. package/dist/react.mjs +6 -0
  276. package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
  277. package/dist/rerankers/http.mjs +2 -2
  278. package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
  279. package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
  280. package/dist/runtime-4GKUIAKV.mjs +595 -0
  281. package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
  282. package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
  283. package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
  284. package/dist/server.d.ts +33 -13
  285. package/dist/server.mjs +81 -100
  286. package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
  287. package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
  288. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  289. package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
  290. package/dist/sync/filesystem.d.ts +3 -3
  291. package/dist/sync/filesystem.mjs +27 -11
  292. package/dist/sync/indexeddb.browser.mjs +287 -0
  293. package/dist/sync/indexeddb.d.ts +3 -3
  294. package/dist/sync/indexeddb.mjs +7 -2
  295. package/dist/sync.browser.mjs +88 -72
  296. package/dist/sync.d.ts +4 -4
  297. package/dist/sync.mjs +89 -59
  298. package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
  299. package/dist/synonyms.mjs +15 -11
  300. package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
  301. package/dist/thread-SNILJXX3.mjs +30 -0
  302. package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
  303. package/dist/vector/search-worker.d.ts +7 -1
  304. package/dist/vector/search-worker.mjs +9 -13
  305. package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
  306. package/dist/workers/entry.mjs +60 -30
  307. package/openapi.json +66 -50
  308. package/package.json +233 -8
  309. package/dist/analysis-change-CKEMM2MC.mjs +0 -55
  310. package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
  311. package/dist/archive-history-E4R73RWD.mjs +0 -29
  312. package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
  313. package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
  314. package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
  315. package/dist/browser/chunk-25MTVDYD.mjs +0 -432
  316. package/dist/browser/chunk-3I27KXQD.mjs +0 -34
  317. package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
  318. package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
  319. package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
  320. package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
  321. package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
  322. package/dist/browser/chunk-AI5UI46D.mjs +0 -12
  323. package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
  324. package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
  325. package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
  326. package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
  327. package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
  328. package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
  329. package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
  330. package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
  331. package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
  332. package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
  333. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  334. package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
  335. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  336. package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
  337. package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
  338. package/dist/browser/chunk-MY2XKH22.mjs +0 -40
  339. package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
  340. package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
  341. package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
  342. package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
  343. package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
  344. package/dist/browser/chunk-TK63WFXK.mjs +0 -90
  345. package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
  346. package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
  347. package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
  348. package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
  349. package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
  350. package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
  351. package/dist/browser/chunk-V43N6TZO.mjs +0 -857
  352. package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
  353. package/dist/browser/chunk-WN464PTL.mjs +0 -375
  354. package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
  355. package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
  356. package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
  357. package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
  358. package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
  359. package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
  360. package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
  361. package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
  362. package/dist/browser/import-5ESVMFDJ.mjs +0 -20
  363. package/dist/browser/methods-KTQCSIN4.mjs +0 -30
  364. package/dist/browser/module-NDSQLPMQ.mjs +0 -25
  365. package/dist/browser/operations-OJ3BXS55.mjs +0 -5
  366. package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
  367. package/dist/browser/pool-3JLET5RO.mjs +0 -297
  368. package/dist/browser/release-HYXPVJVS.mjs +0 -32
  369. package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
  370. package/dist/browser/retention-TY3L3BFN.mjs +0 -211
  371. package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
  372. package/dist/browser/start-2UZTOBG5.mjs +0 -26
  373. package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
  374. package/dist/browser/write-ULXAN5B4.mjs +0 -196
  375. package/dist/chunk-2NE7VCYE.mjs +0 -212
  376. package/dist/chunk-2ROUFMQX.mjs +0 -543
  377. package/dist/chunk-37CD3OKS.mjs +0 -87
  378. package/dist/chunk-3MXV3VHR.mjs +0 -46
  379. package/dist/chunk-3OH67G7R.mjs +0 -214
  380. package/dist/chunk-4B7JIGQD.mjs +0 -106
  381. package/dist/chunk-5YNRKI2C.mjs +0 -81
  382. package/dist/chunk-6HTGOXIK.mjs +0 -875
  383. package/dist/chunk-6OEGXLBX.mjs +0 -57
  384. package/dist/chunk-6RC4K46J.mjs +0 -86
  385. package/dist/chunk-7GFE3PEY.mjs +0 -40
  386. package/dist/chunk-7GXJUJ24.mjs +0 -49
  387. package/dist/chunk-7V5IUP2T.mjs +0 -63
  388. package/dist/chunk-A4L7YTKX.mjs +0 -458
  389. package/dist/chunk-BEZH7VEH.mjs +0 -23
  390. package/dist/chunk-BNL5OPRV.mjs +0 -634
  391. package/dist/chunk-BTPGQC6Z.mjs +0 -27
  392. package/dist/chunk-C7XQYG6U.mjs +0 -50
  393. package/dist/chunk-CQZXZ75V.mjs +0 -562
  394. package/dist/chunk-CWS55TEX.mjs +0 -242
  395. package/dist/chunk-CZQ33DLW.mjs +0 -78
  396. package/dist/chunk-EEBCDP36.mjs +0 -32
  397. package/dist/chunk-FB6463IN.mjs +0 -1499
  398. package/dist/chunk-FCJ553HS.mjs +0 -655
  399. package/dist/chunk-FEZVDOWX.mjs +0 -6996
  400. package/dist/chunk-FOCV4CBZ.mjs +0 -87
  401. package/dist/chunk-FQBFZAWG.mjs +0 -27
  402. package/dist/chunk-G6HRSGOU.mjs +0 -74
  403. package/dist/chunk-GES7U2PM.mjs +0 -68
  404. package/dist/chunk-GH4WQ22V.mjs +0 -34
  405. package/dist/chunk-GPWMUPCA.mjs +0 -1126
  406. package/dist/chunk-GQPTDQCL.mjs +0 -54
  407. package/dist/chunk-HJSN7UYG.mjs +0 -10
  408. package/dist/chunk-I54T6CQP.mjs +0 -117
  409. package/dist/chunk-IPN7S4KX.mjs +0 -130
  410. package/dist/chunk-J4IJTTEX.mjs +0 -226
  411. package/dist/chunk-JSV6OQ4C.mjs +0 -74
  412. package/dist/chunk-K4ZIGL2O.mjs +0 -290
  413. package/dist/chunk-KAKQ74LS.mjs +0 -53
  414. package/dist/chunk-KCQYFO4K.mjs +0 -62
  415. package/dist/chunk-KHKIRS5D.mjs +0 -113
  416. package/dist/chunk-KOKM4PQN.mjs +0 -52
  417. package/dist/chunk-KP7TYHWT.mjs +0 -92
  418. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  419. package/dist/chunk-L5ORED7V.mjs +0 -25
  420. package/dist/chunk-LDNCLRZU.mjs +0 -218
  421. package/dist/chunk-LHKOSI2J.mjs +0 -58
  422. package/dist/chunk-LQG5G2LY.mjs +0 -39
  423. package/dist/chunk-MATMVH65.mjs +0 -9
  424. package/dist/chunk-MP7ABPTD.mjs +0 -189
  425. package/dist/chunk-MW6BR4QP.mjs +0 -316
  426. package/dist/chunk-MWD5VPN7.mjs +0 -21
  427. package/dist/chunk-ODYKO5TO.mjs +0 -101
  428. package/dist/chunk-OG7UGJ6F.mjs +0 -63
  429. package/dist/chunk-ORN5BLL7.mjs +0 -11
  430. package/dist/chunk-OYABFKTW.mjs +0 -1974
  431. package/dist/chunk-P5LANSH6.mjs +0 -121
  432. package/dist/chunk-PELHVTO7.mjs +0 -24
  433. package/dist/chunk-PIH5SJ4N.mjs +0 -7481
  434. package/dist/chunk-PSZI4O24.mjs +0 -253
  435. package/dist/chunk-PV2CG7EJ.mjs +0 -886
  436. package/dist/chunk-PWGYQXML.mjs +0 -470
  437. package/dist/chunk-Q2UL2TPC.mjs +0 -21
  438. package/dist/chunk-QHPYS427.mjs +0 -66
  439. package/dist/chunk-QJVOLKU6.mjs +0 -111
  440. package/dist/chunk-QLDABJZM.mjs +0 -20
  441. package/dist/chunk-QYRNLLLI.mjs +0 -99
  442. package/dist/chunk-RCX4WNLC.mjs +0 -151
  443. package/dist/chunk-RFBLQLLZ.mjs +0 -16
  444. package/dist/chunk-RIHIVFIQ.mjs +0 -240
  445. package/dist/chunk-RLOES5SX.mjs +0 -31
  446. package/dist/chunk-RWURUDSP.mjs +0 -2639
  447. package/dist/chunk-RZWWB2XR.mjs +0 -96
  448. package/dist/chunk-S5LN746F.mjs +0 -10
  449. package/dist/chunk-SOI6LTHM.mjs +0 -216
  450. package/dist/chunk-SQJWAIF4.mjs +0 -12
  451. package/dist/chunk-TDIK6NTV.mjs +0 -10
  452. package/dist/chunk-TQXHPFWT.mjs +0 -177
  453. package/dist/chunk-U55XJEIG.mjs +0 -18
  454. package/dist/chunk-UD5HIS4U.mjs +0 -382
  455. package/dist/chunk-UHICS3JI.mjs +0 -83
  456. package/dist/chunk-UJDJB3NE.mjs +0 -913
  457. package/dist/chunk-VD7J552I.mjs +0 -28
  458. package/dist/chunk-VSLSGASN.mjs +0 -30
  459. package/dist/chunk-VZ2ONHEE.mjs +0 -3715
  460. package/dist/chunk-W7OSSKH5.mjs +0 -1427
  461. package/dist/chunk-WZ7YV54N.mjs +0 -152
  462. package/dist/chunk-X5CV2E2B.mjs +0 -1
  463. package/dist/chunk-Y4WPNHMC.mjs +0 -50
  464. package/dist/chunk-YEKL4JBK.mjs +0 -96
  465. package/dist/chunk-ZXNQEW43.mjs +0 -18
  466. package/dist/chunk-ZYAY26KG.mjs +0 -29
  467. package/dist/direct-executor-V5DRIECI.mjs +0 -34
  468. package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
  469. package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
  470. package/dist/gzip-F4HMIEEQ.mjs +0 -1
  471. package/dist/import-GPKLTSKA.mjs +0 -56
  472. package/dist/methods-7IOINERF.mjs +0 -30
  473. package/dist/module-JYHFTVPS.mjs +0 -63
  474. package/dist/operations-G5EP6JPX.mjs +0 -5
  475. package/dist/phrase-module-XW5MKWP5.mjs +0 -330
  476. package/dist/pool-Y4A46T36.mjs +0 -298
  477. package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
  478. package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
  479. package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
  480. package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
  481. package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
  482. package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
  483. package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
  484. package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
  485. package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
  486. package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
  487. package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
  488. package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
  489. package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
  490. package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
  491. package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
  492. package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
  493. package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
  494. package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
  495. package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
  496. package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
  497. package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
  498. package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
  499. package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
  500. package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
  501. package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
  502. package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
  503. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
  504. package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
  505. package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
  506. package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
  507. package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
  508. package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
  509. package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
  510. package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
  511. package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
  512. package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
  513. package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
  514. package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
  515. package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
  516. package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
  517. package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
  518. package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
  519. package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
  520. package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
  521. package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
  522. package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
  523. package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
  524. package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
  525. package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
  526. package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
  527. package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
  528. package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
  529. package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
  530. package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
  531. package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
  532. package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
  533. package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
  534. package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
  535. package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
  536. package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
  537. package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
  538. package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
  539. package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
  540. package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
  541. package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
  542. package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
  543. package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
  544. package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
  545. package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
  546. package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
  547. package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
  548. package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
  549. package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
  550. package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
  551. package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
  552. package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
  553. package/dist/release-SMR4WJD4.mjs +0 -32
  554. package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
  555. package/dist/restore-queue-LZH7LMLR.mjs +0 -55
  556. package/dist/retention-3K7ARGAG.mjs +0 -66
  557. package/dist/runtime-COIUORCN.mjs +0 -177
  558. package/dist/snapshot-O7ICP7NJ.mjs +0 -783
  559. package/dist/start-S4PFP6RS.mjs +0 -66
  560. package/dist/sync-state-ZWHLAFLE.mjs +0 -15
  561. package/dist/thread-YLJ3ZPQ3.mjs +0 -50
  562. package/dist/write-2OBJGLFS.mjs +0 -233
  563. /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
  564. /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
  565. /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
@@ -1,548 +1,201 @@
1
- import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-K3IPRX6D.mjs';
2
- import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-G64Q6OJF.mjs';
3
- import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-BLT4RHCJ.mjs';
4
- import './chunk-YJHGOJAC.mjs';
5
- import { sha256 } from './chunk-M3MQE2X7.mjs';
6
- import './chunk-HUCSHDTB.mjs';
7
- import './chunk-25MTVDYD.mjs';
8
- import { QUERY_HISTORY_FOLDER } from './chunk-7IGKGWSQ.mjs';
9
- import './chunk-3I27KXQD.mjs';
10
- import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
11
- import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-V43N6TZO.mjs';
12
- import './chunk-Y6IBAUNO.mjs';
13
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-ZI2V4VD7.mjs';
14
- import { restoreVectorFields } from './chunk-EKMGCRSR.mjs';
15
- import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION, manifestWithinPartitionCount } from './chunk-5YWB3E2W.mjs';
16
- import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment, removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-N23GL72Q.mjs';
17
- import './chunk-ML7THVLJ.mjs';
18
- import { createRequestId } from './chunk-VI7BQR4U.mjs';
19
- import './chunk-JPTQZWJQ.mjs';
20
- import { encodeDocument, openChecksummedEnvelope, crc32, requireValidFieldTypes, HEADER_SIZE } from './chunk-DUH6QUVU.mjs';
21
- import { fnv1a } from './chunk-RA6LSBCM.mjs';
22
- import './chunk-MY2XKH22.mjs';
23
- import './chunk-AI5UI46D.mjs';
24
- import './chunk-TK63WFXK.mjs';
25
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WN464PTL.mjs';
26
- import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime, resolveWorkerEntry } from './chunk-XAWNZ4G2.mjs';
27
- import './chunk-TY7L6OXO.mjs';
28
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
29
- import { compareCodePoints, MIN_M } from './chunk-NXQG7DRA.mjs';
30
- import { MAX_FILE_BYTES } from './chunk-GPPOG6PT.mjs';
31
- import './chunk-WXLBZJMT.mjs';
32
- import { NarsilError, ErrorCodes } from './chunk-SRMZXJSL.mjs';
33
- import './chunk-JNOAKDUM.mjs';
34
- import { decode, encode } from '@msgpack/msgpack';
1
+ import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES, scheduleSyncRetention } from './chunk-FE3MRIBN.mjs';
2
+ export { scheduleSyncRetention, startSyncRetentionSweep } from './chunk-FE3MRIBN.mjs';
3
+ import { openPagedSegmentScan, countsAsLatestHolder, buildArchiveFromMemory, indexMetadataOf, archiveStream, createByteReader, admitDurableWrite, chunksOf, requireRestoreMemory, memoryCheckpointStore, createIndexLoads, closePartitionWriter, switchInTurn, writeMetadataEnvelope, listPersistedIndexes, removeLeftoverRewriteFiles, unpackIndexSnapshotEnvelope, earlierSnapshotFootprint, writeArchiveFile, storedBytesOfSegments, archiveFootprint, bytesThatSegmentsHoldInMemory, loadSegmentedCheckpoint, requireCheckpointLayout, holdScheduledCheckpoint, checkpointInTurn, metadataFromConfig, createDetachedIndex, checkPagedSegmentFully, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, drainIndexStateForUnload, readMetadataEnvelope, snapshotCheckpointFor, captureAppliedPrefix, metadataPartitionCount, deserializePayloadV1, createSeqOwner, loadMetadata, readSegmentManifest, reclaimUnlistedKeys, writeLiveVectors, requiresLayoutCheckpoint, transitionKey, readTransition, invalidTransition, unpackEnvelopeBytes, dropStoredVectorValues, replayWal, sanitizeSurfaceForms, patternIndexesFromWire, replayWalUpTo, createRebalancer, growPartitionsTo, createMarkerWriter, metadataConfigOf, durableMetadataConfigOf } from './chunk-SXIRM4MX.mjs';
4
+ export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedSegmentFully, countsAsLatestHolder, createByteReader, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, openPagedSegmentScan, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration, startReadingOpeningSectionsAhead, targetOfPending, transitionWalLayout } from './chunk-SXIRM4MX.mjs';
5
+ import { openColdVectorField, readVectorFile, readGraphParts, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-TSLV5P2J.mjs';
6
+ export { loadFilesByParts, openColdVectorField, waitBeforeColdLoad } from './chunk-TSLV5P2J.mjs';
7
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration, updateIndexSync, requireConfigurationShape, decodeIndexConfiguration } from './chunk-2JRERUNZ.mjs';
8
+ export { decodeIndexConfiguration, encodeIndexConfiguration, holdRenewedSyncEpoch, indexConfigurationOf, requireNoQuerySuggestions, requireSyncSetting, syncStorageOf, updateIndexSync } from './chunk-2JRERUNZ.mjs';
9
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-4CAMH7FO.mjs';
10
+ import { readSnapshotArchive, decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-OOO54OVC.mjs';
11
+ export { readSnapshotArchive } from './chunk-OOO54OVC.mjs';
12
+ import { createDurableDirectory } from './chunk-TW543F33.mjs';
13
+ import { isPagedSegmentFile, openPagedSnapshot, pagedListingOf, createPagedFrozenSegment, writePagedSegment, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, readSegmentFile, SEGMENT_HEADER_SIZE, checkpointLastSeqNo, partitionContentsOf, writeCheckpointSegments, commitCheckpointManifest, removeCheckpointGarbage, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, captureWholePartition, frameRecord, checkpointPrimaryTerm, writeSegmentHeader, readWalSegments, buildEntry } from './chunk-W34VJXY7.mjs';
14
+ export { checkPagedContents, checkPagedSelections, createPagedFrozenSegment, isPagedSegmentFile, mergePagedWindow, openPagedSnapshot, pagedListingOf, pagedTombstonesFor, writePagedSegment } from './chunk-W34VJXY7.mjs';
15
+ import { REWRITE_SPILL_BYTES, TOMBSTONES_BETWEEN_YIELDS, encodeSegmentManifest, adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-U6B7DUQW.mjs';
16
+ import './chunk-5FLRXF73.mjs';
17
+ import './chunk-JDDXV6UF.mjs';
18
+ import { manifestKey, manifestFileRefs, manifestReferencedKeys, manifestWithinPartitionCount, snapshotBundleKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-7MWYORQC.mjs';
19
+ import { getFs } from './chunk-HLPRHOSO.mjs';
20
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
21
+ import { encode } from './chunk-77AUVDQ5.mjs';
22
+ export { bindStorageHost } from './chunk-77AUVDQ5.mjs';
23
+ import { sha256 } from './chunk-LNGE7SLY.mjs';
24
+ import { setNestedValue } from './chunk-QBVTYY4J.mjs';
25
+ import { rewriteSpillOf } from './chunk-IB4BYLE3.mjs';
26
+ import { deserializePartition } from './chunk-2RIS76RJ.mjs';
27
+ import { decodeVectorIndexParts, decodeVectorFilePayload, isDead, bytesToVectors, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LAWIJRMJ.mjs';
28
+ export { createHNSWIndex, osqBitsOf } from './chunk-LAWIJRMJ.mjs';
29
+ import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
30
+ import { hasNrslMagic, HEADER_SIZE, readHeader, storedStateOf, attachedRuntimeOf, historyFilesOf, listHistoryFolder, historyFileOf, QUERY_HISTORY_ARCHIVE_PREFIX, QUERY_HISTORY_FOLDER, openChecksummedEnvelope, crc32, packChecksummedEnvelope } from './chunk-6EZW2GQL.mjs';
31
+ import { isCompositePartition, generateId, mergePartitionStats, resolvePartitionInsertOptions, createPartitionIndex, createCompositePartition, fnv1a, ARRAY_POSITION_GAP, freezeSegmentShared, createFrozenSegment, createSharedFrozenSegment, toError, decodeStoredDocument, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, emitSyncChange, prepareStoredSynonyms, loadSchemaCodeFor, decodeEveryStoredDocument, arrayPositionsAreStale, insertDocumentVectors, createRequestId, createPartitionRouter, buildSegmentPayloads } from './chunk-TINW4BSH.mjs';
32
+ import { encodeDocument } from './chunk-KLVMK3R5.mjs';
33
+ import './chunk-AB27FWZM.mjs';
34
+ import { decode } from './chunk-GXCDWLQI.mjs';
35
+ import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
36
+ import './chunk-7T4LAYT4.mjs';
37
+ import './chunk-JV74RKCY.mjs';
38
+ import { flattenSchema, parseVectorDimension, raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, MAX_DOCUMENT_NESTING_DEPTH, requireValidFieldTypes } from './chunk-R4CE4IVU.mjs';
39
+ import './chunk-U2BKYEC7.mjs';
40
+ import { detectRuntime, spawnNodeWorker, resolveWorkerEntry } from './chunk-OUWZKNO3.mjs';
41
+ import { STATISTICS_BROADCAST_INTERVAL_MS, turnsAreBudgeted, turnIsSpent, yieldTurn, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED, waitOutsideCountedWrite, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-3B775F3C.mjs';
42
+ import './chunk-E2F6LZR3.mjs';
43
+ import { MIN_M } from './chunk-ZXX5GZEX.mjs';
44
+ import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-4PY4JYMY.mjs';
45
+ import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
46
+ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
35
47
 
36
- // src/persistence/durability/node-fs.browser.ts
37
- function filesystemUnavailable() {
38
- throw new NarsilError(
39
- ErrorCodes.CONFIG_INVALID,
40
- "Filesystem durability requires a Node.js runtime; use the memory or IndexedDB persistence adapters in the browser"
41
- );
42
- }
43
- async function getFs() {
44
- return filesystemUnavailable();
45
- }
46
- async function getPath() {
47
- return filesystemUnavailable();
48
- }
49
-
50
- // src/persistence/durability/lock-holder.browser.ts
51
- async function holderIsLive(_holder, _self, _lockPath) {
52
- return true;
53
- }
54
- function holdDirectoryOpen(_directory) {
55
- return () => void 0;
56
- }
57
- async function removeStaleLock(_lockPath, _asidePath, _isStale) {
58
- }
59
-
60
- // src/persistence/durability/directory-lock.ts
61
- var LOCK_FILE_NAME = ".narsil.lock";
62
- var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EACCES", "EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
63
- var lockedDirectories = /* @__PURE__ */ new Set();
64
- function errnoOf(err) {
65
- return err?.code;
66
- }
67
- function integerOrNull(text) {
68
- const value = Number.parseInt(text ?? "", 10);
69
- return Number.isSafeInteger(value) ? value : null;
70
- }
71
- function parseHolder(text) {
72
- const [pidText, threadText, startText, nonceText] = text.split("\n");
73
- const pid = integerOrNull(pidText);
74
- if (pid === null || pid <= 0) return null;
75
- const nonce = nonceText?.trim() ?? "";
76
- return {
77
- pid,
78
- threadId: integerOrNull(threadText),
79
- processStartMs: integerOrNull(startText),
80
- nonce: nonce.length > 0 ? nonce : null
81
- };
82
- }
83
- function formatHolder(holder) {
84
- return `${holder.pid}
85
- ${holder.threadId ?? ""}
86
- ${holder.processStartMs ?? ""}
87
- ${holder.nonce ?? ""}
88
- `;
89
- }
90
- async function readHolder(fs, lockPath) {
91
- try {
92
- return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
93
- } catch (err) {
94
- if (errnoOf(err) === "ENOENT") return "absent";
95
- throw err;
96
- }
97
- }
98
- async function settledHolder(fs, lockPath) {
99
- const first = await readHolder(fs, lockPath);
100
- if (first !== "unreadable") return first;
101
- await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
102
- return readHolder(fs, lockPath);
103
- }
104
- function heldElsewhere(root, holder, lockPath) {
105
- return new NarsilError(
106
- ErrorCodes.CONFIG_INVALID,
107
- `The durability directory "${root}" is in use by ${holder}, and only one engine may write a write-ahead log directory. Point this engine at another directory, or shut the other engine down first. Where no engine uses the directory, delete "${lockPath}"`,
108
- { directory: root }
109
- );
110
- }
111
- function holderLabel(holder, self) {
112
- if (holder.pid !== self.pid) return `process ${holder.pid}`;
113
- return holder.threadId === self.threadId ? "another copy of Narsil in this thread" : "another thread of this process";
114
- }
115
- async function createLockFileExclusively(fs, lockPath, content) {
116
- try {
117
- await fs.writeFile(lockPath, content, { flag: "wx" });
118
- return true;
119
- } catch (err) {
120
- if (errnoOf(err) === "EEXIST") return false;
121
- throw err;
122
- }
123
- }
124
- async function createLockFile(fs, lockPath, draftPath, content) {
125
- await fs.writeFile(draftPath, content, { flag: "w" });
126
- try {
127
- await fs.link(draftPath, lockPath);
128
- return true;
129
- } catch (err) {
130
- const code = errnoOf(err);
131
- if (code === "EEXIST") return false;
132
- if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
133
- return createLockFileExclusively(fs, lockPath, content);
48
+ // src/persistence/paged/adopt.ts
49
+ function yieldToEventLoop() {
50
+ return new Promise((resolve) => setTimeout(resolve, 0));
51
+ }
52
+ function applyNewerTombstones(prepared) {
53
+ for (let at = 0; at < prepared.length; at++) {
54
+ if (!prepared[at].isNew) continue;
55
+ for (let newer = at + 1; newer < prepared.length; newer++) {
56
+ const access = prepared[newer].segment.paged;
57
+ if (access === null) continue;
58
+ for (const docId of access.snapshot.tombstones) prepared[at].segment.tombstoneDocument(docId);
59
+ }
60
+ }
61
+ }
62
+ async function applyNewerTombstonesInTurns(prepared) {
63
+ let applied = 0;
64
+ for (let at = 0; at < prepared.length; at++) {
65
+ if (!prepared[at].isNew) continue;
66
+ for (let newer = at + 1; newer < prepared.length; newer++) {
67
+ const access = prepared[newer].segment.paged;
68
+ if (access === null) continue;
69
+ for (const docId of access.snapshot.tombstones) {
70
+ prepared[at].segment.tombstoneDocument(docId);
71
+ applied += 1;
72
+ if (applied % TOMBSTONES_BETWEEN_YIELDS === 0) await yieldToEventLoop();
73
+ }
134
74
  }
135
- throw err;
136
- } finally {
137
- await fs.unlink(draftPath).catch(() => void 0);
138
75
  }
139
76
  }
140
- async function acquire(fs, root, lockPath, self) {
141
- const draftPath = `${lockPath}.${self.pid}.${self.threadId ?? 0}`;
142
- const content = formatHolder(self);
143
- for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
144
- if (await createLockFile(fs, lockPath, draftPath, content)) return;
145
- const holder = await settledHolder(fs, lockPath);
146
- if (typeof holder === "object" && await holderIsLive()) {
147
- throw heldElsewhere(root, holderLabel(holder, self), lockPath);
77
+ async function prepareDiskAdoption(directory, manager, partitions) {
78
+ for (const entry of partitions) {
79
+ for (const ref of entry.segments) {
80
+ if (!await isPagedSegmentFile(directory, ref.key)) return null;
148
81
  }
149
- await removeStaleLock();
150
- }
151
- throw heldElsewhere(root, "another process", lockPath);
152
- }
153
- async function describeSelf() {
154
- return {
155
- pid: process.pid,
156
- threadId: await nodeThreadId(),
157
- processStartMs: Math.round(Date.now() - process.uptime() * 1e3),
158
- nonce: globalThis.crypto.randomUUID()
159
- };
160
- }
161
- async function lockDirectory(root) {
162
- const fs = await getFs();
163
- const pathMod = await getPath();
164
- const resolvedRoot = pathMod.resolve(root);
165
- const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
166
- const self = await describeSelf();
167
- async function holds() {
168
- const holder = await readHolder(fs, lockPath);
169
- return typeof holder === "object" && holder.nonce === self.nonce;
170
- }
171
- async function removeOwnLockFile() {
172
- if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
173
- }
174
- let directoryKey = null;
175
- let closeDirectory = () => void 0;
176
- try {
177
- await fs.mkdir(resolvedRoot, { recursive: true });
178
- const stat = await fs.stat(resolvedRoot, { bigint: true });
179
- const key = `${stat.dev}:${stat.ino}`;
180
- if (lockedDirectories.has(key)) throw heldElsewhere(root, "another engine in this process", lockPath);
181
- lockedDirectories.add(key);
182
- directoryKey = key;
183
- await acquire(fs, root, lockPath, self);
184
- closeDirectory = holdDirectoryOpen(resolvedRoot);
185
- } catch (err) {
186
- if (directoryKey !== null) lockedDirectories.delete(directoryKey);
187
- await removeOwnLockFile();
188
- if (err instanceof NarsilError) throw err;
189
- throw new NarsilError(
190
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
191
- `The engine cannot lock the durability directory "${root}"`,
192
- { directory: root, cause: err instanceof Error ? err.message : String(err) }
193
- );
194
82
  }
195
- const heldKey = directoryKey;
196
- let released = false;
197
- return {
198
- holds,
199
- async release() {
200
- if (released) return;
201
- released = true;
202
- lockedDirectories.delete(heldKey);
203
- await removeOwnLockFile();
204
- closeDirectory();
205
- }
206
- };
207
- }
208
-
209
- // src/persistence/durability/list-recursive.ts
210
- async function listRecursive(dir, base, pathMod, fs) {
211
- const results = [];
212
- let entries;
213
- try {
214
- entries = await fs.readdir(dir, { withFileTypes: true });
215
- } catch (err) {
216
- if (err.code === "ENOENT") {
217
- return results;
83
+ const prepared = [];
84
+ const droppedKeys = [];
85
+ for (const entry of partitions) {
86
+ if (entry.partitionId >= manager.partitionCount) continue;
87
+ const partition = manager.getPartition(entry.partitionId);
88
+ const held = /* @__PURE__ */ new Map();
89
+ if (isCompositePartition(partition)) {
90
+ for (const segment of partition.frozenSegments()) {
91
+ if (segment.paged !== null) held.set(segment.paged.key, segment);
92
+ }
218
93
  }
219
- throw err;
220
- }
221
- for (const entry of entries) {
222
- const full = pathMod.join(dir, entry.name);
223
- if (entry.isDirectory()) {
224
- results.push(...await listRecursive(full, base, pathMod, fs));
225
- } else {
226
- results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
94
+ const list = [];
95
+ for (const ref of entry.segments) {
96
+ const existing = held.get(ref.key);
97
+ if (existing !== void 0) {
98
+ held.delete(ref.key);
99
+ list.push({ segment: existing, isNew: false });
100
+ continue;
101
+ }
102
+ const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
103
+ list.push({ segment: createPagedFrozenSegment(snapshot), isNew: true });
227
104
  }
105
+ droppedKeys.push(...held.keys());
106
+ await applyNewerTombstonesInTurns(list);
107
+ prepared.push({ partitionId: entry.partitionId, prepared: list });
228
108
  }
229
- return results;
230
- }
231
-
232
- // src/persistence/durability/durable-filesystem.ts
233
- function validateRelativeKey(key) {
234
- if (key.length === 0) {
235
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
236
- }
237
- if (key.includes("\0")) {
238
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: null byte detected", { key });
239
- }
240
- }
241
- function wrapFsyncError(err, key) {
242
- throw new NarsilError(
243
- ErrorCodes.PERSISTENCE_FSYNC_FAILED,
244
- `fsync failed for "${key}"; the write is not acknowledged and the node must recover from the durable log`,
245
- { key, cause: err instanceof Error ? err.message : String(err) }
246
- );
109
+ return { partitions: prepared, droppedKeys };
247
110
  }
248
- async function openReadWriteCreating(filePath, fs) {
249
- try {
250
- return await fs.open(filePath, "r+");
251
- } catch (err) {
252
- if (err.code !== "ENOENT") {
253
- throw err;
254
- }
255
- }
256
- try {
257
- return await fs.open(filePath, "wx+");
258
- } catch (err) {
259
- if (err.code !== "EEXIST") {
260
- throw err;
111
+ function keysThatSegmentsStillRead(manager) {
112
+ const keys = /* @__PURE__ */ new Set();
113
+ for (const partition of manager.getAllPartitions()) {
114
+ if (!isCompositePartition(partition)) continue;
115
+ for (const segment of partition.frozenSegments()) {
116
+ if (segment.paged !== null) keys.add(segment.paged.key);
261
117
  }
262
- return fs.open(filePath, "r+");
263
118
  }
119
+ return keys;
264
120
  }
265
- async function writeFully(handle, buffer, key) {
266
- let offset = 0;
267
- while (offset < buffer.length) {
268
- let bytesWritten;
269
- try {
270
- ;
271
- ({ bytesWritten } = await handle.write(buffer, offset, buffer.length - offset));
272
- } catch (err) {
273
- throw new NarsilError(
274
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
275
- `Write to "${key}" failed; the write is not acknowledged`,
276
- {
277
- key,
278
- written: offset,
279
- total: buffer.length,
280
- cause: err instanceof Error ? err.message : String(err)
281
- }
282
- );
283
- }
284
- if (bytesWritten <= 0) {
285
- throw new NarsilError(
286
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
287
- `Write to "${key}" made no progress; the file would be truncated and the write is not acknowledged`,
288
- { key, written: offset, total: buffer.length }
289
- );
121
+ function closeUnusedSegments(adoption) {
122
+ for (const { prepared } of adoption.partitions) {
123
+ for (const { segment, isNew } of prepared) {
124
+ if (isNew) segment.paged?.close();
290
125
  }
291
- offset += bytesWritten;
292
126
  }
293
127
  }
294
- async function fsyncDirectory(dirPath, fs) {
295
- let handle = null;
128
+ function actionPartitionsOf(adoption) {
129
+ return adoption.partitions.map(({ partitionId, prepared }) => ({
130
+ partitionId,
131
+ segments: prepared.map(({ segment, isNew }) => {
132
+ const access = segment.paged;
133
+ if (access !== null && isNew) return { snapshot: access.snapshot };
134
+ return { segmentId: segment.segmentId };
135
+ })
136
+ }));
137
+ }
138
+ function swapPreparedPartitions(manager, partitions, changed, onSwapped) {
139
+ for (const { partitionId, prepared } of partitions) {
140
+ const kept = [];
141
+ for (const { segment } of prepared) {
142
+ if (segment.arrays.documentCount > 0) kept.push(segment);
143
+ else segment.paged?.close();
144
+ }
145
+ adoptDiskSegments(manager, partitionId, kept, changed);
146
+ onSwapped?.(partitionId, kept);
147
+ }
148
+ }
149
+ function adoptDiskSegments(manager, partitionId, segments, changed) {
150
+ const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
151
+ const before = composite.frozenSegments();
152
+ const kept = [];
153
+ for (const docId of changed) {
154
+ if (docPartitionMap.get(docId) !== partitionId) continue;
155
+ const document = composite.getRef(docId);
156
+ if (document !== void 0) kept.push([docId, document]);
157
+ }
158
+ const dropped = before.filter((segment) => !segments.includes(segment));
159
+ for (const segment of segments) {
160
+ if (before.includes(segment)) continue;
161
+ for (const docId of changed) segment.tombstoneDocument(docId);
162
+ }
163
+ composite.replaceFrozenSegments(segments);
164
+ composite.live.clear();
165
+ for (const [docId, pid] of docPartitionMap) {
166
+ if (pid === partitionId) docPartitionMap.delete(docId);
167
+ }
168
+ composite.live.beginBatch();
296
169
  try {
297
- handle = await fs.open(dirPath, "r");
298
- await handle.sync();
299
- } catch (err) {
300
- const code = err.code;
301
- if (code === "EISDIR" || code === "EPERM" || code === "EINVAL" || code === "ENOTSUP") {
302
- return;
170
+ for (const [docId, document] of kept) {
171
+ composite.live.insert(docId, document, manager.config.schema, manager.language, insertOptions);
172
+ docPartitionMap.set(docId, partitionId);
303
173
  }
304
- wrapFsyncError(err, dirPath);
305
174
  } finally {
306
- if (handle !== null) {
307
- await handle.close().catch(() => void 0);
308
- }
309
- }
310
- }
311
- function createDurableDirectory(root) {
312
- if (root.trim().length === 0) {
313
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Durable directory requires a non-empty root path");
314
- }
315
- async function resolve(key) {
316
- validateRelativeKey(key);
317
- const pathMod = await getPath();
318
- const resolvedBase = pathMod.resolve(root);
319
- const resolvedPath = pathMod.resolve(root, key);
320
- if (resolvedPath !== resolvedBase && !resolvedPath.startsWith(resolvedBase + pathMod.sep)) {
321
- throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: path traversal detected", { key });
322
- }
323
- return { path: resolvedPath, resolvedBase, pathMod };
324
- }
325
- async function ensureDir(filePath, pathMod, fs) {
326
- await fs.mkdir(pathMod.dirname(filePath), { recursive: true });
327
- }
328
- return {
329
- get root() {
330
- return root;
331
- },
332
- lock() {
333
- return lockDirectory(root);
334
- },
335
- async identity() {
336
- const fs = await getFs();
337
- const pathMod = await getPath();
338
- try {
339
- const stat = await fs.stat(pathMod.resolve(root), { bigint: true });
340
- return `${stat.dev}:${stat.ino}`;
341
- } catch (err) {
342
- if (err.code === "ENOENT") return null;
343
- throw err;
344
- }
345
- },
346
- async appendHandle(key) {
347
- const { path: filePath, pathMod } = await resolve(key);
348
- const fs = await getFs();
349
- await ensureDir(filePath, pathMod, fs);
350
- const handle = await fs.open(filePath, "a");
351
- return {
352
- async append(bytes) {
353
- await writeFully(handle, bytes, key);
354
- },
355
- async sync() {
356
- try {
357
- await handle.sync();
358
- } catch (err) {
359
- wrapFsyncError(err, key);
360
- }
361
- },
362
- async truncate(byteLength) {
363
- await handle.truncate(byteLength);
364
- try {
365
- await handle.sync();
366
- } catch (err) {
367
- wrapFsyncError(err, key);
368
- }
369
- },
370
- async size() {
371
- const stat = await handle.stat();
372
- return stat.size;
373
- },
374
- async close() {
375
- await handle.close();
376
- }
377
- };
378
- },
379
- async markerHandle(key) {
380
- const { path: filePath, pathMod } = await resolve(key);
381
- const fs = await getFs();
382
- await ensureDir(filePath, pathMod, fs);
383
- const handle = await openReadWriteCreating(filePath, fs);
384
- return {
385
- async read() {
386
- const stat = await handle.stat();
387
- if (stat.size === 0) {
388
- return new Uint8Array(0);
389
- }
390
- const bytes = new Uint8Array(stat.size);
391
- await handle.read(bytes, 0, stat.size, 0);
392
- return bytes;
393
- },
394
- async writeSlot(offset, slot) {
395
- try {
396
- await handle.write(slot, 0, slot.length, offset);
397
- } catch (err) {
398
- throw new NarsilError(
399
- ErrorCodes.PERSISTENCE_SAVE_FAILED,
400
- `Write to "${key}" failed; the write is not acknowledged`,
401
- { key, cause: err instanceof Error ? err.message : String(err) }
402
- );
403
- }
404
- try {
405
- await handle.sync();
406
- } catch (err) {
407
- wrapFsyncError(err, key);
408
- }
409
- },
410
- async close() {
411
- await handle.close();
412
- }
413
- };
414
- },
415
- async syncDirectoryOf(key) {
416
- const { path: filePath, pathMod } = await resolve(key);
417
- const fs = await getFs();
418
- await fsyncDirectory(pathMod.dirname(filePath), fs);
419
- },
420
- async atomicWrite(key, data) {
421
- const { path: filePath, pathMod } = await resolve(key);
422
- const fs = await getFs();
423
- await ensureDir(filePath, pathMod, fs);
424
- const dir = pathMod.dirname(filePath);
425
- const tempName = `${pathMod.basename(filePath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
426
- const tempPath = pathMod.join(dir, tempName);
427
- const buffers = Array.isArray(data) ? data : [data];
428
- let handle = null;
429
- try {
430
- handle = await fs.open(tempPath, "w");
431
- for (const buffer of buffers) {
432
- await writeFully(handle, buffer, tempPath);
433
- }
434
- try {
435
- await handle.sync();
436
- } catch (err) {
437
- wrapFsyncError(err, tempPath);
438
- }
439
- await handle.close();
440
- handle = null;
441
- await fs.rename(tempPath, filePath);
442
- await fsyncDirectory(dir, fs);
443
- } catch (err) {
444
- if (handle !== null) {
445
- await handle.close().catch(() => void 0);
446
- }
447
- await fs.unlink(tempPath).catch(() => void 0);
448
- throw err;
449
- }
450
- },
451
- async read(key) {
452
- const { path: filePath } = await resolve(key);
453
- const fs = await getFs();
454
- try {
455
- const data = await fs.readFile(filePath);
456
- return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
457
- } catch (err) {
458
- if (err.code === "ENOENT") {
459
- return null;
460
- }
461
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read "${key}"`, {
462
- key,
463
- cause: err instanceof Error ? err.message : String(err)
464
- });
465
- }
466
- },
467
- async readRange(key, offset, length) {
468
- const { path: filePath } = await resolve(key);
469
- const fs = await getFs();
470
- let handle = null;
471
- try {
472
- handle = await fs.open(filePath, "r");
473
- const bytes = new Uint8Array(length);
474
- let filled = 0;
475
- while (filled < length) {
476
- const { bytesRead } = await handle.read(bytes, filled, length - filled, offset + filled);
477
- if (bytesRead === 0) break;
478
- filled += bytesRead;
479
- }
480
- if (filled < length) {
481
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `"${key}" ends before the bytes a read asked for`, {
482
- key,
483
- offset,
484
- length
485
- });
486
- }
487
- return bytes;
488
- } finally {
489
- await handle?.close();
490
- }
491
- },
492
- async byteLength(key) {
493
- const { path: filePath } = await resolve(key);
494
- const fs = await getFs();
495
- try {
496
- return (await fs.stat(filePath)).size;
497
- } catch (err) {
498
- if (err.code === "ENOENT") {
499
- return null;
500
- }
501
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Failed to read the length of "${key}"`, {
502
- key,
503
- cause: err instanceof Error ? err.message : String(err)
504
- });
505
- }
506
- },
507
- async remove(key) {
508
- const { path: filePath, pathMod } = await resolve(key);
509
- const fs = await getFs();
510
- try {
511
- await fs.unlink(filePath);
512
- } catch (err) {
513
- if (err.code === "ENOENT") {
514
- return;
515
- }
516
- throw new NarsilError(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, {
517
- key,
518
- cause: err instanceof Error ? err.message : String(err)
519
- });
175
+ composite.live.endBatch();
176
+ }
177
+ for (const segment of dropped) segment.paged?.close();
178
+ }
179
+ function adoptSegmentsOnWorker(manager, action) {
180
+ const partitions = action.partitions.map(({ partitionId, segments }) => {
181
+ const partition = manager.getPartition(partitionId);
182
+ const held = isCompositePartition(partition) ? partition.frozenSegments() : [];
183
+ const prepared = segments.map((item) => {
184
+ if ("snapshot" in item) return { segment: createPagedFrozenSegment(item.snapshot), isNew: true };
185
+ const existing = held.find((segment) => segment.segmentId === item.segmentId);
186
+ if (existing === void 0) {
187
+ throw new NarsilError(
188
+ ErrorCodes.PARTITION_CORRUPTED,
189
+ `Partition ${partitionId} of "${action.indexName}" holds no segment "${item.segmentId}" to keep`,
190
+ { indexName: action.indexName, partitionId, segmentId: item.segmentId }
191
+ );
520
192
  }
521
- await fsyncDirectory(pathMod.dirname(filePath), fs);
522
- },
523
- async move(from, to) {
524
- const source = await resolve(from);
525
- const target = await resolve(to);
526
- const fs = await getFs();
527
- await ensureDir(target.path, target.pathMod, fs);
528
- await fs.rename(source.path, target.path);
529
- const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
530
- for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
531
- },
532
- async list(prefix) {
533
- const fs = await getFs();
534
- const pathMod = await getPath();
535
- const resolvedBase = pathMod.resolve(root);
536
- const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
537
- const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
538
- const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
539
- return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
540
- },
541
- async pathOf(key) {
542
- const { path: filePath } = await resolve(key);
543
- return filePath;
544
- }
545
- };
193
+ return { segment: existing, isNew: false };
194
+ });
195
+ applyNewerTombstones(prepared);
196
+ return { partitionId, prepared };
197
+ });
198
+ swapPreparedPartitions(manager, partitions, new Set(action.changed));
546
199
  }
547
200
 
548
201
  // src/persistence/durability/sync/layout.ts
@@ -656,6 +309,7 @@ async function reclaimWalBeyondCount(directory, indexName, partitionCount, parti
656
309
  await partition.walWriter.close();
657
310
  } catch (err) {
658
311
  onCloseError(err instanceof Error ? err : new Error(String(err)));
312
+ throw err;
659
313
  }
660
314
  partitions.delete(partitionId);
661
315
  }
@@ -693,170 +347,6 @@ function parseSegmentStartSeqNo(key, prefix) {
693
347
  function formatSegmentStart(startSeqNo) {
694
348
  return startSeqNo.toString().padStart(16, "0");
695
349
  }
696
- function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes, schedulesGraphBuild = true) {
697
- if (entry.document === null) {
698
- return;
699
- }
700
- const document = decode(entry.document);
701
- restoreVectorFields(document, vectorFieldPaths);
702
- const documentId = entry.documentId;
703
- if (manager.has(documentId)) {
704
- removeDocumentVectors(documentId, vecIndexes);
705
- manager.remove(documentId);
706
- }
707
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(document, vectorFieldPaths);
708
- manager.insert(documentId, partitionDoc);
709
- try {
710
- insertDocumentVectors(documentId, extractedVectors, vecIndexes, manager.partitionIdOf(documentId));
711
- } catch (err) {
712
- manager.remove(documentId);
713
- throw err;
714
- }
715
- if (!schedulesGraphBuild) return;
716
- for (const fieldPath of extractedVectors.keys()) {
717
- const vecIndex = vecIndexes.get(fieldPath);
718
- if (vecIndex) {
719
- vecIndex.scheduleBuild();
720
- }
721
- }
722
- }
723
- function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
724
- removeDocumentVectors(entry.documentId, vecIndexes);
725
- if (!manager.has(entry.documentId)) {
726
- return;
727
- }
728
- manager.remove(entry.documentId, removedSpellings);
729
- }
730
-
731
- // src/persistence/durability/marker-writer.ts
732
- function markerKey(indexName, partitionId) {
733
- return `${indexName}/wal/${partitionId}/commit`;
734
- }
735
- async function createMarkerWriter(directory, indexName, partitionId) {
736
- const key = markerKey(indexName, partitionId);
737
- const handle = await directory.markerHandle(key);
738
- const existing = await handle.read();
739
- const current = readCommitMarker(existing);
740
- const createdFresh = existing.length === 0;
741
- let lastSlot = current?.slotIndex ?? null;
742
- let writeSeq = current?.state.writeSeq ?? 0;
743
- return {
744
- created: createdFresh,
745
- existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
746
- async commit(state) {
747
- const slotIndex = nextMarkerSlot(lastSlot);
748
- writeSeq += 1;
749
- const slot = encodeMarkerSlot({ ...state, writeSeq });
750
- await handle.writeSlot(markerSlotOffset(slotIndex), slot);
751
- lastSlot = slotIndex;
752
- },
753
- async close() {
754
- await handle.close();
755
- }
756
- };
757
- }
758
-
759
- // src/persistence/durability/recovery.ts
760
- async function removeLeftoverRewriteFiles(directory, names) {
761
- for (const indexName of names) {
762
- for (const place of ["rebuilt", "received"]) {
763
- for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
764
- }
765
- }
766
- }
767
- async function listPersistedIndexes(directory) {
768
- const metaKeys = await directory.list("");
769
- const names = [];
770
- for (const key of metaKeys) {
771
- if (key.endsWith("/meta")) {
772
- names.push(key.slice(0, -"/meta".length));
773
- }
774
- }
775
- return names;
776
- }
777
- async function loadMetadata(directory, indexName) {
778
- const bytes = await directory.read(`${indexName}/meta`);
779
- if (bytes === null) {
780
- return null;
781
- }
782
- const { metadata } = await readMetadataEnvelope(bytes);
783
- return metadata;
784
- }
785
- async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
786
- await directory.remove(nextSegmentKey(indexName, partitionId));
787
- const read = await readWalSegments(directory, indexName, partitionId);
788
- if (read === null) {
789
- return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
790
- }
791
- const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
792
- if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
793
- throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
794
- }
795
- let highestSeqNo = fromSeqNoExclusive;
796
- let highestPrimaryTerm = 0;
797
- const replay = async (entries) => {
798
- for (const entry of entries) {
799
- if (entry.seqNo <= highestSeqNo) {
800
- continue;
801
- }
802
- if (turnIsSpent()) await yieldTurn();
803
- applyEntry(entry, deps);
804
- highestSeqNo = entry.seqNo;
805
- if (entry.primaryTerm > highestPrimaryTerm) {
806
- highestPrimaryTerm = entry.primaryTerm;
807
- }
808
- }
809
- };
810
- await replay(durableEntries);
811
- if (activeTail !== null) {
812
- await replay(activeTail.entries);
813
- }
814
- await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
815
- if (activeTail !== null) {
816
- await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
817
- }
818
- return { highestSeqNo, highestPrimaryTerm };
819
- }
820
- async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
821
- const lastReplayed = tail.entries.at(-1);
822
- if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
823
- const handle = await directory.appendHandle(tail.key);
824
- try {
825
- if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
826
- else await handle.sync();
827
- } finally {
828
- await handle.close();
829
- }
830
- if (lastReplayed === void 0) return;
831
- const writer = await createMarkerWriter(directory, indexName, partitionId);
832
- try {
833
- await writer.commit({
834
- activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
835
- durableByteLength: tail.cleanEnd,
836
- highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
837
- });
838
- } finally {
839
- await writer.close();
840
- }
841
- }
842
- async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
843
- for (const { key, startSeqNo } of segments) {
844
- if (startSeqNo > activeSegmentSeqNo) {
845
- await directory.remove(key);
846
- }
847
- }
848
- }
849
- function applyEntry(entry, deps) {
850
- deps.writeRecords?.replayed(entry);
851
- if (entry.operation === "DELETE") {
852
- applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
853
- return;
854
- }
855
- applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
856
- }
857
- function snapshotCheckpointFor(checkpoint, partitionId) {
858
- return checkpointLastSeqNo(checkpoint, partitionId);
859
- }
860
350
 
861
351
  // src/persistence/durability/checkpoint-capture.ts
862
352
  function whileNoMutationApplies(partitions, capture2) {
@@ -865,9 +355,9 @@ function whileNoMutationApplies(partitions, capture2) {
865
355
  const released = new Promise((settle) => {
866
356
  release = settle;
867
357
  });
868
- function captureNow() {
358
+ async function captureNow() {
869
359
  try {
870
- resolve(capture2());
360
+ resolve(await capture2());
871
361
  } catch (error) {
872
362
  reject(error);
873
363
  } finally {
@@ -876,13 +366,13 @@ function whileNoMutationApplies(partitions, capture2) {
876
366
  }
877
367
  let waiting = partitions.length;
878
368
  if (waiting === 0) {
879
- captureNow();
369
+ void captureNow();
880
370
  return;
881
371
  }
882
372
  for (const partition of partitions) {
883
373
  partition.appendChain = partition.appendChain.then(() => {
884
374
  waiting -= 1;
885
- if (waiting === 0) captureNow();
375
+ if (waiting === 0) void captureNow();
886
376
  return released;
887
377
  });
888
378
  }
@@ -922,9 +412,11 @@ function noWholePartitions() {
922
412
  return { contents: [], captured: [] };
923
413
  }
924
414
  function wholePartitionsWhereMostChanged(manager, rules) {
925
- const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
415
+ const { priorCheckpoint, priorPartitionIds, aWorkerCanSerialise, captureRecords } = rules;
926
416
  return (targets) => {
927
417
  const whole = { contents: [], captured: [] };
418
+ const everyPartition = rules.everyPartition || requiresLayoutCheckpoint(manager) || priorPartitionIds.length > 0 && priorPartitionIds.length !== manager.partitionCount;
419
+ if (rules.usesLogForUnchangedLayout && !everyPartition) return whole;
928
420
  const changed = targets.filter(
929
421
  (target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
930
422
  );
@@ -947,45 +439,57 @@ function wholePartitionsWhereMostChanged(manager, rules) {
947
439
  return whole;
948
440
  };
949
441
  }
950
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
442
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
951
443
  for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
952
- return whileNoMutationApplies([...indexState.partitions.values()], () => {
953
- onCaptured?.();
954
- const documentCount = manager.countDocuments();
955
- const targets = [];
956
- let recordsLeftInLog = 0;
957
- for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
958
- const partition = indexState.partitions.get(partitionId);
959
- if (partition !== void 0 && partition.failed !== null) continue;
960
- const appliedSeqNo = partition?.appliedSeqNo ?? 0;
961
- const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
962
- appliedSeqNo,
963
- snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
964
- );
965
- recordsLeftInLog += appliedSeqNo - lastSeqNo;
966
- const target = {
967
- partitionId,
968
- lastSeqNo,
969
- primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
970
- };
971
- if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
972
- targets.push(target);
973
- }
974
- const vectorPlans = /* @__PURE__ */ new Map();
975
- try {
976
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
977
- const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
978
- const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
979
- vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, pinsRetiredVectors));
444
+ return whileNoMutationApplies(
445
+ [...indexState.partitions.values()],
446
+ () => captureAppliedPrefix(manager, (cutoffs) => {
447
+ onCaptured?.();
448
+ const partitionCount = manager.partitionCount;
449
+ const previous = priorCheckpointThatLimitsTheRecords;
450
+ const changedLayout = requiresLayoutCheckpoint(manager) || previous !== void 0 && previous.length > 0 && previous.length !== partitionCount;
451
+ const bounded = changedLayout ? void 0 : previous;
452
+ const documentCount = manager.countDocuments();
453
+ const targets = [];
454
+ let recordsLeftInLog = 0;
455
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
456
+ const partition = indexState.partitions.get(partitionId);
457
+ if (partition !== void 0 && partition.failed !== null) continue;
458
+ const appliedSeqNo = Math.min(partition?.appliedSeqNo ?? 0, cutoffs.get(partitionId) ?? Infinity);
459
+ const lastSeqNo = bounded === void 0 ? appliedSeqNo : Math.min(appliedSeqNo, snapshotCheckpointFor(bounded, partitionId) + LOG_RECORDS_PER_CHECKPOINT);
460
+ recordsLeftInLog += appliedSeqNo - lastSeqNo;
461
+ const target = {
462
+ partitionId,
463
+ lastSeqNo,
464
+ primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
465
+ };
466
+ if (termHistoryOf !== void 0) target.termHistory = termHistoryOf(partitionId, lastSeqNo);
467
+ targets.push(target);
980
468
  }
981
- const memoryMatchesTheTargets = recordsLeftInLog === 0;
982
- const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
983
- return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
984
- } catch (error) {
985
- for (const plan of vectorPlans.values()) plan.release();
986
- throw error;
987
- }
988
- });
469
+ const vectorPlans = /* @__PURE__ */ new Map();
470
+ try {
471
+ for (const [fieldPath, vectorIndex] of vectorIndexes) {
472
+ const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
473
+ const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
474
+ vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2, false, true));
475
+ }
476
+ const memoryMatchesTheTargets = recordsLeftInLog === 0;
477
+ const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
478
+ return {
479
+ partitionCount,
480
+ targets,
481
+ documentCount,
482
+ vectorPlans,
483
+ wholePartitions,
484
+ recordsLeftInLog,
485
+ unreplayedCutoffs: cutoffs
486
+ };
487
+ } catch (error) {
488
+ for (const plan of vectorPlans.values()) plan.release();
489
+ throw error;
490
+ }
491
+ })
492
+ );
989
493
  }
990
494
  async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
991
495
  if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
@@ -1005,6 +509,20 @@ async function writeCapturedVectors(store, indexName, capture2, priorManifest, r
1005
509
  return { vectors: written.refs, layouts: written.layouts, written: written.written };
1006
510
  }
1007
511
 
512
+ // src/persistence/durability/checkpoint-prior.ts
513
+ function priorForLayout(manifest, partitionCount) {
514
+ const checkpoints = new Map(manifest?.checkpoint.map((entry) => [entry.partitionId, entry]));
515
+ const partitions = new Map(manifest?.partitions.map((entry) => [entry.partitionId, entry]));
516
+ const prior = { checkpoint: [], partitions: [] };
517
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
518
+ prior.checkpoint.push(
519
+ checkpoints.get(partitionId) ?? { partitionId, lastSeqNo: 0, primaryTerm: SINGLE_NODE_PRIMARY_TERM }
520
+ );
521
+ prior.partitions.push(partitions.get(partitionId) ?? { partitionId, nextSegmentId: 0, segments: [] });
522
+ }
523
+ return prior;
524
+ }
525
+
1008
526
  // src/persistence/durability/checkpoint-worker-entry.ts
1009
527
  function resolveCheckpointWorkerEntry() {
1010
528
  return resolveWorkerEntry(
@@ -1325,6 +843,7 @@ async function writeOneBoundedCheckpoint(input) {
1325
843
  priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1326
844
  everyPartition: fromMemory,
1327
845
  aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
846
+ usesLogForUnchangedLayout: textOnDisk,
1328
847
  ...writeRecords === null ? {} : {
1329
848
  captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1330
849
  }
@@ -1334,20 +853,28 @@ async function writeOneBoundedCheckpoint(input) {
1334
853
  input.indexState,
1335
854
  manager,
1336
855
  vectorIndexes,
1337
- textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
856
+ wholePartitionsOf,
1338
857
  fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1339
858
  priorManifest?.vectors ?? [],
1340
859
  writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1341
860
  textOnDisk ? () => {
1342
861
  changesSinceCapture = manager.openCheckpointChangeLog();
1343
- } : void 0,
1344
- input.pinsRetiredVectors
862
+ } : void 0
1345
863
  );
1346
864
  try {
1347
865
  await openNewLogSegments(input.indexState, input.failPartition);
1348
- return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1349
- return changesSinceCapture;
1350
- });
866
+ return await finishBoundedCheckpoint(
867
+ input,
868
+ manager,
869
+ store,
870
+ { ...metadata, partitionCount: capture2.partitionCount },
871
+ priorManifest,
872
+ capture2,
873
+ fromMemory,
874
+ () => {
875
+ return changesSinceCapture;
876
+ }
877
+ );
1351
878
  } finally {
1352
879
  for (const plan of capture2.vectorPlans.values()) plan.release();
1353
880
  changesSinceCapture?.close();
@@ -1355,24 +882,26 @@ async function writeOneBoundedCheckpoint(input) {
1355
882
  }
1356
883
  async function adoptWrittenSegments(input, manager, partitions, changes) {
1357
884
  const { indexName, indexState, hooks } = input;
1358
- const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-NDSQLPMQ.mjs');
1359
885
  const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1360
886
  if (adoption === null) return [];
1361
- const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1362
- if (changes.partitionsReplaced) return false;
1363
- const changed = changes.takeWrittenDocIds();
1364
- swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1365
- hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1366
- });
1367
- hooks.queueDiskAdoption?.({
1368
- type: "adoptDiskSegments",
1369
- indexName,
1370
- partitions: actionPartitionsOf(adoption),
1371
- changed: [...changed],
1372
- requestId: createRequestId()
1373
- });
1374
- return true;
1375
- });
887
+ const adopted = await whileNoMutationApplies(
888
+ [...indexState.partitions.values()],
889
+ () => manager.publication.write(() => {
890
+ if (changes.partitionsReplaced) return false;
891
+ const changed = changes.takeWrittenDocIds();
892
+ swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
893
+ hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
894
+ });
895
+ hooks.queueDiskAdoption?.({
896
+ type: "adoptDiskSegments",
897
+ indexName,
898
+ partitions: actionPartitionsOf(adoption),
899
+ changed: [...changed],
900
+ requestId: createRequestId()
901
+ });
902
+ return true;
903
+ })
904
+ );
1376
905
  if (!adopted) {
1377
906
  closeUnusedSegments(adoption);
1378
907
  return [];
@@ -1394,7 +923,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1394
923
  targets,
1395
924
  compactionThreshold: input.compactionThreshold,
1396
925
  canOffload: input.canOffload,
1397
- prior: priorManifest,
926
+ prior: priorForLayout(priorManifest, capture2.partitionCount),
1398
927
  wholePartitions: capture2.wholePartitions.contents,
1399
928
  capturedPartitions: capture2.wholePartitions.captured,
1400
929
  keepsWriteRecords: writeRecords !== null
@@ -1410,7 +939,7 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1410
939
  const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1411
940
  if (!input.holdsCheckpointFiles()) {
1412
941
  const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1413
- const stillRead = metadata.textStorage === "disk" ? (await import('./module-NDSQLPMQ.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
942
+ const stillRead = metadata.textStorage === "disk" ? keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1414
943
  const keptForSync = await input.keepsCheckpointKey();
1415
944
  await removeCheckpointGarbage(
1416
945
  store,
@@ -1423,9 +952,12 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1423
952
  if (checkpointMetadata === void 0) {
1424
953
  return;
1425
954
  }
1426
- const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
955
+ const bytes = await writeMetadataEnvelope(
956
+ { ...checkpointMetadata, partitionCount: capture2.partitionCount },
957
+ { checksum: true }
958
+ );
1427
959
  await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
1428
- input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
960
+ input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, capture2.partitionCount);
1429
961
  });
1430
962
  const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
1431
963
  await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
@@ -1433,69 +965,36 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
1433
965
  await reclaimWalBeyondCount(
1434
966
  input.directory,
1435
967
  input.indexName,
1436
- manager.partitionCount,
968
+ capture2.partitionCount,
1437
969
  input.indexState.partitions,
1438
970
  input.markFatal
1439
971
  );
972
+ await input.directory.remove(transitionKey(input.indexName));
1440
973
  const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
1441
974
  noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
1442
- const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
1443
- return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
975
+ const eligibleBeyond = recordsBeyond(input.indexState, targets, capture2.unreplayedCutoffs);
976
+ const anotherIsDue = input.mutationThreshold > 0 && eligibleBeyond >= input.mutationThreshold;
977
+ return anotherIsDue ? eligibleBeyond : capture2.recordsLeftInLog;
1444
978
  }
1445
- function recordsBeyond(indexState, targets) {
979
+ function recordsBeyond(indexState, targets, cutoffs) {
1446
980
  let records = 0;
1447
981
  for (const [partitionId, partition] of indexState.partitions) {
1448
982
  if (partition.failed !== null) continue;
1449
- records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
983
+ records += Math.max(
984
+ 0,
985
+ Math.min(partition.appliedSeqNo, cutoffs?.get(partitionId) ?? Infinity) - snapshotCheckpointFor(targets, partitionId)
986
+ );
1450
987
  }
1451
988
  return records;
1452
989
  }
1453
990
 
1454
- // src/persistence/durability/checkpoint-turn.ts
1455
- function createIndexLoads(vectorIndexesOf2) {
1456
- const loads = /* @__PURE__ */ new Map();
1457
- const startGraphRepairs = (indexName) => {
1458
- for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
1459
- vectorIndex.scheduleBuild();
1460
- vectorIndex.startShuffledRebuildWhenDue();
1461
- }
1462
- };
1463
- return {
1464
- track(indexName, load, startsGraphRepairs = true) {
1465
- const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
1466
- loads.set(
1467
- indexName,
1468
- loaded.catch(() => void 0)
1469
- );
1470
- return loaded;
1471
- },
1472
- settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
1473
- startGraphRepairs
1474
- };
991
+ // src/persistence/durability/deferred-mutations.ts
992
+ var deferred = /* @__PURE__ */ new WeakSet();
993
+ function deferMutationPublication(record) {
994
+ deferred.add(record);
1475
995
  }
1476
- async function switchInTurn(turns, turn) {
1477
- let swapped = false;
1478
- await checkpointInTurn(turns, {
1479
- joinsInFlight: false,
1480
- mayStart: turn.mayStart,
1481
- perform: async () => {
1482
- swapped = turn.swap();
1483
- if (swapped) await turn.checkpoint();
1484
- },
1485
- afterSettling: turn.afterSettling
1486
- });
1487
- return swapped;
1488
- }
1489
- async function checkpointInTurn(turns, turn) {
1490
- if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
1491
- while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
1492
- if (!turn.mayStart()) return;
1493
- const run = turn.perform().finally(() => {
1494
- turns.checkpointInFlight = null;
1495
- turn.afterSettling();
1496
- });
1497
- turns.checkpointInFlight = run;
1498
- return run;
996
+ function hasDeferredPublication(record) {
997
+ return deferred.has(record);
1499
998
  }
1500
999
 
1501
1000
  // src/persistence/durability/held-checkpoints.ts
@@ -1553,24 +1052,6 @@ function createHeldCheckpoints(hooks) {
1553
1052
  };
1554
1053
  }
1555
1054
 
1556
- // src/persistence/durability/manager-state.ts
1557
- function closePartitionWriter(partition) {
1558
- return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
1559
- }
1560
- async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
1561
- indexState.unloading = true;
1562
- try {
1563
- if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
1564
- await pendingMetadataWrite;
1565
- for (const partition of indexState.partitions.values()) {
1566
- await closeWalWriter(closePartitionWriter(partition));
1567
- }
1568
- } catch (error) {
1569
- indexState.unloading = false;
1570
- throw error;
1571
- }
1572
- }
1573
-
1574
1055
  // src/persistence/durability/restore-journal.ts
1575
1056
  var RESTORE_JOURNAL_PREFIX = ".swap-";
1576
1057
  var RESTORE_STAGING_PREFIX = ".restore-";
@@ -2211,6 +1692,8 @@ function createFilePins(remove) {
2211
1692
  }
2212
1693
  };
2213
1694
  }
1695
+
1696
+ // src/persistence/durability/sync/published.ts
2214
1697
  var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
2215
1698
  var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
2216
1699
  function hexOf(bytes) {
@@ -2879,13 +2362,13 @@ function createWalWriter(directory, config) {
2879
2362
  for (const frame of frames) {
2880
2363
  totalBytes += frame.length;
2881
2364
  }
2882
- const joined = new Uint8Array(totalBytes);
2365
+ const joined2 = new Uint8Array(totalBytes);
2883
2366
  let offset = 0;
2884
2367
  for (const frame of frames) {
2885
- joined.set(frame, offset);
2368
+ joined2.set(frame, offset);
2886
2369
  offset += frame.length;
2887
2370
  }
2888
- return joined;
2371
+ return joined2;
2889
2372
  }
2890
2373
  function framesThatFit(frames, start) {
2891
2374
  const room = segmentMaxBytes - activeBytes;
@@ -2906,16 +2389,16 @@ function createWalWriter(directory, config) {
2906
2389
  { indexName: config.indexName, partitionId: config.partitionId }
2907
2390
  );
2908
2391
  }
2909
- const joined = joinFrames([...frames]);
2910
- if (activeBytes + joined.length > MAX_FILE_BYTES) {
2392
+ const joined2 = joinFrames([...frames]);
2393
+ if (activeBytes + joined2.length > MAX_FILE_BYTES) {
2911
2394
  throw new NarsilError(
2912
2395
  ErrorCodes.PERSISTENCE_SAVE_FAILED,
2913
- `A log record of ${joined.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
2914
- { indexName: config.indexName, partitionId: config.partitionId, bytes: joined.length }
2396
+ `A log record of ${joined2.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
2397
+ { indexName: config.indexName, partitionId: config.partitionId, bytes: joined2.length }
2915
2398
  );
2916
2399
  }
2917
- await activeHandle.append(joined);
2918
- activeBytes += joined.length;
2400
+ await activeHandle.append(joined2);
2401
+ activeBytes += joined2.length;
2919
2402
  const lastSeqNo = entries[entries.length - 1].seqNo;
2920
2403
  if (lastSeqNo > highestAppendedSeqNo) {
2921
2404
  highestAppendedSeqNo = lastSeqNo;
@@ -3036,6 +2519,109 @@ function createPartitionRegistry(directory, segmentMaxBytes) {
3036
2519
  }
3037
2520
  };
3038
2521
  }
2522
+
2523
+ // src/serialization/payload-v2.ts
2524
+ function validateHnswMetric(value) {
2525
+ if (value === void 0 || value === null) return void 0;
2526
+ if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
2527
+ return value;
2528
+ }
2529
+ return void 0;
2530
+ }
2531
+ function storedDocumentsOf(raw) {
2532
+ requireValidFieldTypes(raw.schema ?? {});
2533
+ const documents = {};
2534
+ for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
2535
+ documents[docId] = {
2536
+ fields: doc.fields,
2537
+ fieldLengths: doc.field_lengths ?? {}
2538
+ };
2539
+ }
2540
+ dropStoredVectorValues(documents, raw.schema ?? {});
2541
+ return documents;
2542
+ }
2543
+ function documentsOfPayloadV2(data) {
2544
+ const raw = decode(data);
2545
+ return { documents: storedDocumentsOf(raw), schema: raw.schema ?? {} };
2546
+ }
2547
+ function deserializePayloadV2(data) {
2548
+ const raw = decode(data);
2549
+ const documents = storedDocumentsOf(raw);
2550
+ const fieldNames = raw.inverted_index?.field_names ?? [];
2551
+ const invertedIndex = {};
2552
+ for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
2553
+ const count = list.ids.length;
2554
+ const postings = new Array(count);
2555
+ for (let i = 0; i < count; i++) {
2556
+ postings[i] = {
2557
+ docId: list.ids[i],
2558
+ termFrequency: list.tf[i],
2559
+ field: fieldNames[list.fi[i]] ?? "",
2560
+ positions: list.pos ? list.pos[i] : []
2561
+ };
2562
+ }
2563
+ invertedIndex[token] = {
2564
+ docFrequency: list.df,
2565
+ postings
2566
+ };
2567
+ }
2568
+ const numeric = {};
2569
+ for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
2570
+ numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
2571
+ }
2572
+ const boolean = {};
2573
+ for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
2574
+ boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
2575
+ }
2576
+ const geopoint = {};
2577
+ for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
2578
+ geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
2579
+ }
2580
+ const vectorData = {};
2581
+ for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
2582
+ vectorData[field] = {
2583
+ dimension: data2.dimension,
2584
+ vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
2585
+ hnswGraph: data2.hnsw_graph ? {
2586
+ entryPoint: data2.hnsw_graph.entry_point,
2587
+ maxLayer: data2.hnsw_graph.max_layer,
2588
+ m: data2.hnsw_graph.m,
2589
+ efConstruction: data2.hnsw_graph.ef_construction,
2590
+ metric: validateHnswMetric(data2.hnsw_graph.metric),
2591
+ nodes: data2.hnsw_graph.nodes
2592
+ } : null,
2593
+ codes: data2.codes ? { bits: data2.codes.bits, centroid: data2.codes.centroid, records: data2.codes.records } : null
2594
+ };
2595
+ }
2596
+ return {
2597
+ indexName: raw.index_name,
2598
+ partitionId: raw.partition_id,
2599
+ totalPartitions: raw.total_partitions,
2600
+ language: raw.language ?? "english",
2601
+ schema: raw.schema ?? {},
2602
+ docCount: raw.doc_count ?? 0,
2603
+ avgDocLength: raw.avg_doc_length ?? 0,
2604
+ documents,
2605
+ invertedIndex,
2606
+ fieldIndexes: {
2607
+ numeric,
2608
+ boolean,
2609
+ enum: raw.field_indexes?.enum ?? {},
2610
+ geopoint,
2611
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
2612
+ },
2613
+ surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
2614
+ vectorData,
2615
+ statistics: {
2616
+ totalDocuments: raw.statistics?.total_documents ?? 0,
2617
+ totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
2618
+ averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
2619
+ docFrequencies: raw.statistics?.doc_frequencies ?? {}
2620
+ }
2621
+ };
2622
+ }
2623
+
2624
+ // src/persistence/durability/earlier-layout/bundle.ts
3039
2625
  var SNAPSHOT_BUNDLE_VERSION = 2;
3040
2626
  function invalid(message) {
3041
2627
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
@@ -3095,7 +2681,7 @@ async function loadSnapshotBundle(bytes, deps) {
3095
2681
  while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
3096
2682
  if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
3097
2683
  for (let i = 0; i < bundle.partitions.length; i += 1) {
3098
- deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
2684
+ deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
3099
2685
  }
3100
2686
  for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
3101
2687
  const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
@@ -3403,6 +2989,8 @@ function validateMergedPartition(merged) {
3403
2989
  }
3404
2990
  }
3405
2991
  }
2992
+
2993
+ // src/persistence/durability/earlier-layout/segment.ts
3406
2994
  function normalizeTombstones(raw) {
3407
2995
  if (raw === void 0) return [];
3408
2996
  if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
@@ -3441,7 +3029,6 @@ var decoder = new TextDecoder();
3441
3029
  async function countPagedPartition(store, partition) {
3442
3030
  const { directory } = store;
3443
3031
  if (directory === void 0 || partition.segments.length === 0) return null;
3444
- const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-NDSQLPMQ.mjs');
3445
3032
  for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
3446
3033
  const scans = [];
3447
3034
  try {
@@ -3493,6 +3080,8 @@ async function countCheckpointDocuments(store, indexName) {
3493
3080
  const bundle = await store.read(snapshotBundleKey(indexName));
3494
3081
  return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
3495
3082
  }
3083
+
3084
+ // src/persistence/durability/earlier-layout/load.ts
3496
3085
  function removeWhatTheSegmentReplaces(manager, segment) {
3497
3086
  manager.beginBatchRemove();
3498
3087
  try {
@@ -3508,20 +3097,21 @@ function removeWhatTheSegmentReplaces(manager, segment) {
3508
3097
  }
3509
3098
  function attachDocuments(manager, partitionId, segment) {
3510
3099
  if (segment.documents.length === 0) return;
3511
- const payload = buildSegmentPayload(
3100
+ const outputs = buildSegmentPayloads(
3512
3101
  segment.documents,
3513
3102
  manager.config.schema,
3514
3103
  manager.language,
3515
3104
  resolvePartitionInsertOptions(manager.config, manager.analysis),
3516
3105
  manager.config.trackPositions ?? true
3517
3106
  );
3518
- const documents = segment.documents.map((entry) => entry.document);
3519
- const snapshot = freezeSegmentShared(payload, documents);
3520
- if (snapshot === null) {
3521
- manager.mergeSegment(partitionId, payload, documents);
3522
- return;
3107
+ const prepared = outputs.map(({ payload, documents: members }) => {
3108
+ const documents = members.map((entry) => entry.document);
3109
+ return { payload, documents, snapshot: freezeSegmentShared(payload, documents) };
3110
+ });
3111
+ for (const { payload, documents, snapshot } of prepared) {
3112
+ if (snapshot === null) manager.mergeSegment(partitionId, payload, documents);
3113
+ else manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
3523
3114
  }
3524
- manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
3525
3115
  }
3526
3116
  async function loadPartition(store, indexName, partition, deps) {
3527
3117
  if (deps.storedTermsAreCurrent) {
@@ -3542,7 +3132,7 @@ async function loadPartition(store, indexName, partition, deps) {
3542
3132
  totalPartitions: deps.manager.partitionCount,
3543
3133
  language: deps.manager.language.name
3544
3134
  });
3545
- deps.manager.deserializePartition(partition.partitionId, merged);
3135
+ deps.manager.deserializePartition(partition.partitionId, merged, deserializePartition);
3546
3136
  }
3547
3137
  async function readGraph(store, ref) {
3548
3138
  if (ref.graphKey === null) return null;
@@ -3741,13 +3331,68 @@ async function recoverCheckpoint(deps) {
3741
3331
  return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
3742
3332
  }
3743
3333
 
3744
- // src/persistence/durability/stored-analysis.ts
3745
- function generationOfStoredTerms(store, indexName) {
3746
- return readSegmentManifest(store, indexName).then((decoded) => {
3747
- if (decoded === null) return null;
3748
- return decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
3749
- });
3334
+ // src/persistence/durability/layout-transition/recovery.ts
3335
+ async function settleStoredLayout(store, metadata) {
3336
+ const transition = await readTransition(store, metadata.indexName);
3337
+ const decoded = await readSegmentManifest(store, metadata.indexName);
3338
+ const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
3339
+ if (transition !== null && partitionCount !== transition.oldPartitionCount && partitionCount !== transition.newPartitionCount) {
3340
+ invalidTransition("matches neither the manifest nor the metadata");
3341
+ }
3342
+ if (transition !== null && decoded === null && partitionCount !== transition.oldPartitionCount) {
3343
+ invalidTransition("has no old checkpoint or matching metadata");
3344
+ }
3345
+ const changedMetadata = partitionCount !== metadata.partitionCount;
3346
+ metadata.partitionCount = partitionCount;
3347
+ return {
3348
+ transition,
3349
+ changedMetadata,
3350
+ replaysOldLayout: transition !== null && partitionCount === transition.oldPartitionCount
3351
+ };
3352
+ }
3353
+ async function checkRetiredHead(directory, indexName, head) {
3354
+ const wal = await readWalSegments(directory, indexName, head.partitionId);
3355
+ if (wal === null) return;
3356
+ if (wal.highestReadFromWal > head.lastSeqNo || wal.marker.state.highestDurableSeqNo > head.lastSeqNo || wal.activeTail?.entries.some((entry) => entry.seqNo > head.lastSeqNo)) {
3357
+ invalidTransition("has records above a retired partition head");
3358
+ }
3359
+ }
3360
+ async function replayAcrossLayout(directory, indexName, transition, checkpoint, deps) {
3361
+ const terms = [];
3362
+ for (const head of transition.oldHeads) {
3363
+ const from = checkpointLastSeqNo(checkpoint, head.partitionId);
3364
+ if (from > head.lastSeqNo) invalidTransition("has an old head below its checkpoint");
3365
+ const replayed2 = await replayWalUpTo(directory, indexName, head.partitionId, from, head.lastSeqNo, deps);
3366
+ if (replayed2.highestSeqNo < head.lastSeqNo) invalidTransition("is missing an admitted old record");
3367
+ terms[head.partitionId] = Math.max(
3368
+ head.primaryTerm,
3369
+ replayed2.highestPrimaryTerm,
3370
+ checkpointPrimaryTerm(checkpoint, head.partitionId)
3371
+ );
3372
+ if (head.partitionId >= transition.newPartitionCount) await checkRetiredHead(directory, indexName, head);
3373
+ }
3374
+ const spill = await rewriteSpillOf({ directory, writeRecords: deps.writeRecords ?? null }, deps.manager);
3375
+ await createRebalancer().rebalance(
3376
+ deps.manager,
3377
+ transition.newPartitionCount,
3378
+ createPartitionRouter(),
3379
+ void 0,
3380
+ spill
3381
+ );
3382
+ const replayed = [];
3383
+ for (let partitionId = 0; partitionId < transition.newPartitionCount; partitionId++) {
3384
+ const head = transition.oldHeads[partitionId];
3385
+ const next = await replayWal(directory, indexName, partitionId, head?.lastSeqNo ?? 0, deps);
3386
+ replayed.push({
3387
+ partitionId,
3388
+ lastSeqNo: next.highestSeqNo,
3389
+ primaryTerm: Math.max(terms[partitionId] ?? SINGLE_NODE_PRIMARY_TERM, next.highestPrimaryTerm)
3390
+ });
3391
+ }
3392
+ return { checkpoint: replayed, removeUnread: async () => spill?.removeUnread(deps.manager) };
3750
3393
  }
3394
+
3395
+ // src/persistence/durability/stored-analysis.ts
3751
3396
  function promotePendingAnalysis(metadata) {
3752
3397
  const pending = metadata.pendingAnalysis;
3753
3398
  if (pending === void 0) return;
@@ -3761,9 +3406,13 @@ function promotePendingAnalysis(metadata) {
3761
3406
  }
3762
3407
  async function settleStoredAnalysis(store, metadata, metadataOnly) {
3763
3408
  if (metadataOnly && metadata.pendingAnalysis === void 0) return false;
3764
- const stored = await generationOfStoredTerms(store, metadata.indexName);
3409
+ const decoded = await readSegmentManifest(store, metadata.indexName);
3410
+ const partitionCount = decoded?.kind === "current" ? decoded.manifest.checkpoint.length : metadata.partitionCount;
3411
+ const changedLayout = partitionCount !== metadata.partitionCount;
3412
+ metadata.partitionCount = partitionCount;
3413
+ const stored = decoded === null ? null : decoded.kind === "current" ? decoded.manifest.analysisGeneration ?? 0 : 0;
3765
3414
  const current = metadata.analysisGeneration ?? 0;
3766
- if (stored === null || stored === current) return false;
3415
+ if (stored === null || stored === current) return changedLayout;
3767
3416
  if (stored === metadata.pendingAnalysis?.generation) {
3768
3417
  promotePendingAnalysis(metadata);
3769
3418
  return true;
@@ -3787,15 +3436,16 @@ async function writeCountedMetadata(deps, indexName, documentCount) {
3787
3436
  if (metadata === void 0) return;
3788
3437
  const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
3789
3438
  await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
3790
- }).catch(() => void 0);
3439
+ });
3791
3440
  }
3792
3441
  async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3793
3442
  const { directory, hooks } = deps;
3794
3443
  const store = directoryCheckpointStore(directory);
3795
3444
  const metadata = await loadMetadata(directory, indexName);
3796
3445
  if (metadata === null) {
3797
- return;
3446
+ return null;
3798
3447
  }
3448
+ const layout = metadataOnly ? { transition: null, changedMetadata: false, replaysOldLayout: false } : await settleStoredLayout(store, metadata);
3799
3449
  let derivedCount;
3800
3450
  if (metadataOnly && metadata.documentCount === void 0) {
3801
3451
  try {
@@ -3808,15 +3458,15 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3808
3458
  const promoted = await settleStoredAnalysis(store, metadata, metadataOnly);
3809
3459
  await hooks.createIndexFromMetadata(metadata, !metadataOnly);
3810
3460
  if (metadataOnly) {
3811
- if (derivedCount !== void 0 || promoted) {
3461
+ if (derivedCount !== void 0 || promoted || layout.changedMetadata) {
3812
3462
  await writeCountedMetadata(deps, indexName, derivedCount ?? metadata.documentCount ?? 0);
3813
3463
  }
3814
- return;
3464
+ return null;
3815
3465
  }
3816
3466
  const manager = hooks.getManager(indexName);
3817
3467
  const current = hooks.buildMetadata(indexName);
3818
3468
  if (manager === void 0 || current === void 0) {
3819
- return;
3469
+ return null;
3820
3470
  }
3821
3471
  const writeRecords = hooks.writeRecords ?? null;
3822
3472
  writeRecords?.dropIndex(indexName);
@@ -3827,7 +3477,13 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3827
3477
  writeRecords,
3828
3478
  defersGraphBuilds: true
3829
3479
  };
3480
+ let removeUnread = async () => void 0;
3830
3481
  const replay = async (checkpoint) => {
3482
+ if (layout.replaysOldLayout && layout.transition !== null) {
3483
+ const replayed2 = await replayAcrossLayout(directory, indexName, layout.transition, checkpoint, replayDeps);
3484
+ removeUnread = replayed2.removeUnread;
3485
+ return replayed2.checkpoint;
3486
+ }
3831
3487
  const replayed = [];
3832
3488
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
3833
3489
  const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
@@ -3871,9 +3527,19 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3871
3527
  for (const { partitionId, lastSeqNo, primaryTerm } of recovered.checkpoint) {
3872
3528
  deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
3873
3529
  }
3874
- if (recovered.upgradedDocumentCount !== null || promoted) {
3530
+ if (layout.transition === null && (recovered.upgradedDocumentCount !== null || promoted || layout.changedMetadata)) {
3875
3531
  await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount ?? manager.countDocuments());
3876
3532
  }
3533
+ return layout.transition === null ? null : removeUnread;
3534
+ }
3535
+ async function recoverWithLayoutCheckpoint(deps, indexName, metadataOnly, checkpoint) {
3536
+ const removeUnread = await recoverPersistedIndex(deps, indexName, metadataOnly);
3537
+ if (removeUnread === null) return;
3538
+ try {
3539
+ await checkpoint();
3540
+ } finally {
3541
+ await removeUnread();
3542
+ }
3877
3543
  }
3878
3544
 
3879
3545
  // src/persistence/durability/manager.ts
@@ -3950,21 +3616,25 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3950
3616
  }
3951
3617
  }
3952
3618
  }
3953
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false) {
3619
+ async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, pinsRetiredVectors = false, requested = false) {
3954
3620
  const indexState = indexes.get(indexName);
3955
3621
  if (indexState === void 0 || indexState.unloading || heldCheckpoints.holds(indexName, indexState)) return;
3956
- return checkpointInTurn(indexState, {
3957
- joinsInFlight: !afterInFlight && !pinsRetiredVectors,
3958
- mayStart: () => !indexState.unloading && !shuttingDown,
3959
- perform: () => performCheckpoint(indexName, indexState, fromMemory, pinsRetiredVectors),
3960
- afterSettling: () => releaseStalledWrites(indexState)
3961
- });
3622
+ return holdScheduledCheckpoint(
3623
+ checkpointInTurn(indexState, {
3624
+ joinsInFlight: !afterInFlight && !pinsRetiredVectors,
3625
+ mayStart: () => !indexState.unloading && !shuttingDown,
3626
+ perform: () => performCheckpoint(indexName, indexState, fromMemory),
3627
+ afterSettling: () => releaseStalledWrites(indexState)
3628
+ }),
3629
+ requested
3630
+ );
3962
3631
  }
3963
3632
  async function requestedCheckpoint(indexName, fromMemory = false, pinsVectors = false) {
3964
3633
  heldCheckpoints.refuse(indexName, indexes.get(indexName));
3965
- return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors);
3634
+ await requireCheckpointLayout(hooks.getManager(indexName));
3635
+ return checkpointIndex(indexName, fromMemory, fromMemory, pinsVectors, true);
3966
3636
  }
3967
- async function performCheckpoint(indexName, indexState, fromMemory = false, pinsRetiredVectors = false) {
3637
+ async function performCheckpoint(indexName, indexState, fromMemory = false) {
3968
3638
  await directoryWatch.verify();
3969
3639
  if (fatalError !== null) throw fatalError;
3970
3640
  await runDurableCheckpoint({
@@ -3975,7 +3645,6 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3975
3645
  compactionThreshold,
3976
3646
  canOffload: canOffloadCheckpoint,
3977
3647
  fromMemory,
3978
- pinsRetiredVectors,
3979
3648
  mutationThreshold: checkpointMutationThreshold,
3980
3649
  mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
3981
3650
  holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
@@ -4007,9 +3676,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4007
3676
  const recordedByIndex = /* @__PURE__ */ new Map();
4008
3677
  for (let i = 0; i < records.length; i++) {
4009
3678
  if (!outcomes[i].ok) continue;
4010
- const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
3679
+ const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0, deferred: false };
4011
3680
  recorded.records += 1;
4012
3681
  recorded.documentBytes += records[i].document?.byteLength ?? 0;
3682
+ recorded.deferred ||= hasDeferredPublication(records[i]);
4013
3683
  recordedByIndex.set(records[i].indexName, recorded);
4014
3684
  }
4015
3685
  for (const [indexName, recorded] of recordedByIndex) {
@@ -4021,7 +3691,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4021
3691
  markFatal(toError(err));
4022
3692
  });
4023
3693
  }
4024
- await stallWhileCheckpointsFallBehind(indexState);
3694
+ if (!recorded.deferred) await stallWhileCheckpointsFallBehind(indexState);
4025
3695
  }
4026
3696
  if (recordedByIndex.size === 0) return outcomes;
4027
3697
  if (mode === "sync") await directoryWatch.verify();
@@ -4056,7 +3726,13 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4056
3726
  syncStorage
4057
3727
  };
4058
3728
  function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
4059
- return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
3729
+ const recovered = recoverWithLayoutCheckpoint(
3730
+ recoveryDeps,
3731
+ indexName,
3732
+ metadataOnly,
3733
+ () => performCheckpoint(indexName, getOrCreateIndexState(indexName), true)
3734
+ );
3735
+ return loads.track(indexName, recovered, startsGraphRepairs);
4060
3736
  }
4061
3737
  return {
4062
3738
  isActive: () => true,
@@ -4138,7 +3814,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4138
3814
  },
4139
3815
  async checkpointAll() {
4140
3816
  for (const indexName of [...indexes.keys()]) {
4141
- await checkpointIndex(indexName);
3817
+ await requestedCheckpoint(indexName);
4142
3818
  }
4143
3819
  },
4144
3820
  ...unloading,
@@ -4164,27 +3840,47 @@ function createDurabilityManager(config, hooks, directoryOverride) {
4164
3840
  }
4165
3841
 
4166
3842
  // src/persistence/durability/snapshot-only/checkpoint.ts
4167
- function capture(input, prior) {
3843
+ function capture(input, prior, cutoffs) {
4168
3844
  const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
4169
3845
  const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
4170
3846
  const targets = [];
4171
3847
  const contents = [];
3848
+ const positions = /* @__PURE__ */ new Map();
4172
3849
  for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
4173
- const lastSeqNo = input.appliedSeqNoByPartition.get(partitionId) ?? 0;
3850
+ const lastSeqNo = Math.min(
3851
+ input.appliedSeqNoByPartition.get(partitionId) ?? 0,
3852
+ cutoffs.get(partitionId) ?? Infinity
3853
+ );
3854
+ positions.set(partitionId, lastSeqNo);
4174
3855
  const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
4175
3856
  if (unchanged) continue;
4176
3857
  targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
4177
3858
  contents.push(partitionContentsOf(input.manager, partitionId));
4178
3859
  }
4179
3860
  const plans = /* @__PURE__ */ new Map();
4180
- for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
4181
- const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
4182
- plans.set(fieldPath, vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key)));
3861
+ try {
3862
+ for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
3863
+ const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
3864
+ plans.set(
3865
+ fieldPath,
3866
+ vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), false, true)
3867
+ );
3868
+ }
3869
+ return {
3870
+ targets,
3871
+ plans,
3872
+ documentCount: input.manager.countDocuments(),
3873
+ contents,
3874
+ positions,
3875
+ partitionCount: input.manager.partitionCount
3876
+ };
3877
+ } catch (error) {
3878
+ for (const plan of plans.values()) plan.release();
3879
+ throw error;
4183
3880
  }
4184
- return { targets, plans, documentCount: input.manager.countDocuments(), contents };
4185
3881
  }
4186
3882
  async function writeCaptured(input, store, prior, captured) {
4187
- const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, input.manager.partitionCount);
3883
+ const priorWithinCount = prior === null ? null : manifestWithinPartitionCount(prior, captured.partitionCount);
4188
3884
  const written = await writeCheckpointSegments({
4189
3885
  store,
4190
3886
  metadata: input.metadata,
@@ -4210,9 +3906,17 @@ async function writeSnapshotOnlyCheckpoint(input) {
4210
3906
  const store = adapterCheckpointStore(input.adapter);
4211
3907
  for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
4212
3908
  const prior = await readCurrentManifest(store, input.indexName);
4213
- const captured = capture(input, prior);
4214
- await writeCaptured(input, store, prior, captured);
4215
- return captured.documentCount;
3909
+ const captured = await captureAppliedPrefix(input.manager, (cutoffs) => capture(input, prior, cutoffs));
3910
+ try {
3911
+ await writeCaptured(input, store, prior, captured);
3912
+ return {
3913
+ documentCount: captured.documentCount,
3914
+ positions: captured.positions,
3915
+ partitionCount: captured.partitionCount
3916
+ };
3917
+ } finally {
3918
+ for (const plan of captured.plans.values()) plan.release();
3919
+ }
4216
3920
  }
4217
3921
 
4218
3922
  // src/persistence/durability/snapshot-only/index-state.ts
@@ -4225,6 +3929,20 @@ function createSnapshotIndexState() {
4225
3929
  appliedSeqNoByPartition: /* @__PURE__ */ new Map()
4226
3930
  };
4227
3931
  }
3932
+ function metadataKey(indexName) {
3933
+ return `${indexName}/meta`;
3934
+ }
3935
+ function noteSnapshotCommitted(state, positions, partitionCount) {
3936
+ let recordsLeft = 0;
3937
+ for (const [partitionId, sequence] of state.appliedSeqNoByPartition) {
3938
+ if (partitionId >= partitionCount) {
3939
+ state.appliedSeqNoByPartition.delete(partitionId);
3940
+ continue;
3941
+ }
3942
+ recordsLeft += Math.max(0, sequence - (positions.get(partitionId) ?? 0));
3943
+ }
3944
+ state.mutationsSinceCheckpoint = recordsLeft;
3945
+ }
4228
3946
 
4229
3947
  // src/persistence/durability/snapshot-only/restores.ts
4230
3948
  function toError5(error) {
@@ -4264,9 +3982,6 @@ function createPendingRestores(onFatalError) {
4264
3982
  }
4265
3983
 
4266
3984
  // src/persistence/durability/snapshot-only/index.ts
4267
- function metadataKey(indexName) {
4268
- return `${indexName}/meta`;
4269
- }
4270
3985
  function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4271
3986
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
4272
3987
  const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
@@ -4312,10 +4027,8 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4312
4027
  await settleRestore(indexName);
4313
4028
  const manager = hooks.getManager(indexName);
4314
4029
  const metadata = hooks.buildTermsMetadata(indexName);
4315
- if (manager === void 0 || metadata === void 0) {
4316
- return;
4317
- }
4318
- const documentCount = await writeSnapshotOnlyCheckpoint({
4030
+ if (manager === void 0 || metadata === void 0) return;
4031
+ const { documentCount, positions, partitionCount } = await writeSnapshotOnlyCheckpoint({
4319
4032
  adapter,
4320
4033
  indexName,
4321
4034
  metadata,
@@ -4328,11 +4041,12 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4328
4041
  await queueMetadataWrite(indexName, async () => {
4329
4042
  const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
4330
4043
  if (checkpointMetadata === void 0) return;
4044
+ checkpointMetadata.partitionCount = partitionCount;
4331
4045
  const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
4332
4046
  await adapter.save(metadataKey(indexName), metadataBytes);
4333
- hooks.recordCheckpoint?.(indexName, documentCount, manager.partitionCount);
4047
+ hooks.recordCheckpoint?.(indexName, documentCount, partitionCount);
4334
4048
  });
4335
- indexState.mutationsSinceCheckpoint = 0;
4049
+ noteSnapshotCommitted(indexState, positions, partitionCount);
4336
4050
  if (publisher !== void 0) {
4337
4051
  const partitionIds = [];
4338
4052
  for (let i = 0; i < manager.partitionCount; i += 1) {
@@ -4341,15 +4055,19 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4341
4055
  await publisher.publishPartitions(indexName, partitionIds);
4342
4056
  }
4343
4057
  }
4344
- async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory) {
4058
+ async function checkpointIndex(indexName, fromMemory = false, afterInFlight = fromMemory, requested = false) {
4345
4059
  const indexState = indexes.get(indexName);
4346
4060
  if (indexState === void 0 || indexState.unloading) return;
4347
- return checkpointInTurn(indexState, {
4348
- joinsInFlight: !afterInFlight,
4349
- mayStart: () => !indexState.unloading && !shuttingDown,
4350
- perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
4351
- afterSettling: () => void 0
4352
- });
4061
+ if (requested) await requireCheckpointLayout(hooks.getManager(indexName));
4062
+ return holdScheduledCheckpoint(
4063
+ checkpointInTurn(indexState, {
4064
+ joinsInFlight: !afterInFlight,
4065
+ mayStart: () => !indexState.unloading && !shuttingDown,
4066
+ perform: () => indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)),
4067
+ afterSettling: () => void 0
4068
+ }),
4069
+ requested
4070
+ );
4353
4071
  }
4354
4072
  function queueMetadataWrite(indexName, write) {
4355
4073
  const previous = metadataWrites.get(indexName) ?? Promise.resolve();
@@ -4379,7 +4097,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4379
4097
  let derivedCount;
4380
4098
  if (metadataOnly && metadata.documentCount === void 0) {
4381
4099
  try {
4382
- derivedCount = await countStoredBundleDocuments(indexName);
4100
+ derivedCount = await countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
4383
4101
  metadata.documentCount = derivedCount;
4384
4102
  } catch {
4385
4103
  derivedCount = void 0;
@@ -4423,9 +4141,6 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4423
4141
  await queueMetadataWrite(indexName, () => saveMetadata(indexName, upgradedCount)).catch(() => void 0);
4424
4142
  }
4425
4143
  }
4426
- function countStoredBundleDocuments(indexName) {
4427
- return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
4428
- }
4429
4144
  async function recordMutation(record) {
4430
4145
  const indexState = getOrCreateIndexState(record.indexName);
4431
4146
  const buffered = indexState.applyChain.then(async () => {
@@ -4505,10 +4220,10 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4505
4220
  });
4506
4221
  },
4507
4222
  checkpoint(indexName) {
4508
- return checkpointIndex(indexName);
4223
+ return checkpointIndex(indexName, false, false, true);
4509
4224
  },
4510
4225
  checkpointFromMemory(indexName) {
4511
- return checkpointIndex(indexName, true);
4226
+ return checkpointIndex(indexName, true, true, true);
4512
4227
  },
4513
4228
  async checkpointAfterSwitch(indexName, swap) {
4514
4229
  const indexState = getOrCreateIndexState(indexName);
@@ -4527,7 +4242,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
4527
4242
  },
4528
4243
  async checkpointAll() {
4529
4244
  for (const indexName of [...indexes.keys()]) {
4530
- await checkpointIndex(indexName);
4245
+ await checkpointIndex(indexName, false, false, true);
4531
4246
  }
4532
4247
  },
4533
4248
  stageRestore(indexName) {
@@ -4587,7 +4302,13 @@ function buildMetadata(indexName, hooks, documentCount) {
4587
4302
  if (config === void 0) {
4588
4303
  return void 0;
4589
4304
  }
4590
- return metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount, documentCount);
4305
+ const manager = hooks.getManager(indexName);
4306
+ return metadataFromConfig(
4307
+ indexName,
4308
+ config,
4309
+ manager === void 0 ? void 0 : metadataPartitionCount(manager),
4310
+ documentCount
4311
+ );
4591
4312
  }
4592
4313
  function createDurabilityIntegration(tier, hooks) {
4593
4314
  const managerHooks = {
@@ -4612,22 +4333,30 @@ function createDurabilityIntegration(tier, hooks) {
4612
4333
  ...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
4613
4334
  };
4614
4335
  const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
4615
- function partitionFor(indexName, docId) {
4616
- const partitionManager = hooks.getManager(indexName);
4617
- const partitionCount = partitionManager?.partitionCount ?? 1;
4618
- return fnv1a(docId) % Math.max(1, partitionCount);
4619
- }
4620
4336
  async function recordMutation(indexName, docId, operation, document, apply) {
4621
- const partitionId = partitionFor(indexName, docId);
4622
- const seqNo = await manager.recordMutation({
4623
- indexName,
4624
- partitionId,
4625
- operation,
4626
- documentId: docId,
4627
- document,
4628
- apply
4337
+ return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
4338
+ const partitionId = fnv1a(docId) % partitionCount;
4339
+ let queued;
4340
+ try {
4341
+ const record = {
4342
+ indexName,
4343
+ partitionId,
4344
+ operation,
4345
+ documentId: docId,
4346
+ document,
4347
+ async apply() {
4348
+ queued = await apply() ?? void 0;
4349
+ if (queued !== void 0) deferMutationPublication(record);
4350
+ }
4351
+ };
4352
+ const seqNo = await manager.recordMutation(record);
4353
+ queued?.commit(partitionId, partitionCount, seqNo);
4354
+ return { indexName, partitionId, seqNo };
4355
+ } catch (error) {
4356
+ queued?.fail(error);
4357
+ throw error;
4358
+ }
4629
4359
  });
4630
- return { indexName, partitionId, seqNo };
4631
4360
  }
4632
4361
  return {
4633
4362
  manager,
@@ -4638,29 +4367,48 @@ function createDurabilityIntegration(tier, hooks) {
4638
4367
  const records = [];
4639
4368
  const outcomes = new Array(mutations.length);
4640
4369
  const recordPositions = [];
4370
+ const queued = [];
4641
4371
  for (let i = 0; i < mutations.length; i++) {
4642
4372
  if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
4643
4373
  const mutation = mutations[i];
4644
4374
  try {
4645
- records.push({
4375
+ const record = {
4646
4376
  indexName,
4647
- partitionId: partitionFor(indexName, mutation.docId),
4377
+ partitionId: 0,
4648
4378
  operation: mutation.document === null ? "DELETE" : "INDEX",
4649
4379
  documentId: mutation.docId,
4650
4380
  document: mutation.document === null ? null : encodeDocument(mutation.document),
4651
- apply: mutation.apply
4652
- });
4381
+ async apply() {
4382
+ queued[i] = await mutation.apply() ?? void 0;
4383
+ if (queued[i] !== void 0) deferMutationPublication(record);
4384
+ }
4385
+ };
4386
+ records.push(record);
4653
4387
  recordPositions.push(i);
4654
4388
  } catch (error) {
4655
4389
  outcomes[i] = { ok: false, error };
4656
4390
  }
4657
4391
  }
4658
- const recorded = await manager.recordMutations(records);
4659
- for (let i = 0; i < recorded.length; i++) {
4660
- const outcome = recorded[i];
4661
- outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
4662
- }
4663
- return outcomes;
4392
+ return admitDurableWrite(hooks.getManager(indexName), async (partitionCount) => {
4393
+ let recorded;
4394
+ try {
4395
+ for (const record of records) {
4396
+ record.partitionId = fnv1a(record.documentId) % partitionCount;
4397
+ }
4398
+ recorded = await manager.recordMutations(records);
4399
+ } catch (error) {
4400
+ for (const entry of queued) entry?.fail(error);
4401
+ throw error;
4402
+ }
4403
+ for (let i = 0; i < recorded.length; i++) {
4404
+ const outcome = recorded[i];
4405
+ const entry = queued[recordPositions[i]];
4406
+ if (outcome.ok) entry?.commit(records[i].partitionId, partitionCount, outcome.seqNo);
4407
+ else entry?.fail(outcome.error);
4408
+ outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
4409
+ }
4410
+ return outcomes;
4411
+ });
4664
4412
  },
4665
4413
  recordRemove(indexName, docId, apply) {
4666
4414
  return recordMutation(indexName, docId, "DELETE", null, apply);
@@ -4668,6 +4416,172 @@ function createDurabilityIntegration(tier, hooks) {
4668
4416
  };
4669
4417
  }
4670
4418
 
4419
+ // src/engine/invalidation.ts
4420
+ function createInvalidationFromConfig(config, tierKind, deps, indexRegistry) {
4421
+ if (config?.invalidation === void 0) {
4422
+ return null;
4423
+ }
4424
+ if (tierKind !== "snapshot") {
4425
+ throw new NarsilError(
4426
+ ErrorCodes.CONFIG_INVALID,
4427
+ "Invalidation requires a shared persistence adapter. The write-ahead log tier owns its directory exclusively, so configure a non-filesystem persistence adapter to share data between instances"
4428
+ );
4429
+ }
4430
+ return createInvalidationIntegration({
4431
+ adapter: config.invalidation,
4432
+ instanceId: generateId(),
4433
+ ...deps,
4434
+ listBroadcastIndexNames() {
4435
+ const indexes = [];
4436
+ for (const [name, entry] of indexRegistry) {
4437
+ if (entry.config.defaultScoring === "broadcast") indexes.push(name);
4438
+ }
4439
+ return indexes;
4440
+ }
4441
+ });
4442
+ }
4443
+ function createInvalidationIntegration(deps) {
4444
+ const { adapter, instanceId, reloadIndex, getManager, listBroadcastIndexNames, onError } = deps;
4445
+ const foreignStats = /* @__PURE__ */ new Map();
4446
+ const reloadChains = /* @__PURE__ */ new Map();
4447
+ let broadcastTimer = null;
4448
+ let stopped = false;
4449
+ function toError6(err) {
4450
+ return err instanceof Error ? err : new Error(String(err));
4451
+ }
4452
+ function pruneStaleForeignStats(entries) {
4453
+ const cutoff = Date.now() - FOREIGN_STATS_TTL_MS;
4454
+ for (const [sourceInstanceId, entry] of entries) {
4455
+ if (entry.receivedAt < cutoff) {
4456
+ entries.delete(sourceInstanceId);
4457
+ }
4458
+ }
4459
+ }
4460
+ function scheduleReload(indexName) {
4461
+ const chain = reloadChains.get(indexName) ?? Promise.resolve();
4462
+ const next = chain.then(() => reloadIndex(indexName)).catch((err) => onError(toError6(err)));
4463
+ reloadChains.set(indexName, next);
4464
+ }
4465
+ function storeForeignStats(indexName, sourceInstanceId, stats) {
4466
+ let entries = foreignStats.get(indexName);
4467
+ if (entries === void 0) {
4468
+ entries = /* @__PURE__ */ new Map();
4469
+ foreignStats.set(indexName, entries);
4470
+ }
4471
+ entries.set(sourceInstanceId, { stats, receivedAt: Date.now() });
4472
+ pruneStaleForeignStats(entries);
4473
+ }
4474
+ function handleEvent(event) {
4475
+ try {
4476
+ if (event.type === "partition") {
4477
+ if (event.sourceInstanceId === instanceId) {
4478
+ return;
4479
+ }
4480
+ scheduleReload(event.indexName);
4481
+ return;
4482
+ }
4483
+ if (event.instanceId === instanceId) {
4484
+ return;
4485
+ }
4486
+ storeForeignStats(event.indexName, event.instanceId, event.stats);
4487
+ } catch (err) {
4488
+ onError(toError6(err));
4489
+ }
4490
+ }
4491
+ async function publishOwnStatistics() {
4492
+ for (const indexName of listBroadcastIndexNames()) {
4493
+ const manager = getManager(indexName);
4494
+ if (manager === void 0) {
4495
+ continue;
4496
+ }
4497
+ const aggregate = manager.statisticsMessage();
4498
+ try {
4499
+ await adapter.publish({
4500
+ type: "statistics",
4501
+ indexName,
4502
+ instanceId,
4503
+ stats: {
4504
+ totalDocs: aggregate.totalDocuments,
4505
+ docFrequencies: aggregate.docFrequencies,
4506
+ totalFieldLengths: aggregate.totalFieldLengths
4507
+ }
4508
+ });
4509
+ } catch (err) {
4510
+ onError(toError6(err));
4511
+ }
4512
+ }
4513
+ }
4514
+ return {
4515
+ instanceId,
4516
+ async publishPartitions(indexName, partitions) {
4517
+ if (stopped || partitions.length === 0) {
4518
+ return;
4519
+ }
4520
+ try {
4521
+ await adapter.publish({
4522
+ type: "partition",
4523
+ indexName,
4524
+ partitions,
4525
+ timestamp: Date.now(),
4526
+ sourceInstanceId: instanceId
4527
+ });
4528
+ } catch (err) {
4529
+ onError(toError6(err));
4530
+ }
4531
+ },
4532
+ broadcastStats(indexName, tokens) {
4533
+ const manager = getManager(indexName);
4534
+ if (manager === void 0) {
4535
+ return void 0;
4536
+ }
4537
+ const own = manager.getAggregateStats(tokens);
4538
+ const entries = foreignStats.get(indexName);
4539
+ if (entries === void 0 || entries.size === 0) {
4540
+ return mergePartitionStats([own]);
4541
+ }
4542
+ pruneStaleForeignStats(entries);
4543
+ const combined = [own];
4544
+ for (const [, entry] of entries) {
4545
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
4546
+ for (const token of tokens) {
4547
+ if (Object.hasOwn(entry.stats.docFrequencies, token))
4548
+ docFrequencies[token] = entry.stats.docFrequencies[token];
4549
+ }
4550
+ combined.push({
4551
+ totalDocuments: entry.stats.totalDocs,
4552
+ docFrequencies,
4553
+ totalFieldLengths: entry.stats.totalFieldLengths
4554
+ });
4555
+ }
4556
+ return mergePartitionStats(combined);
4557
+ },
4558
+ async start() {
4559
+ await adapter.subscribe(handleEvent);
4560
+ if (broadcastTimer !== null || stopped) {
4561
+ return;
4562
+ }
4563
+ broadcastTimer = setInterval(() => {
4564
+ void publishOwnStatistics();
4565
+ }, STATISTICS_BROADCAST_INTERVAL_MS);
4566
+ if (typeof broadcastTimer.unref === "function") {
4567
+ broadcastTimer.unref();
4568
+ }
4569
+ },
4570
+ async shutdown() {
4571
+ stopped = true;
4572
+ if (broadcastTimer !== null) {
4573
+ clearInterval(broadcastTimer);
4574
+ broadcastTimer = null;
4575
+ }
4576
+ try {
4577
+ await adapter.shutdown();
4578
+ } catch (err) {
4579
+ onError(toError6(err));
4580
+ }
4581
+ }
4582
+ };
4583
+ }
4584
+
4671
4585
  // src/engine/durability-wiring.ts
4672
4586
  function createDurabilityFromTier(tier, wiring) {
4673
4587
  return createDurabilityIntegration(tier, {
@@ -4698,7 +4612,11 @@ function createDurabilityFromTier(tier, wiring) {
4698
4612
  keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
4699
4613
  onHistoryMoved(indexName) {
4700
4614
  if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
4701
- import('./retention-TY3L3BFN.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
4615
+ try {
4616
+ scheduleSyncRetention(wiring.engineCore(), indexName);
4617
+ } catch (error) {
4618
+ wiring.emitFatalError(error instanceof Error ? error : new Error(String(error)));
4619
+ }
4702
4620
  }
4703
4621
  });
4704
4622
  }
@@ -4713,4 +4631,1489 @@ async function reloadIndexAndCopies(orchestrator, durability, indexName) {
4713
4631
  }
4714
4632
  }
4715
4633
 
4716
- export { createDurabilityFromTier, reloadIndexAndCopies };
4634
+ // src/engine/export/lines.ts
4635
+ var NEWLINE = 10;
4636
+ function joined(parts, total) {
4637
+ if (parts.length === 1) return parts[0];
4638
+ const bytes = new Uint8Array(total);
4639
+ let offset = 0;
4640
+ for (const part of parts) {
4641
+ bytes.set(part, offset);
4642
+ offset += part.byteLength;
4643
+ }
4644
+ return bytes;
4645
+ }
4646
+ async function* exportLines(source, maxLineBytes) {
4647
+ const decoder3 = new TextDecoder("utf-8", { fatal: true });
4648
+ let parts = [];
4649
+ let pending = 0;
4650
+ let overflowing = false;
4651
+ let line = 0;
4652
+ const finishLine = () => {
4653
+ line += 1;
4654
+ if (overflowing) {
4655
+ overflowing = false;
4656
+ parts = [];
4657
+ pending = 0;
4658
+ return { line, refused: "too-long" };
4659
+ }
4660
+ const bytes = joined(parts, pending);
4661
+ parts = [];
4662
+ pending = 0;
4663
+ try {
4664
+ return { line, text: decoder3.decode(bytes) };
4665
+ } catch {
4666
+ return { line, refused: "not-utf8" };
4667
+ }
4668
+ };
4669
+ for await (const chunk of chunksOf(source)) {
4670
+ let start = 0;
4671
+ for (; ; ) {
4672
+ const newline = chunk.indexOf(NEWLINE, start);
4673
+ const end = newline < 0 ? chunk.byteLength : newline;
4674
+ const piece = chunk.subarray(start, end);
4675
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
4676
+ if (piece.byteLength > 0) parts.push(piece.slice());
4677
+ pending += piece.byteLength;
4678
+ } else {
4679
+ overflowing = true;
4680
+ parts = [];
4681
+ }
4682
+ if (newline < 0) break;
4683
+ yield finishLine();
4684
+ start = newline + 1;
4685
+ }
4686
+ }
4687
+ if (pending > 0 || overflowing) yield finishLine();
4688
+ }
4689
+
4690
+ // src/serialization/base64.ts
4691
+ var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4692
+ var PAD = 61;
4693
+ var BYTES_PER_GROUP = 3;
4694
+ var CHARACTERS_PER_GROUP = 4;
4695
+ var ASCII_LIMIT = 128;
4696
+ var ENCODE = new Uint8Array(ALPHABET.length);
4697
+ var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
4698
+ for (let at = 0; at < ALPHABET.length; at++) {
4699
+ ENCODE[at] = ALPHABET.charCodeAt(at);
4700
+ DECODE[ALPHABET.charCodeAt(at)] = at;
4701
+ }
4702
+ var asciiDecoder = new TextDecoder();
4703
+ function encodeBase64(bytes) {
4704
+ const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
4705
+ let write = 0;
4706
+ let read = 0;
4707
+ for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
4708
+ const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
4709
+ out[write++] = ENCODE[group >>> 18];
4710
+ out[write++] = ENCODE[group >>> 12 & 63];
4711
+ out[write++] = ENCODE[group >>> 6 & 63];
4712
+ out[write++] = ENCODE[group & 63];
4713
+ }
4714
+ const left = bytes.byteLength - read;
4715
+ if (left > 0) {
4716
+ const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
4717
+ out[write++] = ENCODE[group >>> 18];
4718
+ out[write++] = ENCODE[group >>> 12 & 63];
4719
+ out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
4720
+ out[write++] = PAD;
4721
+ }
4722
+ return asciiDecoder.decode(out);
4723
+ }
4724
+ function sextetAt(text, at) {
4725
+ const code = text.charCodeAt(at);
4726
+ return code < ASCII_LIMIT ? DECODE[code] : -1;
4727
+ }
4728
+ function decodeExactBase64(text) {
4729
+ if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
4730
+ const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
4731
+ const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
4732
+ const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
4733
+ let write = 0;
4734
+ for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
4735
+ const a2 = sextetAt(text, at);
4736
+ const b2 = sextetAt(text, at + 1);
4737
+ const c2 = sextetAt(text, at + 2);
4738
+ const d = sextetAt(text, at + 3);
4739
+ if ((a2 | b2 | c2 | d) < 0) return null;
4740
+ const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
4741
+ out[write++] = group >>> 16;
4742
+ out[write++] = group >>> 8 & 255;
4743
+ out[write++] = group & 255;
4744
+ }
4745
+ if (padding === 0) return out;
4746
+ const a = sextetAt(text, fullEnd);
4747
+ const b = sextetAt(text, fullEnd + 1);
4748
+ const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
4749
+ if ((a | b | c) < 0) return null;
4750
+ if (padding === 2 && (b & 15) !== 0) return null;
4751
+ if (padding === 1 && (c & 3) !== 0) return null;
4752
+ out[write++] = a << 2 | b >>> 4;
4753
+ if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
4754
+ return out;
4755
+ }
4756
+
4757
+ // src/engine/export/tags.ts
4758
+ var TAG_PREFIX = "$";
4759
+ var DATE_TAG = "$date";
4760
+ var BYTES_TAG = "$bytes";
4761
+ var NUMBER_TAG = "$number";
4762
+ var OBJECT_TAG = "$object";
4763
+ var MILLISECONDS_PER_SECOND = 1e3;
4764
+ var NANOSECONDS_PER_MILLISECOND = 1e6;
4765
+ var MAX_NANOSECONDS = 999999999;
4766
+ var MAX_DATE_MILLISECONDS = 864e13;
4767
+ var ESCAPED_TAG_PREFIX = "\\u0024";
4768
+ var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
4769
+ ["NaN", Number.NaN],
4770
+ ["Infinity", Number.POSITIVE_INFINITY],
4771
+ ["-Infinity", Number.NEGATIVE_INFINITY]
4772
+ ]);
4773
+ function refuse(path, reason) {
4774
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
4775
+ field: path
4776
+ });
4777
+ }
4778
+ function isRecord2(value) {
4779
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4780
+ }
4781
+ function startsWithTagPrefix(key) {
4782
+ return key.startsWith(TAG_PREFIX);
4783
+ }
4784
+ function soleTagKey(record) {
4785
+ let sole = null;
4786
+ for (const key of Object.keys(record)) {
4787
+ if (sole !== null) return null;
4788
+ sole = key;
4789
+ }
4790
+ return sole !== null && startsWithTagPrefix(sole) ? sole : null;
4791
+ }
4792
+ function dateOf(value, path) {
4793
+ if (!Array.isArray(value) || value.length !== 2)
4794
+ refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
4795
+ const seconds = value[0];
4796
+ const nanoseconds = value[1];
4797
+ if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
4798
+ refuse(
4799
+ path,
4800
+ `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
4801
+ );
4802
+ }
4803
+ if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
4804
+ refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
4805
+ }
4806
+ const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
4807
+ if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
4808
+ refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
4809
+ return new Date(milliseconds);
4810
+ }
4811
+ function bytesOf(value, path) {
4812
+ const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
4813
+ if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
4814
+ return bytes;
4815
+ }
4816
+ function numberOf(value, path) {
4817
+ const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
4818
+ if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
4819
+ return number;
4820
+ }
4821
+ function requireDepth(path, depth) {
4822
+ if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
4823
+ }
4824
+ function untagMap(record, path, depth) {
4825
+ requireDepth(path, depth);
4826
+ for (const key of Object.keys(record)) {
4827
+ record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
4828
+ }
4829
+ return record;
4830
+ }
4831
+ function untagList(list, path, depth) {
4832
+ requireDepth(path, depth);
4833
+ for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
4834
+ return list;
4835
+ }
4836
+ function untagTag(tag, value, path, depth) {
4837
+ switch (tag) {
4838
+ case DATE_TAG:
4839
+ return dateOf(value, path);
4840
+ case BYTES_TAG:
4841
+ return bytesOf(value, path);
4842
+ case NUMBER_TAG:
4843
+ return numberOf(value, path);
4844
+ case OBJECT_TAG:
4845
+ if (!isRecord2(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
4846
+ return untagMap(value, path, depth);
4847
+ default:
4848
+ return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
4849
+ }
4850
+ }
4851
+ function untag(value, path, depth) {
4852
+ if (typeof value !== "object" || value === null) return value;
4853
+ if (Array.isArray(value)) return untagList(value, path, depth);
4854
+ const record = value;
4855
+ const tag = soleTagKey(record);
4856
+ return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
4857
+ }
4858
+ function untagDocumentLine(text, parsed) {
4859
+ if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
4860
+ return untagMap(parsed, "", 0);
4861
+ }
4862
+
4863
+ // src/engine/export/import.ts
4864
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
4865
+ var DOCUMENT_EXPORT_VERSION = 1;
4866
+ function invalidHeader(message) {
4867
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
4868
+ }
4869
+ function isRecord3(value) {
4870
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4871
+ }
4872
+ function configOfHeader(line) {
4873
+ if (line === void 0) invalidHeader("is missing");
4874
+ if (!("text" in line)) {
4875
+ invalidHeader(
4876
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
4877
+ );
4878
+ }
4879
+ let raw;
4880
+ try {
4881
+ raw = JSON.parse(line.text);
4882
+ } catch {
4883
+ invalidHeader("is not JSON");
4884
+ }
4885
+ if (!isRecord3(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
4886
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
4887
+ }
4888
+ const configuration = requireConfigurationShape(raw.index, "document export header");
4889
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
4890
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
4891
+ return {
4892
+ schema: configuration.schema,
4893
+ language: configuration.language,
4894
+ ...fields,
4895
+ ...embedding === void 0 ? {} : {
4896
+ embedding: {
4897
+ fields: embedding.fields,
4898
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
4899
+ }
4900
+ }
4901
+ };
4902
+ }
4903
+ async function prepareIndex(engine, indexName, config) {
4904
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
4905
+ await engine.createIndex(indexName, config);
4906
+ return;
4907
+ }
4908
+ const existing = flattenSchema(engine.getStats(indexName).schema);
4909
+ const exported = flattenSchema(config.schema);
4910
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
4911
+ for (const field of fields) {
4912
+ if (existing[field] === exported[field]) continue;
4913
+ throw new NarsilError(
4914
+ ErrorCodes.CONFIG_INVALID,
4915
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
4916
+ { indexName, field }
4917
+ );
4918
+ }
4919
+ }
4920
+ function highestReportedAt(refused) {
4921
+ let highest = 0;
4922
+ for (let at = 1; at < refused.length; at++) {
4923
+ if (refused[at].line > refused[highest].line) highest = at;
4924
+ }
4925
+ return highest;
4926
+ }
4927
+ function refuse2(state, line, code, message) {
4928
+ state.result.refusedCount += 1;
4929
+ const refused = state.result.refused;
4930
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
4931
+ refused.push({ line, code, message });
4932
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
4933
+ return;
4934
+ }
4935
+ if (line >= refused[state.highestReported].line) return;
4936
+ refused[state.highestReported] = { line, code, message };
4937
+ state.highestReported = highestReportedAt(refused);
4938
+ }
4939
+ async function flush(state) {
4940
+ if (state.batch.length === 0) return;
4941
+ const batch = state.batch;
4942
+ state.batch = [];
4943
+ state.batchIds = /* @__PURE__ */ new Set();
4944
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
4945
+ const inserts = [];
4946
+ const updates = [];
4947
+ for (const entry of batch) {
4948
+ if (await state.engine.has(state.indexName, entry.docId))
4949
+ updates.push({ docId: entry.docId, document: entry.document });
4950
+ else inserts.push(entry.document);
4951
+ }
4952
+ for (const outcome of [
4953
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
4954
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
4955
+ ]) {
4956
+ if (outcome === null) continue;
4957
+ state.result.imported += outcome.succeeded.length;
4958
+ for (const failure of outcome.failed) {
4959
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
4960
+ refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
4961
+ }
4962
+ }
4963
+ }
4964
+ async function takeLine(state, entry) {
4965
+ if (!("text" in entry)) {
4966
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
4967
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
4968
+ return;
4969
+ }
4970
+ if (entry.text.length === 0) return;
4971
+ let document;
4972
+ try {
4973
+ document = JSON.parse(entry.text);
4974
+ } catch {
4975
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
4976
+ return;
4977
+ }
4978
+ if (!isRecord3(document) || typeof document.id !== "string" || document.id.length === 0) {
4979
+ refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
4980
+ return;
4981
+ }
4982
+ try {
4983
+ untagDocumentLine(entry.text, document);
4984
+ } catch (error) {
4985
+ if (!(error instanceof NarsilError)) throw error;
4986
+ refuse2(state, entry.line, error.code, error.message);
4987
+ return;
4988
+ }
4989
+ if (state.batchIds.has(document.id)) await flush(state);
4990
+ state.batch.push({ line: entry.line, docId: document.id, document });
4991
+ state.batchIds.add(document.id);
4992
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
4993
+ }
4994
+ async function readDocumentExport(engine, indexName, source) {
4995
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
4996
+ const header = await lines.next();
4997
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
4998
+ const state = {
4999
+ engine,
5000
+ indexName,
5001
+ batch: [],
5002
+ batchIds: /* @__PURE__ */ new Set(),
5003
+ result: { imported: 0, refusedCount: 0, refused: [] },
5004
+ highestReported: 0
5005
+ };
5006
+ for await (const entry of lines) {
5007
+ await takeLine(state, entry);
5008
+ if (turnsAreBudgeted && turnIsSpent()) await yieldTurn();
5009
+ }
5010
+ await flush(state);
5011
+ state.result.refused.sort((a, b) => a.line - b.line);
5012
+ return state.result;
5013
+ }
5014
+
5015
+ // src/engine/export/json-line.ts
5016
+ function refuse3(path, kind, docId) {
5017
+ throw new NarsilError(
5018
+ ErrorCodes.DOC_VALIDATION_FAILED,
5019
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
5020
+ { docId, field: path }
5021
+ );
5022
+ }
5023
+ function encodeNumber(value, path, target) {
5024
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
5025
+ if (!target.tagged) refuse3(path, String(value), target.docId);
5026
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
5027
+ }
5028
+ function encodeDate(value, path, target) {
5029
+ const milliseconds = value.getTime();
5030
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse3(path, "a date", target.docId);
5031
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
5032
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
5033
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
5034
+ }
5035
+ function encodeBytes(value, path, target) {
5036
+ if (!target.tagged) refuse3(path, "bytes", target.docId);
5037
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
5038
+ }
5039
+ function encodeList(value, path, target) {
5040
+ target.parts.push("[");
5041
+ for (let at = 0; at < value.length; at++) {
5042
+ if (at > 0) target.parts.push(",");
5043
+ encodeValue(value[at], `${path}[${at}]`, target);
5044
+ }
5045
+ target.parts.push("]");
5046
+ }
5047
+ function encodeMap(value, path, target) {
5048
+ const proto = Object.getPrototypeOf(value);
5049
+ if (proto !== Object.prototype && proto !== null) refuse3(path, "a value other than a plain object", target.docId);
5050
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
5051
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
5052
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
5053
+ for (let at = 0; at < entries.length; at++) {
5054
+ const [key, entry] = entries[at];
5055
+ if (at > 0) target.parts.push(",");
5056
+ target.parts.push(JSON.stringify(key), ":");
5057
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
5058
+ }
5059
+ target.parts.push(wrapped ? "}}" : "}");
5060
+ }
5061
+ function encodeValue(value, path, target) {
5062
+ if (value === null) {
5063
+ target.parts.push("null");
5064
+ return;
5065
+ }
5066
+ switch (typeof value) {
5067
+ case "string":
5068
+ target.parts.push(JSON.stringify(value));
5069
+ return;
5070
+ case "boolean":
5071
+ target.parts.push(value ? "true" : "false");
5072
+ return;
5073
+ case "number":
5074
+ target.parts.push(encodeNumber(value, path, target));
5075
+ return;
5076
+ case "object":
5077
+ break;
5078
+ default:
5079
+ refuse3(path, `a ${typeof value}`, target.docId);
5080
+ }
5081
+ if (Array.isArray(value)) encodeList(value, path, target);
5082
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
5083
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
5084
+ else encodeMap(value, path, target);
5085
+ }
5086
+ function encodeExactJson(value, docId) {
5087
+ const target = { docId, tagged: false, parts: [] };
5088
+ encodeValue(value, "", target);
5089
+ return target.parts.join("");
5090
+ }
5091
+ function encodeDocumentJson(document, docId) {
5092
+ const target = { docId, tagged: true, parts: [] };
5093
+ encodeValue(document, "", target);
5094
+ return target.parts.join("");
5095
+ }
5096
+
5097
+ // src/engine/export/paged-lines.ts
5098
+ async function coldFieldOf(checkpoint, field) {
5099
+ const { directory } = checkpoint;
5100
+ if (directory === void 0 || field.files.length === 0 || field.graphParts.length === 0) return null;
5101
+ if (![...field.files, ...field.graphParts].every((file) => file.headOffset !== void 0)) return null;
5102
+ const fieldType = checkpoint.manifest.schema[field.fieldPath];
5103
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
5104
+ if (dimension === null) return null;
5105
+ const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
5106
+ return openColdVectorField({
5107
+ fieldPath: field.fieldPath,
5108
+ dimension,
5109
+ metric: "cosine",
5110
+ codeBits: null,
5111
+ files: await Promise.all(field.files.map(withPath)),
5112
+ graphParts: await Promise.all(field.graphParts.map(withPath)),
5113
+ resolvePartition: () => void 0
5114
+ });
5115
+ }
5116
+ async function openFileVectorReads(checkpoint, wholeFileReads) {
5117
+ const fields = /* @__PURE__ */ new Map();
5118
+ for (const field of checkpoint.manifest.vectors) {
5119
+ const opened = await coldFieldOf(checkpoint, field);
5120
+ if (opened !== null) fields.set(field.fieldPath, opened);
5121
+ }
5122
+ const fallback = fields.size === checkpoint.manifest.vectors.length ? null : await wholeFileReads();
5123
+ return {
5124
+ async vectorOf(fieldPath, docId) {
5125
+ const field = fields.get(fieldPath);
5126
+ if (field === void 0) return fallback === null ? null : fallback.vectorOf(fieldPath, docId);
5127
+ const number = field.numberOf(docId);
5128
+ return number === void 0 ? null : Array.from(field.vectorOfNumber(number));
5129
+ },
5130
+ close() {
5131
+ for (const field of fields.values()) field.close();
5132
+ fallback?.close();
5133
+ }
5134
+ };
5135
+ }
5136
+ async function* pagedPartitionLines(checkpoint, partition, vectors, openWholeSegment) {
5137
+ const { directory } = checkpoint;
5138
+ if (directory === void 0) return;
5139
+ const scans = [];
5140
+ try {
5141
+ for (const ref of partition.segments) {
5142
+ const paged = await isPagedSegmentFile(directory, ref.key);
5143
+ scans.push(paged ? await openPagedSegmentScan(directory, ref) : await openWholeSegment(ref));
5144
+ }
5145
+ for (let at = 0; at < scans.length; at++) {
5146
+ const scan = scans[at];
5147
+ for (let ordinal = 0; ordinal < scan.count; ordinal++) {
5148
+ const docId = scan.idAt(ordinal);
5149
+ if (!countsAsLatestHolder(scans, at, docId)) continue;
5150
+ const document = { id: docId, ...scan.storedDocumentAt(ordinal) };
5151
+ document.id = docId;
5152
+ for (const field of checkpoint.manifest.vectors) {
5153
+ const vector = await vectors.vectorOf(field.fieldPath, docId);
5154
+ if (vector !== null) setNestedValue(document, field.fieldPath, vector);
5155
+ }
5156
+ yield `${encodeDocumentJson(document, docId)}
5157
+ `;
5158
+ }
5159
+ }
5160
+ } finally {
5161
+ for (const scan of scans) scan.close();
5162
+ }
5163
+ }
5164
+
5165
+ // src/engine/export/write.ts
5166
+ var VECTOR_FILE_ENVELOPE_VERSION2 = 2;
5167
+ var FLOAT32_BYTES = 4;
5168
+ var ENVELOPE_VERSION_OFFSET = 4;
5169
+ async function locateVectors(checkpoint) {
5170
+ const located = /* @__PURE__ */ new Map();
5171
+ for (const field of checkpoint.manifest.vectors) {
5172
+ const byDocId = /* @__PURE__ */ new Map();
5173
+ for (const ref of field.files) {
5174
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
5175
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION2,
5176
+ what: `vector file "${ref.key}"`,
5177
+ allowsCompression: false
5178
+ });
5179
+ const decoded = decodeVectorFilePayload(decode(payload));
5180
+ for (let position = 0; position < decoded.docIds.length; position++) {
5181
+ if (isDead(ref.dead, position)) continue;
5182
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
5183
+ }
5184
+ }
5185
+ located.set(field.fieldPath, byDocId);
5186
+ }
5187
+ return located;
5188
+ }
5189
+ async function readVector(checkpoint, location) {
5190
+ const width = location.dimension * FLOAT32_BYTES;
5191
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
5192
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
5193
+ }
5194
+ async function readSegment(checkpoint, ref) {
5195
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
5196
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
5197
+ what: `segment "${ref.key}"`,
5198
+ allowsCompression: true
5199
+ });
5200
+ return decodeKeywordSegment(payload, false);
5201
+ }
5202
+ var decoder2 = new TextDecoder();
5203
+ function idAt(contents, ordinal) {
5204
+ const ids = contents.arrays.idTable;
5205
+ return decoder2.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
5206
+ }
5207
+ async function winnersOf(checkpoint, partition) {
5208
+ const winners = /* @__PURE__ */ new Map();
5209
+ for (let at = 0; at < partition.segments.length; at++) {
5210
+ const contents = await readSegment(checkpoint, partition.segments[at]);
5211
+ for (const docId of contents.tombstones) winners.delete(docId);
5212
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
5213
+ }
5214
+ return winners;
5215
+ }
5216
+ async function wholeSegmentScan(checkpoint, ref) {
5217
+ const contents = await readSegment(checkpoint, ref);
5218
+ const table = contents.arrays.documentTable;
5219
+ const ids = /* @__PURE__ */ new Set();
5220
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
5221
+ return {
5222
+ count: contents.arrays.documentCount,
5223
+ schema: contents.schema,
5224
+ tombstones: new Set(contents.tombstones),
5225
+ idAt: (ordinal) => idAt(contents, ordinal),
5226
+ holds: (docId) => ids.has(docId),
5227
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
5228
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
5229
+ table.discardedFields
5230
+ ),
5231
+ close: () => void 0
5232
+ };
5233
+ }
5234
+ async function wholeFileVectorReads(checkpoint) {
5235
+ const located = await locateVectors(checkpoint);
5236
+ return {
5237
+ async vectorOf(fieldPath, docId) {
5238
+ const location = located.get(fieldPath)?.get(docId);
5239
+ return location === void 0 ? null : readVector(checkpoint, location);
5240
+ },
5241
+ close: () => located.clear()
5242
+ };
5243
+ }
5244
+ async function* pagedDocumentLines(checkpoint) {
5245
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
5246
+ try {
5247
+ for (const partition of checkpoint.manifest.partitions) {
5248
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
5249
+ }
5250
+ } finally {
5251
+ vectors.close();
5252
+ }
5253
+ }
5254
+ async function anyPartitionIsPaged(checkpoint) {
5255
+ for (const partition of checkpoint.manifest.partitions) {
5256
+ for (const ref of partition.segments) {
5257
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
5258
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
5259
+ }
5260
+ }
5261
+ return false;
5262
+ }
5263
+ async function* documentLines(checkpoint) {
5264
+ if (await anyPartitionIsPaged(checkpoint)) {
5265
+ yield* pagedDocumentLines(checkpoint);
5266
+ return;
5267
+ }
5268
+ const vectors = await locateVectors(checkpoint);
5269
+ for (const partition of checkpoint.manifest.partitions) {
5270
+ const winners = await winnersOf(checkpoint, partition);
5271
+ for (let at = 0; at < partition.segments.length; at++) {
5272
+ const contents = await readSegment(checkpoint, partition.segments[at]);
5273
+ const table = contents.arrays.documentTable;
5274
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
5275
+ const docId = idAt(contents, ordinal);
5276
+ if (winners.get(docId) !== at) continue;
5277
+ const stored = decodeStoredDocument(
5278
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
5279
+ table.discardedFields
5280
+ );
5281
+ const document = { id: docId, ...stored };
5282
+ document.id = docId;
5283
+ for (const [fieldPath, located] of vectors) {
5284
+ const location = located.get(docId);
5285
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
5286
+ }
5287
+ yield `${encodeDocumentJson(document, docId)}
5288
+ `;
5289
+ }
5290
+ }
5291
+ }
5292
+ }
5293
+ async function* exportLines2(header, checkpoint) {
5294
+ yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
5295
+ `;
5296
+ yield* documentLines(checkpoint);
5297
+ }
5298
+ function documentExportStream(header, checkpoint) {
5299
+ const lines = exportLines2(header, checkpoint);
5300
+ const encoder = new TextEncoder();
5301
+ let released = false;
5302
+ const release = () => {
5303
+ if (released) return;
5304
+ released = true;
5305
+ checkpoint.release();
5306
+ };
5307
+ return new ReadableStream({
5308
+ async pull(controller) {
5309
+ try {
5310
+ let text = "";
5311
+ while (text.length < EXPORT_CHUNK_BYTES) {
5312
+ const step = await lines.next();
5313
+ if (step.done) {
5314
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
5315
+ release();
5316
+ controller.close();
5317
+ return;
5318
+ }
5319
+ text += step.value;
5320
+ }
5321
+ controller.enqueue(encoder.encode(text));
5322
+ } catch (error) {
5323
+ release();
5324
+ controller.error(error);
5325
+ }
5326
+ },
5327
+ async cancel() {
5328
+ release();
5329
+ await lines.return(void 0);
5330
+ }
5331
+ });
5332
+ }
5333
+
5334
+ // src/query-suggestions/archive-history.ts
5335
+ function inDayOrder(left, right) {
5336
+ if (left.day !== right.day) return left.day < right.day ? -1 : 1;
5337
+ return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
5338
+ }
5339
+ async function* historyArchiveEntries(core, indexName) {
5340
+ const runtime = attachedRuntimeOf(core);
5341
+ const files = runtime?.host.files ?? historyFilesOf(core.durability?.manager.historyStorage?.() ?? null);
5342
+ await runtime?.saveDay(indexName);
5343
+ const names = [];
5344
+ for (const key of await listHistoryFolder(files, indexName)) {
5345
+ const name = historyFileOf(indexName, key);
5346
+ if (name !== null) names.push(name);
5347
+ }
5348
+ names.sort(inDayOrder);
5349
+ let position = 0;
5350
+ for (const name of names) {
5351
+ const bytes = await files.read(name.key);
5352
+ if (bytes === null || bytes === "too-long") continue;
5353
+ yield { name: `${QUERY_HISTORY_ARCHIVE_PREFIX}${position}`, bytes };
5354
+ position += 1;
5355
+ }
5356
+ }
5357
+
5358
+ // src/persistence/archive/validate.ts
5359
+ function requireDimension(key, found, expected) {
5360
+ if (expected === found) return;
5361
+ throw new NarsilError(
5362
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5363
+ `The vector file "${key}" holds vectors of ${found} dimensions where the schema gives ${String(expected)}`,
5364
+ { key, found, expected }
5365
+ );
5366
+ }
5367
+ function requireSchemaOfManifest(key, schema, manifest) {
5368
+ for (const [field, type] of Object.entries(schema)) {
5369
+ const listed = manifest.schema[field];
5370
+ if (listed === void 0 || listed === type) continue;
5371
+ throw new NarsilError(
5372
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5373
+ `The segment "${key}" gives field "${field}" the type "${type}", while the manifest gives it "${listed}"`,
5374
+ { key, field }
5375
+ );
5376
+ }
5377
+ }
5378
+ async function validateStagedCheckpoint(store, manifest) {
5379
+ for (const partition of manifest.partitions) {
5380
+ for (const ref of partition.segments) {
5381
+ const { directory } = store;
5382
+ if (directory !== void 0) {
5383
+ if (await isPagedSegmentFile(directory, ref.key)) {
5384
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
5385
+ continue;
5386
+ }
5387
+ }
5388
+ const contents = await readSegmentFile(store, ref);
5389
+ requireSchemaOfManifest(ref.key, contents.schema, manifest);
5390
+ decodeEveryStoredDocument(contents.arrays.documentTable);
5391
+ }
5392
+ }
5393
+ for (const field of manifest.vectors) {
5394
+ const fieldType = manifest.schema[field.fieldPath];
5395
+ const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
5396
+ for (const file of field.files) {
5397
+ const { payload } = await readVectorFile(store, file, field.fieldPath);
5398
+ requireDimension(file.key, payload.dimension, dimension);
5399
+ }
5400
+ await readGraphParts(store, field);
5401
+ }
5402
+ }
5403
+
5404
+ // src/engine/snapshot/restore-setup.ts
5405
+ function prepareRestoredIndex(configuration, deps, reject) {
5406
+ const configFields = restoredConfigFields(configuration, reject);
5407
+ const embedding = restoredEmbedding(configuration.embedding, reject);
5408
+ const languageName = configuration.language ?? "";
5409
+ const language = getLanguage(languageName);
5410
+ const schema = validateSchema(configuration.schema);
5411
+ validatePatternValueLimit(configFields.patternValueLimit);
5412
+ requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
5413
+ validateVectorPromotion(configFields.vectorPromotion);
5414
+ validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
5415
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
5416
+ if (configFields.required !== void 0 && configFields.required.length > 0) {
5417
+ validateRequiredFieldsInSchema(configFields.required, schema);
5418
+ }
5419
+ const config = {
5420
+ schema,
5421
+ language: languageName,
5422
+ ...configFields,
5423
+ ...embedding !== void 0 ? {
5424
+ embedding: {
5425
+ fields: embedding.fields,
5426
+ ...embedding.adapter !== void 0 ? { adapter: embedding.adapter } : {}
5427
+ }
5428
+ } : {}
5429
+ };
5430
+ resolveIndexAnalysis(config);
5431
+ const adapterName = embedding?.adapter ?? null;
5432
+ let embeddingAdapter = null;
5433
+ if (embedding !== void 0) {
5434
+ const candidate = adapterName !== null ? deps.embeddingAdapters.get(adapterName) ?? null : deps.defaultEmbeddingAdapter;
5435
+ if (candidate) {
5436
+ validateEmbeddingConfig({ fields: embedding.fields, adapter: candidate }, schema, void 0);
5437
+ embeddingAdapter = candidate;
5438
+ }
5439
+ }
5440
+ const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
5441
+ const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
5442
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
5443
+ return {
5444
+ config,
5445
+ language,
5446
+ embeddingAdapter,
5447
+ adapterName,
5448
+ storedRevision,
5449
+ analysisGeneration,
5450
+ storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
5451
+ synonyms,
5452
+ querySuggestions: storedStateOf(configuration)
5453
+ };
5454
+ }
5455
+ function restoredSynonyms(raw, config, language, reject) {
5456
+ if (raw === void 0 || raw === null) return null;
5457
+ return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
5458
+ }
5459
+ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
5460
+ const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema);
5461
+ if (loadingSchemaCode !== null) await loadingSchemaCode;
5462
+ const detached = createDetachedIndex(
5463
+ indexName,
5464
+ prepared.config,
5465
+ prepared.language,
5466
+ prepared.synonyms?.lookup ?? null,
5467
+ deps.executor
5468
+ );
5469
+ while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
5470
+ return detached;
5471
+ }
5472
+ function restoredRegistryEntry(prepared, manager, deps) {
5473
+ return {
5474
+ config: prepared.config,
5475
+ language: prepared.language,
5476
+ embeddingAdapter: prepared.embeddingAdapter,
5477
+ embeddingAdapterName: prepared.adapterName,
5478
+ vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
5479
+ indexUuid: null,
5480
+ heldPartitions: null,
5481
+ documentCount: manager.countDocuments(),
5482
+ partitionCount: manager.partitionCount,
5483
+ analysisRevision: prepared.storedRevision,
5484
+ arrayPositionGap: prepared.storedArrayPositionGap,
5485
+ synonyms: prepared.synonyms?.encoded ?? null,
5486
+ analysisGeneration: prepared.analysisGeneration,
5487
+ pendingAnalysis: null,
5488
+ durableAnalysis: null,
5489
+ sync: null,
5490
+ querySuggestions: prepared.querySuggestions
5491
+ };
5492
+ }
5493
+
5494
+ // src/engine/snapshot/replace.ts
5495
+ function releaseTimedOut(indexName, timeoutMs) {
5496
+ return new NarsilError(
5497
+ ErrorCodes.WORKER_TIMEOUT,
5498
+ `The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
5499
+ { indexName, timeoutMs }
5500
+ );
5501
+ }
5502
+ async function withinLimit(indexName, release, work, deps) {
5503
+ release.settled = work.catch(() => {
5504
+ release.reloads = true;
5505
+ });
5506
+ let timer;
5507
+ const limit = new Promise((_, reject) => {
5508
+ timer = setTimeout(() => {
5509
+ release.timedOut = true;
5510
+ reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
5511
+ }, deps.releaseTimeoutMs);
5512
+ });
5513
+ try {
5514
+ await Promise.race([work, limit]);
5515
+ } finally {
5516
+ clearTimeout(timer);
5517
+ }
5518
+ }
5519
+ function vectorIndexesOf2(indexName, deps) {
5520
+ const manager = deps.executor.getManager(indexName);
5521
+ return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
5522
+ }
5523
+ async function holdVectorCopies(vectorIndexes, release) {
5524
+ for (const vectorIndex of vectorIndexes) {
5525
+ if (release.held.includes(vectorIndex)) continue;
5526
+ release.held.push(vectorIndex);
5527
+ await vectorIndex.holdWorkerCopies();
5528
+ }
5529
+ }
5530
+ async function releaseCopies(indexName, release, deps) {
5531
+ release.reloads = await deps.copies.releaseIndexCopies(indexName);
5532
+ await holdVectorCopies(vectorIndexesOf2(indexName, deps), release);
5533
+ }
5534
+ async function endRelease(indexName, release, deps) {
5535
+ await release.settled;
5536
+ for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
5537
+ await deps.copies.endCopyRelease(indexName, release.reloads);
5538
+ }
5539
+ function persistedCheckpoint(indexName, partitionCount, deps) {
5540
+ const durable = deps.durability?.manager;
5541
+ if (durable === void 0) return [];
5542
+ return Array.from({ length: partitionCount }, (_, partitionId) => ({
5543
+ partitionId,
5544
+ lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
5545
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
5546
+ }));
5547
+ }
5548
+ async function commitToStorage(indexName, restored, previousPartitions, deps) {
5549
+ const durable = deps.durability?.manager;
5550
+ const staging = restored.staging;
5551
+ if (durable === void 0 || staging === null) return;
5552
+ const previous = persistedCheckpoint(indexName, previousPartitions, deps);
5553
+ if (previousPartitions > 0) await durable.unloadIndex(indexName);
5554
+ const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
5555
+ try {
5556
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
5557
+ await staging.commit();
5558
+ } catch (error) {
5559
+ if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
5560
+ throw error;
5561
+ }
5562
+ await staging.finish();
5563
+ const relocate = await staging.relocation().catch(() => null);
5564
+ if (relocate === null) return;
5565
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
5566
+ const { manager } = restored.detached;
5567
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
5568
+ const partition = manager.getPartition(partitionId);
5569
+ if (!isCompositePartition(partition)) continue;
5570
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
5571
+ }
5572
+ }
5573
+ function activate(indexName, restored, entry, deps) {
5574
+ deps.executor.installDetachedIndex(restored.detached);
5575
+ deps.indexRegistry.set(indexName, entry);
5576
+ restored.detached.attach();
5577
+ deps.clearAnalysisStale(indexName);
5578
+ const { prepared } = restored;
5579
+ if (prepared.storedRevision !== prepared.language.revision || arrayPositionsAreStale(prepared.config, prepared.storedArrayPositionGap)) {
5580
+ deps.markAnalysisStale({
5581
+ indexName,
5582
+ language: prepared.language.name,
5583
+ storedRevision: prepared.storedRevision,
5584
+ currentRevision: prepared.language.revision
5585
+ });
5586
+ }
5587
+ if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
5588
+ deps.forgetWatermark(indexName);
5589
+ }
5590
+ async function replaceExisting(indexName, restored, entry, deps) {
5591
+ const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
5592
+ try {
5593
+ await deps.abandonAnalysisChange(indexName);
5594
+ await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
5595
+ await deps.indexState.replace(indexName, async (wasOpen) => {
5596
+ await deps.abandonAnalysisRebuild(indexName);
5597
+ const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
5598
+ const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
5599
+ await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
5600
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
5601
+ await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
5602
+ if (previous === void 0) activate(indexName, restored, entry, deps);
5603
+ else await previous.publication.write(() => activate(indexName, restored, entry, deps));
5604
+ await deps.querySuggestions?.afterActivate(entry);
5605
+ });
5606
+ } finally {
5607
+ const ending = endRelease(indexName, release, deps).catch(() => void 0);
5608
+ if (!release.timedOut) await ending;
5609
+ }
5610
+ }
5611
+ async function createFresh(indexName, restored, entry, deps) {
5612
+ deps.copies.desyncIndex(indexName);
5613
+ deps.executor.installDetachedIndex(restored.detached);
5614
+ deps.indexRegistry.set(indexName, entry);
5615
+ try {
5616
+ await deps.indexState.registerReplacing(indexName, async () => {
5617
+ await commitToStorage(indexName, restored, 0, deps);
5618
+ activate(indexName, restored, entry, deps);
5619
+ await deps.querySuggestions?.afterActivate(entry);
5620
+ });
5621
+ } catch (error) {
5622
+ if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
5623
+ throw error;
5624
+ } finally {
5625
+ await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
5626
+ }
5627
+ }
5628
+ async function stageMetadata(indexName, staging, entry) {
5629
+ const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
5630
+ await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
5631
+ }
5632
+ async function installRestoredIndex(indexName, restored, deps, history = null) {
5633
+ const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
5634
+ const previousSync = deps.indexRegistry.get(indexName)?.sync ?? null;
5635
+ if (previousSync !== null) entry.sync = { ...previousSync, active: false, epoch: newSyncEpoch() };
5636
+ let replaced;
5637
+ try {
5638
+ if (deps.indexRegistry.has(indexName)) await deps.abandonAnalysisChange(indexName);
5639
+ await deps.querySuggestions?.beforeInstall(entry, deps.indexRegistry.get(indexName));
5640
+ if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
5641
+ replaced = deps.indexRegistry.get(indexName);
5642
+ if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
5643
+ else await createFresh(indexName, restored, entry, deps);
5644
+ } catch (error) {
5645
+ if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
5646
+ else deps.executor.discardDetachedIndex(restored.detached);
5647
+ throw error;
5648
+ }
5649
+ await restored.staging?.removeRetired().catch(() => void 0);
5650
+ const durable = deps.durability?.manager;
5651
+ if (restored.staging === null && durable !== void 0) {
5652
+ await durable.persistMetadata(indexName);
5653
+ await durable.checkpoint(indexName);
5654
+ }
5655
+ if (previousSync !== null) {
5656
+ const storage = durable?.syncStorage?.();
5657
+ if (storage !== void 0)
5658
+ await oneSyncChangeAtATime(deps.indexRegistry, indexName, () => storage.release(indexName));
5659
+ emitSyncChange(deps.indexRegistry, { kind: "epoch", indexName });
5660
+ }
5661
+ await deps.querySuggestions?.afterInstall(entry, history);
5662
+ if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
5663
+ await deps.announceIndexCreated(indexName, restored.prepared.config);
5664
+ for (const vectorIndex of restored.detached.manager.getVectorIndexes().values()) {
5665
+ vectorIndex.scheduleBuild();
5666
+ vectorIndex.startShuffledRebuildWhenDue();
5667
+ }
5668
+ }
5669
+
5670
+ // src/engine/snapshot/archive-restore.ts
5671
+ function rejectConfiguration(message) {
5672
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
5673
+ }
5674
+ function renamed(manifest, sourceName, indexName) {
5675
+ const rename = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
5676
+ return {
5677
+ ...manifest,
5678
+ partitions: manifest.partitions.map((partition) => ({
5679
+ ...partition,
5680
+ segments: partition.segments.map((segment) => ({ ...segment, key: rename(segment.key) }))
5681
+ })),
5682
+ vectors: manifest.vectors.map((field) => ({
5683
+ ...field,
5684
+ files: field.files.map((file) => ({ ...file, key: rename(file.key) })),
5685
+ graphParts: field.graphParts.map((part) => ({ ...part, key: rename(part.key) }))
5686
+ }))
5687
+ };
5688
+ }
5689
+ function requireSameSchema(prepared, manifest) {
5690
+ const configured = flattenSchema(prepared.config.schema);
5691
+ const keys = /* @__PURE__ */ new Set([...Object.keys(configured), ...Object.keys(manifest.schema)]);
5692
+ for (const field of keys) {
5693
+ if (configured[field] === manifest.schema[field]) continue;
5694
+ throw new NarsilError(
5695
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5696
+ `The archive's configuration gives field "${field}" another type than its manifest does`,
5697
+ { field }
5698
+ );
5699
+ }
5700
+ }
5701
+ function partitionCountOf(manifest) {
5702
+ let count = 1;
5703
+ for (const partition of manifest.partitions) count = Math.max(count, partition.partitionId + 1);
5704
+ return count;
5705
+ }
5706
+ function missingPart(what) {
5707
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
5708
+ }
5709
+ function keepsTextOnDisk(prepared, staging) {
5710
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
5711
+ }
5712
+ async function stageArchive(indexName, reader, staging, deps, staged) {
5713
+ await readSnapshotArchive(reader, {
5714
+ history: async (chunks) => {
5715
+ if (staged.history === void 0) {
5716
+ staged.history = await deps.querySuggestions?.historySink(staged.prepared?.querySuggestions ?? null) ?? null;
5717
+ }
5718
+ await staged.history?.add(chunks);
5719
+ },
5720
+ config: async (bytes) => {
5721
+ staged.prepared = prepareRestoredIndex(await decodeIndexConfiguration(bytes), deps, rejectConfiguration);
5722
+ },
5723
+ manifest: async (manifest2, sourceName) => {
5724
+ if (staged.prepared === null) missingPart("configuration");
5725
+ requireSameSchema(staged.prepared, manifest2);
5726
+ const replacesIndex = deps.indexRegistry.has(indexName);
5727
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
5728
+ requireRestoreMemory(
5729
+ indexName,
5730
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
5731
+ replacesIndex
5732
+ );
5733
+ staged.manifest = renamed(manifest2, sourceName, indexName);
5734
+ },
5735
+ file: async (entryName, _ref, chunks) => {
5736
+ if (staged.manifest === null) missingPart("manifest");
5737
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
5738
+ }
5739
+ });
5740
+ const { prepared, manifest } = staged;
5741
+ if (prepared === null) missingPart("configuration");
5742
+ if (manifest === null) missingPart("manifest");
5743
+ const { directory } = staging;
5744
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
5745
+ const replacesIndex = deps.indexRegistry.has(indexName);
5746
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
5747
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
5748
+ }
5749
+ await validateStagedCheckpoint(staging, manifest);
5750
+ return { prepared, manifest };
5751
+ }
5752
+ async function loadStagedIndex(indexName, staged, staging, store, deps) {
5753
+ const { prepared, manifest } = staged;
5754
+ await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
5755
+ const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
5756
+ try {
5757
+ await loadSegmentedCheckpoint(
5758
+ store,
5759
+ indexName,
5760
+ manifest,
5761
+ { manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
5762
+ true
5763
+ );
5764
+ } catch (error) {
5765
+ deps.executor.discardDetachedIndex(detached);
5766
+ throw error;
5767
+ }
5768
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint };
5769
+ }
5770
+ async function restoreArchive(indexName, reader, deps) {
5771
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
5772
+ const store = staging?.store ?? memoryCheckpointStore();
5773
+ const archive = { prepared: null, manifest: null, history: void 0 };
5774
+ try {
5775
+ const staged = await stageArchive(indexName, reader, store, deps, archive);
5776
+ const restored = await loadStagedIndex(indexName, staged, staging, store, deps);
5777
+ await installRestoredIndex(indexName, restored, deps, archive.history ?? null);
5778
+ } catch (error) {
5779
+ await archive.history?.abandon().catch(() => void 0);
5780
+ throw error;
5781
+ } finally {
5782
+ await staging?.discard().catch(() => void 0);
5783
+ await reader.close();
5784
+ }
5785
+ }
5786
+
5787
+ // src/engine/snapshot/earlier-restore.ts
5788
+ function invalidSnapshot(message, details) {
5789
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, message, details);
5790
+ }
5791
+ async function decodeEarlierSnapshot(data) {
5792
+ const payloadBytes = await unpackIndexSnapshotEnvelope(data);
5793
+ let decoded;
5794
+ try {
5795
+ decoded = decode(payloadBytes);
5796
+ } catch (err) {
5797
+ invalidSnapshot(`Snapshot data is not a Narsil snapshot: ${err instanceof Error ? err.message : String(err)}`, {
5798
+ bytes: data.length
5799
+ });
5800
+ }
5801
+ if (typeof decoded !== "object" || decoded === null || Array.isArray(decoded)) {
5802
+ invalidSnapshot("Snapshot data does not hold a snapshot envelope", { bytes: data.length });
5803
+ }
5804
+ const envelope = decoded;
5805
+ if (envelope.version !== 1 && envelope.version !== 2) {
5806
+ invalidSnapshot(`Unsupported snapshot version: ${envelope.version}. Expected version 1 or 2`, {
5807
+ version: envelope.version
5808
+ });
5809
+ }
5810
+ if (!envelope.schema || typeof envelope.schema !== "object")
5811
+ invalidSnapshot("Invalid snapshot: missing or invalid schema");
5812
+ if (!envelope.language || typeof envelope.language !== "string") {
5813
+ invalidSnapshot("Invalid snapshot: missing or invalid language");
5814
+ }
5815
+ if (!Array.isArray(envelope.partitions)) invalidSnapshot("Invalid snapshot: missing partitions");
5816
+ return { ...envelope, vectorPromotion: raiseEarlierReleaseGraphSetting(envelope.vectorPromotion) };
5817
+ }
5818
+ function vectorsOfSnapshot(envelope) {
5819
+ const byDocument = /* @__PURE__ */ new Map();
5820
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
5821
+ for (const part of decodeVectorIndexParts(rawParts)) {
5822
+ const values = bytesToVectors(part.vectors);
5823
+ for (let i = 0; i < part.docIds.length; i++) {
5824
+ let fields = byDocument.get(part.docIds[i]);
5825
+ if (fields === void 0) {
5826
+ fields = /* @__PURE__ */ new Map();
5827
+ byDocument.set(part.docIds[i], fields);
5828
+ }
5829
+ fields.set(fieldPath, values.slice(i * part.dimension, (i + 1) * part.dimension));
5830
+ }
5831
+ }
5832
+ }
5833
+ return byDocument;
5834
+ }
5835
+ function footprintOf(envelope, prepared, replacesIndex) {
5836
+ let partitionBytes = 0;
5837
+ for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
5838
+ const vectorCounts = /* @__PURE__ */ new Map();
5839
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
5840
+ let count = 0;
5841
+ for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
5842
+ vectorCounts.set(fieldPath, count);
5843
+ }
5844
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
5845
+ }
5846
+ function documentsOfPartition(envelope, payload) {
5847
+ return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
5848
+ }
5849
+ function insertRestoredDocument(manager, docId, fields, vectors) {
5850
+ manager.insert(docId, fields, { skipClone: true });
5851
+ if (vectors === void 0) return;
5852
+ try {
5853
+ insertDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
5854
+ } catch (err) {
5855
+ manager.remove(docId);
5856
+ throw err;
5857
+ }
5858
+ }
5859
+ function insertSnapshotDocuments(envelope, manager) {
5860
+ const vectors = vectorsOfSnapshot(envelope);
5861
+ for (const payload of envelope.partitions ?? []) {
5862
+ for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
5863
+ insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
5864
+ }
5865
+ }
5866
+ for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
5867
+ }
5868
+ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
5869
+ const entry = restoredRegistryEntry(prepared, built.manager, deps);
5870
+ const archive = await buildArchiveFromMemory({
5871
+ indexName,
5872
+ metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
5873
+ manager: built.manager,
5874
+ vectorIndexes: built.manager.getVectorIndexes(),
5875
+ appliedSeqNo: () => 0,
5876
+ partitionId: null
5877
+ });
5878
+ try {
5879
+ for (const ref of manifestFileRefs(archive.manifest))
5880
+ await staging.store.write(ref.key, [await archive.readFile(ref)]);
5881
+ } finally {
5882
+ archive.release();
5883
+ }
5884
+ await validateStagedCheckpoint(staging.store, archive.manifest);
5885
+ return archive.manifest;
5886
+ }
5887
+ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
5888
+ const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
5889
+ let manifest;
5890
+ try {
5891
+ insertSnapshotDocuments(envelope, built.manager);
5892
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
5893
+ manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
5894
+ } catch (err) {
5895
+ deps.executor.discardDetachedIndex(built);
5896
+ throw err;
5897
+ }
5898
+ deps.executor.discardDetachedIndex(built);
5899
+ return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
5900
+ }
5901
+ async function restoreEarlierSnapshot(indexName, data, deps) {
5902
+ const envelope = await decodeEarlierSnapshot(data);
5903
+ const prepared = prepareRestoredIndex(envelope, deps);
5904
+ const replacesIndex = deps.indexRegistry.has(indexName);
5905
+ requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
5906
+ const analysedNow = {
5907
+ ...prepared,
5908
+ storedRevision: prepared.language.revision,
5909
+ storedArrayPositionGap: ARRAY_POSITION_GAP
5910
+ };
5911
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
5912
+ try {
5913
+ await installRestoredIndex(
5914
+ indexName,
5915
+ await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
5916
+ deps
5917
+ );
5918
+ } finally {
5919
+ await staging?.discard().catch(() => void 0);
5920
+ }
5921
+ }
5922
+
5923
+ // src/engine/snapshot/index.ts
5924
+ var NRSL_MAGIC_BYTES = 4;
5925
+ async function checkpointForArchive(deps) {
5926
+ const durable = deps.durability?.manager;
5927
+ if (durable?.openArchiveCheckpoint !== void 0) return durable.openArchiveCheckpoint(deps.indexName);
5928
+ return buildArchiveFromMemory({
5929
+ indexName: deps.indexName,
5930
+ metadata: indexMetadataOf(deps.indexName, deps.entry, deps.manager.partitionCount),
5931
+ manager: deps.manager,
5932
+ vectorIndexes: deps.manager.getVectorIndexes(),
5933
+ appliedSeqNo: (partitionId) => durable?.highestPersistedSeqNo(deps.indexName, partitionId) ?? 0,
5934
+ partitionId: null
5935
+ });
5936
+ }
5937
+ async function* queryHistoryEntries(core, indexName) {
5938
+ yield* historyArchiveEntries(core, indexName);
5939
+ }
5940
+ async function createSnapshot(deps, coreWithAddedHistory) {
5941
+ const trailingEntries = coreWithAddedHistory === null ? void 0 : () => queryHistoryEntries(coreWithAddedHistory, deps.indexName);
5942
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
5943
+ const checkpoint = await checkpointForArchive(deps);
5944
+ try {
5945
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
5946
+ const config = await encodeIndexConfiguration(deps.entry);
5947
+ return archiveStream({
5948
+ ...checkpoint,
5949
+ config,
5950
+ sourceIndexName: deps.indexName,
5951
+ ...trailingEntries !== void 0 ? { trailingEntries } : {}
5952
+ });
5953
+ }
5954
+ } catch (error) {
5955
+ checkpoint.release();
5956
+ throw error;
5957
+ }
5958
+ checkpoint.release();
5959
+ }
5960
+ throw new NarsilError(
5961
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
5962
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
5963
+ { indexName: deps.indexName }
5964
+ );
5965
+ }
5966
+ function requireSnapshotSource(data) {
5967
+ if (data instanceof Uint8Array) return data;
5968
+ if (data !== null && typeof data === "object") {
5969
+ if (typeof data.getReader === "function") return data;
5970
+ if (typeof data[Symbol.asyncIterator] === "function") {
5971
+ return data;
5972
+ }
5973
+ }
5974
+ throw new NarsilError(
5975
+ ErrorCodes.DOC_VALIDATION_FAILED,
5976
+ "Snapshot data must be a Uint8Array, a ReadableStream of bytes, or an async iterable of bytes"
5977
+ );
5978
+ }
5979
+ async function readEarlierSnapshotFile(reader) {
5980
+ const length = HEADER_SIZE + readHeader(await reader.peek(HEADER_SIZE)).payloadLength;
5981
+ if (length > MAX_FILE_BYTES) {
5982
+ throw new NarsilError(
5983
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5984
+ `The snapshot declares ${length} bytes, above the file size limit of ${MAX_FILE_BYTES}`,
5985
+ { bytes: length }
5986
+ );
5987
+ }
5988
+ const bytes = await reader.readExactly(length) ?? new Uint8Array(0);
5989
+ if (bytes.byteLength < length) {
5990
+ throw new NarsilError(
5991
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5992
+ "The snapshot ends before the payload its header declares"
5993
+ );
5994
+ }
5995
+ const trailing = await reader.readExactly(1);
5996
+ if (trailing !== null && trailing.byteLength > 0) {
5997
+ throw new NarsilError(
5998
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
5999
+ "The snapshot holds bytes after the payload its header declares"
6000
+ );
6001
+ }
6002
+ return bytes;
6003
+ }
6004
+ async function restoreFromSnapshot(indexName, data, deps) {
6005
+ const reader = createByteReader(requireSnapshotSource(data));
6006
+ const head = await reader.peek(NRSL_MAGIC_BYTES);
6007
+ if (head.byteLength >= NRSL_MAGIC_BYTES && hasNrslMagic(head)) {
6008
+ let bytes;
6009
+ try {
6010
+ bytes = await readEarlierSnapshotFile(reader);
6011
+ } finally {
6012
+ await reader.close();
6013
+ }
6014
+ await restoreEarlierSnapshot(indexName, bytes, deps);
6015
+ return;
6016
+ }
6017
+ if (head.byteLength === 0) {
6018
+ await reader.close();
6019
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Snapshot data is empty");
6020
+ }
6021
+ await restoreArchive(indexName, reader, deps);
6022
+ }
6023
+
6024
+ // src/engine/sync/status.ts
6025
+ function syncStatusOf(sync) {
6026
+ if (sync === null) return { enabled: false, active: false, epoch: null };
6027
+ return {
6028
+ enabled: true,
6029
+ ...sync.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: sync.historyMaxAgeMs } : {},
6030
+ ...sync.historyMaxBytes !== void 0 ? { historyMaxBytes: sync.historyMaxBytes } : {},
6031
+ active: sync.active,
6032
+ epoch: sync.epoch
6033
+ };
6034
+ }
6035
+
6036
+ // src/engine/sync/methods.ts
6037
+ async function updateSyncOf(core, indexName, setting) {
6038
+ core.guardShutdown();
6039
+ core.requireIndex(indexName);
6040
+ await updateIndexSync(core, indexName, setting);
6041
+ }
6042
+ function syncOf(core, indexName) {
6043
+ core.guardShutdown();
6044
+ return syncStatusOf(core.requireIndex(indexName).sync);
6045
+ }
6046
+
6047
+ // src/persistence/paged/rewrite-spill.ts
6048
+ function openRewriteSpill(storage, manager, target) {
6049
+ const config = target?.config ?? manager.config;
6050
+ return createRewriteSpill(storage, manager.indexName, {
6051
+ schema: flattenSchema(config.schema),
6052
+ trackPositions: config.trackPositions ?? true,
6053
+ partialWordAnalysis: target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis,
6054
+ textAnalysis: {
6055
+ language: target?.language ?? manager.language,
6056
+ options: resolvePartitionInsertOptions(config, target?.analysis ?? manager.analysis)
6057
+ }
6058
+ });
6059
+ }
6060
+ function createRewriteSpill(storage, indexName, analysis) {
6061
+ const { directory, writeRecords } = storage;
6062
+ const prefix = `${indexName}/rebuilt/${generateId()}/`;
6063
+ const opened = [];
6064
+ let nextFile = 0;
6065
+ async function removeFiles() {
6066
+ for (const key of await directory.list(prefix)) await directory.remove(key);
6067
+ }
6068
+ function freezeLive(partition) {
6069
+ if (!isCompositePartition(partition)) return null;
6070
+ return partition.freezeLiveTail((payload, documents) => {
6071
+ const snapshot = freezeSegmentShared(payload, documents, void 0, analysis.partialWordAnalysis);
6072
+ return snapshot === null ? createFrozenSegment(payload, documents) : createSharedFrozenSegment(snapshot);
6073
+ });
6074
+ }
6075
+ return {
6076
+ newPartition(partitionId) {
6077
+ const live = createPartitionIndex(partitionId, analysis.trackPositions);
6078
+ return createCompositePartition(partitionId, analysis.trackPositions, live, analysis.textAnalysis);
6079
+ },
6080
+ async spillIfFull(partition, partitionId) {
6081
+ if (!isCompositePartition(partition)) return;
6082
+ if (partition.live.estimateMemoryBytes() < REWRITE_SPILL_BYTES) return;
6083
+ const frozen = freezeLive(partition);
6084
+ if (frozen === null) return;
6085
+ const key = `${prefix}${nextFile}`;
6086
+ nextFile += 1;
6087
+ const written = await writePagedSegment({
6088
+ directory,
6089
+ key,
6090
+ schema: analysis.schema,
6091
+ tombstones: [],
6092
+ selections: [{ segment: frozen, keep: (ordinal) => !frozen.isTombstoned(ordinal) }],
6093
+ partialWordAnalysis: analysis.partialWordAnalysis,
6094
+ textAnalysis: analysis.textAnalysis,
6095
+ ...writeRecords === null ? {} : { writeRecords: writeRecords.captureSegments(indexName, partitionId, [frozen]) }
6096
+ });
6097
+ const listing = await pagedListingOf(directory, { key, bytes: written.bytes, checksum: written.checksum });
6098
+ const segment = createPagedFrozenSegment(openPagedSnapshot(listing));
6099
+ opened.push(segment);
6100
+ partition.swapFrozenSegments([frozen.segmentId], segment);
6101
+ },
6102
+ async abandon() {
6103
+ for (const segment of opened.splice(0)) segment.paged?.close();
6104
+ await removeFiles();
6105
+ },
6106
+ async removeUnread(manager) {
6107
+ for (const partition of manager.getAllPartitions()) {
6108
+ if (!isCompositePartition(partition)) continue;
6109
+ for (const segment of partition.frozenSegments()) {
6110
+ if (segment.paged?.key.startsWith(prefix)) return;
6111
+ }
6112
+ }
6113
+ for (const segment of opened.splice(0)) segment.paged?.close();
6114
+ await removeFiles();
6115
+ }
6116
+ };
6117
+ }
6118
+
6119
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, actionPartitionsOf, adoptSegmentsOnWorker, checkpointForArchive, closeUnusedSegments, createDurabilityFromTier, createInvalidationFromConfig, createSnapshot, documentExportStream, keysThatSegmentsStillRead, openFileVectorReads, openRewriteSpill, pagedPartitionLines, prepareDiskAdoption, readDocumentExport, reloadIndexAndCopies, restoreFromSnapshot, swapPreparedPartitions, syncOf, updateSyncOf };