@delali/narsil 0.3.1-next.98 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +45 -0
  2. package/README.md +27 -8
  3. package/dist/adapters/filesystem.d.ts +1 -1
  4. package/dist/adapters/filesystem.mjs +101 -13
  5. package/dist/adapters/gcs.d.ts +49 -0
  6. package/dist/adapters/gcs.mjs +213 -0
  7. package/dist/adapters/indexeddb.d.ts +1 -1
  8. package/dist/adapters/indexeddb.mjs +2 -7
  9. package/dist/adapters/memory.d.ts +1 -1
  10. package/dist/adapters/memory.mjs +5 -0
  11. package/dist/adapters/s3.d.ts +53 -0
  12. package/dist/adapters/s3.mjs +263 -0
  13. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  14. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  15. package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
  16. package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
  17. package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
  18. package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
  19. package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
  20. package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
  21. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  22. package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
  23. package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
  24. package/dist/browser/embed-GG6PS3EA.mjs +264 -0
  25. package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
  26. package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
  27. package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
  28. package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
  29. package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
  30. package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
  31. package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
  32. package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
  33. package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
  34. package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
  35. package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
  36. package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
  37. package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
  38. package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
  39. package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
  40. package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
  41. package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
  42. package/dist/chunk-2AFZE7Y3.mjs +72 -0
  43. package/dist/chunk-2CKDCC65.mjs +47 -0
  44. package/dist/chunk-3ECIRFS5.mjs +61 -0
  45. package/dist/chunk-44W3CBNG.mjs +7 -0
  46. package/dist/chunk-4H4RG2FR.mjs +13 -0
  47. package/dist/chunk-4PY4JYMY.mjs +11 -0
  48. package/dist/chunk-4ZPVAHWF.mjs +1289 -0
  49. package/dist/chunk-5EI5CXO6.mjs +197 -0
  50. package/dist/chunk-5F5MXWS5.mjs +283 -0
  51. package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
  52. package/dist/chunk-6FOENO4E.mjs +273 -0
  53. package/dist/chunk-6GHX35ZM.mjs +108 -0
  54. package/dist/chunk-6MSZ543Y.mjs +32 -0
  55. package/dist/chunk-6N6HHBCQ.mjs +4596 -0
  56. package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
  57. package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
  58. package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
  59. package/dist/chunk-AVBPB3MI.mjs +602 -0
  60. package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
  61. package/dist/chunk-BUMREPLU.mjs +15 -0
  62. package/dist/chunk-BY6NALRK.mjs +31921 -0
  63. package/dist/chunk-C4ZMXXTL.mjs +92 -0
  64. package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
  65. package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
  66. package/dist/chunk-DXZ6JDWG.mjs +797 -0
  67. package/dist/chunk-FJXJZUWF.mjs +209 -0
  68. package/dist/chunk-FS3HPPSC.mjs +212 -0
  69. package/dist/chunk-FT4KLZXW.mjs +1164 -0
  70. package/dist/chunk-GR5GMFJJ.mjs +104 -0
  71. package/dist/chunk-GXCDWLQI.mjs +193 -0
  72. package/dist/chunk-GXED6R3Q.mjs +271 -0
  73. package/dist/chunk-HLPRHOSO.mjs +17 -0
  74. package/dist/chunk-IL5DRCNU.mjs +3041 -0
  75. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  76. package/dist/chunk-K5HSUPYS.mjs +11 -0
  77. package/dist/chunk-KYIRXTCA.mjs +100 -0
  78. package/dist/chunk-LFXS6OF7.mjs +289 -0
  79. package/dist/chunk-LNGE7SLY.mjs +151 -0
  80. package/dist/chunk-M2Z5WVWW.mjs +194 -0
  81. package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
  82. package/dist/chunk-M3OW2XSP.mjs +40 -0
  83. package/dist/chunk-MFPSQZ6P.mjs +127 -0
  84. package/dist/chunk-MRCJBWJZ.mjs +1292 -0
  85. package/dist/chunk-MSIN4NM3.mjs +25 -0
  86. package/dist/chunk-MW3CJVGP.mjs +219 -0
  87. package/dist/chunk-MYM6CAVK.mjs +76 -0
  88. package/dist/chunk-NJJN4FMM.mjs +84 -0
  89. package/dist/chunk-NM7I3BM7.mjs +59 -0
  90. package/dist/chunk-OIJ64LGT.mjs +19 -0
  91. package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
  92. package/dist/chunk-Q2AP2FSN.mjs +15 -0
  93. package/dist/chunk-R2PJKSNL.mjs +54 -0
  94. package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
  95. package/dist/chunk-S2POFTLN.mjs +10 -0
  96. package/dist/chunk-S5QMWECL.mjs +7 -0
  97. package/dist/chunk-SLAC5PAB.mjs +5008 -0
  98. package/dist/chunk-T4HMSZCX.mjs +5 -0
  99. package/dist/chunk-TACP6VK6.mjs +111 -0
  100. package/dist/chunk-TGPKS2UM.mjs +125 -0
  101. package/dist/chunk-TKKXPKEO.mjs +979 -0
  102. package/dist/chunk-V6EUBAIT.mjs +19 -0
  103. package/dist/chunk-XL56L4IR.mjs +228 -0
  104. package/dist/chunk-XQIAMDAG.mjs +3076 -0
  105. package/dist/chunk-YTUZ4R22.mjs +1063 -0
  106. package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
  107. package/dist/chunk-ZPJFGGGI.mjs +638 -0
  108. package/dist/chunk-ZXX5GZEX.mjs +17 -0
  109. package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
  110. package/dist/client.d.ts +11 -10
  111. package/dist/client.mjs +8 -898
  112. package/dist/config-Bnl7L-0H.d.ts +608 -0
  113. package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
  114. package/dist/core-leaf-6JVHEVFB.mjs +8 -0
  115. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  116. package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
  117. package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
  118. package/dist/distribution/coordinator/etcd.d.ts +63 -7
  119. package/dist/distribution/coordinator/etcd.mjs +725 -81
  120. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  121. package/dist/distribution/coordinator/in-memory.mjs +65 -10
  122. package/dist/distribution/transport/grpc.d.ts +2 -2
  123. package/dist/distribution/transport/grpc.mjs +39 -10
  124. package/dist/distribution/transport/in-memory.d.ts +2 -2
  125. package/dist/distribution/transport/in-memory.mjs +9 -6
  126. package/dist/distribution/transport/tcp.d.ts +2 -2
  127. package/dist/distribution/transport/tcp.mjs +20 -15
  128. package/dist/distribution.d.ts +267 -56
  129. package/dist/distribution.mjs +23156 -11362
  130. package/dist/durability-config-VWS2K24E.mjs +96 -0
  131. package/dist/embed-4PHHKGBK.mjs +263 -0
  132. package/dist/embeddings/openai.d.ts +1 -1
  133. package/dist/embeddings/openai.mjs +2 -2
  134. package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
  135. package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
  136. package/dist/gzip-2UCZW2MM.mjs +31 -0
  137. package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
  138. package/dist/index.browser.mjs +4 -4
  139. package/dist/index.d.ts +2133 -57
  140. package/dist/index.mjs +16 -32
  141. package/dist/index.react-native.mjs +12 -0
  142. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  143. package/dist/invalidation/filesystem.d.ts +1 -1
  144. package/dist/invalidation/filesystem.mjs +1 -2
  145. package/dist/invalidation/noop.d.ts +1 -1
  146. package/dist/languages/albanian.mjs +1 -1
  147. package/dist/languages/amharic.mjs +1 -1
  148. package/dist/languages/arabic.mjs +1 -1
  149. package/dist/languages/armenian.mjs +1 -1
  150. package/dist/languages/azerbaijani.mjs +1 -1
  151. package/dist/languages/bambara.mjs +1 -1
  152. package/dist/languages/basque.mjs +1 -1
  153. package/dist/languages/belarusian.mjs +1 -1
  154. package/dist/languages/bengali.mjs +1 -1
  155. package/dist/languages/bosnian.mjs +1 -1
  156. package/dist/languages/breton.mjs +1 -1
  157. package/dist/languages/bulgarian.mjs +1 -1
  158. package/dist/languages/burmese.mjs +1 -1
  159. package/dist/languages/catalan.mjs +1 -1
  160. package/dist/languages/chinese.mjs +1 -1
  161. package/dist/languages/croatian.mjs +1 -1
  162. package/dist/languages/czech.mjs +1 -1
  163. package/dist/languages/dagbani.mjs +1 -1
  164. package/dist/languages/danish.mjs +1 -1
  165. package/dist/languages/dutch.mjs +1 -1
  166. package/dist/languages/english.mjs +1 -1
  167. package/dist/languages/esperanto.mjs +1 -1
  168. package/dist/languages/estonian.mjs +1 -1
  169. package/dist/languages/ewe.mjs +1 -1
  170. package/dist/languages/faroese.mjs +1 -1
  171. package/dist/languages/fijian.mjs +1 -1
  172. package/dist/languages/finnish.mjs +1 -1
  173. package/dist/languages/french.mjs +1 -1
  174. package/dist/languages/ga.mjs +1 -1
  175. package/dist/languages/galician.mjs +1 -1
  176. package/dist/languages/georgian.mjs +1 -1
  177. package/dist/languages/german.mjs +1 -1
  178. package/dist/languages/greek.mjs +1 -1
  179. package/dist/languages/guarani.mjs +1 -1
  180. package/dist/languages/gujarati.mjs +1 -1
  181. package/dist/languages/haitian-creole.mjs +1 -1
  182. package/dist/languages/hausa.mjs +1 -1
  183. package/dist/languages/hawaiian.mjs +1 -1
  184. package/dist/languages/hebrew.mjs +1 -1
  185. package/dist/languages/hindi.mjs +1 -1
  186. package/dist/languages/hungarian.mjs +1 -1
  187. package/dist/languages/icelandic.mjs +1 -1
  188. package/dist/languages/igbo.mjs +1 -1
  189. package/dist/languages/indonesian.mjs +1 -1
  190. package/dist/languages/irish.mjs +1 -1
  191. package/dist/languages/italian.mjs +1 -1
  192. package/dist/languages/japanese.mjs +1 -1
  193. package/dist/languages/kannada.mjs +1 -1
  194. package/dist/languages/kazakh.mjs +1 -1
  195. package/dist/languages/khmer.mjs +1 -1
  196. package/dist/languages/kinyarwanda.mjs +1 -1
  197. package/dist/languages/kirundi.mjs +1 -1
  198. package/dist/languages/korean.mjs +1 -1
  199. package/dist/languages/kurmanji.mjs +1 -1
  200. package/dist/languages/kyrgyz.mjs +1 -1
  201. package/dist/languages/lao.mjs +1 -1
  202. package/dist/languages/latin.mjs +1 -1
  203. package/dist/languages/latvian.mjs +1 -1
  204. package/dist/languages/lingala.mjs +1 -1
  205. package/dist/languages/lithuanian.mjs +1 -1
  206. package/dist/languages/luxembourgish.mjs +1 -1
  207. package/dist/languages/macedonian.mjs +1 -1
  208. package/dist/languages/malagasy.mjs +1 -1
  209. package/dist/languages/malay.mjs +1 -1
  210. package/dist/languages/malayalam.mjs +1 -1
  211. package/dist/languages/maltese.mjs +1 -1
  212. package/dist/languages/maori.mjs +1 -1
  213. package/dist/languages/marathi.mjs +1 -1
  214. package/dist/languages/nepali.mjs +1 -1
  215. package/dist/languages/norwegian.mjs +1 -1
  216. package/dist/languages/oromo.mjs +1 -1
  217. package/dist/languages/persian.mjs +1 -1
  218. package/dist/languages/polish.mjs +1 -1
  219. package/dist/languages/portuguese.mjs +1 -1
  220. package/dist/languages/punjabi.mjs +1 -1
  221. package/dist/languages/romanian.mjs +1 -1
  222. package/dist/languages/russian.mjs +1 -1
  223. package/dist/languages/samoan.mjs +1 -1
  224. package/dist/languages/sanskrit.mjs +1 -1
  225. package/dist/languages/scottish-gaelic.mjs +1 -1
  226. package/dist/languages/serbian.mjs +1 -1
  227. package/dist/languages/shona.mjs +1 -1
  228. package/dist/languages/sinhala.mjs +1 -1
  229. package/dist/languages/slovak.mjs +1 -1
  230. package/dist/languages/slovenian.mjs +1 -1
  231. package/dist/languages/sorani.mjs +1 -1
  232. package/dist/languages/spanish.mjs +1 -1
  233. package/dist/languages/swahili.mjs +1 -1
  234. package/dist/languages/swedish.mjs +1 -1
  235. package/dist/languages/tagalog.mjs +1 -1
  236. package/dist/languages/tamil.mjs +1 -1
  237. package/dist/languages/tatar.mjs +1 -1
  238. package/dist/languages/telugu.mjs +1 -1
  239. package/dist/languages/thai.mjs +1 -1
  240. package/dist/languages/tibetan.mjs +1 -1
  241. package/dist/languages/tigrinya.mjs +1 -1
  242. package/dist/languages/tongan.mjs +1 -1
  243. package/dist/languages/turkish.mjs +1 -1
  244. package/dist/languages/twi.mjs +1 -1
  245. package/dist/languages/ukrainian.mjs +1 -1
  246. package/dist/languages/urdu.mjs +1 -1
  247. package/dist/languages/vietnamese.mjs +1 -1
  248. package/dist/languages/welsh.mjs +1 -1
  249. package/dist/languages/wolof.mjs +1 -1
  250. package/dist/languages/xhosa.mjs +1 -1
  251. package/dist/languages/yoruba.mjs +1 -1
  252. package/dist/languages/zulu.mjs +1 -1
  253. package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
  254. package/dist/module-5ZSLUQII.mjs +3729 -0
  255. package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
  256. package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
  257. package/dist/optional-writes-4UOQKETU.mjs +247 -0
  258. package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
  259. package/dist/partial-words.browser.mjs +1650 -0
  260. package/dist/partial-words.mjs +17 -11
  261. package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
  262. package/dist/pattern.mjs +6 -7
  263. package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
  264. package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
  265. package/dist/phrase-module-EFESWOCR.mjs +635 -0
  266. package/dist/query-actions-HMJUVWGT.mjs +119 -0
  267. package/dist/query-suggestions/merge-worker.d.ts +55 -0
  268. package/dist/query-suggestions/merge-worker.mjs +53 -0
  269. package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
  270. package/dist/query-suggestions.browser.mjs +47 -0
  271. package/dist/query-suggestions.d.ts +20 -0
  272. package/dist/query-suggestions.mjs +19 -0
  273. package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
  274. package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
  275. package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
  276. package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
  277. package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
  278. package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
  279. package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
  280. package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
  281. package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
  282. package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
  283. package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
  284. package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
  285. package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
  286. package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
  287. package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
  288. package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
  289. package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
  290. package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
  291. package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
  292. package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
  293. package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
  294. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  295. package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
  296. package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
  297. package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
  298. package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
  299. package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
  300. package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
  301. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  302. package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
  303. package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
  304. package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
  305. package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
  306. package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
  307. package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
  308. package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
  309. package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
  310. package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
  311. package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
  312. package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
  313. package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
  314. package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
  315. package/dist/react-native/partial-words.mjs +1658 -0
  316. package/dist/react-native/pattern.mjs +2570 -0
  317. package/dist/react-native/query-suggestions.mjs +55 -0
  318. package/dist/react-native/sync.mjs +2759 -0
  319. package/dist/react-native/synonyms.mjs +676 -0
  320. package/dist/react.d.ts +12 -8
  321. package/dist/react.mjs +45 -1
  322. package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
  323. package/dist/rerankers/http.d.ts +1 -1
  324. package/dist/rerankers/http.mjs +2 -2
  325. package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
  326. package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
  327. package/dist/runtime-USQYKN3I.mjs +615 -0
  328. package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
  329. package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
  330. package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
  331. package/dist/server.d.ts +348 -22
  332. package/dist/server.mjs +2211 -327
  333. package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
  334. package/dist/store-leaf-6VMMSVBE.mjs +20 -0
  335. package/dist/stored-values-BWIMpiT_.d.ts +25 -0
  336. package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
  337. package/dist/sync/filesystem.d.ts +30 -0
  338. package/dist/sync/filesystem.mjs +288 -0
  339. package/dist/sync/indexeddb.browser.mjs +287 -0
  340. package/dist/sync/indexeddb.d.ts +30 -0
  341. package/dist/sync/indexeddb.mjs +242 -0
  342. package/dist/sync.browser.mjs +1271 -0
  343. package/dist/sync.d.ts +23 -0
  344. package/dist/sync.mjs +2712 -0
  345. package/dist/synonyms.browser.mjs +658 -0
  346. package/dist/synonyms.mjs +17 -7
  347. package/dist/tasks-65b6t6lk.d.ts +782 -0
  348. package/dist/thread-UJ3W5LVI.mjs +31 -0
  349. package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
  350. package/dist/types-CpCg-Gq-.d.ts +227 -0
  351. package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
  352. package/dist/vector/search-worker.d.ts +10 -2
  353. package/dist/vector/search-worker.mjs +11 -11
  354. package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
  355. package/dist/workers/entry.mjs +75 -15
  356. package/openapi.json +6840 -0
  357. package/package.json +406 -19
  358. package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
  359. package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
  360. package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
  361. package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
  362. package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
  363. package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
  364. package/dist/browser/chunk-QWES44AW.mjs +0 -473
  365. package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
  366. package/dist/browser/module-PDXYHT75.mjs +0 -2822
  367. package/dist/browser/release-76MXRPGV.mjs +0 -33
  368. package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
  369. package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
  370. package/dist/browser/write-L7FAQA47.mjs +0 -208
  371. package/dist/chunk-37CD3OKS.mjs +0 -87
  372. package/dist/chunk-5XNIRLFC.mjs +0 -96
  373. package/dist/chunk-7X5JST5L.mjs +0 -114
  374. package/dist/chunk-AV4IJKD2.mjs +0 -35
  375. package/dist/chunk-AY3AQ3UH.mjs +0 -73
  376. package/dist/chunk-BKB3XTMI.mjs +0 -707
  377. package/dist/chunk-C7SLA6CM.mjs +0 -19
  378. package/dist/chunk-COVCQ73F.mjs +0 -587
  379. package/dist/chunk-DJHHF2L5.mjs +0 -17
  380. package/dist/chunk-E74YLAIG.mjs +0 -24
  381. package/dist/chunk-FMWRCP2C.mjs +0 -57
  382. package/dist/chunk-GES7U2PM.mjs +0 -68
  383. package/dist/chunk-HJSN7UYG.mjs +0 -10
  384. package/dist/chunk-JFRNNBOM.mjs +0 -35
  385. package/dist/chunk-K25LZA54.mjs +0 -8484
  386. package/dist/chunk-KEHK2O6P.mjs +0 -13
  387. package/dist/chunk-KOKM4PQN.mjs +0 -52
  388. package/dist/chunk-KZ3NX2P7.mjs +0 -21
  389. package/dist/chunk-L3L5YFXC.mjs +0 -614
  390. package/dist/chunk-L5ORED7V.mjs +0 -25
  391. package/dist/chunk-LH4NZNJJ.mjs +0 -1731
  392. package/dist/chunk-LU2CE5AE.mjs +0 -507
  393. package/dist/chunk-LXWDQLTW.mjs +0 -30
  394. package/dist/chunk-MHOSCQCQ.mjs +0 -948
  395. package/dist/chunk-NKRBEGJZ.mjs +0 -1705
  396. package/dist/chunk-NTRMFARG.mjs +0 -462
  397. package/dist/chunk-NW4XDSP2.mjs +0 -212
  398. package/dist/chunk-OOEXLUGR.mjs +0 -117
  399. package/dist/chunk-ORBLD3EL.mjs +0 -6
  400. package/dist/chunk-OWNZYMRE.mjs +0 -3591
  401. package/dist/chunk-PPY2GSIB.mjs +0 -44
  402. package/dist/chunk-QY7GC2MW.mjs +0 -33
  403. package/dist/chunk-R6A773WK.mjs +0 -226
  404. package/dist/chunk-R6PR4WYZ.mjs +0 -68
  405. package/dist/chunk-R7EEKAJE.mjs +0 -593
  406. package/dist/chunk-RONEOEON.mjs +0 -426
  407. package/dist/chunk-VWULPWRJ.mjs +0 -7469
  408. package/dist/chunk-WCJKUGHB.mjs +0 -262
  409. package/dist/chunk-WQBCBKY3.mjs +0 -8
  410. package/dist/chunk-X5CV2E2B.mjs +0 -1
  411. package/dist/chunk-YA7XLE2R.mjs +0 -1256
  412. package/dist/chunk-YTJCGKM4.mjs +0 -56
  413. package/dist/chunk-ZG477IPV.mjs +0 -526
  414. package/dist/chunk-ZYNAMIYX.mjs +0 -113
  415. package/dist/config-BGB_fDtq.d.ts +0 -237
  416. package/dist/direct-executor-CXZ66ACP.mjs +0 -22
  417. package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
  418. package/dist/engine-BGCEm9E_.d.ts +0 -577
  419. package/dist/module-3NONKBZX.mjs +0 -1860
  420. package/dist/release-BH3DKW3T.mjs +0 -33
  421. package/dist/restore-queue-JPRBUYII.mjs +0 -35
  422. package/dist/runtime-ZKYZFJJC.mjs +0 -177
  423. package/dist/snapshot-HOGXNO5W.mjs +0 -793
  424. package/dist/tasks-HG3rlxx6.d.ts +0 -302
  425. package/dist/thread-B657LQXK.mjs +0 -692
  426. package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/server.mjs CHANGED
@@ -1,35 +1,1263 @@
1
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-YA7XLE2R.mjs';
2
- export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-YA7XLE2R.mjs';
3
- import { rerankOnHost } from './chunk-ZYNAMIYX.mjs';
4
- import { validateIndexName } from './chunk-2WYMG3OZ.mjs';
5
- import './chunk-DJHHF2L5.mjs';
6
- import './chunk-P36MUFEJ.mjs';
7
- import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-R7EEKAJE.mjs';
8
- import { detectRuntime } from './chunk-AV4IJKD2.mjs';
9
- import './chunk-JFRNNBOM.mjs';
10
- import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
11
- import './chunk-LXWDQLTW.mjs';
12
- import './chunk-TGP5M5R7.mjs';
1
+ import { answerRequest } from './chunk-MW3CJVGP.mjs';
2
+ import { resolveVectorText } from './chunk-NJJN4FMM.mjs';
3
+ import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, refusedRegex, toHttpError, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, requestSignal, withServerRerankWindow, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, isThreadFeedNote, clearGateSlot, clearBodyBudgetSlot, confirmDeparture, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-XQIAMDAG.mjs';
4
+ export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-XQIAMDAG.mjs';
5
+ import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-5F5MXWS5.mjs';
6
+ import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, CHANGES_CLOSE_RESYNC, CHANGES_CLOSE_POLICY, MAX_SUBSCRIBER_MESSAGE_BYTES, CHANGES_CLOSE_TOO_LARGE, closeReasonOf, CHANGES_CLOSE_CREDENTIAL_EXPIRED, CHANGES_CLOSE_SETTING, CHANGES_CLOSE_SERVER_ERROR, CHANGES_CLOSE_GOING_AWAY, SYNC_CLOSE_GOING_AWAY } from './chunk-TACP6VK6.mjs';
7
+ import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-LFXS6OF7.mjs';
8
+ import { isRecord, validateIndexName } from './chunk-C74KPRR7.mjs';
9
+ import { executeUntilAnswered } from './chunk-4H4RG2FR.mjs';
10
+ import './chunk-XL56L4IR.mjs';
11
+ import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-AVBPB3MI.mjs';
12
+ import './chunk-M2Z5WVWW.mjs';
13
+ import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-ZPJFGGGI.mjs';
14
+ import './chunk-JDDXV6UF.mjs';
15
+ import { manifestFileRefs } from './chunk-C4ZMXXTL.mjs';
16
+ import './chunk-MSIN4NM3.mjs';
17
+ import { encode as encode$1 } from './chunk-S5QMWECL.mjs';
18
+ import './chunk-LNGE7SLY.mjs';
19
+ import './chunk-R2PJKSNL.mjs';
20
+ import './chunk-MFPSQZ6P.mjs';
21
+ import './chunk-FS3HPPSC.mjs';
22
+ import './chunk-R724DJSG.mjs';
23
+ import './chunk-V6EUBAIT.mjs';
24
+ import './chunk-TKKXPKEO.mjs';
25
+ import { querySuggestions } from './chunk-M3OW2XSP.mjs';
26
+ import './chunk-S2POFTLN.mjs';
27
+ import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-BY6NALRK.mjs';
28
+ import './chunk-2CKDCC65.mjs';
29
+ import './chunk-2AFZE7Y3.mjs';
30
+ import { decode } from './chunk-GXCDWLQI.mjs';
31
+ import './chunk-BUMREPLU.mjs';
32
+ import './chunk-7T4LAYT4.mjs';
33
+ import './chunk-IL5DRCNU.mjs';
34
+ import './chunk-7P5YWDRQ.mjs';
13
35
  import './chunk-U2BKYEC7.mjs';
14
- import './chunk-GGSH6KBQ.mjs';
15
- import './chunk-WQBCBKY3.mjs';
16
- import './chunk-LXPIXDUJ.mjs';
17
- import './chunk-E74YLAIG.mjs';
18
- import './chunk-KEHK2O6P.mjs';
19
- import './chunk-OOEXLUGR.mjs';
20
- import './chunk-7X5JST5L.mjs';
21
- import './chunk-R6A773WK.mjs';
22
- import './chunk-YTJCGKM4.mjs';
23
- import { startIdleHeapCleanup } from './chunk-5XNIRLFC.mjs';
24
- import './chunk-L5ORED7V.mjs';
25
- import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
26
- export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
36
+ import { detectRuntime } from './chunk-FJXJZUWF.mjs';
37
+ import { startIdleHeapCleanup } from './chunk-5EI5CXO6.mjs';
38
+ import './chunk-3ECIRFS5.mjs';
39
+ import './chunk-ASZLNVWW.mjs';
40
+ import './chunk-ZXX5GZEX.mjs';
41
+ import './chunk-GR5GMFJJ.mjs';
42
+ import './chunk-4PY4JYMY.mjs';
43
+ import { ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
44
+ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
45
+ import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
27
46
  import { randomUUID } from 'crypto';
47
+ import { encode } from '@msgpack/msgpack';
28
48
  import { createReadStream } from 'fs';
29
- import { rm } from 'fs/promises';
30
- import path from 'path';
31
49
  import { MessageChannel } from 'worker_threads';
32
50
 
51
+ // src/server/changes/setting-handlers.ts
52
+ function createChangesSettingHandlers(engine) {
53
+ async function getChanges(ctx) {
54
+ try {
55
+ respondJson(ctx, await engine.getChanges(ctx.params[0]));
56
+ } catch (err) {
57
+ respondError(ctx, err);
58
+ }
59
+ }
60
+ async function updateChanges(ctx) {
61
+ const body = parseJson(ctx);
62
+ if (body === void 0) return;
63
+ try {
64
+ await engine.updateChanges(ctx.params[0], body);
65
+ respondJson(ctx, { ok: true });
66
+ } catch (err) {
67
+ respondError(ctx, err);
68
+ }
69
+ }
70
+ return { getChanges, updateChanges };
71
+ }
72
+
73
+ // src/changes/encode.ts
74
+ var FIXMAP_TAG = 128;
75
+ var FIXARRAY_LIMIT = 16;
76
+ var ARRAY16_LIMIT = 65536;
77
+ var FIXARRAY_TAG = 144;
78
+ var ARRAY16_TAG = 220;
79
+ var ARRAY32_TAG = 221;
80
+ var CHANGES_MESSAGE_KEY_COUNT = 4;
81
+ var CHANGE_FIXED_KEY_COUNT = 3;
82
+ var keys = null;
83
+ function keyBytes(name) {
84
+ keys ??= {};
85
+ let bytes = keys[name];
86
+ if (bytes === void 0) {
87
+ bytes = encode$1(name);
88
+ keys[name] = bytes;
89
+ }
90
+ return bytes;
91
+ }
92
+ function arrayHeader(count) {
93
+ if (count < FIXARRAY_LIMIT) return Uint8Array.of(FIXARRAY_TAG | count);
94
+ if (count < ARRAY16_LIMIT) return Uint8Array.of(ARRAY16_TAG, count >>> 8, count & 255);
95
+ return Uint8Array.of(ARRAY32_TAG, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
96
+ }
97
+ function concat(parts) {
98
+ let length = 0;
99
+ for (const part of parts) length += part.byteLength;
100
+ const out = new Uint8Array(length);
101
+ let at = 0;
102
+ for (const part of parts) {
103
+ out.set(part, at);
104
+ at += part.byteLength;
105
+ }
106
+ return out;
107
+ }
108
+ function encodedChangeParts(change) {
109
+ const keyCount = CHANGE_FIXED_KEY_COUNT + (change.document === null ? 0 : 1) + (change.previousDocument === null ? 0 : 1);
110
+ const parts = [
111
+ Uint8Array.of(FIXMAP_TAG | keyCount),
112
+ keyBytes("seqNo"),
113
+ encode$1(change.seqNo),
114
+ keyBytes("type"),
115
+ keyBytes(change.type),
116
+ keyBytes("documentId"),
117
+ encode$1(change.documentId)
118
+ ];
119
+ if (change.document !== null) parts.push(keyBytes("document"), change.document);
120
+ if (change.previousDocument !== null) parts.push(keyBytes("previousDocument"), change.previousDocument);
121
+ return parts;
122
+ }
123
+ function encodeChangesMessage(partitionId, changes, scannedSeqNo) {
124
+ const parts = [
125
+ Uint8Array.of(FIXMAP_TAG | CHANGES_MESSAGE_KEY_COUNT),
126
+ keyBytes("type"),
127
+ keyBytes("changes"),
128
+ keyBytes("partitionId"),
129
+ encode$1(partitionId),
130
+ keyBytes("scannedSeqNo"),
131
+ encode$1(scannedSeqNo),
132
+ keyBytes("changes"),
133
+ arrayHeader(changes.length)
134
+ ];
135
+ for (const change of changes) parts.push(...encodedChangeParts(change));
136
+ return concat(parts);
137
+ }
138
+ function encodeFeedMessage(message) {
139
+ return encode$1(message, { ignoreUndefined: true });
140
+ }
141
+
142
+ // src/changes/session.ts
143
+ var POLICY_CODES = /* @__PURE__ */ new Set([
144
+ ErrorCodes.CHANGES_INVALID_POSITIONS,
145
+ ErrorCodes.SEARCH_INVALID_FILTER,
146
+ ErrorCodes.SEARCH_INVALID_FIELD
147
+ ]);
148
+ var ChangesSession = class {
149
+ constructor(deps) {
150
+ this.deps = deps;
151
+ this.stopBudgetWatch = deps.buffered.onFreed(() => {
152
+ if (this.state === "subscribed" && this.charged === 0) this.advance();
153
+ });
154
+ this.armExpiry();
155
+ }
156
+ state = "awaiting";
157
+ cursor = null;
158
+ sentSeqNos = [];
159
+ silentBytes = [];
160
+ unacked = [];
161
+ unackedBytes = 0;
162
+ charged = 0;
163
+ pumping = false;
164
+ pumpAgain = false;
165
+ retry = null;
166
+ expiry = null;
167
+ incoming = Promise.resolve();
168
+ stopBudgetWatch;
169
+ get indexName() {
170
+ return this.deps.indexName;
171
+ }
172
+ receive(bytes, binary = true) {
173
+ if (this.state === "closed") return;
174
+ if (!binary) {
175
+ this.close(CHANGES_CLOSE_POLICY, "feed messages travel in binary frames");
176
+ return;
177
+ }
178
+ if (bytes.byteLength > MAX_SUBSCRIBER_MESSAGE_BYTES) {
179
+ this.close(CHANGES_CLOSE_TOO_LARGE, "message too large");
180
+ return;
181
+ }
182
+ let message;
183
+ try {
184
+ message = subscriberMessageOf(decode(bytes));
185
+ } catch {
186
+ message = null;
187
+ }
188
+ if (message === null) {
189
+ this.close(CHANGES_CLOSE_POLICY, "invalid message");
190
+ return;
191
+ }
192
+ const received = message;
193
+ this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
194
+ }
195
+ advance() {
196
+ if (this.state !== "subscribed") return;
197
+ void this.pump();
198
+ }
199
+ drained() {
200
+ this.settleBuffered();
201
+ this.advance();
202
+ }
203
+ resync(reason) {
204
+ if (this.state === "closed") return;
205
+ this.send(encodeFeedMessage({ type: "resync", reason }));
206
+ this.close(CHANGES_CLOSE_RESYNC, `resync ${reason}`);
207
+ }
208
+ previousDocumentOff() {
209
+ if (this.cursor?.filtered === true) this.refuseFilterSetting();
210
+ }
211
+ close(code, reason) {
212
+ if (this.state === "closed") return;
213
+ this.state = "closed";
214
+ this.cleanUp();
215
+ this.deps.socket.close(code, closeReasonOf(reason));
216
+ this.deps.onClosed(this);
217
+ }
218
+ closedByPeer() {
219
+ if (this.state === "closed") return;
220
+ this.state = "closed";
221
+ this.cleanUp();
222
+ this.deps.onClosed(this);
223
+ }
224
+ armExpiry() {
225
+ const { expiresAt } = this.deps;
226
+ if (expiresAt === null || this.state === "closed") return;
227
+ const remaining = expiresAt - Date.now();
228
+ if (remaining <= 0) {
229
+ queueMicrotask(() => this.close(CHANGES_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
230
+ return;
231
+ }
232
+ this.expiry = setTimeout(
233
+ () => {
234
+ this.expiry = null;
235
+ this.armExpiry();
236
+ },
237
+ Math.min(remaining, TIMER_DELAY_LIMIT_MS)
238
+ );
239
+ }
240
+ refuseFilterSetting() {
241
+ this.close(CHANGES_CLOSE_SETTING, ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED);
242
+ }
243
+ cleanUp() {
244
+ if (this.retry !== null) clearTimeout(this.retry);
245
+ this.retry = null;
246
+ if (this.expiry !== null) clearTimeout(this.expiry);
247
+ this.expiry = null;
248
+ this.stopBudgetWatch();
249
+ this.deps.buffered.give(this.charged);
250
+ this.charged = 0;
251
+ }
252
+ fail(error) {
253
+ this.deps.onError(error);
254
+ this.close(CHANGES_CLOSE_SERVER_ERROR, "feed read failed");
255
+ }
256
+ send(bytes) {
257
+ if (this.state === "closed") return;
258
+ this.deps.socket.send(bytes);
259
+ this.settleBuffered();
260
+ }
261
+ settleBuffered() {
262
+ const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
263
+ if (now < this.charged) this.deps.buffered.give(this.charged - now);
264
+ this.charged = Math.min(this.charged, now);
265
+ }
266
+ async handle(message) {
267
+ if (this.state === "closed") return;
268
+ if (message.type === "subscribe") {
269
+ if (this.state !== "awaiting" || this.cursor !== null) {
270
+ return this.close(CHANGES_CLOSE_POLICY, "subscribe after the first message");
271
+ }
272
+ return this.subscribe(message);
273
+ }
274
+ if (this.state !== "subscribed") return this.close(CHANGES_CLOSE_POLICY, "subscribe must come first");
275
+ this.acknowledge(message.positions);
276
+ }
277
+ closeOver(error) {
278
+ if (!(error instanceof NarsilError)) this.fail(error);
279
+ else if (error.code === ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED) this.refuseFilterSetting();
280
+ else if (error.code === ErrorCodes.CHANGES_NOT_ENABLED) this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
281
+ else if (POLICY_CODES.has(error.code)) this.close(CHANGES_CLOSE_POLICY, error.message);
282
+ else this.fail(error);
283
+ }
284
+ async subscribe(message) {
285
+ const { source, indexName, reads } = this.deps;
286
+ if (message.filter !== void 0) {
287
+ const refused = this.deps.refuseFilter(message.filter);
288
+ if (refused !== null) return this.close(CHANGES_CLOSE_POLICY, refused.message);
289
+ }
290
+ let opened;
291
+ let vectorFields = [];
292
+ try {
293
+ opened = await ChangeCursor.open({ source, indexName, reads }, message);
294
+ if (typeof opened !== "string") vectorFields = [...await source.vectorFieldPaths(indexName)].sort();
295
+ } catch (error) {
296
+ return this.closeOver(error);
297
+ }
298
+ if (this.state === "closed") return;
299
+ if (typeof opened === "string") return this.resync(opened);
300
+ const cursor = opened;
301
+ this.cursor = cursor;
302
+ this.sentSeqNos = [...cursor.subscribedSeqNos];
303
+ this.silentBytes = new Array(cursor.state.partitionCount).fill(0);
304
+ for (let i = 0; i < cursor.state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
305
+ this.state = "subscribed";
306
+ this.send(
307
+ encodeFeedMessage({
308
+ type: "subscribed",
309
+ epoch: cursor.state.epoch,
310
+ partitionCount: cursor.state.partitionCount,
311
+ positions: changePositionList(cursor.subscribedSeqNos),
312
+ heads: changePositionList(cursor.heads),
313
+ bytesBehind: cursor.bytesBehind,
314
+ windowBytes: this.deps.windowBytes,
315
+ previousDocument: cursor.state.previousDocument,
316
+ vectorFields
317
+ })
318
+ );
319
+ void this.pump();
320
+ }
321
+ acknowledge(positions) {
322
+ const seqNos = seqNosByPartition(positions, this.sentSeqNos.length);
323
+ if (seqNos === null) {
324
+ this.close(CHANGES_CLOSE_POLICY, "positions must name each partition once");
325
+ return;
326
+ }
327
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
328
+ if (seqNos[partitionId] > this.sentSeqNos[partitionId]) {
329
+ this.close(CHANGES_CLOSE_POLICY, "acknowledgement above a sent position");
330
+ return;
331
+ }
332
+ }
333
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
334
+ const queue = this.unacked[partitionId];
335
+ while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
336
+ this.unackedBytes -= queue.items[queue.head].bytes;
337
+ queue.head += 1;
338
+ }
339
+ if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
340
+ queue.items = queue.items.slice(queue.head);
341
+ queue.head = 0;
342
+ }
343
+ }
344
+ void this.pump();
345
+ }
346
+ windowAllows(nextBytes) {
347
+ return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
348
+ }
349
+ settingAllows(cursor) {
350
+ const current = this.deps.source.changesState(this.deps.indexName);
351
+ if (current === null) {
352
+ this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
353
+ return false;
354
+ }
355
+ if (current.epoch !== cursor.state.epoch) {
356
+ this.resync("epoch");
357
+ return false;
358
+ }
359
+ if (cursor.filtered && !current.previousDocument) {
360
+ this.refuseFilterSetting();
361
+ return false;
362
+ }
363
+ return true;
364
+ }
365
+ async pump() {
366
+ const { cursor } = this;
367
+ if (cursor === null) return;
368
+ if (this.pumping) {
369
+ this.pumpAgain = true;
370
+ return;
371
+ }
372
+ this.pumping = true;
373
+ try {
374
+ do {
375
+ this.pumpAgain = false;
376
+ if (!this.settingAllows(cursor)) return;
377
+ await cursor.refreshHeads();
378
+ await this.sendBehind(cursor);
379
+ } while (this.pumpAgain && this.state === "subscribed");
380
+ this.scheduleRetryWhileUndurable(cursor);
381
+ } catch (error) {
382
+ this.fail(error);
383
+ } finally {
384
+ this.pumping = false;
385
+ }
386
+ }
387
+ async sendBehind(cursor) {
388
+ const { windowBytes } = this.deps;
389
+ let progressed = true;
390
+ while (progressed && this.state === "subscribed") {
391
+ progressed = false;
392
+ for (let partitionId = 0; partitionId < this.sentSeqNos.length; partitionId++) {
393
+ if (this.state !== "subscribed" || !cursor.behind(partitionId)) continue;
394
+ if (!this.windowAllows(1) || this.charged > 0) return;
395
+ const room = this.unackedBytes === 0 ? windowBytes : windowBytes - this.unackedBytes;
396
+ const batch = await cursor.readBatch(partitionId, Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES)));
397
+ if (this.state !== "subscribed") return;
398
+ if (batch === "history") return this.resync("history");
399
+ if (batch === null) continue;
400
+ if (!this.settingAllows(cursor)) return;
401
+ progressed = true;
402
+ if (batch.changes.length === 0 && this.silentBytes[partitionId] + batch.scannedBytes < windowBytes) {
403
+ this.silentBytes[partitionId] += batch.scannedBytes;
404
+ cursor.commit(batch);
405
+ continue;
406
+ }
407
+ const message = encodeChangesMessage(partitionId, batch.changes, batch.scannedSeqNo);
408
+ if (!this.windowAllows(message.byteLength)) return;
409
+ if (!this.deps.buffered.take(message.byteLength)) return;
410
+ this.charged += message.byteLength;
411
+ this.unacked[partitionId].items.push({ seqNo: batch.scannedSeqNo, bytes: message.byteLength });
412
+ this.unackedBytes += message.byteLength;
413
+ this.silentBytes[partitionId] = 0;
414
+ this.sentSeqNos[partitionId] = batch.scannedSeqNo;
415
+ cursor.commit(batch);
416
+ this.send(message);
417
+ }
418
+ }
419
+ }
420
+ scheduleRetryWhileUndurable(cursor) {
421
+ if (this.state !== "subscribed" || this.retry !== null || !cursor.awaitsDurability()) return;
422
+ this.retry = setTimeout(() => {
423
+ this.retry = null;
424
+ this.advance();
425
+ }, FEED_HEAD_RETRY_MS);
426
+ }
427
+ };
428
+
429
+ // src/changes/hub.ts
430
+ function createChangesHub(source, options = {}) {
431
+ const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
432
+ const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
433
+ const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
434
+ const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
435
+ const refuseFilter = options.refuseFilter ?? (() => null);
436
+ const sessionsByIndex = /* @__PURE__ */ new Map();
437
+ const unwatchByIndex = /* @__PURE__ */ new Map();
438
+ let connections = 0;
439
+ let stopListening = null;
440
+ let closed = false;
441
+ function each(indexName, action) {
442
+ const sessions = sessionsByIndex.get(indexName);
443
+ if (sessions === void 0) return;
444
+ for (const session of [...sessions]) action(session);
445
+ }
446
+ function onChange(change) {
447
+ switch (change.kind) {
448
+ case "advanced":
449
+ each(change.indexName, (session) => session.advance());
450
+ return;
451
+ case "epoch":
452
+ each(change.indexName, (session) => session.resync("epoch"));
453
+ return;
454
+ case "previous-document-off":
455
+ each(change.indexName, (session) => session.previousDocumentOff());
456
+ return;
457
+ case "changes-off":
458
+ case "dropped":
459
+ each(change.indexName, (session) => session.close(CHANGES_CLOSE_RESYNC, "change capture is off"));
460
+ return;
461
+ case "off":
462
+ case "settings":
463
+ return;
464
+ }
465
+ }
466
+ function release() {
467
+ connections -= 1;
468
+ if (connections === 0 && stopListening !== null) {
469
+ stopListening();
470
+ stopListening = null;
471
+ }
472
+ }
473
+ function forget(session) {
474
+ const sessions = sessionsByIndex.get(session.indexName);
475
+ if (sessions === void 0 || !sessions.delete(session)) return;
476
+ if (sessions.size === 0) {
477
+ sessionsByIndex.delete(session.indexName);
478
+ unwatchByIndex.get(session.indexName)?.();
479
+ unwatchByIndex.delete(session.indexName);
480
+ }
481
+ release();
482
+ }
483
+ function open(indexName, expiresAt, socket) {
484
+ if (stopListening === null) stopListening = source.listen(onChange);
485
+ const session = new ChangesSession({
486
+ source,
487
+ indexName,
488
+ socket,
489
+ windowBytes,
490
+ expiresAt,
491
+ reads,
492
+ buffered,
493
+ refuseFilter,
494
+ onClosed: forget,
495
+ onError: (error) => options.onError?.(error)
496
+ });
497
+ let sessions = sessionsByIndex.get(indexName);
498
+ if (sessions === void 0) {
499
+ sessions = /* @__PURE__ */ new Set();
500
+ sessionsByIndex.set(indexName, sessions);
501
+ const unwatch = source.watchIndex?.(indexName);
502
+ if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
503
+ }
504
+ sessions.add(session);
505
+ return session;
506
+ }
507
+ return {
508
+ admit(indexName, expiresAt) {
509
+ if (closed) {
510
+ throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
511
+ }
512
+ if (source.changesState(indexName) === null) throw changesNotEnabled(indexName);
513
+ if (connections >= maxConnections) {
514
+ throw new NarsilError(
515
+ ServerErrorCodes.TOO_MANY_REQUESTS,
516
+ `The server holds ${connections} change feed connections, the most that it accepts`,
517
+ { maximum: maxConnections }
518
+ );
519
+ }
520
+ connections += 1;
521
+ let settled = false;
522
+ return {
523
+ open(socket) {
524
+ if (settled) throw new Error("A change feed admission opens one connection");
525
+ settled = true;
526
+ return open(indexName, expiresAt, socket);
527
+ },
528
+ cancel() {
529
+ if (settled) return;
530
+ settled = true;
531
+ release();
532
+ }
533
+ };
534
+ },
535
+ connectionCount: () => connections,
536
+ close() {
537
+ closed = true;
538
+ for (const indexName of [...sessionsByIndex.keys()]) {
539
+ each(indexName, (session) => session.close(CHANGES_CLOSE_GOING_AWAY, "server closing"));
540
+ }
541
+ stopListening?.();
542
+ stopListening = null;
543
+ }
544
+ };
545
+ }
546
+ var FIXARRAY_LIMIT2 = 16;
547
+ var ARRAY16_LIMIT2 = 65536;
548
+ var FIXARRAY_TAG2 = 144;
549
+ var ARRAY16_TAG2 = 220;
550
+ var ARRAY32_TAG2 = 221;
551
+ var FIXMAP_OF_THREE = 131;
552
+ var entriesKeyPrefix = (() => {
553
+ const parts = [Uint8Array.of(FIXMAP_OF_THREE), encode("type"), encode("entries"), encode("partitionId")];
554
+ return concat2(parts);
555
+ })();
556
+ var entriesKey = encode("entries");
557
+ function concat2(parts) {
558
+ let length = 0;
559
+ for (const part of parts) length += part.byteLength;
560
+ const out = new Uint8Array(length);
561
+ let at = 0;
562
+ for (const part of parts) {
563
+ out.set(part, at);
564
+ at += part.byteLength;
565
+ }
566
+ return out;
567
+ }
568
+ function arrayHeader2(count) {
569
+ if (count < FIXARRAY_LIMIT2) return Uint8Array.of(FIXARRAY_TAG2 | count);
570
+ if (count < ARRAY16_LIMIT2) return Uint8Array.of(ARRAY16_TAG2, count >>> 8, count & 255);
571
+ return Uint8Array.of(ARRAY32_TAG2, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
572
+ }
573
+ function encodeEntriesMessage(partitionId, payloads) {
574
+ return concat2([entriesKeyPrefix, encode(partitionId), entriesKey, arrayHeader2(payloads.length), ...payloads]);
575
+ }
576
+ function encodeServerMessage(message) {
577
+ return encode(message, { ignoreUndefined: true });
578
+ }
579
+
580
+ // src/sync/feed/session.ts
581
+ var FeedSession = class {
582
+ constructor(deps) {
583
+ this.deps = deps;
584
+ this.stopBudgetWatch = deps.buffered.onFreed(() => {
585
+ if (this.state === "subscribed" && this.charged === 0) this.advance();
586
+ });
587
+ this.armExpiry();
588
+ }
589
+ state = "awaiting";
590
+ epoch = "";
591
+ sent = [];
592
+ heads = [];
593
+ unacked = [];
594
+ unackedBytes = 0;
595
+ charged = 0;
596
+ pumping = false;
597
+ pumpAgain = false;
598
+ retry = null;
599
+ expiry = null;
600
+ incoming = Promise.resolve();
601
+ stopBudgetWatch;
602
+ armExpiry() {
603
+ const { expiresAt } = this.deps;
604
+ if (expiresAt === null || this.state === "closed") return;
605
+ const remaining = expiresAt - Date.now();
606
+ if (remaining <= 0) {
607
+ queueMicrotask(() => this.close(SYNC_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
608
+ return;
609
+ }
610
+ this.expiry = setTimeout(
611
+ () => {
612
+ this.expiry = null;
613
+ this.armExpiry();
614
+ },
615
+ Math.min(remaining, TIMER_DELAY_LIMIT_MS)
616
+ );
617
+ }
618
+ get indexName() {
619
+ return this.deps.indexName;
620
+ }
621
+ receive(bytes, binary = true) {
622
+ if (this.state === "closed") return;
623
+ if (!binary) {
624
+ this.close(SYNC_CLOSE_POLICY, "feed messages travel in binary frames");
625
+ return;
626
+ }
627
+ if (bytes.byteLength > MAX_DEVICE_MESSAGE_BYTES) {
628
+ this.close(SYNC_CLOSE_TOO_LARGE, "message too large");
629
+ return;
630
+ }
631
+ let message;
632
+ try {
633
+ message = deviceMessageOf(decode(bytes));
634
+ } catch {
635
+ message = null;
636
+ }
637
+ if (message === null) {
638
+ this.close(SYNC_CLOSE_POLICY, "invalid message");
639
+ return;
640
+ }
641
+ const received = message;
642
+ this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
643
+ }
644
+ advance() {
645
+ if (this.state !== "subscribed") return;
646
+ void this.pump();
647
+ }
648
+ sendSettings() {
649
+ if (this.state !== "subscribed") return;
650
+ const state = this.deps.source.indexState(this.deps.indexName);
651
+ if (state === null) return;
652
+ this.send(encodeServerMessage({ type: "settings", settings: state.settings }));
653
+ }
654
+ drained() {
655
+ this.settleBuffered();
656
+ this.advance();
657
+ }
658
+ resync(reason) {
659
+ if (this.state === "closed") return;
660
+ this.send(encodeServerMessage({ type: "resync", reason }));
661
+ this.close(SYNC_CLOSE_RESYNC, `resync ${reason}`);
662
+ }
663
+ close(code, reason) {
664
+ if (this.state === "closed") return;
665
+ this.state = "closed";
666
+ this.cleanUp();
667
+ this.deps.socket.close(code, reason);
668
+ this.deps.onClosed(this);
669
+ }
670
+ closedByPeer() {
671
+ if (this.state === "closed") return;
672
+ this.state = "closed";
673
+ this.cleanUp();
674
+ this.deps.onClosed(this);
675
+ }
676
+ cleanUp() {
677
+ if (this.retry !== null) clearTimeout(this.retry);
678
+ this.retry = null;
679
+ if (this.expiry !== null) clearTimeout(this.expiry);
680
+ this.expiry = null;
681
+ this.stopBudgetWatch();
682
+ this.deps.buffered.give(this.charged);
683
+ this.charged = 0;
684
+ }
685
+ fail(error) {
686
+ this.deps.onError(error);
687
+ this.close(SYNC_CLOSE_SERVER_ERROR, "feed read failed");
688
+ }
689
+ send(bytes) {
690
+ if (this.state === "closed") return;
691
+ this.deps.socket.send(bytes);
692
+ this.settleBuffered();
693
+ }
694
+ settleBuffered() {
695
+ const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
696
+ if (now < this.charged) this.deps.buffered.give(this.charged - now);
697
+ this.charged = Math.min(this.charged, now);
698
+ }
699
+ async handle(message) {
700
+ if (this.state === "closed") return;
701
+ if (message.type === "subscribe") {
702
+ if (this.state !== "awaiting") return this.close(SYNC_CLOSE_POLICY, "subscribe after the first message");
703
+ return this.subscribe(message.epoch, message.positions);
704
+ }
705
+ if (this.state !== "subscribed") return this.close(SYNC_CLOSE_POLICY, "subscribe must come first");
706
+ if (message.type === "ack") return this.acknowledge(message.positions);
707
+ this.answerCheckpoint(message.id).catch((error) => this.fail(error));
708
+ }
709
+ async currentBounds() {
710
+ const all = [];
711
+ for (let partitionId = 0; partitionId < this.sent.length; partitionId++) {
712
+ all.push(await this.deps.source.bounds(this.deps.indexName, partitionId, "current"));
713
+ }
714
+ return all;
715
+ }
716
+ async answerCheckpoint(id) {
717
+ const covering = (await this.currentBounds()).map((bounds) => bounds.highestSeqNo);
718
+ while (this.state === "subscribed") {
719
+ const all = await this.currentBounds();
720
+ let covered = true;
721
+ for (let partitionId = 0; partitionId < all.length; partitionId++) {
722
+ covering[partitionId] = Math.min(covering[partitionId], all[partitionId].highestSeqNo);
723
+ const reachable = Math.max(all[partitionId].head, this.sent[partitionId]);
724
+ if (reachable < covering[partitionId]) covered = false;
725
+ }
726
+ if (covered) {
727
+ const heads = all.map((bounds) => bounds.head);
728
+ this.send(encodeServerMessage({ type: "checkpoint", id, heads: positionList(heads) }));
729
+ this.advance();
730
+ return;
731
+ }
732
+ await new Promise((resolve) => setTimeout(resolve, FEED_HEAD_RETRY_MS));
733
+ }
734
+ }
735
+ async subscribe(epoch, positions) {
736
+ const { source, indexName } = this.deps;
737
+ const state = source.indexState(indexName);
738
+ if (state === null) return this.close(SYNC_CLOSE_RESYNC, "sync is off");
739
+ if (epoch !== state.epoch) return this.resync("epoch");
740
+ const seqNos = positionsByPartition(positions, state.partitionCount);
741
+ if (seqNos === null) return this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
742
+ const heads = [];
743
+ for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
744
+ const bounds = await this.deps.reads.run(() => source.bounds(indexName, partitionId, "current"));
745
+ if (seqNos[partitionId] < bounds.oldestSeqNo - 1) return this.resync("history");
746
+ if (seqNos[partitionId] > bounds.highestSeqNo) return this.resync("ahead");
747
+ heads.push(bounds.head);
748
+ }
749
+ const bytesBehind = await this.deps.reads.run(() => source.bytesBehind(indexName, seqNos));
750
+ const snapshotBytes = await this.deps.reads.run(() => source.snapshotBytes(indexName));
751
+ if (this.state === "closed") return;
752
+ if (source.indexState(indexName)?.epoch !== epoch) return this.resync("epoch");
753
+ this.epoch = epoch;
754
+ this.sent = seqNos;
755
+ this.heads = heads;
756
+ for (let i = 0; i < state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
757
+ this.state = "subscribed";
758
+ this.send(
759
+ encodeServerMessage({
760
+ type: "subscribed",
761
+ epoch,
762
+ partitionCount: state.partitionCount,
763
+ heads: positionList(heads),
764
+ bytesBehind,
765
+ snapshotBytes,
766
+ windowBytes: this.deps.windowBytes,
767
+ settings: state.settings
768
+ })
769
+ );
770
+ void this.pump();
771
+ }
772
+ acknowledge(positions) {
773
+ const seqNos = positionsByPartition(positions, this.sent.length);
774
+ if (seqNos === null) {
775
+ this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
776
+ return;
777
+ }
778
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
779
+ if (seqNos[partitionId] > this.sent[partitionId]) {
780
+ this.close(SYNC_CLOSE_POLICY, "acknowledgement above a sent entry");
781
+ return;
782
+ }
783
+ }
784
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
785
+ const queue = this.unacked[partitionId];
786
+ while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
787
+ this.unackedBytes -= queue.items[queue.head].bytes;
788
+ queue.head += 1;
789
+ }
790
+ if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
791
+ queue.items = queue.items.slice(queue.head);
792
+ queue.head = 0;
793
+ }
794
+ }
795
+ void this.pump();
796
+ }
797
+ windowAllows(nextBytes) {
798
+ return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
799
+ }
800
+ async pump() {
801
+ if (this.pumping) {
802
+ this.pumpAgain = true;
803
+ return;
804
+ }
805
+ this.pumping = true;
806
+ try {
807
+ do {
808
+ this.pumpAgain = false;
809
+ await this.refreshHeads();
810
+ await this.sendBehind();
811
+ } while (this.pumpAgain && this.state === "subscribed");
812
+ this.scheduleRetryWhileUndurable();
813
+ } catch (error) {
814
+ this.fail(error);
815
+ } finally {
816
+ this.pumping = false;
817
+ }
818
+ }
819
+ async refreshHeads() {
820
+ const { source, indexName } = this.deps;
821
+ if (source.indexState(indexName)?.epoch !== this.epoch) return this.resync("epoch");
822
+ for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
823
+ this.heads[partitionId] = Math.max(this.heads[partitionId], (await source.bounds(indexName, partitionId)).head);
824
+ }
825
+ }
826
+ async sendBehind() {
827
+ const { source, indexName } = this.deps;
828
+ let progressed = true;
829
+ while (progressed && this.state === "subscribed") {
830
+ progressed = false;
831
+ for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
832
+ if (this.state !== "subscribed" || this.sent[partitionId] >= this.heads[partitionId]) continue;
833
+ if (!this.windowAllows(1) || this.charged > 0) return;
834
+ const room = this.unackedBytes === 0 ? this.deps.windowBytes : this.deps.windowBytes - this.unackedBytes;
835
+ const maxBytes = Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES));
836
+ const after = this.sent[partitionId];
837
+ const read = await this.deps.reads.run(
838
+ () => source.read(indexName, partitionId, after, this.heads[partitionId], maxBytes)
839
+ );
840
+ if (this.state !== "subscribed") return;
841
+ if (read.gone) return this.resync("history");
842
+ const frames = read.frames.filter((frame) => frame.seqNo > this.sent[partitionId]);
843
+ if (frames.length === 0) continue;
844
+ let bytes = 0;
845
+ for (const frame of frames) bytes += frame.payload.byteLength;
846
+ if (!this.windowAllows(bytes)) return;
847
+ const message = encodeEntriesMessage(
848
+ partitionId,
849
+ frames.map((frame) => frame.payload)
850
+ );
851
+ if (!this.deps.buffered.take(message.byteLength)) return;
852
+ this.charged += message.byteLength;
853
+ const queue = this.unacked[partitionId].items;
854
+ for (const frame of frames) queue.push({ seqNo: frame.seqNo, bytes: frame.payload.byteLength });
855
+ this.unackedBytes += bytes;
856
+ this.sent[partitionId] = frames[frames.length - 1].seqNo;
857
+ this.send(message);
858
+ progressed = true;
859
+ }
860
+ }
861
+ }
862
+ scheduleRetryWhileUndurable() {
863
+ if (this.state !== "subscribed" || this.retry !== null) return;
864
+ const { source, indexName } = this.deps;
865
+ const waiting = this.heads.some((head, partitionId) => source.pendingBeyondHead(indexName, partitionId, head));
866
+ if (!waiting) return;
867
+ this.retry = setTimeout(() => {
868
+ this.retry = null;
869
+ this.advance();
870
+ }, FEED_HEAD_RETRY_MS);
871
+ }
872
+ };
873
+
874
+ // src/sync/feed/hub.ts
875
+ function createFeedHub(source, options = {}) {
876
+ const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
877
+ const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
878
+ const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
879
+ const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
880
+ const sessionsByIndex = /* @__PURE__ */ new Map();
881
+ const unwatchByIndex = /* @__PURE__ */ new Map();
882
+ let connections = 0;
883
+ let stopListening = null;
884
+ let closed = false;
885
+ function each(indexName, action) {
886
+ const sessions = sessionsByIndex.get(indexName);
887
+ if (sessions === void 0) return;
888
+ for (const session of [...sessions]) action(session);
889
+ }
890
+ function onChange(change) {
891
+ switch (change.kind) {
892
+ case "advanced":
893
+ each(change.indexName, (session) => session.advance());
894
+ return;
895
+ case "settings":
896
+ each(change.indexName, (session) => session.sendSettings());
897
+ return;
898
+ case "epoch":
899
+ each(change.indexName, (session) => session.resync("epoch"));
900
+ return;
901
+ case "off":
902
+ case "dropped":
903
+ each(change.indexName, (session) => session.close(SYNC_CLOSE_RESYNC, "sync is off"));
904
+ return;
905
+ }
906
+ }
907
+ function forget(session) {
908
+ const sessions = sessionsByIndex.get(session.indexName);
909
+ if (sessions === void 0 || !sessions.delete(session)) return;
910
+ if (sessions.size === 0) {
911
+ sessionsByIndex.delete(session.indexName);
912
+ unwatchByIndex.get(session.indexName)?.();
913
+ unwatchByIndex.delete(session.indexName);
914
+ }
915
+ release();
916
+ }
917
+ function release() {
918
+ connections -= 1;
919
+ if (connections === 0 && stopListening !== null) {
920
+ stopListening();
921
+ stopListening = null;
922
+ }
923
+ }
924
+ function open(indexName, expiresAt, socket) {
925
+ if (stopListening === null) stopListening = source.listen(onChange);
926
+ const session = new FeedSession({
927
+ source,
928
+ indexName,
929
+ socket,
930
+ windowBytes,
931
+ expiresAt,
932
+ reads,
933
+ buffered,
934
+ onClosed: forget,
935
+ onError: (error) => options.onError?.(error)
936
+ });
937
+ let sessions = sessionsByIndex.get(indexName);
938
+ if (sessions === void 0) {
939
+ sessions = /* @__PURE__ */ new Set();
940
+ sessionsByIndex.set(indexName, sessions);
941
+ const unwatch = source.watchIndex?.(indexName);
942
+ if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
943
+ }
944
+ sessions.add(session);
945
+ return session;
946
+ }
947
+ return {
948
+ admit(indexName, expiresAt) {
949
+ if (closed) {
950
+ throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
951
+ }
952
+ if (source.indexState(indexName) === null) {
953
+ throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
954
+ }
955
+ if (connections >= maxConnections) {
956
+ throw new NarsilError(
957
+ ServerErrorCodes.TOO_MANY_REQUESTS,
958
+ `The server holds ${connections} feed connections, the most that it accepts`,
959
+ { maximum: maxConnections }
960
+ );
961
+ }
962
+ connections += 1;
963
+ let settled = false;
964
+ return {
965
+ open(socket) {
966
+ if (settled) throw new Error("A feed admission opens one connection");
967
+ settled = true;
968
+ return open(indexName, expiresAt, socket);
969
+ },
970
+ cancel() {
971
+ if (settled) return;
972
+ settled = true;
973
+ release();
974
+ }
975
+ };
976
+ },
977
+ connectionCount: () => connections,
978
+ close() {
979
+ closed = true;
980
+ for (const indexName of [...sessionsByIndex.keys()]) {
981
+ each(indexName, (session) => session.close(SYNC_CLOSE_GOING_AWAY, "server closing"));
982
+ }
983
+ stopListening?.();
984
+ stopListening = null;
985
+ }
986
+ };
987
+ }
988
+
989
+ // src/sync/feed/engine-source.ts
990
+ function syncSettingsOf(entry) {
991
+ const { config } = entry;
992
+ const synonyms = entry.synonyms === null ? [] : rulesOfStoredSynonyms(entry.synonyms);
993
+ return {
994
+ language: entry.language.name,
995
+ ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
996
+ ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
997
+ ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
998
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
999
+ ...config.bm25 !== void 0 ? { bm25: config.bm25 } : {},
1000
+ ...synonyms.length > 0 ? { synonyms } : {}
1001
+ };
1002
+ }
1003
+ function storageOf(core) {
1004
+ return core.durability?.manager.syncStorage?.() ?? null;
1005
+ }
1006
+ function engineFeedSource(engine, core) {
1007
+ const requireStorage = () => {
1008
+ const storage = storageOf(core);
1009
+ if (storage === null) {
1010
+ throw new NarsilError(ErrorCodes.SYNC_DURABILITY_REQUIRED, "Device sync needs write-ahead log durability");
1011
+ }
1012
+ return storage;
1013
+ };
1014
+ return {
1015
+ indexState(indexName) {
1016
+ const entry = core.indexRegistry.get(indexName);
1017
+ if (entry === void 0 || entry.sync === null || storageOf(core) === null) return null;
1018
+ return { epoch: entry.sync.epoch, partitionCount: entry.partitionCount, settings: syncSettingsOf(entry) };
1019
+ },
1020
+ async bounds(indexName, partitionId) {
1021
+ const log = await requireStorage().openPartitionLog(indexName, partitionId);
1022
+ const applied = core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0;
1023
+ return { oldestSeqNo: log.oldestSeqNo, head: log.head, highestSeqNo: Math.max(log.head, applied) };
1024
+ },
1025
+ async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
1026
+ const read = await requireStorage().readPartitionLog(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes);
1027
+ return { frames: read.frames, gone: read.gone };
1028
+ },
1029
+ async bytesBehind(indexName, seqNos) {
1030
+ const storage = requireStorage();
1031
+ let bytes = 0;
1032
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
1033
+ const log = await storage.openPartitionLog(indexName, partitionId);
1034
+ bytes += await storage.logBytesAfter(log, seqNos[partitionId]);
1035
+ }
1036
+ return bytes;
1037
+ },
1038
+ async snapshotBytes(indexName) {
1039
+ const storage = requireStorage();
1040
+ const published = await storage.published(indexName);
1041
+ if (published !== null) return publishedArchiveLayout(indexName, published, storage).byteLength;
1042
+ const manifest = await storage.currentManifest(indexName);
1043
+ if (manifest === null) return 0;
1044
+ return estimatedArchiveBytes(manifestFileRefs(manifest));
1045
+ },
1046
+ pendingBeyondHead(indexName, partitionId, head) {
1047
+ return (core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0) > head;
1048
+ },
1049
+ listen(onChange) {
1050
+ const onWrite = (payload) => onChange({ kind: "advanced", indexName: payload.indexName });
1051
+ engine.on("write", onWrite);
1052
+ const stopSyncChanges = listenForSyncChanges(core.indexRegistry, onChange);
1053
+ return () => {
1054
+ engine.off("write", onWrite);
1055
+ stopSyncChanges();
1056
+ };
1057
+ }
1058
+ };
1059
+ }
1060
+
1061
+ // src/server/sync/engine-serving.ts
1062
+ function engineSyncServing(engine, core) {
1063
+ startSyncRetentionSweep(core);
1064
+ return {
1065
+ async snapshot(indexName) {
1066
+ const published = await publishedSyncSnapshot(core, indexName);
1067
+ try {
1068
+ return {
1069
+ entityTag: published.snapshot.entityTag,
1070
+ layout: publishedArchiveLayout(indexName, published.snapshot, published.storage),
1071
+ release: published.unpin
1072
+ };
1073
+ } catch (error) {
1074
+ published.unpin();
1075
+ throw error;
1076
+ }
1077
+ },
1078
+ feed: engineFeedSource(engine, core),
1079
+ admitFeed: async () => void 0
1080
+ };
1081
+ }
1082
+
1083
+ // src/server/sync/feed-host.ts
1084
+ var RelayedFeedSocket = class {
1085
+ constructor(port, feedId) {
1086
+ this.port = port;
1087
+ this.feedId = feedId;
1088
+ }
1089
+ inTransit = 0;
1090
+ threadBuffered = 0;
1091
+ closed = false;
1092
+ send(bytes) {
1093
+ if (this.closed) return;
1094
+ const copy = bytes.slice();
1095
+ this.inTransit += copy.byteLength;
1096
+ this.post({ type: "feed-send", feedId: this.feedId, bytes: copy }, [copy.buffer]);
1097
+ }
1098
+ bufferedAmount() {
1099
+ return this.inTransit + this.threadBuffered;
1100
+ }
1101
+ close(code, reason) {
1102
+ if (this.closed) return;
1103
+ this.closed = true;
1104
+ this.post({ type: "feed-close", feedId: this.feedId, code, reason }, []);
1105
+ }
1106
+ sent(bytes, buffered) {
1107
+ this.inTransit = Math.max(0, this.inTransit - bytes);
1108
+ this.threadBuffered = buffered;
1109
+ }
1110
+ markClosed() {
1111
+ this.closed = true;
1112
+ }
1113
+ post(note, transfer) {
1114
+ try {
1115
+ this.port.postMessage(note, transfer);
1116
+ } catch {
1117
+ this.closed = true;
1118
+ }
1119
+ }
1120
+ };
1121
+ function routeMissing() {
1122
+ return new NarsilError(ServerErrorCodes.NOT_FOUND, "Route not found");
1123
+ }
1124
+ function createFeedHost(feeds, authorize) {
1125
+ const authorizeContext = (context) => authorize === void 0 ? Promise.resolve({ allowed: true, expiresAt: null }) : authorize(context);
1126
+ function admit(feed, indexName, expiresAt) {
1127
+ const hosted = feeds[feed];
1128
+ if (hosted === void 0) return Promise.reject(routeMissing());
1129
+ return hosted.admit(indexName, expiresAt);
1130
+ }
1131
+ const gateways = {};
1132
+ for (const feed of Object.keys(feeds)) {
1133
+ gateways[feed] = {
1134
+ authorize: authorizeContext,
1135
+ async admit(indexName, expiresAt) {
1136
+ const admission = await admit(feed, indexName, expiresAt);
1137
+ return { open: (socket) => admission.open(socket), cancel: () => admission.cancel() };
1138
+ }
1139
+ };
1140
+ }
1141
+ function forThread(port) {
1142
+ const relayed = /* @__PURE__ */ new Map();
1143
+ let nextFeedId = 0;
1144
+ let closed = false;
1145
+ function closeFeed(feedId) {
1146
+ const feed = relayed.get(feedId);
1147
+ if (feed === void 0) return;
1148
+ relayed.delete(feedId);
1149
+ feed.socket.markClosed();
1150
+ if (feed.endpoint === null) feed.admission.cancel();
1151
+ else feed.endpoint.closedByPeer();
1152
+ }
1153
+ return {
1154
+ async admit(request) {
1155
+ try {
1156
+ const admission = await admit(request.feed, request.indexName, request.expiresAt);
1157
+ if (closed) {
1158
+ admission.cancel();
1159
+ return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: null };
1160
+ }
1161
+ nextFeedId += 1;
1162
+ const feedId = nextFeedId;
1163
+ relayed.set(feedId, { admission, socket: new RelayedFeedSocket(port, feedId), endpoint: null });
1164
+ return { type: "feed-admitted", requestId: request.requestId, feedId, error: null };
1165
+ } catch (err) {
1166
+ return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: toHttpError(err).body };
1167
+ }
1168
+ },
1169
+ handle(note) {
1170
+ const feed = relayed.get(note.feedId);
1171
+ if (feed === void 0) return;
1172
+ switch (note.type) {
1173
+ case "feed-open":
1174
+ if (feed.endpoint === null) feed.endpoint = feed.admission.open(feed.socket);
1175
+ return;
1176
+ case "feed-message":
1177
+ feed.endpoint?.receive(note.bytes, note.binary);
1178
+ return;
1179
+ case "feed-sent":
1180
+ feed.socket.sent(note.bytes, note.buffered);
1181
+ feed.endpoint?.drained();
1182
+ return;
1183
+ case "feed-cancel":
1184
+ case "feed-closed":
1185
+ closeFeed(note.feedId);
1186
+ return;
1187
+ }
1188
+ },
1189
+ closeAll() {
1190
+ closed = true;
1191
+ for (const feedId of [...relayed.keys()]) closeFeed(feedId);
1192
+ }
1193
+ };
1194
+ }
1195
+ return {
1196
+ gateways,
1197
+ forThread,
1198
+ close() {
1199
+ for (const hosted of Object.values(feeds)) hosted?.close();
1200
+ }
1201
+ };
1202
+ }
1203
+
1204
+ // src/server/feeds.ts
1205
+ function syncServingOf(engine, core) {
1206
+ const ofNode = syncServingSourceOf(engine);
1207
+ if (ofNode !== void 0) return ofNode();
1208
+ return core === void 0 ? null : engineSyncServing(engine, core);
1209
+ }
1210
+ function changesServingOf(engine, core) {
1211
+ const ofNode = changesServingSourceOf(engine);
1212
+ if (ofNode !== void 0) return ofNode();
1213
+ if (core === void 0) return null;
1214
+ return { source: engineChangeSource(core, listenForSyncChanges), admitFeed: async () => void 0 };
1215
+ }
1216
+ function hostedSyncFeed(serving, options) {
1217
+ const hub = createFeedHub(serving.feed, {
1218
+ ...options,
1219
+ onError: (error) => console.warn("A device sync feed failed to read its index:", error)
1220
+ });
1221
+ return {
1222
+ async admit(indexName, expiresAt) {
1223
+ await serving.admitFeed(indexName);
1224
+ return hub.admit(indexName, expiresAt);
1225
+ },
1226
+ close: () => hub.close()
1227
+ };
1228
+ }
1229
+ function hostedChangesFeed(serving, options, allowRegex) {
1230
+ const hub = createChangesHub(serving.source, {
1231
+ ...options,
1232
+ refuseFilter: (filter) => refusedRegex(filter, allowRegex),
1233
+ onError: (error) => console.warn("A change feed failed to read its index:", error)
1234
+ });
1235
+ return {
1236
+ async admit(indexName, expiresAt) {
1237
+ await serving.admitFeed(indexName);
1238
+ return hub.admit(indexName, expiresAt);
1239
+ },
1240
+ close: () => hub.close()
1241
+ };
1242
+ }
1243
+ function startFeeds(engine, core, settings) {
1244
+ const hosted = {};
1245
+ let syncServing = null;
1246
+ if (settings.sync !== null) {
1247
+ syncServing = syncServingOf(engine, core);
1248
+ if (syncServing !== null) hosted.sync = hostedSyncFeed(syncServing, settings.sync);
1249
+ }
1250
+ if (settings.changes !== null) {
1251
+ const serving = changesServingOf(engine, core);
1252
+ if (serving !== null) hosted.changes = hostedChangesFeed(serving, settings.changes, settings.allowRegex);
1253
+ }
1254
+ if (hosted.sync === void 0 && hosted.changes === void 0) return null;
1255
+ return { host: createFeedHost(hosted, settings.authorize), syncServing };
1256
+ }
1257
+ function snapshotHistoryOf(raw) {
1258
+ if (raw === null || raw === "false") return false;
1259
+ return raw === "true" ? true : null;
1260
+ }
33
1261
  function createAdminHandlers(deps) {
34
1262
  const { engine, tasks, limits } = deps;
35
1263
  async function checkpoint(ctx) {
@@ -41,8 +1269,15 @@ function createAdminHandlers(deps) {
41
1269
  }
42
1270
  }
43
1271
  async function snapshot(ctx) {
1272
+ const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
1273
+ if (queryHistory === null) {
1274
+ badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
1275
+ value: ctx.query.get("queryHistory")
1276
+ });
1277
+ return;
1278
+ }
44
1279
  try {
45
- const archive = await engine.snapshot(ctx.params[0]);
1280
+ const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
46
1281
  if (ctx.abort.aborted) {
47
1282
  await archive.cancel();
48
1283
  return;
@@ -100,6 +1335,16 @@ function createAdminHandlers(deps) {
100
1335
  respondError(ctx, err);
101
1336
  }
102
1337
  }
1338
+ async function updateBm25(ctx) {
1339
+ const body = parseJson(ctx);
1340
+ if (body === void 0) return;
1341
+ try {
1342
+ await engine.updateBM25(ctx.params[0], body);
1343
+ respondJson(ctx, { ok: true });
1344
+ } catch (err) {
1345
+ respondError(ctx, err);
1346
+ }
1347
+ }
103
1348
  async function memory(ctx) {
104
1349
  try {
105
1350
  const stats = await engine.getMemoryStats();
@@ -109,15 +1354,15 @@ function createAdminHandlers(deps) {
109
1354
  respondError(ctx, err);
110
1355
  }
111
1356
  }
112
- function requireIndexExists(name) {
113
- engine.getStats(name);
1357
+ async function requireIndexExists(name) {
1358
+ await indexReadsOf(engine).getStats(name);
114
1359
  }
115
1360
  async function optimize(ctx) {
116
1361
  const body = parseJsonOptional(ctx);
117
1362
  if (!body) return;
118
1363
  const name = ctx.params[0];
119
1364
  try {
120
- requireIndexExists(name);
1365
+ await requireIndexExists(name);
121
1366
  } catch (err) {
122
1367
  respondError(ctx, err);
123
1368
  return;
@@ -128,7 +1373,7 @@ function createAdminHandlers(deps) {
128
1373
  async function rebuildAnalysis(ctx) {
129
1374
  const name = ctx.params[0];
130
1375
  try {
131
- requireIndexExists(name);
1376
+ await requireIndexExists(name);
132
1377
  } catch (err) {
133
1378
  respondError(ctx, err);
134
1379
  return;
@@ -136,6 +1381,24 @@ function createAdminHandlers(deps) {
136
1381
  const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
137
1382
  respondJson(ctx, record, 202);
138
1383
  }
1384
+ async function changeAnalysis(ctx) {
1385
+ const body = parseJsonOptional(ctx);
1386
+ if (!body) return;
1387
+ const failure = validateChangeAnalysis(body);
1388
+ if (failure) {
1389
+ rejectInvalid(ctx, failure);
1390
+ return;
1391
+ }
1392
+ const name = ctx.params[0];
1393
+ try {
1394
+ await requireIndexExists(name);
1395
+ } catch (err) {
1396
+ respondError(ctx, err);
1397
+ return;
1398
+ }
1399
+ const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
1400
+ respondJson(ctx, record, 202);
1401
+ }
139
1402
  async function rebalance(ctx) {
140
1403
  const body = parseJsonOptional(ctx);
141
1404
  if (!body) return;
@@ -147,7 +1410,7 @@ function createAdminHandlers(deps) {
147
1410
  const target = body.targetPartitionCount;
148
1411
  const name = ctx.params[0];
149
1412
  try {
150
- requireIndexExists(name);
1413
+ await requireIndexExists(name);
151
1414
  } catch (err) {
152
1415
  respondError(ctx, err);
153
1416
  return;
@@ -230,10 +1493,12 @@ function createAdminHandlers(deps) {
230
1493
  partitionConfig,
231
1494
  readSynonyms,
232
1495
  replaceSynonyms,
1496
+ updateBm25,
233
1497
  memory,
234
1498
  optimize,
235
1499
  rebalance,
236
1500
  rebuildAnalysis,
1501
+ changeAnalysis,
237
1502
  restore,
238
1503
  getTask,
239
1504
  listTasks,
@@ -241,14 +1506,6 @@ function createAdminHandlers(deps) {
241
1506
  };
242
1507
  }
243
1508
 
244
- // src/server/handlers/capabilities.ts
245
- function createCapabilitiesHandler() {
246
- function report(ctx) {
247
- respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
248
- }
249
- return { report };
250
- }
251
-
252
1509
  // src/server/handlers/cluster.ts
253
1510
  function requireCluster(deps) {
254
1511
  if (deps.cluster === void 0) {
@@ -301,7 +1558,12 @@ function createClusterHandlers(deps) {
301
1558
  }
302
1559
  const table = await cluster.getAllocation(indexName);
303
1560
  if (table === null) {
304
- respondJson(ctx, { indexName, allocated: false, version: null, partitions: [] });
1561
+ respondJson(ctx, {
1562
+ indexName,
1563
+ allocated: false,
1564
+ version: null,
1565
+ partitions: []
1566
+ });
305
1567
  return;
306
1568
  }
307
1569
  const partitions = [];
@@ -319,12 +1581,158 @@ function createClusterHandlers(deps) {
319
1581
  });
320
1582
  }
321
1583
  partitions.sort((left, right) => left.partitionId - right.partitionId);
322
- respondJson(ctx, { indexName, allocated: partitions.length > 0, version: table.version, partitions });
1584
+ respondJson(ctx, {
1585
+ indexName,
1586
+ allocated: partitions.length > 0,
1587
+ version: table.version,
1588
+ partitions
1589
+ });
323
1590
  } catch (err) {
324
1591
  respondError(ctx, err);
325
1592
  }
326
1593
  }
327
- return { topology, allocation };
1594
+ async function replicationFactor(ctx) {
1595
+ try {
1596
+ const cluster = requireCluster(deps);
1597
+ const indexName = ctx.params[0] ?? "";
1598
+ const malformed = malformedIndexName(indexName);
1599
+ if (malformed !== null) {
1600
+ badRequest(ctx.res, malformed, { indexName });
1601
+ return;
1602
+ }
1603
+ const body = parseJson(ctx);
1604
+ if (body === void 0) return;
1605
+ const factor = isRecord(body) ? body.replicationFactor : void 0;
1606
+ if (typeof factor !== "number") {
1607
+ badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
1608
+ return;
1609
+ }
1610
+ await cluster.setReplicationFactor(indexName, factor);
1611
+ respondJson(ctx, { indexName, replicationFactor: factor });
1612
+ } catch (err) {
1613
+ respondError(ctx, err);
1614
+ }
1615
+ }
1616
+ async function health(ctx) {
1617
+ try {
1618
+ respondJson(ctx, await requireCluster(deps).getHealth());
1619
+ } catch (err) {
1620
+ respondError(ctx, err);
1621
+ }
1622
+ }
1623
+ return { topology, allocation, replicationFactor, health };
1624
+ }
1625
+
1626
+ // src/server/handlers/backups.ts
1627
+ function stepsOf(context) {
1628
+ return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
1629
+ }
1630
+ function restoreOptionsOf(body) {
1631
+ const { indexName } = body;
1632
+ if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
1633
+ const placement = readRestorePlacement(body);
1634
+ return typeof placement === "string" ? placement : { indexName, ...placement };
1635
+ }
1636
+ function readBody(ctx) {
1637
+ const body = parseJson(ctx);
1638
+ if (body === void 0) return void 0;
1639
+ if (!isRecord(body)) {
1640
+ badRequest(ctx.res, "The body must be a JSON object");
1641
+ return void 0;
1642
+ }
1643
+ return body;
1644
+ }
1645
+ function createBackupHandlers(deps) {
1646
+ async function create(ctx) {
1647
+ try {
1648
+ const cluster = requireCluster(deps);
1649
+ const indexName = ctx.params[0] ?? "";
1650
+ const body = readBody(ctx);
1651
+ if (body === void 0) return;
1652
+ if (typeof body.backup !== "string") {
1653
+ badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
1654
+ return;
1655
+ }
1656
+ if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
1657
+ badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
1658
+ return;
1659
+ }
1660
+ const backupName = validateBackupName(body.backup);
1661
+ const queryHistory = body.queryHistory === true;
1662
+ const record = await deps.tasks.start(
1663
+ "backup",
1664
+ indexName,
1665
+ async (context) => {
1666
+ await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
1667
+ },
1668
+ void 0,
1669
+ backupName
1670
+ );
1671
+ respondJson(ctx, record, 202);
1672
+ } catch (err) {
1673
+ respondError(ctx, err);
1674
+ }
1675
+ }
1676
+ async function list(ctx) {
1677
+ try {
1678
+ const cluster = requireCluster(deps);
1679
+ respondJson(ctx, { backups: await cluster.listBackups() });
1680
+ } catch (err) {
1681
+ respondError(ctx, err);
1682
+ }
1683
+ }
1684
+ async function restore(ctx) {
1685
+ try {
1686
+ const cluster = requireCluster(deps);
1687
+ const backupName = validateBackupName(ctx.params[0]);
1688
+ const body = readBody(ctx);
1689
+ if (body === void 0) return;
1690
+ const options = restoreOptionsOf(body);
1691
+ if (typeof options === "string") {
1692
+ badRequest(ctx.res, options, { backupName });
1693
+ return;
1694
+ }
1695
+ const record = await deps.tasks.start(
1696
+ "backupRestore",
1697
+ options.indexName,
1698
+ async (context) => {
1699
+ await cluster.restoreBackup(backupName, { ...options, ...stepsOf(context) });
1700
+ },
1701
+ void 0,
1702
+ backupName
1703
+ );
1704
+ respondJson(ctx, record, 202);
1705
+ } catch (err) {
1706
+ respondError(ctx, err);
1707
+ }
1708
+ }
1709
+ async function remove(ctx) {
1710
+ try {
1711
+ const cluster = requireCluster(deps);
1712
+ const backupName = validateBackupName(ctx.params[0]);
1713
+ const record = await deps.tasks.start(
1714
+ "backupDelete",
1715
+ "",
1716
+ async (context) => {
1717
+ await cluster.deleteBackup(backupName, stepsOf(context));
1718
+ },
1719
+ void 0,
1720
+ backupName
1721
+ );
1722
+ respondJson(ctx, record, 202);
1723
+ } catch (err) {
1724
+ respondError(ctx, err);
1725
+ }
1726
+ }
1727
+ return { create, list, restore, remove };
1728
+ }
1729
+
1730
+ // src/server/handlers/capabilities.ts
1731
+ function createCapabilitiesHandler() {
1732
+ function report(ctx) {
1733
+ respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
1734
+ }
1735
+ return { report };
328
1736
  }
329
1737
  function importResultOf(result, maxErrors) {
330
1738
  const errors = result.refused.slice(0, maxErrors);
@@ -346,7 +1754,7 @@ function createDocumentExportHandlers(deps) {
346
1754
  }
347
1755
  await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
348
1756
  } catch (err) {
349
- respondError(ctx, err);
1757
+ respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
350
1758
  }
351
1759
  }
352
1760
  async function importExport(ctx) {
@@ -378,9 +1786,10 @@ function createDocumentExportHandlers(deps) {
378
1786
  // src/server/handlers/documents.ts
379
1787
  function createDocumentHandlers(deps) {
380
1788
  const { engine } = deps;
1789
+ const writeBlock = writeBlockLookupOf(engine);
381
1790
  const reads = createDocumentReadHandlers(deps);
382
1791
  async function insert(ctx) {
383
- const body = parseJson(ctx);
1792
+ const body = await parseWriteJson(ctx, writeBlock);
384
1793
  if (!body) return;
385
1794
  const failure = validateDocumentBody(body);
386
1795
  if (failure) {
@@ -395,7 +1804,7 @@ function createDocumentHandlers(deps) {
395
1804
  }
396
1805
  }
397
1806
  async function put(ctx) {
398
- const body = parseJson(ctx);
1807
+ const body = await parseWriteJson(ctx, writeBlock);
399
1808
  if (!body) return;
400
1809
  const failure = validateDocumentBody(body);
401
1810
  if (failure) {
@@ -416,7 +1825,7 @@ function createDocumentHandlers(deps) {
416
1825
  }
417
1826
  }
418
1827
  async function patch(ctx) {
419
- const body = parseJson(ctx);
1828
+ const body = await parseWriteJson(ctx, writeBlock);
420
1829
  if (!body) return;
421
1830
  const failure = validateDocumentBody(body);
422
1831
  if (failure) {
@@ -447,7 +1856,7 @@ function createDocumentHandlers(deps) {
447
1856
  }
448
1857
  }
449
1858
  async function batch(ctx) {
450
- const body = parseJson(ctx);
1859
+ const body = parseJson(ctx, ctx.rawBody && isRemoveOnlyBatch(ctx.rawBody) ? "read" : "write");
451
1860
  if (!body) return;
452
1861
  const failure = validateBatch(body);
453
1862
  if (failure) {
@@ -528,7 +1937,7 @@ function* iterateNdjson(buffer, maxLineBytes) {
528
1937
  if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
529
1938
  if (contentEnd > start) {
530
1939
  lineNumber++;
531
- yield { lineNumber, text: buffer.toString("utf8", start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
1940
+ yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
532
1941
  }
533
1942
  start = end + 1;
534
1943
  }
@@ -544,6 +1953,7 @@ function isPlainObject(value) {
544
1953
  async function runImport(engine, options) {
545
1954
  const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
546
1955
  const bytesTotal = body.length;
1956
+ const writeBlock = writeBlockLookupOf(engine);
547
1957
  const errors = [];
548
1958
  let indexed = 0;
549
1959
  let failed = 0;
@@ -558,7 +1968,12 @@ async function runImport(engine, options) {
558
1968
  indexed += result.succeeded.length;
559
1969
  for (const failure of result.failed) {
560
1970
  const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
561
- recordFailure({ docId: failure.docId, code: serialized.code, message: serialized.message });
1971
+ recordFailure({
1972
+ docId: failure.docId,
1973
+ code: serialized.code,
1974
+ message: serialized.message,
1975
+ ...serialized.details === void 0 ? {} : { details: serialized.details }
1976
+ });
562
1977
  }
563
1978
  };
564
1979
  const inFlight = /* @__PURE__ */ new Set();
@@ -584,9 +1999,17 @@ async function runImport(engine, options) {
584
1999
  try {
585
2000
  for (const line of iterateNdjson(body, maxLineBytes)) {
586
2001
  bytesProcessed = line.bytesConsumed;
2002
+ const refusal = parseRefusalOf(line.bytes);
2003
+ if (refusal !== null) {
2004
+ const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
2005
+ recordFailure(
2006
+ block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
2007
+ );
2008
+ continue;
2009
+ }
587
2010
  let parsed;
588
2011
  try {
589
- parsed = JSON.parse(line.text);
2012
+ parsed = JSON.parse(line.bytes.toString("utf8"));
590
2013
  } catch {
591
2014
  recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
592
2015
  continue;
@@ -697,15 +2120,27 @@ function mapHttpIndexConfig(http) {
697
2120
  if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
698
2121
  if (http.bm25 !== void 0) config.bm25 = http.bm25;
699
2122
  if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
2123
+ if (http.textStorage !== void 0) config.textStorage = http.textStorage;
700
2124
  if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
701
2125
  if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
702
2126
  if (http.strict !== void 0) config.strict = http.strict;
703
2127
  if (http.required !== void 0) config.required = http.required;
704
2128
  if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
705
2129
  if (http.synonyms !== void 0) config.synonyms = http.synonyms;
706
- if (http.stopWords !== void 0) {
2130
+ if (http.tokenizer !== void 0) {
2131
+ if (typeof http.tokenizer !== "string") {
2132
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
2133
+ }
2134
+ config.tokenizer = http.tokenizer;
2135
+ }
2136
+ if (typeof http.stopWords === "string") {
2137
+ config.stopWords = http.stopWords;
2138
+ } else if (http.stopWords !== void 0) {
707
2139
  if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
708
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "stopWords" must be an array of strings');
2140
+ throw new NarsilError(
2141
+ ErrorCodes.CONFIG_INVALID,
2142
+ 'Field "stopWords" must be a registered name or an array of strings'
2143
+ );
709
2144
  }
710
2145
  config.stopWords = new Set(http.stopWords);
711
2146
  }
@@ -734,86 +2169,385 @@ function mapEmbedding(embedding) {
734
2169
  return mapped;
735
2170
  }
736
2171
 
737
- // src/server/handlers/indexes.ts
738
- function createIndexHandlers(deps) {
739
- const { engine } = deps;
740
- async function create(ctx) {
2172
+ // src/server/handlers/indexes.ts
2173
+ function createIndexHandlers(deps) {
2174
+ const { engine } = deps;
2175
+ async function create(ctx) {
2176
+ const body = parseJson(ctx);
2177
+ if (!body) return;
2178
+ const failure = validateCreateIndex(body);
2179
+ if (failure) {
2180
+ rejectInvalid(ctx, failure);
2181
+ return;
2182
+ }
2183
+ try {
2184
+ const config = mapHttpIndexConfig(body.config);
2185
+ await engine.createIndex(body.name, config);
2186
+ respondJson(ctx, { name: body.name }, 201);
2187
+ } catch (err) {
2188
+ respondError(ctx, err);
2189
+ }
2190
+ }
2191
+ async function list(ctx) {
2192
+ try {
2193
+ respondJson(ctx, { indexes: await indexReadsOf(engine).listIndexes() });
2194
+ } catch (err) {
2195
+ respondError(ctx, err);
2196
+ }
2197
+ }
2198
+ async function drop(ctx) {
2199
+ try {
2200
+ await engine.dropIndex(ctx.params[0]);
2201
+ respondJson(ctx, { name: ctx.params[0], dropped: true });
2202
+ } catch (err) {
2203
+ respondError(ctx, err);
2204
+ }
2205
+ }
2206
+ async function stats(ctx) {
2207
+ try {
2208
+ respondJson(ctx, await indexReadsOf(engine).getStats(ctx.params[0]));
2209
+ } catch (err) {
2210
+ respondError(ctx, err);
2211
+ }
2212
+ }
2213
+ async function partitions(ctx) {
2214
+ try {
2215
+ respondJson(ctx, {
2216
+ partitions: await indexReadsOf(engine).getPartitionStats(ctx.params[0])
2217
+ });
2218
+ } catch (err) {
2219
+ respondError(ctx, err);
2220
+ }
2221
+ }
2222
+ async function clear(ctx) {
2223
+ try {
2224
+ await engine.clear(ctx.params[0]);
2225
+ respondJson(ctx, { name: ctx.params[0], cleared: true });
2226
+ } catch (err) {
2227
+ respondError(ctx, err);
2228
+ }
2229
+ }
2230
+ async function open(ctx) {
2231
+ try {
2232
+ await engine.open(ctx.params[0]);
2233
+ respondJson(ctx, { name: ctx.params[0], state: "open" });
2234
+ } catch (err) {
2235
+ respondError(ctx, err);
2236
+ }
2237
+ }
2238
+ async function close(ctx) {
2239
+ try {
2240
+ await engine.close(ctx.params[0]);
2241
+ respondJson(ctx, { name: ctx.params[0], state: "closed" });
2242
+ } catch (err) {
2243
+ respondError(ctx, err);
2244
+ }
2245
+ }
2246
+ return { create, list, drop, stats, partitions, clear, open, close };
2247
+ }
2248
+
2249
+ // src/server/handlers/metrics.ts
2250
+ function labelValue(value) {
2251
+ return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
2252
+ }
2253
+ function labels(pairs) {
2254
+ return `{${pairs.map(([name, value]) => `${name}="${labelValue(value)}"`).join(",")}}`;
2255
+ }
2256
+ function family(name, type, help, samples) {
2257
+ return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
2258
+ }
2259
+ function prometheusText(metrics) {
2260
+ const lines = [
2261
+ ...family(
2262
+ "narsil_replication_lag",
2263
+ "gauge",
2264
+ "Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
2265
+ metrics.replicationLag.map(
2266
+ (figure) => `narsil_replication_lag${labels([
2267
+ ["index", figure.indexName],
2268
+ ["partition", String(figure.partitionId)],
2269
+ ["replica", figure.replica]
2270
+ ])} ${figure.lag}`
2271
+ )
2272
+ ),
2273
+ ...family(
2274
+ "narsil_commit_point",
2275
+ "gauge",
2276
+ "The highest sequence number that the leader of the partition has acknowledged to a client.",
2277
+ metrics.commitPoints.map(
2278
+ (figure) => `narsil_commit_point${labels([
2279
+ ["index", figure.indexName],
2280
+ ["partition", String(figure.partitionId)]
2281
+ ])} ${figure.commitPoint}`
2282
+ )
2283
+ ),
2284
+ ...family(
2285
+ "narsil_commit_point_floor",
2286
+ "gauge",
2287
+ "The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
2288
+ metrics.partitions.map(
2289
+ (figure) => `narsil_commit_point_floor${labels([
2290
+ ["index", figure.indexName],
2291
+ ["partition", String(figure.partitionId)]
2292
+ ])} ${figure.commitPointFloor}`
2293
+ )
2294
+ ),
2295
+ ...family(
2296
+ "narsil_in_sync_replicas",
2297
+ "gauge",
2298
+ "The replicas besides the primary that are in the in-sync set of the partition.",
2299
+ metrics.partitions.map(
2300
+ (figure) => `narsil_in_sync_replicas${labels([
2301
+ ["index", figure.indexName],
2302
+ ["partition", String(figure.partitionId)]
2303
+ ])} ${figure.inSyncReplicas}`
2304
+ )
2305
+ ),
2306
+ ...family(
2307
+ "narsil_vector_rebuild_waiting_for_memory",
2308
+ "gauge",
2309
+ "A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
2310
+ metrics.vectorRebuildsWaitingForMemory.map(
2311
+ (figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
2312
+ ["index", figure.indexName],
2313
+ ["field", figure.fieldName]
2314
+ ])} 1`
2315
+ )
2316
+ ),
2317
+ ...family(
2318
+ "narsil_coordinator_request_seconds",
2319
+ "histogram",
2320
+ "How long the calls from this node to the cluster coordinator took.",
2321
+ metrics.coordinatorRequests.flatMap((figure) => {
2322
+ const operation = ["operation", figure.operation];
2323
+ return [
2324
+ ...figure.bucketBoundsSeconds.map(
2325
+ (bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
2326
+ ),
2327
+ `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
2328
+ `narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
2329
+ `narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
2330
+ ];
2331
+ })
2332
+ )
2333
+ ];
2334
+ return `${lines.join("\n")}
2335
+ `;
2336
+ }
2337
+ function createMetricsHandler(deps) {
2338
+ return async function metrics(ctx) {
2339
+ try {
2340
+ if (deps.cluster === void 0) {
2341
+ throw new NarsilError(
2342
+ ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
2343
+ "This server fronts a single engine, so it reports no cluster metrics",
2344
+ {}
2345
+ );
2346
+ }
2347
+ const text = prometheusText(await deps.cluster.getMetrics());
2348
+ if (ctx.abort.aborted) return;
2349
+ sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
2350
+ } catch (err) {
2351
+ respondError(ctx, err);
2352
+ }
2353
+ };
2354
+ }
2355
+
2356
+ // src/server/handlers/query-suggestions.ts
2357
+ var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
2358
+ function readObject(ctx) {
2359
+ const body = parseJson(ctx);
2360
+ if (body === void 0) return void 0;
2361
+ if (!isRecord(body)) {
2362
+ badRequest(ctx.res, "The body must be a JSON object");
2363
+ return void 0;
2364
+ }
2365
+ return body;
2366
+ }
2367
+ function readQuery(ctx, body) {
2368
+ if (typeof body.query === "string") return body.query;
2369
+ badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
2370
+ return void 0;
2371
+ }
2372
+ function isStringList(value) {
2373
+ return Array.isArray(value) && value.every((entry) => typeof entry === "string");
2374
+ }
2375
+ function isIdentity(value) {
2376
+ return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
2377
+ }
2378
+ function previewLimitOf(raw) {
2379
+ if (raw === null) return void 0;
2380
+ return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
2381
+ }
2382
+ async function identityOf(options, ctx) {
2383
+ if (ctx.hookContext === null) return void 0;
2384
+ try {
2385
+ const identity = await options.identify(ctx.hookContext);
2386
+ if (identity === null) return null;
2387
+ return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
2388
+ } catch {
2389
+ return void 0;
2390
+ }
2391
+ }
2392
+ function noIdentify(indexName) {
2393
+ return new NarsilError(
2394
+ ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
2395
+ "This server has no identify function for popular query suggestions, so it counts no search",
2396
+ { indexName }
2397
+ );
2398
+ }
2399
+ function createQuerySuggestionHandlers(deps, options) {
2400
+ const { engine, tasks } = deps;
2401
+ const suggestions = () => querySuggestions(engine);
2402
+ async function update(ctx) {
741
2403
  const body = parseJson(ctx);
742
- if (!body) return;
743
- const failure = validateCreateIndex(body);
744
- if (failure) {
745
- rejectInvalid(ctx, failure);
746
- return;
2404
+ if (body === void 0) return;
2405
+ try {
2406
+ await suggestions().updateSetting(ctx.params[0], body);
2407
+ respondJson(ctx, { ok: true });
2408
+ } catch (err) {
2409
+ respondError(ctx, err);
747
2410
  }
2411
+ }
2412
+ async function status(ctx) {
748
2413
  try {
749
- const config = mapHttpIndexConfig(body.config);
750
- await engine.createIndex(body.name, config);
751
- respondJson(ctx, { name: body.name }, 201);
2414
+ respondJson(ctx, await suggestions().status(ctx.params[0]));
752
2415
  } catch (err) {
753
2416
  respondError(ctx, err);
754
2417
  }
755
2418
  }
756
- function list(ctx) {
2419
+ async function record(ctx) {
2420
+ const body = readObject(ctx);
2421
+ if (body === void 0) return;
2422
+ const extra = Object.keys(body).filter((key) => key !== "query");
2423
+ if (extra.length > 0) {
2424
+ badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
2425
+ fields: extra
2426
+ });
2427
+ return;
2428
+ }
2429
+ const query = readQuery(ctx, body);
2430
+ if (query === void 0) return;
2431
+ const indexName = ctx.params[0];
757
2432
  try {
758
- respondJson(ctx, { indexes: engine.listIndexes() });
2433
+ if (options === void 0) throw noIdentify(indexName);
2434
+ const identity = await identityOf(options, ctx);
2435
+ if (identity === void 0) {
2436
+ if (!ctx.abort.aborted) {
2437
+ sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
2438
+ }
2439
+ return;
2440
+ }
2441
+ if (identity === null) {
2442
+ respondJson(ctx, { recorded: false, outcome: "unidentified" });
2443
+ return;
2444
+ }
2445
+ respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
759
2446
  } catch (err) {
760
2447
  respondError(ctx, err);
761
2448
  }
762
2449
  }
763
- async function drop(ctx) {
2450
+ async function build(ctx) {
2451
+ const indexName = ctx.params[0];
764
2452
  try {
765
- await engine.dropIndex(ctx.params[0]);
766
- respondJson(ctx, { name: ctx.params[0], dropped: true });
2453
+ const operations = suggestions();
2454
+ const current = await operations.status(indexName);
2455
+ if (current.setting?.enabled !== true) {
2456
+ throw new NarsilError(
2457
+ ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
2458
+ `Index "${indexName}" has popular query suggestions off`,
2459
+ { indexName }
2460
+ );
2461
+ }
2462
+ const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
2463
+ const finished = await operations.build(indexName);
2464
+ context.reportBuild(finished);
2465
+ if (finished.status === "failed") {
2466
+ throw new NarsilError(
2467
+ finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
2468
+ "The build of popular query suggestions failed",
2469
+ { indexName, buildId: finished.buildId }
2470
+ );
2471
+ }
2472
+ });
2473
+ respondJson(ctx, task, 202);
767
2474
  } catch (err) {
768
2475
  respondError(ctx, err);
769
2476
  }
770
2477
  }
771
- function stats(ctx) {
2478
+ async function preview(ctx) {
2479
+ const limit = previewLimitOf(ctx.query.get("limit"));
2480
+ if (limit === null) {
2481
+ badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
2482
+ return;
2483
+ }
772
2484
  try {
773
- respondJson(ctx, engine.getStats(ctx.params[0]));
2485
+ respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
774
2486
  } catch (err) {
775
2487
  respondError(ctx, err);
776
2488
  }
777
2489
  }
778
- function partitions(ctx) {
2490
+ async function readBans(ctx) {
779
2491
  try {
780
- respondJson(ctx, { partitions: engine.getPartitionStats(ctx.params[0]) });
2492
+ respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
781
2493
  } catch (err) {
782
2494
  respondError(ctx, err);
783
2495
  }
784
2496
  }
785
- async function clear(ctx) {
2497
+ async function replaceBans(ctx) {
2498
+ const body = readObject(ctx);
2499
+ if (body === void 0) return;
2500
+ if (!isStringList(body.bans)) {
2501
+ badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
2502
+ return;
2503
+ }
786
2504
  try {
787
- await engine.clear(ctx.params[0]);
788
- respondJson(ctx, { name: ctx.params[0], cleared: true });
2505
+ respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
789
2506
  } catch (err) {
790
2507
  respondError(ctx, err);
791
2508
  }
792
2509
  }
793
- async function open(ctx) {
2510
+ async function remove(ctx) {
2511
+ const body = readObject(ctx);
2512
+ if (body === void 0) return;
2513
+ const query = readQuery(ctx, body);
2514
+ if (query === void 0) return;
794
2515
  try {
795
- await engine.open(ctx.params[0]);
796
- respondJson(ctx, { name: ctx.params[0], state: "open" });
2516
+ const held = await suggestions().removeQuery(ctx.params[0], query);
2517
+ respondJson(ctx, { suggestionIndexes: held });
797
2518
  } catch (err) {
798
2519
  respondError(ctx, err);
799
2520
  }
800
2521
  }
801
- async function close(ctx) {
2522
+ async function manual(ctx) {
2523
+ const body = readObject(ctx);
2524
+ if (body === void 0) return;
2525
+ const query = readQuery(ctx, body);
2526
+ if (query === void 0) return;
2527
+ const { group, count } = body;
2528
+ if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
2529
+ badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
2530
+ return;
2531
+ }
802
2532
  try {
803
- await engine.close(ctx.params[0]);
804
- respondJson(ctx, { name: ctx.params[0], state: "closed" });
2533
+ const name = await suggestions().addSuggestion(ctx.params[0], {
2534
+ group,
2535
+ query,
2536
+ ...count === void 0 ? {} : { count }
2537
+ });
2538
+ respondJson(ctx, { indexName: name });
805
2539
  } catch (err) {
806
2540
  respondError(ctx, err);
807
2541
  }
808
2542
  }
809
- return { create, list, drop, stats, partitions, clear, open, close };
2543
+ return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
810
2544
  }
811
2545
 
812
2546
  // src/server/handlers/search.ts
813
2547
  function createSearchHandlers(deps) {
814
2548
  const { engine, limits, allowRegex } = deps;
815
2549
  async function search(ctx) {
816
- const params = parseJson(ctx);
2550
+ const params = parseJson(ctx, "read");
817
2551
  if (!params) return;
818
2552
  const failure = validateQuery(params, limits);
819
2553
  if (failure) {
@@ -822,13 +2556,23 @@ function createSearchHandlers(deps) {
822
2556
  }
823
2557
  if (refuseRegex(ctx, params.filters, allowRegex)) return;
824
2558
  try {
825
- respondJson(ctx, await engine.query(ctx.params[0], withServerRerankWindow(params, limits)));
2559
+ const query = withServerRerankWindow(params, limits);
2560
+ const result = await searchWithSignal(ctx, query);
2561
+ await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
826
2562
  } catch (err) {
827
2563
  respondError(ctx, err);
828
2564
  }
829
2565
  }
2566
+ async function searchWithSignal(ctx, params) {
2567
+ bindQuerySignal(params, requestSignal(ctx.abort));
2568
+ try {
2569
+ return await engine.query(ctx.params[0], params);
2570
+ } finally {
2571
+ clearQuerySignal(params);
2572
+ }
2573
+ }
830
2574
  async function preflight(ctx) {
831
- const params = parseJson(ctx);
2575
+ const params = parseJson(ctx, "read");
832
2576
  if (!params) return;
833
2577
  const failure = validateQuery(params, limits);
834
2578
  if (failure) {
@@ -837,13 +2581,18 @@ function createSearchHandlers(deps) {
837
2581
  }
838
2582
  if (refuseRegex(ctx, params.filters, allowRegex)) return;
839
2583
  try {
840
- respondJson(ctx, await engine.preflight(ctx.params[0], params));
2584
+ bindQuerySignal(params, requestSignal(ctx.abort));
2585
+ try {
2586
+ respondJson(ctx, await engine.preflight(ctx.params[0], params));
2587
+ } finally {
2588
+ clearQuerySignal(params);
2589
+ }
841
2590
  } catch (err) {
842
2591
  respondError(ctx, err);
843
2592
  }
844
2593
  }
845
2594
  async function suggest(ctx) {
846
- const params = parseJson(ctx);
2595
+ const params = parseJson(ctx, "read");
847
2596
  if (!params) return;
848
2597
  const failure = validateSuggest(params);
849
2598
  if (failure) {
@@ -904,6 +2653,15 @@ function assertLimit(field, value, minimum) {
904
2653
  { field, value, minimum }
905
2654
  );
906
2655
  }
2656
+ function assertFraction(field, value) {
2657
+ if (value === void 0) return;
2658
+ if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
2659
+ throw new NarsilError(
2660
+ ErrorCodes.CONFIG_INVALID,
2661
+ `limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
2662
+ { field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
2663
+ );
2664
+ }
907
2665
  function assertRerankWindowCap(value, engineCap) {
908
2666
  if (value === void 0) return;
909
2667
  const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
@@ -918,8 +2676,19 @@ function assertServerLimits(limits, engineRerankWindow) {
918
2676
  if (limits === void 0) return;
919
2677
  for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
920
2678
  for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
2679
+ assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
921
2680
  assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
922
2681
  }
2682
+ function assertQuerySuggestionsOptions(options) {
2683
+ if (options === void 0) return;
2684
+ const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
2685
+ if (typeof identify === "function") return;
2686
+ throw new NarsilError(
2687
+ ErrorCodes.CONFIG_INVALID,
2688
+ "querySuggestions must carry an identify function that returns the person and the group of a request",
2689
+ { field: "querySuggestions.identify" }
2690
+ );
2691
+ }
923
2692
  function assertAllowRegex(allowRegex) {
924
2693
  if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
925
2694
  throw new NarsilError(
@@ -929,231 +2698,36 @@ function assertAllowRegex(allowRegex) {
929
2698
  );
930
2699
  }
931
2700
 
932
- // src/server/request-threads/captured-response.ts
933
- var DEFAULT_STATUS = 200;
934
- var NO_RESPONSE_STATUS = 500;
935
- function statusOf(line) {
936
- const parsed = Number.parseInt(line, 10);
937
- return Number.isFinite(parsed) ? parsed : DEFAULT_STATUS;
938
- }
939
- function bodyBytes(body) {
940
- if (body === void 0) return new Uint8Array(0);
941
- if (typeof body === "string") return Buffer.from(body, "utf8");
942
- return body;
943
- }
944
- function createRelayAbort() {
945
- const listeners = [];
946
- let aborted = false;
947
- return {
948
- get aborted() {
949
- return aborted;
950
- },
951
- onAbort(fn) {
952
- if (aborted) fn();
953
- else listeners.push(fn);
954
- },
955
- trigger() {
956
- if (aborted) return;
957
- aborted = true;
958
- for (const fn of listeners) fn();
959
- }
960
- };
961
- }
962
- function createCapturingResponse(remoteAddress, requestId, sendPart) {
963
- let status = DEFAULT_STATUS;
964
- const headers = [];
965
- let finished = false;
966
- let streaming = false;
967
- let inFlight = 0;
968
- let writable = null;
969
- const abort = createRelayAbort();
970
- let resolveDone = () => void 0;
971
- const done = new Promise((resolve) => {
972
- resolveDone = resolve;
973
- });
974
- function finish(body) {
975
- if (finished) return;
976
- finished = true;
977
- if (streaming) {
978
- sendPart({ type: "stream-end", requestId, failed: false }, []);
979
- resolveDone(null);
980
- return;
981
- }
982
- resolveDone({ status, headers, body });
983
- }
984
- function startStreaming() {
985
- if (streaming) return;
986
- streaming = true;
987
- sendPart({ type: "stream-start", requestId, status, headers }, []);
988
- }
989
- const sink = {
990
- cork(callback) {
991
- callback();
992
- return sink;
993
- },
994
- writeStatus(line) {
995
- if (!finished && !streaming) status = statusOf(line);
996
- return sink;
997
- },
998
- writeHeader(key, value) {
999
- if (!finished && !streaming) headers.push([key, value]);
1000
- return sink;
1001
- },
1002
- end(body) {
1003
- finish(bodyBytes(body));
1004
- return sink;
1005
- },
1006
- write(chunk) {
1007
- if (finished || abort.aborted) return true;
1008
- startStreaming();
1009
- const copy = chunk.slice();
1010
- inFlight += 1;
1011
- sendPart({ type: "stream-chunk", requestId, chunk: copy }, [copy.buffer]);
1012
- return inFlight < RELAY_STREAM_CHUNKS_IN_FLIGHT;
1013
- },
1014
- pause: () => void 0,
1015
- resume: () => void 0,
1016
- close() {
1017
- if (finished) return sink;
1018
- finished = true;
1019
- if (streaming) {
1020
- sendPart({ type: "stream-end", requestId, failed: true }, []);
1021
- resolveDone(null);
1022
- } else {
1023
- status = NO_RESPONSE_STATUS;
1024
- resolveDone({ status, headers, body: new Uint8Array(0) });
1025
- }
1026
- return sink;
1027
- },
1028
- endWithoutBody() {
1029
- finish(new Uint8Array(0));
1030
- return sink;
1031
- },
1032
- tryEnd(body) {
1033
- finish(body);
1034
- return [true, true];
1035
- },
1036
- getWriteOffset: () => 0,
1037
- onWritable(handler) {
1038
- writable = handler;
1039
- return sink;
1040
- },
1041
- onAborted: () => sink,
1042
- onData: () => sink,
1043
- getRemoteAddressAsText: () => new TextEncoder().encode(remoteAddress).buffer
1044
- };
1045
- return {
1046
- sink,
1047
- abort,
1048
- done,
1049
- credit() {
1050
- inFlight = Math.max(0, inFlight - 1);
1051
- if (inFlight >= RELAY_STREAM_CHUNKS_IN_FLIGHT || writable === null) return;
1052
- const handler = writable;
1053
- writable = null;
1054
- handler(0);
1055
- },
1056
- cancel() {
1057
- abort.trigger();
1058
- if (writable !== null) {
1059
- const handler = writable;
1060
- writable = null;
1061
- handler(0);
1062
- }
1063
- if (!finished) {
1064
- finished = true;
1065
- resolveDone(null);
1066
- }
1067
- },
1068
- finishUnanswered() {
1069
- if (finished) return;
1070
- if (streaming) {
1071
- finished = true;
1072
- sendPart({ type: "stream-end", requestId, failed: true }, []);
1073
- resolveDone(null);
1074
- return;
1075
- }
1076
- status = NO_RESPONSE_STATUS;
1077
- finish(new Uint8Array(0));
1078
- }
1079
- };
1080
- }
1081
- function toRelayedResponse(requestId, captured) {
1082
- return { type: "response", requestId, status: captured.status, headers: captured.headers, body: captured.body };
1083
- }
1084
-
1085
- // src/server/request-threads/host.ts
1086
- var RELAYED_TYPES = /* @__PURE__ */ new Set([
1087
- "request",
1088
- "authorize",
1089
- "embed",
1090
- "rerank",
1091
- "stream-credit",
1092
- "stream-cancel"
1093
- ]);
1094
- function isRelayRequest(message) {
1095
- if (typeof message !== "object" || message === null) return false;
1096
- const { type, requestId } = message;
1097
- if (type === "touch" || type === "leaving") return true;
1098
- return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
1099
- }
1100
- function spooledBodyOf(relayed) {
1101
- if (relayed === null) return null;
1102
- return {
1103
- path: relayed.path,
1104
- bytes: relayed.bytes,
1105
- remove: () => rm(path.dirname(relayed.path), { recursive: true, force: true })
1106
- };
1107
- }
1108
- async function answerRequest(handlers, request, port, streams) {
1109
- const captured = createCapturingResponse(
1110
- request.hookContext?.remoteAddress ?? "",
1111
- request.requestId,
1112
- (part, moved) => port.postMessage(part, moved)
1113
- );
1114
- streams.set(request.requestId, captured);
1115
- const ctx = {
1116
- res: captured.sink,
1117
- params: request.params,
1118
- query: new URLSearchParams(request.query),
1119
- contentType: request.contentType,
1120
- rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody.buffer, request.rawBody.byteOffset, request.rawBody.byteLength),
1121
- spooledBody: spooledBodyOf(request.spooledBody),
1122
- abort: captured.abort,
1123
- hookContext: request.hookContext
1124
- };
1125
- try {
1126
- await handlers[request.route](ctx);
1127
- } catch {
1128
- sendError(ctx.res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
1129
- }
1130
- captured.finishUnanswered();
1131
- const answered = await captured.done;
1132
- streams.delete(request.requestId);
1133
- return answered === null ? null : toRelayedResponse(request.requestId, answered);
1134
- }
2701
+ // src/server/request-threads/relay-answers.ts
1135
2702
  async function answerAuthorize(authorize, request) {
1136
2703
  if (authorize === void 0) {
1137
- return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
2704
+ return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
1138
2705
  }
1139
2706
  const authorization = await authorize(request.context);
1140
2707
  if (authorization.allowed) {
1141
- return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
2708
+ return {
2709
+ type: "authorized",
2710
+ requestId: request.requestId,
2711
+ denial: null,
2712
+ hookError: false,
2713
+ expiresAt: authorization.expiresAt
2714
+ };
1142
2715
  }
1143
2716
  return {
1144
2717
  type: "authorized",
1145
2718
  requestId: request.requestId,
1146
2719
  denial: authorization.denial,
1147
- hookError: authorization.denial === null
2720
+ hookError: authorization.denial === null,
2721
+ expiresAt: null
1148
2722
  };
1149
2723
  }
1150
- async function answerEmbed(core, request) {
2724
+ async function answerEmbed(core, request, signal) {
1151
2725
  try {
1152
2726
  const entry = core.requireIndex(request.indexName);
1153
2727
  const resolved = await resolveVectorText(
1154
2728
  { vector: request.vector },
1155
2729
  entry.embeddingAdapter,
1156
- core.abortController.signal,
2730
+ signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
1157
2731
  entry.embeddingAdapterName
1158
2732
  );
1159
2733
  const value = resolved.vector?.value;
@@ -1168,15 +2742,56 @@ async function answerEmbed(core, request) {
1168
2742
  return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
1169
2743
  }
1170
2744
  }
1171
- async function answerRerank(core, request) {
2745
+ async function answerRerank(core, request, signal) {
1172
2746
  try {
1173
- const scores = await rerankOnHost(core, request.adapter, request.query, request.documents);
2747
+ const { rerankOnHost } = await loadRerankRuntime();
2748
+ const scores = await rerankOnHost(core, request.adapter, request.query, request.documents, signal);
1174
2749
  return { type: "reranked", requestId: request.requestId, scores, error: null };
1175
2750
  } catch (err) {
1176
2751
  if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
1177
2752
  return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
1178
2753
  }
1179
2754
  }
2755
+ async function answerAwaitWrites(core, request) {
2756
+ await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
2757
+ return { type: "writes-applied", requestId: request.requestId };
2758
+ }
2759
+ async function answerFeedAdmit(feeds, request) {
2760
+ if (feeds !== null) return feeds.admit(request);
2761
+ return {
2762
+ type: "feed-admitted",
2763
+ requestId: request.requestId,
2764
+ feedId: null,
2765
+ error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
2766
+ };
2767
+ }
2768
+
2769
+ // src/server/request-threads/host.ts
2770
+ function threadClosed(port, executor, clearSlots) {
2771
+ return new Promise((resolve) => {
2772
+ port.once("close", () => {
2773
+ clearSlots();
2774
+ resolve();
2775
+ });
2776
+ void executor.whenAbandoned?.().then(resolve);
2777
+ });
2778
+ }
2779
+ var RELAYED_TYPES = /* @__PURE__ */ new Set([
2780
+ "request",
2781
+ "authorize",
2782
+ "embed",
2783
+ "rerank",
2784
+ "await-writes",
2785
+ "stream-credit",
2786
+ "stream-cancel",
2787
+ "feed-admit"
2788
+ ]);
2789
+ function isRelayRequest(message) {
2790
+ if (typeof message !== "object" || message === null) return false;
2791
+ const { type, requestId } = message;
2792
+ if (type === "touch" || type === "leaving") return true;
2793
+ return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
2794
+ }
1180
2795
  function touchIntervalFor(copyIdleTimeoutMs) {
1181
2796
  return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
1182
2797
  }
@@ -1188,20 +2803,42 @@ async function startRequestThreads(options) {
1188
2803
  const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
1189
2804
  const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
1190
2805
  const threads = /* @__PURE__ */ new Map();
2806
+ const slotUsers = /* @__PURE__ */ new Map();
2807
+ const longQueriesInFlight = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
2808
+ let longQueryThread = null;
2809
+ const closeAtExit = /* @__PURE__ */ new Set();
2810
+ const retiring = /* @__PURE__ */ new Set();
1191
2811
  let stopped = false;
2812
+ function closeThreadsAtExit() {
2813
+ closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
2814
+ }
1192
2815
  function releaseDescriptor(thread) {
2816
+ thread.feeds?.closeAll();
2817
+ closeAtExit.delete(thread.closeAtExit);
1193
2818
  if (thread.descriptor !== null) {
1194
2819
  try {
1195
2820
  acceptor.removeChildAppDescriptor(thread.descriptor);
1196
2821
  } catch {
1197
2822
  }
2823
+ thread.descriptor = null;
1198
2824
  }
1199
2825
  confirmDeparture(thread.departure);
1200
2826
  }
1201
2827
  function relayFrom(thread) {
1202
2828
  const { port } = thread;
1203
2829
  const streams = /* @__PURE__ */ new Map();
2830
+ const calls = /* @__PURE__ */ new Map();
2831
+ port.on("close", () => {
2832
+ thread.feeds?.closeAll();
2833
+ for (const stream of streams.values()) stream.cancel();
2834
+ for (const call of calls.values()) call.abort();
2835
+ calls.clear();
2836
+ });
1204
2837
  port.on("message", (message) => {
2838
+ if (isThreadFeedNote(message)) {
2839
+ thread.feeds?.handle(message);
2840
+ return;
2841
+ }
1205
2842
  if (!isRelayRequest(message)) return;
1206
2843
  if (message.type === "leaving") {
1207
2844
  releaseDescriptor(thread);
@@ -1217,31 +2854,116 @@ async function startRequestThreads(options) {
1217
2854
  }
1218
2855
  if (message.type === "stream-cancel") {
1219
2856
  streams.get(message.requestId)?.cancel();
2857
+ calls.get(message.requestId)?.abort();
2858
+ calls.delete(message.requestId);
1220
2859
  return;
1221
2860
  }
1222
- const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "embed" ? answerEmbed(core, message) : answerRerank(core, message);
1223
- void reply.then((answer) => {
1224
- if (answer !== null) port.postMessage(answer);
1225
- });
2861
+ const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
2862
+ if (call !== void 0) calls.set(message.requestId, call);
2863
+ const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
2864
+ void reply.then(
2865
+ (answer) => {
2866
+ calls.delete(message.requestId);
2867
+ if (answer !== null) port.postMessage(answer);
2868
+ },
2869
+ () => {
2870
+ calls.delete(message.requestId);
2871
+ }
2872
+ );
1226
2873
  });
1227
2874
  }
1228
- async function stopThread(workerId, thread) {
2875
+ function connectToLongQueries(thread) {
2876
+ if (longQueryThread === null) return;
2877
+ const channel = new MessageChannel();
2878
+ thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
2879
+ longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
2880
+ }
2881
+ function dropLongQueryThread() {
2882
+ if (longQueryThread === null) return;
2883
+ longQueryThread.port.close();
2884
+ longQueryThread = null;
2885
+ }
2886
+ function retireThread(workerId, thread) {
1229
2887
  releaseDescriptor(thread);
1230
- thread.port.close();
1231
- clearGateSlot(options.gate, gateSlotOfWorker(workerId));
1232
- await thread.executor.execute({ type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(() => void 0);
2888
+ thread.port.unref();
2889
+ retiring.add(thread.closed);
2890
+ void thread.closed.then(() => retiring.delete(thread.closed));
2891
+ void executeUntilAnswered(thread.executor, { type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(
2892
+ () => void 0
2893
+ );
2894
+ return thread.closed;
1233
2895
  }
1234
2896
  const listener = {
2897
+ async onLongQueryWorkerReady(workerId, executor) {
2898
+ if (stopped) return;
2899
+ dropLongQueryThread();
2900
+ const channel = new MessageChannel();
2901
+ const relaying = {
2902
+ feeds: null,
2903
+ descriptor: null,
2904
+ port: channel.port2,
2905
+ executor,
2906
+ departure: createDepartureWord(),
2907
+ closeAtExit: createCloseAtExitWords(),
2908
+ closed: new Promise((resolve) => channel.port2.once("close", resolve))
2909
+ };
2910
+ relayFrom(relaying);
2911
+ try {
2912
+ await executor.execute(
2913
+ {
2914
+ type: "serveLongQueries",
2915
+ requestId: `serve-long-queries-${workerId}`,
2916
+ settings: {
2917
+ port: channel.port1,
2918
+ touchIntervalMs,
2919
+ limits: options.limits,
2920
+ patternWorkCap: options.core.patternWorkCap,
2921
+ maxRerankWindow: options.core.rerankSettings.maxWindow,
2922
+ allowRegex: options.allowRegex
2923
+ }
2924
+ },
2925
+ [channel.port1]
2926
+ );
2927
+ } catch (err) {
2928
+ channel.port2.close();
2929
+ console.warn(`Worker ${workerId} could not start answering long queries:`, err);
2930
+ return;
2931
+ }
2932
+ if (stopped) {
2933
+ channel.port2.close();
2934
+ return;
2935
+ }
2936
+ longQueryThread = { workerId, port: channel.port2, executor };
2937
+ for (const thread of threads.values()) connectToLongQueries(thread);
2938
+ },
1235
2939
  async onWorkerReady(workerId, executor) {
1236
2940
  if (stopped) return;
1237
2941
  const channel = new MessageChannel();
2942
+ const slot = gateSlotOfWorker(workerId);
2943
+ slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
1238
2944
  const thread = {
2945
+ feeds: options.feeds?.forThread(channel.port2) ?? null,
1239
2946
  descriptor: null,
1240
2947
  port: channel.port2,
1241
2948
  executor,
1242
- departure: createDepartureWord()
2949
+ departure: createDepartureWord(),
2950
+ closeAtExit: createCloseAtExitWords(),
2951
+ closed: threadClosed(channel.port2, executor, () => {
2952
+ const remaining = (slotUsers.get(slot) ?? 1) - 1;
2953
+ if (remaining > 0) {
2954
+ slotUsers.set(slot, remaining);
2955
+ return;
2956
+ }
2957
+ slotUsers.delete(slot);
2958
+ clearGateSlot(options.gate, slot);
2959
+ clearBodyBudgetSlot(options.bodyBudget, slot);
2960
+ })
1243
2961
  };
1244
2962
  relayFrom(thread);
2963
+ closeAtExit.add(thread.closeAtExit);
2964
+ const longQueries = longQueryThread;
2965
+ const longQueryChannel = longQueries === null ? null : new MessageChannel();
2966
+ const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
1245
2967
  let result;
1246
2968
  try {
1247
2969
  result = await executor.execute(
@@ -1251,9 +2973,11 @@ async function startRequestThreads(options) {
1251
2973
  settings: {
1252
2974
  port: channel.port1,
1253
2975
  departure: thread.departure.buffer,
2976
+ closeAtExit: thread.closeAtExit.buffer,
1254
2977
  gate: options.gate,
1255
2978
  gateSlot: gateSlotOfWorker(workerId),
1256
2979
  maxConcurrentRequests: options.limits.maxConcurrentRequests,
2980
+ bodyBudget: options.bodyBudget,
1257
2981
  touchIntervalMs,
1258
2982
  cors: options.cors,
1259
2983
  limits: options.limits,
@@ -1262,33 +2986,50 @@ async function startRequestThreads(options) {
1262
2986
  searchHooks,
1263
2987
  patternWorkCap: options.core.patternWorkCap,
1264
2988
  maxRerankWindow: options.core.rerankSettings.maxWindow,
1265
- allowRegex: options.allowRegex
2989
+ allowRegex: options.allowRegex,
2990
+ feeds: Object.keys(options.feeds?.gateways ?? {}),
2991
+ longQueriesInFlight,
2992
+ longQueryPort: longQueryChannel?.port1 ?? null
1266
2993
  }
1267
2994
  },
1268
- [channel.port1]
2995
+ transfer
1269
2996
  );
1270
2997
  } catch (err) {
2998
+ closeAtExit.delete(thread.closeAtExit);
1271
2999
  channel.port2.close();
3000
+ longQueryChannel?.port2.close();
1272
3001
  console.warn(`Worker ${workerId} could not start taking requests:`, err);
1273
3002
  return;
1274
3003
  }
1275
3004
  thread.descriptor = result.descriptor;
1276
3005
  if (stopped || departureConfirmed(thread.departure)) {
1277
- await stopThread(workerId, thread);
3006
+ longQueryChannel?.port2.close();
3007
+ await retireThread(workerId, thread);
1278
3008
  return;
1279
3009
  }
3010
+ if (longQueryChannel !== null && longQueries !== null && longQueryThread === longQueries) {
3011
+ longQueries.port.postMessage({ type: "long-query-port", port: longQueryChannel.port2 }, [
3012
+ longQueryChannel.port2
3013
+ ]);
3014
+ } else {
3015
+ longQueryChannel?.port2.close();
3016
+ connectToLongQueries(thread);
3017
+ }
1280
3018
  acceptor.addChildAppDescriptor(result.descriptor);
1281
3019
  threads.set(workerId, thread);
1282
3020
  },
1283
3021
  onWorkerGone(workerId) {
3022
+ if (longQueryThread?.workerId === workerId) {
3023
+ dropLongQueryThread();
3024
+ return;
3025
+ }
1284
3026
  const thread = threads.get(workerId);
1285
3027
  if (thread === void 0) return;
1286
3028
  threads.delete(workerId);
1287
- releaseDescriptor(thread);
1288
- thread.port.close();
1289
- clearGateSlot(options.gate, gateSlotOfWorker(workerId));
3029
+ void retireThread(workerId, thread);
1290
3030
  }
1291
3031
  };
3032
+ process.on("exit", closeThreadsAtExit);
1292
3033
  await core.orchestrator.serveRequestsOnWorkers(listener);
1293
3034
  return {
1294
3035
  get threadCount() {
@@ -1296,16 +3037,87 @@ async function startRequestThreads(options) {
1296
3037
  },
1297
3038
  async stop() {
1298
3039
  stopped = true;
1299
- core.orchestrator.stopRequestThreads();
3040
+ process.off("exit", closeThreadsAtExit);
1300
3041
  const stopping = [...threads.entries()].map(([workerId, thread]) => {
1301
3042
  threads.delete(workerId);
1302
- return stopThread(workerId, thread);
3043
+ return retireThread(workerId, thread);
1303
3044
  });
1304
- await Promise.all(stopping);
3045
+ core.orchestrator.stopRequestThreads();
3046
+ const longQueries = longQueryThread;
3047
+ dropLongQueryThread();
3048
+ if (longQueries !== null) {
3049
+ stopping.push(
3050
+ longQueries.executor.execute({ type: "stopServing", requestId: `stop-long-queries-${longQueries.workerId}` }).then(
3051
+ () => void 0,
3052
+ () => void 0
3053
+ )
3054
+ );
3055
+ }
3056
+ await Promise.all([...stopping, ...retiring]);
1305
3057
  }
1306
3058
  };
1307
3059
  }
1308
3060
 
3061
+ // src/server/sync/options.ts
3062
+ var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
3063
+ var FEED_REFUSALS = {
3064
+ sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
3065
+ changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
3066
+ };
3067
+ function assertCount(feed, field, value) {
3068
+ if (value === void 0) return;
3069
+ if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
3070
+ throw new NarsilError(
3071
+ ErrorCodes.CONFIG_INVALID,
3072
+ `${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
3073
+ { field, value: String(value) }
3074
+ );
3075
+ }
3076
+ function resolveFeedOptions(feed, value) {
3077
+ if (value === void 0 || value === false) return null;
3078
+ if (value !== true && (typeof value !== "object" || value === null)) {
3079
+ throw new NarsilError(
3080
+ ErrorCodes.CONFIG_INVALID,
3081
+ `${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
3082
+ { field: feed, value: String(value) }
3083
+ );
3084
+ }
3085
+ const settings = value === true ? {} : value;
3086
+ for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
3087
+ return {
3088
+ windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
3089
+ maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
3090
+ maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
3091
+ maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
3092
+ };
3093
+ }
3094
+ function assertFeedHasAuthentication(feed, resolved, options) {
3095
+ if (resolved === null || options.onRequest !== void 0) return;
3096
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
3097
+ }
3098
+
3099
+ // src/server/sync/setting-handlers.ts
3100
+ function createSyncSettingHandlers(engine) {
3101
+ async function getSync(ctx) {
3102
+ try {
3103
+ respondJson(ctx, await engine.getSync(ctx.params[0]));
3104
+ } catch (err) {
3105
+ respondError(ctx, err);
3106
+ }
3107
+ }
3108
+ async function updateSync(ctx) {
3109
+ const body = parseJson(ctx);
3110
+ if (body === void 0) return;
3111
+ try {
3112
+ await engine.updateSync(ctx.params[0], body);
3113
+ respondJson(ctx, { ok: true });
3114
+ } catch (err) {
3115
+ respondError(ctx, err);
3116
+ }
3117
+ }
3118
+ return { getSync, updateSync };
3119
+ }
3120
+
1309
3121
  // src/server/task-status.ts
1310
3122
  var TERMINAL_TASK_STATUSES = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled"]);
1311
3123
  function isTerminalTaskStatus(status) {
@@ -1390,7 +3202,7 @@ var TaskRegistry = class {
1390
3202
  * @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
1391
3203
  * already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
1392
3204
  * one holds its own working set until it finishes. */
1393
- async start(type, indexName, op, progress) {
3205
+ async start(type, indexName, op, progress, backupName) {
1394
3206
  if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
1395
3207
  throw new NarsilError(
1396
3208
  ServerErrorCodes.TOO_MANY_REQUESTS,
@@ -1403,6 +3215,7 @@ var TaskRegistry = class {
1403
3215
  id: randomUUID(),
1404
3216
  type,
1405
3217
  indexName,
3218
+ ...backupName === void 0 ? {} : { backupName },
1406
3219
  owner: this.instanceId,
1407
3220
  status: "running",
1408
3221
  createdAt: now,
@@ -1491,9 +3304,9 @@ var TaskRegistry = class {
1491
3304
  return task.writes.catch(() => {
1492
3305
  });
1493
3306
  }
1494
- reportProgress(task, progress) {
3307
+ reportProgress(task, change) {
1495
3308
  if (task.settled) return;
1496
- task.record = { ...task.record, progress };
3309
+ task.record = { ...task.record, ...change };
1497
3310
  const now = Date.now();
1498
3311
  if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
1499
3312
  task.lastProgressWriteAt = now;
@@ -1502,9 +3315,13 @@ var TaskRegistry = class {
1502
3315
  async drive(task, op) {
1503
3316
  const context = {
1504
3317
  signal: task.controller.signal,
1505
- reportProgress: (progress) => this.reportProgress(task, progress),
3318
+ reportProgress: (progress) => this.reportProgress(task, { progress }),
3319
+ reportSteps: (steps) => this.reportProgress(task, { steps }),
1506
3320
  reportResult: (result) => {
1507
3321
  task.record = { ...task.record, result };
3322
+ },
3323
+ reportBuild: (build) => {
3324
+ task.record = { ...task.record, build };
1508
3325
  }
1509
3326
  };
1510
3327
  let settled;
@@ -1526,8 +3343,16 @@ var TaskRegistry = class {
1526
3343
  };
1527
3344
 
1528
3345
  // src/server/create-server.ts
3346
+ function bodyBytesInFlightOf(fraction) {
3347
+ const processLimitBytes = readMemoryFigures()?.processLimitBytes;
3348
+ if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
3349
+ return Math.floor(fraction * processLimitBytes);
3350
+ }
1529
3351
  function resolveLimits(limits, engineRerankWindow) {
1530
3352
  return {
3353
+ maxBodyBytesInFlight: bodyBytesInFlightOf(
3354
+ limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
3355
+ ),
1531
3356
  maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
1532
3357
  maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
1533
3358
  maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
@@ -1560,7 +3385,7 @@ function assertSecureBinding(host, options) {
1560
3385
  { host }
1561
3386
  );
1562
3387
  }
1563
- function mainThreadHandlers(deps) {
3388
+ function mainThreadHandlers(deps, syncServing) {
1564
3389
  const idx = createIndexHandlers(deps);
1565
3390
  const doc = createDocumentHandlers(deps);
1566
3391
  const search = createSearchHandlers(deps);
@@ -1568,8 +3393,12 @@ function mainThreadHandlers(deps) {
1568
3393
  const documentExport = createDocumentExportHandlers(deps);
1569
3394
  const health = createHealthHandlers(deps);
1570
3395
  const cluster = createClusterHandlers(deps);
3396
+ const backups = createBackupHandlers(deps);
1571
3397
  const version = createVersionHandler(deps);
1572
3398
  const capabilities = createCapabilitiesHandler();
3399
+ const syncSetting = createSyncSettingHandlers(deps.engine);
3400
+ const changesSetting = createChangesSettingHandlers(deps.engine);
3401
+ const suggestions = createQuerySuggestionHandlers(deps, deps.querySuggestions);
1573
3402
  return {
1574
3403
  livez: health.livez,
1575
3404
  readyz: health.readyz,
@@ -1577,6 +3406,13 @@ function mainThreadHandlers(deps) {
1577
3406
  capabilities: capabilities.report,
1578
3407
  clusterTopology: cluster.topology,
1579
3408
  clusterAllocation: cluster.allocation,
3409
+ clusterReplicationFactor: cluster.replicationFactor,
3410
+ clusterHealth: cluster.health,
3411
+ createBackup: backups.create,
3412
+ listBackups: backups.list,
3413
+ restoreBackup: backups.restore,
3414
+ deleteBackup: backups.remove,
3415
+ metrics: createMetricsHandler(deps),
1580
3416
  memory: admin.memory,
1581
3417
  listTasks: admin.listTasks,
1582
3418
  getTask: admin.getTask,
@@ -1606,6 +3442,7 @@ function mainThreadHandlers(deps) {
1606
3442
  multiGet: doc.multiGet,
1607
3443
  checkpoint: admin.checkpoint,
1608
3444
  rebuildAnalysis: admin.rebuildAnalysis,
3445
+ changeAnalysis: admin.changeAnalysis,
1609
3446
  snapshot: admin.snapshot,
1610
3447
  vectorMaintenance: admin.vectorMaintenance,
1611
3448
  compact: admin.compact,
@@ -1614,9 +3451,24 @@ function mainThreadHandlers(deps) {
1614
3451
  partitionConfig: admin.partitionConfig,
1615
3452
  readSynonyms: admin.readSynonyms,
1616
3453
  replaceSynonyms: admin.replaceSynonyms,
3454
+ updateBm25: admin.updateBm25,
1617
3455
  restore: admin.restore,
1618
3456
  exportDocuments: documentExport.exportDocuments,
1619
- importExport: documentExport.importExport
3457
+ importExport: documentExport.importExport,
3458
+ getSync: syncSetting.getSync,
3459
+ updateSync: syncSetting.updateSync,
3460
+ syncSnapshot: createSyncSnapshotHandler(syncServing),
3461
+ getChanges: changesSetting.getChanges,
3462
+ updateChanges: changesSetting.updateChanges,
3463
+ updateQuerySuggestions: suggestions.update,
3464
+ querySuggestionsStatus: suggestions.status,
3465
+ recordQuery: suggestions.record,
3466
+ buildQuerySuggestions: suggestions.build,
3467
+ previewQuerySuggestions: suggestions.preview,
3468
+ readQueryBans: suggestions.readBans,
3469
+ replaceQueryBans: suggestions.replaceBans,
3470
+ removeQuery: suggestions.remove,
3471
+ addManualSuggestion: suggestions.manual
1620
3472
  };
1621
3473
  }
1622
3474
  var NarsilHttpServer = class {
@@ -1629,12 +3481,19 @@ var NarsilHttpServer = class {
1629
3481
  listenSocket = null;
1630
3482
  ready = false;
1631
3483
  threads = null;
3484
+ sync;
3485
+ changes;
3486
+ syncServing = null;
3487
+ feeds = null;
1632
3488
  stopIdleHeapCleanup = () => void 0;
1633
3489
  constructor(engine, options = {}) {
1634
3490
  assertServerEngine(engine);
1635
3491
  const engineRerankWindow = rerankWindowCapOf(engine);
1636
3492
  assertServerLimits(options.limits, engineRerankWindow);
1637
3493
  assertAllowRegex(options.allowRegex);
3494
+ assertQuerySuggestionsOptions(options.querySuggestions);
3495
+ this.sync = resolveFeedOptions("sync", options.sync);
3496
+ this.changes = resolveFeedOptions("changes", options.changes);
1638
3497
  this.host = options.host ?? "127.0.0.1";
1639
3498
  this.port = options.port ?? 9876;
1640
3499
  this.options = options;
@@ -1657,10 +3516,13 @@ var NarsilHttpServer = class {
1657
3516
  isReady: () => this.ready,
1658
3517
  build: resolveBuild(options.build),
1659
3518
  cluster: options.cluster,
1660
- requestThreadCount: () => this.requestThreadCount
3519
+ requestThreadCount: () => this.requestThreadCount,
3520
+ querySuggestions: options.querySuggestions
1661
3521
  };
1662
3522
  }
1663
3523
  async listen() {
3524
+ assertFeedHasAuthentication("sync", this.sync, this.options);
3525
+ assertFeedHasAuthentication("changes", this.changes, this.options);
1664
3526
  assertSecureBinding(this.host, this.options);
1665
3527
  const uws = await loadUWebSockets();
1666
3528
  this.uws = uws;
@@ -1688,6 +3550,9 @@ var NarsilHttpServer = class {
1688
3550
  this.uws.us_listen_socket_close(this.listenSocket);
1689
3551
  this.listenSocket = null;
1690
3552
  }
3553
+ this.feeds?.close();
3554
+ this.feeds = null;
3555
+ this.syncServing = null;
1691
3556
  if (this.threads !== null) {
1692
3557
  const threads = this.threads;
1693
3558
  this.threads = null;
@@ -1705,17 +3570,34 @@ var NarsilHttpServer = class {
1705
3570
  get requestThreadCount() {
1706
3571
  return this.threads?.threadCount ?? 0;
1707
3572
  }
3573
+ startFeeds(app, authorize, core) {
3574
+ const started = startFeeds(this.engine, core, {
3575
+ sync: this.sync,
3576
+ changes: this.changes,
3577
+ allowRegex: this.deps.allowRegex,
3578
+ authorize
3579
+ });
3580
+ if (started === null) return;
3581
+ this.syncServing = started.syncServing;
3582
+ this.feeds = started.host;
3583
+ const { sync, changes } = started.host.gateways;
3584
+ if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE);
3585
+ if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE);
3586
+ }
1708
3587
  async registerRoutes(app) {
1709
3588
  const cors = resolveCors(this.options.cors);
1710
3589
  const authorize = this.options.onRequest ? authorizerFor(this.options.onRequest) : void 0;
1711
3590
  const core = engineCoreOf(this.engine);
1712
3591
  const threadCount = core === void 0 ? 0 : core.orchestrator.requestThreadCount();
1713
3592
  const gate = createSharedGateBuffer(gateSlotOfWorker(threadCount));
3593
+ const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
1714
3594
  const { limits, build, allowRegex } = this.deps;
1715
- const handlers = mainThreadHandlers(this.deps);
3595
+ const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
3596
+ this.startFeeds(app, authorize, core);
1716
3597
  const run = createRouteRunner({
1717
3598
  authorize,
1718
3599
  gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
3600
+ bodyBudget: createSharedBodyBudget(bodyBudget, limits.maxBodyBytesInFlight, MAIN_THREAD_GATE_SLOT),
1719
3601
  corsHeaders: cors ? corsHeaderReader(cors) : void 0
1720
3602
  });
1721
3603
  registerServerRoutes(app, run, handlers, limits, cors);
@@ -1727,10 +3609,12 @@ var NarsilHttpServer = class {
1727
3609
  handlers,
1728
3610
  authorize,
1729
3611
  gate,
3612
+ bodyBudget,
1730
3613
  limits,
1731
3614
  allowRegex,
1732
3615
  build,
1733
- cors
3616
+ cors,
3617
+ feeds: this.feeds
1734
3618
  });
1735
3619
  } catch (err) {
1736
3620
  console.warn("The server could not start request threads, so the main thread answers every request:", err);