@delali/narsil 0.3.1-next.99 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
package/dist/server.mjs
CHANGED
|
@@ -1,35 +1,1263 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import './chunk-
|
|
10
|
-
import
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
1
|
+
import { answerRequest } from './chunk-MW3CJVGP.mjs';
|
|
2
|
+
import { resolveVectorText } from './chunk-NJJN4FMM.mjs';
|
|
3
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, refusedRegex, toHttpError, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, requestSignal, withServerRerankWindow, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, isThreadFeedNote, clearGateSlot, clearBodyBudgetSlot, confirmDeparture, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-XQIAMDAG.mjs';
|
|
4
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-XQIAMDAG.mjs';
|
|
5
|
+
import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-5F5MXWS5.mjs';
|
|
6
|
+
import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, CHANGES_CLOSE_RESYNC, CHANGES_CLOSE_POLICY, MAX_SUBSCRIBER_MESSAGE_BYTES, CHANGES_CLOSE_TOO_LARGE, closeReasonOf, CHANGES_CLOSE_CREDENTIAL_EXPIRED, CHANGES_CLOSE_SETTING, CHANGES_CLOSE_SERVER_ERROR, CHANGES_CLOSE_GOING_AWAY, SYNC_CLOSE_GOING_AWAY } from './chunk-TACP6VK6.mjs';
|
|
7
|
+
import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-LFXS6OF7.mjs';
|
|
8
|
+
import { isRecord, validateIndexName } from './chunk-C74KPRR7.mjs';
|
|
9
|
+
import { executeUntilAnswered } from './chunk-4H4RG2FR.mjs';
|
|
10
|
+
import './chunk-XL56L4IR.mjs';
|
|
11
|
+
import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-AVBPB3MI.mjs';
|
|
12
|
+
import './chunk-M2Z5WVWW.mjs';
|
|
13
|
+
import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-ZPJFGGGI.mjs';
|
|
14
|
+
import './chunk-JDDXV6UF.mjs';
|
|
15
|
+
import { manifestFileRefs } from './chunk-C4ZMXXTL.mjs';
|
|
16
|
+
import './chunk-MSIN4NM3.mjs';
|
|
17
|
+
import { encode as encode$1 } from './chunk-S5QMWECL.mjs';
|
|
18
|
+
import './chunk-LNGE7SLY.mjs';
|
|
19
|
+
import './chunk-R2PJKSNL.mjs';
|
|
20
|
+
import './chunk-MFPSQZ6P.mjs';
|
|
21
|
+
import './chunk-FS3HPPSC.mjs';
|
|
22
|
+
import './chunk-R724DJSG.mjs';
|
|
23
|
+
import './chunk-V6EUBAIT.mjs';
|
|
24
|
+
import './chunk-TKKXPKEO.mjs';
|
|
25
|
+
import { querySuggestions } from './chunk-M3OW2XSP.mjs';
|
|
26
|
+
import './chunk-S2POFTLN.mjs';
|
|
27
|
+
import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-BY6NALRK.mjs';
|
|
28
|
+
import './chunk-2CKDCC65.mjs';
|
|
29
|
+
import './chunk-2AFZE7Y3.mjs';
|
|
30
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
31
|
+
import './chunk-BUMREPLU.mjs';
|
|
32
|
+
import './chunk-7T4LAYT4.mjs';
|
|
33
|
+
import './chunk-IL5DRCNU.mjs';
|
|
34
|
+
import './chunk-7P5YWDRQ.mjs';
|
|
13
35
|
import './chunk-U2BKYEC7.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-
|
|
17
|
-
import './chunk-
|
|
18
|
-
import './chunk-
|
|
19
|
-
import './chunk-
|
|
20
|
-
import './chunk-
|
|
21
|
-
import './chunk-
|
|
22
|
-
|
|
23
|
-
import {
|
|
24
|
-
import './chunk-L5ORED7V.mjs';
|
|
25
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
26
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-KV3AZFT3.mjs';
|
|
36
|
+
import { detectRuntime } from './chunk-FJXJZUWF.mjs';
|
|
37
|
+
import { startIdleHeapCleanup } from './chunk-5EI5CXO6.mjs';
|
|
38
|
+
import './chunk-3ECIRFS5.mjs';
|
|
39
|
+
import './chunk-ASZLNVWW.mjs';
|
|
40
|
+
import './chunk-ZXX5GZEX.mjs';
|
|
41
|
+
import './chunk-GR5GMFJJ.mjs';
|
|
42
|
+
import './chunk-4PY4JYMY.mjs';
|
|
43
|
+
import { ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
|
|
44
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
|
|
45
|
+
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
27
46
|
import { randomUUID } from 'crypto';
|
|
47
|
+
import { encode } from '@msgpack/msgpack';
|
|
28
48
|
import { createReadStream } from 'fs';
|
|
29
|
-
import { rm } from 'fs/promises';
|
|
30
|
-
import path from 'path';
|
|
31
49
|
import { MessageChannel } from 'worker_threads';
|
|
32
50
|
|
|
51
|
+
// src/server/changes/setting-handlers.ts
|
|
52
|
+
function createChangesSettingHandlers(engine) {
|
|
53
|
+
async function getChanges(ctx) {
|
|
54
|
+
try {
|
|
55
|
+
respondJson(ctx, await engine.getChanges(ctx.params[0]));
|
|
56
|
+
} catch (err) {
|
|
57
|
+
respondError(ctx, err);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async function updateChanges(ctx) {
|
|
61
|
+
const body = parseJson(ctx);
|
|
62
|
+
if (body === void 0) return;
|
|
63
|
+
try {
|
|
64
|
+
await engine.updateChanges(ctx.params[0], body);
|
|
65
|
+
respondJson(ctx, { ok: true });
|
|
66
|
+
} catch (err) {
|
|
67
|
+
respondError(ctx, err);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return { getChanges, updateChanges };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// src/changes/encode.ts
|
|
74
|
+
var FIXMAP_TAG = 128;
|
|
75
|
+
var FIXARRAY_LIMIT = 16;
|
|
76
|
+
var ARRAY16_LIMIT = 65536;
|
|
77
|
+
var FIXARRAY_TAG = 144;
|
|
78
|
+
var ARRAY16_TAG = 220;
|
|
79
|
+
var ARRAY32_TAG = 221;
|
|
80
|
+
var CHANGES_MESSAGE_KEY_COUNT = 4;
|
|
81
|
+
var CHANGE_FIXED_KEY_COUNT = 3;
|
|
82
|
+
var keys = null;
|
|
83
|
+
function keyBytes(name) {
|
|
84
|
+
keys ??= {};
|
|
85
|
+
let bytes = keys[name];
|
|
86
|
+
if (bytes === void 0) {
|
|
87
|
+
bytes = encode$1(name);
|
|
88
|
+
keys[name] = bytes;
|
|
89
|
+
}
|
|
90
|
+
return bytes;
|
|
91
|
+
}
|
|
92
|
+
function arrayHeader(count) {
|
|
93
|
+
if (count < FIXARRAY_LIMIT) return Uint8Array.of(FIXARRAY_TAG | count);
|
|
94
|
+
if (count < ARRAY16_LIMIT) return Uint8Array.of(ARRAY16_TAG, count >>> 8, count & 255);
|
|
95
|
+
return Uint8Array.of(ARRAY32_TAG, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
|
|
96
|
+
}
|
|
97
|
+
function concat(parts) {
|
|
98
|
+
let length = 0;
|
|
99
|
+
for (const part of parts) length += part.byteLength;
|
|
100
|
+
const out = new Uint8Array(length);
|
|
101
|
+
let at = 0;
|
|
102
|
+
for (const part of parts) {
|
|
103
|
+
out.set(part, at);
|
|
104
|
+
at += part.byteLength;
|
|
105
|
+
}
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
function encodedChangeParts(change) {
|
|
109
|
+
const keyCount = CHANGE_FIXED_KEY_COUNT + (change.document === null ? 0 : 1) + (change.previousDocument === null ? 0 : 1);
|
|
110
|
+
const parts = [
|
|
111
|
+
Uint8Array.of(FIXMAP_TAG | keyCount),
|
|
112
|
+
keyBytes("seqNo"),
|
|
113
|
+
encode$1(change.seqNo),
|
|
114
|
+
keyBytes("type"),
|
|
115
|
+
keyBytes(change.type),
|
|
116
|
+
keyBytes("documentId"),
|
|
117
|
+
encode$1(change.documentId)
|
|
118
|
+
];
|
|
119
|
+
if (change.document !== null) parts.push(keyBytes("document"), change.document);
|
|
120
|
+
if (change.previousDocument !== null) parts.push(keyBytes("previousDocument"), change.previousDocument);
|
|
121
|
+
return parts;
|
|
122
|
+
}
|
|
123
|
+
function encodeChangesMessage(partitionId, changes, scannedSeqNo) {
|
|
124
|
+
const parts = [
|
|
125
|
+
Uint8Array.of(FIXMAP_TAG | CHANGES_MESSAGE_KEY_COUNT),
|
|
126
|
+
keyBytes("type"),
|
|
127
|
+
keyBytes("changes"),
|
|
128
|
+
keyBytes("partitionId"),
|
|
129
|
+
encode$1(partitionId),
|
|
130
|
+
keyBytes("scannedSeqNo"),
|
|
131
|
+
encode$1(scannedSeqNo),
|
|
132
|
+
keyBytes("changes"),
|
|
133
|
+
arrayHeader(changes.length)
|
|
134
|
+
];
|
|
135
|
+
for (const change of changes) parts.push(...encodedChangeParts(change));
|
|
136
|
+
return concat(parts);
|
|
137
|
+
}
|
|
138
|
+
function encodeFeedMessage(message) {
|
|
139
|
+
return encode$1(message, { ignoreUndefined: true });
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// src/changes/session.ts
|
|
143
|
+
var POLICY_CODES = /* @__PURE__ */ new Set([
|
|
144
|
+
ErrorCodes.CHANGES_INVALID_POSITIONS,
|
|
145
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
146
|
+
ErrorCodes.SEARCH_INVALID_FIELD
|
|
147
|
+
]);
|
|
148
|
+
var ChangesSession = class {
|
|
149
|
+
constructor(deps) {
|
|
150
|
+
this.deps = deps;
|
|
151
|
+
this.stopBudgetWatch = deps.buffered.onFreed(() => {
|
|
152
|
+
if (this.state === "subscribed" && this.charged === 0) this.advance();
|
|
153
|
+
});
|
|
154
|
+
this.armExpiry();
|
|
155
|
+
}
|
|
156
|
+
state = "awaiting";
|
|
157
|
+
cursor = null;
|
|
158
|
+
sentSeqNos = [];
|
|
159
|
+
silentBytes = [];
|
|
160
|
+
unacked = [];
|
|
161
|
+
unackedBytes = 0;
|
|
162
|
+
charged = 0;
|
|
163
|
+
pumping = false;
|
|
164
|
+
pumpAgain = false;
|
|
165
|
+
retry = null;
|
|
166
|
+
expiry = null;
|
|
167
|
+
incoming = Promise.resolve();
|
|
168
|
+
stopBudgetWatch;
|
|
169
|
+
get indexName() {
|
|
170
|
+
return this.deps.indexName;
|
|
171
|
+
}
|
|
172
|
+
receive(bytes, binary = true) {
|
|
173
|
+
if (this.state === "closed") return;
|
|
174
|
+
if (!binary) {
|
|
175
|
+
this.close(CHANGES_CLOSE_POLICY, "feed messages travel in binary frames");
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (bytes.byteLength > MAX_SUBSCRIBER_MESSAGE_BYTES) {
|
|
179
|
+
this.close(CHANGES_CLOSE_TOO_LARGE, "message too large");
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
let message;
|
|
183
|
+
try {
|
|
184
|
+
message = subscriberMessageOf(decode(bytes));
|
|
185
|
+
} catch {
|
|
186
|
+
message = null;
|
|
187
|
+
}
|
|
188
|
+
if (message === null) {
|
|
189
|
+
this.close(CHANGES_CLOSE_POLICY, "invalid message");
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
const received = message;
|
|
193
|
+
this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
|
|
194
|
+
}
|
|
195
|
+
advance() {
|
|
196
|
+
if (this.state !== "subscribed") return;
|
|
197
|
+
void this.pump();
|
|
198
|
+
}
|
|
199
|
+
drained() {
|
|
200
|
+
this.settleBuffered();
|
|
201
|
+
this.advance();
|
|
202
|
+
}
|
|
203
|
+
resync(reason) {
|
|
204
|
+
if (this.state === "closed") return;
|
|
205
|
+
this.send(encodeFeedMessage({ type: "resync", reason }));
|
|
206
|
+
this.close(CHANGES_CLOSE_RESYNC, `resync ${reason}`);
|
|
207
|
+
}
|
|
208
|
+
previousDocumentOff() {
|
|
209
|
+
if (this.cursor?.filtered === true) this.refuseFilterSetting();
|
|
210
|
+
}
|
|
211
|
+
close(code, reason) {
|
|
212
|
+
if (this.state === "closed") return;
|
|
213
|
+
this.state = "closed";
|
|
214
|
+
this.cleanUp();
|
|
215
|
+
this.deps.socket.close(code, closeReasonOf(reason));
|
|
216
|
+
this.deps.onClosed(this);
|
|
217
|
+
}
|
|
218
|
+
closedByPeer() {
|
|
219
|
+
if (this.state === "closed") return;
|
|
220
|
+
this.state = "closed";
|
|
221
|
+
this.cleanUp();
|
|
222
|
+
this.deps.onClosed(this);
|
|
223
|
+
}
|
|
224
|
+
armExpiry() {
|
|
225
|
+
const { expiresAt } = this.deps;
|
|
226
|
+
if (expiresAt === null || this.state === "closed") return;
|
|
227
|
+
const remaining = expiresAt - Date.now();
|
|
228
|
+
if (remaining <= 0) {
|
|
229
|
+
queueMicrotask(() => this.close(CHANGES_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
this.expiry = setTimeout(
|
|
233
|
+
() => {
|
|
234
|
+
this.expiry = null;
|
|
235
|
+
this.armExpiry();
|
|
236
|
+
},
|
|
237
|
+
Math.min(remaining, TIMER_DELAY_LIMIT_MS)
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
refuseFilterSetting() {
|
|
241
|
+
this.close(CHANGES_CLOSE_SETTING, ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED);
|
|
242
|
+
}
|
|
243
|
+
cleanUp() {
|
|
244
|
+
if (this.retry !== null) clearTimeout(this.retry);
|
|
245
|
+
this.retry = null;
|
|
246
|
+
if (this.expiry !== null) clearTimeout(this.expiry);
|
|
247
|
+
this.expiry = null;
|
|
248
|
+
this.stopBudgetWatch();
|
|
249
|
+
this.deps.buffered.give(this.charged);
|
|
250
|
+
this.charged = 0;
|
|
251
|
+
}
|
|
252
|
+
fail(error) {
|
|
253
|
+
this.deps.onError(error);
|
|
254
|
+
this.close(CHANGES_CLOSE_SERVER_ERROR, "feed read failed");
|
|
255
|
+
}
|
|
256
|
+
send(bytes) {
|
|
257
|
+
if (this.state === "closed") return;
|
|
258
|
+
this.deps.socket.send(bytes);
|
|
259
|
+
this.settleBuffered();
|
|
260
|
+
}
|
|
261
|
+
settleBuffered() {
|
|
262
|
+
const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
|
|
263
|
+
if (now < this.charged) this.deps.buffered.give(this.charged - now);
|
|
264
|
+
this.charged = Math.min(this.charged, now);
|
|
265
|
+
}
|
|
266
|
+
async handle(message) {
|
|
267
|
+
if (this.state === "closed") return;
|
|
268
|
+
if (message.type === "subscribe") {
|
|
269
|
+
if (this.state !== "awaiting" || this.cursor !== null) {
|
|
270
|
+
return this.close(CHANGES_CLOSE_POLICY, "subscribe after the first message");
|
|
271
|
+
}
|
|
272
|
+
return this.subscribe(message);
|
|
273
|
+
}
|
|
274
|
+
if (this.state !== "subscribed") return this.close(CHANGES_CLOSE_POLICY, "subscribe must come first");
|
|
275
|
+
this.acknowledge(message.positions);
|
|
276
|
+
}
|
|
277
|
+
closeOver(error) {
|
|
278
|
+
if (!(error instanceof NarsilError)) this.fail(error);
|
|
279
|
+
else if (error.code === ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED) this.refuseFilterSetting();
|
|
280
|
+
else if (error.code === ErrorCodes.CHANGES_NOT_ENABLED) this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
|
|
281
|
+
else if (POLICY_CODES.has(error.code)) this.close(CHANGES_CLOSE_POLICY, error.message);
|
|
282
|
+
else this.fail(error);
|
|
283
|
+
}
|
|
284
|
+
async subscribe(message) {
|
|
285
|
+
const { source, indexName, reads } = this.deps;
|
|
286
|
+
if (message.filter !== void 0) {
|
|
287
|
+
const refused = this.deps.refuseFilter(message.filter);
|
|
288
|
+
if (refused !== null) return this.close(CHANGES_CLOSE_POLICY, refused.message);
|
|
289
|
+
}
|
|
290
|
+
let opened;
|
|
291
|
+
let vectorFields = [];
|
|
292
|
+
try {
|
|
293
|
+
opened = await ChangeCursor.open({ source, indexName, reads }, message);
|
|
294
|
+
if (typeof opened !== "string") vectorFields = [...await source.vectorFieldPaths(indexName)].sort();
|
|
295
|
+
} catch (error) {
|
|
296
|
+
return this.closeOver(error);
|
|
297
|
+
}
|
|
298
|
+
if (this.state === "closed") return;
|
|
299
|
+
if (typeof opened === "string") return this.resync(opened);
|
|
300
|
+
const cursor = opened;
|
|
301
|
+
this.cursor = cursor;
|
|
302
|
+
this.sentSeqNos = [...cursor.subscribedSeqNos];
|
|
303
|
+
this.silentBytes = new Array(cursor.state.partitionCount).fill(0);
|
|
304
|
+
for (let i = 0; i < cursor.state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
|
|
305
|
+
this.state = "subscribed";
|
|
306
|
+
this.send(
|
|
307
|
+
encodeFeedMessage({
|
|
308
|
+
type: "subscribed",
|
|
309
|
+
epoch: cursor.state.epoch,
|
|
310
|
+
partitionCount: cursor.state.partitionCount,
|
|
311
|
+
positions: changePositionList(cursor.subscribedSeqNos),
|
|
312
|
+
heads: changePositionList(cursor.heads),
|
|
313
|
+
bytesBehind: cursor.bytesBehind,
|
|
314
|
+
windowBytes: this.deps.windowBytes,
|
|
315
|
+
previousDocument: cursor.state.previousDocument,
|
|
316
|
+
vectorFields
|
|
317
|
+
})
|
|
318
|
+
);
|
|
319
|
+
void this.pump();
|
|
320
|
+
}
|
|
321
|
+
acknowledge(positions) {
|
|
322
|
+
const seqNos = seqNosByPartition(positions, this.sentSeqNos.length);
|
|
323
|
+
if (seqNos === null) {
|
|
324
|
+
this.close(CHANGES_CLOSE_POLICY, "positions must name each partition once");
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
328
|
+
if (seqNos[partitionId] > this.sentSeqNos[partitionId]) {
|
|
329
|
+
this.close(CHANGES_CLOSE_POLICY, "acknowledgement above a sent position");
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
334
|
+
const queue = this.unacked[partitionId];
|
|
335
|
+
while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
|
|
336
|
+
this.unackedBytes -= queue.items[queue.head].bytes;
|
|
337
|
+
queue.head += 1;
|
|
338
|
+
}
|
|
339
|
+
if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
|
|
340
|
+
queue.items = queue.items.slice(queue.head);
|
|
341
|
+
queue.head = 0;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
void this.pump();
|
|
345
|
+
}
|
|
346
|
+
windowAllows(nextBytes) {
|
|
347
|
+
return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
|
|
348
|
+
}
|
|
349
|
+
settingAllows(cursor) {
|
|
350
|
+
const current = this.deps.source.changesState(this.deps.indexName);
|
|
351
|
+
if (current === null) {
|
|
352
|
+
this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
|
|
353
|
+
return false;
|
|
354
|
+
}
|
|
355
|
+
if (current.epoch !== cursor.state.epoch) {
|
|
356
|
+
this.resync("epoch");
|
|
357
|
+
return false;
|
|
358
|
+
}
|
|
359
|
+
if (cursor.filtered && !current.previousDocument) {
|
|
360
|
+
this.refuseFilterSetting();
|
|
361
|
+
return false;
|
|
362
|
+
}
|
|
363
|
+
return true;
|
|
364
|
+
}
|
|
365
|
+
async pump() {
|
|
366
|
+
const { cursor } = this;
|
|
367
|
+
if (cursor === null) return;
|
|
368
|
+
if (this.pumping) {
|
|
369
|
+
this.pumpAgain = true;
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
this.pumping = true;
|
|
373
|
+
try {
|
|
374
|
+
do {
|
|
375
|
+
this.pumpAgain = false;
|
|
376
|
+
if (!this.settingAllows(cursor)) return;
|
|
377
|
+
await cursor.refreshHeads();
|
|
378
|
+
await this.sendBehind(cursor);
|
|
379
|
+
} while (this.pumpAgain && this.state === "subscribed");
|
|
380
|
+
this.scheduleRetryWhileUndurable(cursor);
|
|
381
|
+
} catch (error) {
|
|
382
|
+
this.fail(error);
|
|
383
|
+
} finally {
|
|
384
|
+
this.pumping = false;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
async sendBehind(cursor) {
|
|
388
|
+
const { windowBytes } = this.deps;
|
|
389
|
+
let progressed = true;
|
|
390
|
+
while (progressed && this.state === "subscribed") {
|
|
391
|
+
progressed = false;
|
|
392
|
+
for (let partitionId = 0; partitionId < this.sentSeqNos.length; partitionId++) {
|
|
393
|
+
if (this.state !== "subscribed" || !cursor.behind(partitionId)) continue;
|
|
394
|
+
if (!this.windowAllows(1) || this.charged > 0) return;
|
|
395
|
+
const room = this.unackedBytes === 0 ? windowBytes : windowBytes - this.unackedBytes;
|
|
396
|
+
const batch = await cursor.readBatch(partitionId, Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES)));
|
|
397
|
+
if (this.state !== "subscribed") return;
|
|
398
|
+
if (batch === "history") return this.resync("history");
|
|
399
|
+
if (batch === null) continue;
|
|
400
|
+
if (!this.settingAllows(cursor)) return;
|
|
401
|
+
progressed = true;
|
|
402
|
+
if (batch.changes.length === 0 && this.silentBytes[partitionId] + batch.scannedBytes < windowBytes) {
|
|
403
|
+
this.silentBytes[partitionId] += batch.scannedBytes;
|
|
404
|
+
cursor.commit(batch);
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
const message = encodeChangesMessage(partitionId, batch.changes, batch.scannedSeqNo);
|
|
408
|
+
if (!this.windowAllows(message.byteLength)) return;
|
|
409
|
+
if (!this.deps.buffered.take(message.byteLength)) return;
|
|
410
|
+
this.charged += message.byteLength;
|
|
411
|
+
this.unacked[partitionId].items.push({ seqNo: batch.scannedSeqNo, bytes: message.byteLength });
|
|
412
|
+
this.unackedBytes += message.byteLength;
|
|
413
|
+
this.silentBytes[partitionId] = 0;
|
|
414
|
+
this.sentSeqNos[partitionId] = batch.scannedSeqNo;
|
|
415
|
+
cursor.commit(batch);
|
|
416
|
+
this.send(message);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
scheduleRetryWhileUndurable(cursor) {
|
|
421
|
+
if (this.state !== "subscribed" || this.retry !== null || !cursor.awaitsDurability()) return;
|
|
422
|
+
this.retry = setTimeout(() => {
|
|
423
|
+
this.retry = null;
|
|
424
|
+
this.advance();
|
|
425
|
+
}, FEED_HEAD_RETRY_MS);
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
|
|
429
|
+
// src/changes/hub.ts
|
|
430
|
+
function createChangesHub(source, options = {}) {
|
|
431
|
+
const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
|
|
432
|
+
const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
|
|
433
|
+
const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
|
|
434
|
+
const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
|
|
435
|
+
const refuseFilter = options.refuseFilter ?? (() => null);
|
|
436
|
+
const sessionsByIndex = /* @__PURE__ */ new Map();
|
|
437
|
+
const unwatchByIndex = /* @__PURE__ */ new Map();
|
|
438
|
+
let connections = 0;
|
|
439
|
+
let stopListening = null;
|
|
440
|
+
let closed = false;
|
|
441
|
+
function each(indexName, action) {
|
|
442
|
+
const sessions = sessionsByIndex.get(indexName);
|
|
443
|
+
if (sessions === void 0) return;
|
|
444
|
+
for (const session of [...sessions]) action(session);
|
|
445
|
+
}
|
|
446
|
+
function onChange(change) {
|
|
447
|
+
switch (change.kind) {
|
|
448
|
+
case "advanced":
|
|
449
|
+
each(change.indexName, (session) => session.advance());
|
|
450
|
+
return;
|
|
451
|
+
case "epoch":
|
|
452
|
+
each(change.indexName, (session) => session.resync("epoch"));
|
|
453
|
+
return;
|
|
454
|
+
case "previous-document-off":
|
|
455
|
+
each(change.indexName, (session) => session.previousDocumentOff());
|
|
456
|
+
return;
|
|
457
|
+
case "changes-off":
|
|
458
|
+
case "dropped":
|
|
459
|
+
each(change.indexName, (session) => session.close(CHANGES_CLOSE_RESYNC, "change capture is off"));
|
|
460
|
+
return;
|
|
461
|
+
case "off":
|
|
462
|
+
case "settings":
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
function release() {
|
|
467
|
+
connections -= 1;
|
|
468
|
+
if (connections === 0 && stopListening !== null) {
|
|
469
|
+
stopListening();
|
|
470
|
+
stopListening = null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
function forget(session) {
|
|
474
|
+
const sessions = sessionsByIndex.get(session.indexName);
|
|
475
|
+
if (sessions === void 0 || !sessions.delete(session)) return;
|
|
476
|
+
if (sessions.size === 0) {
|
|
477
|
+
sessionsByIndex.delete(session.indexName);
|
|
478
|
+
unwatchByIndex.get(session.indexName)?.();
|
|
479
|
+
unwatchByIndex.delete(session.indexName);
|
|
480
|
+
}
|
|
481
|
+
release();
|
|
482
|
+
}
|
|
483
|
+
function open(indexName, expiresAt, socket) {
|
|
484
|
+
if (stopListening === null) stopListening = source.listen(onChange);
|
|
485
|
+
const session = new ChangesSession({
|
|
486
|
+
source,
|
|
487
|
+
indexName,
|
|
488
|
+
socket,
|
|
489
|
+
windowBytes,
|
|
490
|
+
expiresAt,
|
|
491
|
+
reads,
|
|
492
|
+
buffered,
|
|
493
|
+
refuseFilter,
|
|
494
|
+
onClosed: forget,
|
|
495
|
+
onError: (error) => options.onError?.(error)
|
|
496
|
+
});
|
|
497
|
+
let sessions = sessionsByIndex.get(indexName);
|
|
498
|
+
if (sessions === void 0) {
|
|
499
|
+
sessions = /* @__PURE__ */ new Set();
|
|
500
|
+
sessionsByIndex.set(indexName, sessions);
|
|
501
|
+
const unwatch = source.watchIndex?.(indexName);
|
|
502
|
+
if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
|
|
503
|
+
}
|
|
504
|
+
sessions.add(session);
|
|
505
|
+
return session;
|
|
506
|
+
}
|
|
507
|
+
return {
|
|
508
|
+
admit(indexName, expiresAt) {
|
|
509
|
+
if (closed) {
|
|
510
|
+
throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
|
|
511
|
+
}
|
|
512
|
+
if (source.changesState(indexName) === null) throw changesNotEnabled(indexName);
|
|
513
|
+
if (connections >= maxConnections) {
|
|
514
|
+
throw new NarsilError(
|
|
515
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
516
|
+
`The server holds ${connections} change feed connections, the most that it accepts`,
|
|
517
|
+
{ maximum: maxConnections }
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
connections += 1;
|
|
521
|
+
let settled = false;
|
|
522
|
+
return {
|
|
523
|
+
open(socket) {
|
|
524
|
+
if (settled) throw new Error("A change feed admission opens one connection");
|
|
525
|
+
settled = true;
|
|
526
|
+
return open(indexName, expiresAt, socket);
|
|
527
|
+
},
|
|
528
|
+
cancel() {
|
|
529
|
+
if (settled) return;
|
|
530
|
+
settled = true;
|
|
531
|
+
release();
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
},
|
|
535
|
+
connectionCount: () => connections,
|
|
536
|
+
close() {
|
|
537
|
+
closed = true;
|
|
538
|
+
for (const indexName of [...sessionsByIndex.keys()]) {
|
|
539
|
+
each(indexName, (session) => session.close(CHANGES_CLOSE_GOING_AWAY, "server closing"));
|
|
540
|
+
}
|
|
541
|
+
stopListening?.();
|
|
542
|
+
stopListening = null;
|
|
543
|
+
}
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
var FIXARRAY_LIMIT2 = 16;
|
|
547
|
+
var ARRAY16_LIMIT2 = 65536;
|
|
548
|
+
var FIXARRAY_TAG2 = 144;
|
|
549
|
+
var ARRAY16_TAG2 = 220;
|
|
550
|
+
var ARRAY32_TAG2 = 221;
|
|
551
|
+
var FIXMAP_OF_THREE = 131;
|
|
552
|
+
var entriesKeyPrefix = (() => {
|
|
553
|
+
const parts = [Uint8Array.of(FIXMAP_OF_THREE), encode("type"), encode("entries"), encode("partitionId")];
|
|
554
|
+
return concat2(parts);
|
|
555
|
+
})();
|
|
556
|
+
var entriesKey = encode("entries");
|
|
557
|
+
function concat2(parts) {
|
|
558
|
+
let length = 0;
|
|
559
|
+
for (const part of parts) length += part.byteLength;
|
|
560
|
+
const out = new Uint8Array(length);
|
|
561
|
+
let at = 0;
|
|
562
|
+
for (const part of parts) {
|
|
563
|
+
out.set(part, at);
|
|
564
|
+
at += part.byteLength;
|
|
565
|
+
}
|
|
566
|
+
return out;
|
|
567
|
+
}
|
|
568
|
+
function arrayHeader2(count) {
|
|
569
|
+
if (count < FIXARRAY_LIMIT2) return Uint8Array.of(FIXARRAY_TAG2 | count);
|
|
570
|
+
if (count < ARRAY16_LIMIT2) return Uint8Array.of(ARRAY16_TAG2, count >>> 8, count & 255);
|
|
571
|
+
return Uint8Array.of(ARRAY32_TAG2, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
|
|
572
|
+
}
|
|
573
|
+
function encodeEntriesMessage(partitionId, payloads) {
|
|
574
|
+
return concat2([entriesKeyPrefix, encode(partitionId), entriesKey, arrayHeader2(payloads.length), ...payloads]);
|
|
575
|
+
}
|
|
576
|
+
function encodeServerMessage(message) {
|
|
577
|
+
return encode(message, { ignoreUndefined: true });
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// src/sync/feed/session.ts
|
|
581
|
+
var FeedSession = class {
|
|
582
|
+
constructor(deps) {
|
|
583
|
+
this.deps = deps;
|
|
584
|
+
this.stopBudgetWatch = deps.buffered.onFreed(() => {
|
|
585
|
+
if (this.state === "subscribed" && this.charged === 0) this.advance();
|
|
586
|
+
});
|
|
587
|
+
this.armExpiry();
|
|
588
|
+
}
|
|
589
|
+
state = "awaiting";
|
|
590
|
+
epoch = "";
|
|
591
|
+
sent = [];
|
|
592
|
+
heads = [];
|
|
593
|
+
unacked = [];
|
|
594
|
+
unackedBytes = 0;
|
|
595
|
+
charged = 0;
|
|
596
|
+
pumping = false;
|
|
597
|
+
pumpAgain = false;
|
|
598
|
+
retry = null;
|
|
599
|
+
expiry = null;
|
|
600
|
+
incoming = Promise.resolve();
|
|
601
|
+
stopBudgetWatch;
|
|
602
|
+
armExpiry() {
|
|
603
|
+
const { expiresAt } = this.deps;
|
|
604
|
+
if (expiresAt === null || this.state === "closed") return;
|
|
605
|
+
const remaining = expiresAt - Date.now();
|
|
606
|
+
if (remaining <= 0) {
|
|
607
|
+
queueMicrotask(() => this.close(SYNC_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
this.expiry = setTimeout(
|
|
611
|
+
() => {
|
|
612
|
+
this.expiry = null;
|
|
613
|
+
this.armExpiry();
|
|
614
|
+
},
|
|
615
|
+
Math.min(remaining, TIMER_DELAY_LIMIT_MS)
|
|
616
|
+
);
|
|
617
|
+
}
|
|
618
|
+
get indexName() {
|
|
619
|
+
return this.deps.indexName;
|
|
620
|
+
}
|
|
621
|
+
receive(bytes, binary = true) {
|
|
622
|
+
if (this.state === "closed") return;
|
|
623
|
+
if (!binary) {
|
|
624
|
+
this.close(SYNC_CLOSE_POLICY, "feed messages travel in binary frames");
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
if (bytes.byteLength > MAX_DEVICE_MESSAGE_BYTES) {
|
|
628
|
+
this.close(SYNC_CLOSE_TOO_LARGE, "message too large");
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
631
|
+
let message;
|
|
632
|
+
try {
|
|
633
|
+
message = deviceMessageOf(decode(bytes));
|
|
634
|
+
} catch {
|
|
635
|
+
message = null;
|
|
636
|
+
}
|
|
637
|
+
if (message === null) {
|
|
638
|
+
this.close(SYNC_CLOSE_POLICY, "invalid message");
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
const received = message;
|
|
642
|
+
this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
|
|
643
|
+
}
|
|
644
|
+
advance() {
|
|
645
|
+
if (this.state !== "subscribed") return;
|
|
646
|
+
void this.pump();
|
|
647
|
+
}
|
|
648
|
+
sendSettings() {
|
|
649
|
+
if (this.state !== "subscribed") return;
|
|
650
|
+
const state = this.deps.source.indexState(this.deps.indexName);
|
|
651
|
+
if (state === null) return;
|
|
652
|
+
this.send(encodeServerMessage({ type: "settings", settings: state.settings }));
|
|
653
|
+
}
|
|
654
|
+
drained() {
|
|
655
|
+
this.settleBuffered();
|
|
656
|
+
this.advance();
|
|
657
|
+
}
|
|
658
|
+
resync(reason) {
|
|
659
|
+
if (this.state === "closed") return;
|
|
660
|
+
this.send(encodeServerMessage({ type: "resync", reason }));
|
|
661
|
+
this.close(SYNC_CLOSE_RESYNC, `resync ${reason}`);
|
|
662
|
+
}
|
|
663
|
+
close(code, reason) {
|
|
664
|
+
if (this.state === "closed") return;
|
|
665
|
+
this.state = "closed";
|
|
666
|
+
this.cleanUp();
|
|
667
|
+
this.deps.socket.close(code, reason);
|
|
668
|
+
this.deps.onClosed(this);
|
|
669
|
+
}
|
|
670
|
+
closedByPeer() {
|
|
671
|
+
if (this.state === "closed") return;
|
|
672
|
+
this.state = "closed";
|
|
673
|
+
this.cleanUp();
|
|
674
|
+
this.deps.onClosed(this);
|
|
675
|
+
}
|
|
676
|
+
cleanUp() {
|
|
677
|
+
if (this.retry !== null) clearTimeout(this.retry);
|
|
678
|
+
this.retry = null;
|
|
679
|
+
if (this.expiry !== null) clearTimeout(this.expiry);
|
|
680
|
+
this.expiry = null;
|
|
681
|
+
this.stopBudgetWatch();
|
|
682
|
+
this.deps.buffered.give(this.charged);
|
|
683
|
+
this.charged = 0;
|
|
684
|
+
}
|
|
685
|
+
fail(error) {
|
|
686
|
+
this.deps.onError(error);
|
|
687
|
+
this.close(SYNC_CLOSE_SERVER_ERROR, "feed read failed");
|
|
688
|
+
}
|
|
689
|
+
send(bytes) {
|
|
690
|
+
if (this.state === "closed") return;
|
|
691
|
+
this.deps.socket.send(bytes);
|
|
692
|
+
this.settleBuffered();
|
|
693
|
+
}
|
|
694
|
+
settleBuffered() {
|
|
695
|
+
const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
|
|
696
|
+
if (now < this.charged) this.deps.buffered.give(this.charged - now);
|
|
697
|
+
this.charged = Math.min(this.charged, now);
|
|
698
|
+
}
|
|
699
|
+
async handle(message) {
|
|
700
|
+
if (this.state === "closed") return;
|
|
701
|
+
if (message.type === "subscribe") {
|
|
702
|
+
if (this.state !== "awaiting") return this.close(SYNC_CLOSE_POLICY, "subscribe after the first message");
|
|
703
|
+
return this.subscribe(message.epoch, message.positions);
|
|
704
|
+
}
|
|
705
|
+
if (this.state !== "subscribed") return this.close(SYNC_CLOSE_POLICY, "subscribe must come first");
|
|
706
|
+
if (message.type === "ack") return this.acknowledge(message.positions);
|
|
707
|
+
this.answerCheckpoint(message.id).catch((error) => this.fail(error));
|
|
708
|
+
}
|
|
709
|
+
async currentBounds() {
|
|
710
|
+
const all = [];
|
|
711
|
+
for (let partitionId = 0; partitionId < this.sent.length; partitionId++) {
|
|
712
|
+
all.push(await this.deps.source.bounds(this.deps.indexName, partitionId, "current"));
|
|
713
|
+
}
|
|
714
|
+
return all;
|
|
715
|
+
}
|
|
716
|
+
async answerCheckpoint(id) {
|
|
717
|
+
const covering = (await this.currentBounds()).map((bounds) => bounds.highestSeqNo);
|
|
718
|
+
while (this.state === "subscribed") {
|
|
719
|
+
const all = await this.currentBounds();
|
|
720
|
+
let covered = true;
|
|
721
|
+
for (let partitionId = 0; partitionId < all.length; partitionId++) {
|
|
722
|
+
covering[partitionId] = Math.min(covering[partitionId], all[partitionId].highestSeqNo);
|
|
723
|
+
const reachable = Math.max(all[partitionId].head, this.sent[partitionId]);
|
|
724
|
+
if (reachable < covering[partitionId]) covered = false;
|
|
725
|
+
}
|
|
726
|
+
if (covered) {
|
|
727
|
+
const heads = all.map((bounds) => bounds.head);
|
|
728
|
+
this.send(encodeServerMessage({ type: "checkpoint", id, heads: positionList(heads) }));
|
|
729
|
+
this.advance();
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
await new Promise((resolve) => setTimeout(resolve, FEED_HEAD_RETRY_MS));
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
async subscribe(epoch, positions) {
|
|
736
|
+
const { source, indexName } = this.deps;
|
|
737
|
+
const state = source.indexState(indexName);
|
|
738
|
+
if (state === null) return this.close(SYNC_CLOSE_RESYNC, "sync is off");
|
|
739
|
+
if (epoch !== state.epoch) return this.resync("epoch");
|
|
740
|
+
const seqNos = positionsByPartition(positions, state.partitionCount);
|
|
741
|
+
if (seqNos === null) return this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
|
|
742
|
+
const heads = [];
|
|
743
|
+
for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
|
|
744
|
+
const bounds = await this.deps.reads.run(() => source.bounds(indexName, partitionId, "current"));
|
|
745
|
+
if (seqNos[partitionId] < bounds.oldestSeqNo - 1) return this.resync("history");
|
|
746
|
+
if (seqNos[partitionId] > bounds.highestSeqNo) return this.resync("ahead");
|
|
747
|
+
heads.push(bounds.head);
|
|
748
|
+
}
|
|
749
|
+
const bytesBehind = await this.deps.reads.run(() => source.bytesBehind(indexName, seqNos));
|
|
750
|
+
const snapshotBytes = await this.deps.reads.run(() => source.snapshotBytes(indexName));
|
|
751
|
+
if (this.state === "closed") return;
|
|
752
|
+
if (source.indexState(indexName)?.epoch !== epoch) return this.resync("epoch");
|
|
753
|
+
this.epoch = epoch;
|
|
754
|
+
this.sent = seqNos;
|
|
755
|
+
this.heads = heads;
|
|
756
|
+
for (let i = 0; i < state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
|
|
757
|
+
this.state = "subscribed";
|
|
758
|
+
this.send(
|
|
759
|
+
encodeServerMessage({
|
|
760
|
+
type: "subscribed",
|
|
761
|
+
epoch,
|
|
762
|
+
partitionCount: state.partitionCount,
|
|
763
|
+
heads: positionList(heads),
|
|
764
|
+
bytesBehind,
|
|
765
|
+
snapshotBytes,
|
|
766
|
+
windowBytes: this.deps.windowBytes,
|
|
767
|
+
settings: state.settings
|
|
768
|
+
})
|
|
769
|
+
);
|
|
770
|
+
void this.pump();
|
|
771
|
+
}
|
|
772
|
+
acknowledge(positions) {
|
|
773
|
+
const seqNos = positionsByPartition(positions, this.sent.length);
|
|
774
|
+
if (seqNos === null) {
|
|
775
|
+
this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
779
|
+
if (seqNos[partitionId] > this.sent[partitionId]) {
|
|
780
|
+
this.close(SYNC_CLOSE_POLICY, "acknowledgement above a sent entry");
|
|
781
|
+
return;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
785
|
+
const queue = this.unacked[partitionId];
|
|
786
|
+
while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
|
|
787
|
+
this.unackedBytes -= queue.items[queue.head].bytes;
|
|
788
|
+
queue.head += 1;
|
|
789
|
+
}
|
|
790
|
+
if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
|
|
791
|
+
queue.items = queue.items.slice(queue.head);
|
|
792
|
+
queue.head = 0;
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
void this.pump();
|
|
796
|
+
}
|
|
797
|
+
windowAllows(nextBytes) {
|
|
798
|
+
return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
|
|
799
|
+
}
|
|
800
|
+
async pump() {
|
|
801
|
+
if (this.pumping) {
|
|
802
|
+
this.pumpAgain = true;
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
this.pumping = true;
|
|
806
|
+
try {
|
|
807
|
+
do {
|
|
808
|
+
this.pumpAgain = false;
|
|
809
|
+
await this.refreshHeads();
|
|
810
|
+
await this.sendBehind();
|
|
811
|
+
} while (this.pumpAgain && this.state === "subscribed");
|
|
812
|
+
this.scheduleRetryWhileUndurable();
|
|
813
|
+
} catch (error) {
|
|
814
|
+
this.fail(error);
|
|
815
|
+
} finally {
|
|
816
|
+
this.pumping = false;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
async refreshHeads() {
|
|
820
|
+
const { source, indexName } = this.deps;
|
|
821
|
+
if (source.indexState(indexName)?.epoch !== this.epoch) return this.resync("epoch");
|
|
822
|
+
for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
|
|
823
|
+
this.heads[partitionId] = Math.max(this.heads[partitionId], (await source.bounds(indexName, partitionId)).head);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
async sendBehind() {
|
|
827
|
+
const { source, indexName } = this.deps;
|
|
828
|
+
let progressed = true;
|
|
829
|
+
while (progressed && this.state === "subscribed") {
|
|
830
|
+
progressed = false;
|
|
831
|
+
for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
|
|
832
|
+
if (this.state !== "subscribed" || this.sent[partitionId] >= this.heads[partitionId]) continue;
|
|
833
|
+
if (!this.windowAllows(1) || this.charged > 0) return;
|
|
834
|
+
const room = this.unackedBytes === 0 ? this.deps.windowBytes : this.deps.windowBytes - this.unackedBytes;
|
|
835
|
+
const maxBytes = Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES));
|
|
836
|
+
const after = this.sent[partitionId];
|
|
837
|
+
const read = await this.deps.reads.run(
|
|
838
|
+
() => source.read(indexName, partitionId, after, this.heads[partitionId], maxBytes)
|
|
839
|
+
);
|
|
840
|
+
if (this.state !== "subscribed") return;
|
|
841
|
+
if (read.gone) return this.resync("history");
|
|
842
|
+
const frames = read.frames.filter((frame) => frame.seqNo > this.sent[partitionId]);
|
|
843
|
+
if (frames.length === 0) continue;
|
|
844
|
+
let bytes = 0;
|
|
845
|
+
for (const frame of frames) bytes += frame.payload.byteLength;
|
|
846
|
+
if (!this.windowAllows(bytes)) return;
|
|
847
|
+
const message = encodeEntriesMessage(
|
|
848
|
+
partitionId,
|
|
849
|
+
frames.map((frame) => frame.payload)
|
|
850
|
+
);
|
|
851
|
+
if (!this.deps.buffered.take(message.byteLength)) return;
|
|
852
|
+
this.charged += message.byteLength;
|
|
853
|
+
const queue = this.unacked[partitionId].items;
|
|
854
|
+
for (const frame of frames) queue.push({ seqNo: frame.seqNo, bytes: frame.payload.byteLength });
|
|
855
|
+
this.unackedBytes += bytes;
|
|
856
|
+
this.sent[partitionId] = frames[frames.length - 1].seqNo;
|
|
857
|
+
this.send(message);
|
|
858
|
+
progressed = true;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
scheduleRetryWhileUndurable() {
|
|
863
|
+
if (this.state !== "subscribed" || this.retry !== null) return;
|
|
864
|
+
const { source, indexName } = this.deps;
|
|
865
|
+
const waiting = this.heads.some((head, partitionId) => source.pendingBeyondHead(indexName, partitionId, head));
|
|
866
|
+
if (!waiting) return;
|
|
867
|
+
this.retry = setTimeout(() => {
|
|
868
|
+
this.retry = null;
|
|
869
|
+
this.advance();
|
|
870
|
+
}, FEED_HEAD_RETRY_MS);
|
|
871
|
+
}
|
|
872
|
+
};
|
|
873
|
+
|
|
874
|
+
// src/sync/feed/hub.ts
|
|
875
|
+
function createFeedHub(source, options = {}) {
|
|
876
|
+
const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
|
|
877
|
+
const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
|
|
878
|
+
const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
|
|
879
|
+
const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
|
|
880
|
+
const sessionsByIndex = /* @__PURE__ */ new Map();
|
|
881
|
+
const unwatchByIndex = /* @__PURE__ */ new Map();
|
|
882
|
+
let connections = 0;
|
|
883
|
+
let stopListening = null;
|
|
884
|
+
let closed = false;
|
|
885
|
+
function each(indexName, action) {
|
|
886
|
+
const sessions = sessionsByIndex.get(indexName);
|
|
887
|
+
if (sessions === void 0) return;
|
|
888
|
+
for (const session of [...sessions]) action(session);
|
|
889
|
+
}
|
|
890
|
+
function onChange(change) {
|
|
891
|
+
switch (change.kind) {
|
|
892
|
+
case "advanced":
|
|
893
|
+
each(change.indexName, (session) => session.advance());
|
|
894
|
+
return;
|
|
895
|
+
case "settings":
|
|
896
|
+
each(change.indexName, (session) => session.sendSettings());
|
|
897
|
+
return;
|
|
898
|
+
case "epoch":
|
|
899
|
+
each(change.indexName, (session) => session.resync("epoch"));
|
|
900
|
+
return;
|
|
901
|
+
case "off":
|
|
902
|
+
case "dropped":
|
|
903
|
+
each(change.indexName, (session) => session.close(SYNC_CLOSE_RESYNC, "sync is off"));
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
function forget(session) {
|
|
908
|
+
const sessions = sessionsByIndex.get(session.indexName);
|
|
909
|
+
if (sessions === void 0 || !sessions.delete(session)) return;
|
|
910
|
+
if (sessions.size === 0) {
|
|
911
|
+
sessionsByIndex.delete(session.indexName);
|
|
912
|
+
unwatchByIndex.get(session.indexName)?.();
|
|
913
|
+
unwatchByIndex.delete(session.indexName);
|
|
914
|
+
}
|
|
915
|
+
release();
|
|
916
|
+
}
|
|
917
|
+
function release() {
|
|
918
|
+
connections -= 1;
|
|
919
|
+
if (connections === 0 && stopListening !== null) {
|
|
920
|
+
stopListening();
|
|
921
|
+
stopListening = null;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
function open(indexName, expiresAt, socket) {
|
|
925
|
+
if (stopListening === null) stopListening = source.listen(onChange);
|
|
926
|
+
const session = new FeedSession({
|
|
927
|
+
source,
|
|
928
|
+
indexName,
|
|
929
|
+
socket,
|
|
930
|
+
windowBytes,
|
|
931
|
+
expiresAt,
|
|
932
|
+
reads,
|
|
933
|
+
buffered,
|
|
934
|
+
onClosed: forget,
|
|
935
|
+
onError: (error) => options.onError?.(error)
|
|
936
|
+
});
|
|
937
|
+
let sessions = sessionsByIndex.get(indexName);
|
|
938
|
+
if (sessions === void 0) {
|
|
939
|
+
sessions = /* @__PURE__ */ new Set();
|
|
940
|
+
sessionsByIndex.set(indexName, sessions);
|
|
941
|
+
const unwatch = source.watchIndex?.(indexName);
|
|
942
|
+
if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
|
|
943
|
+
}
|
|
944
|
+
sessions.add(session);
|
|
945
|
+
return session;
|
|
946
|
+
}
|
|
947
|
+
return {
|
|
948
|
+
admit(indexName, expiresAt) {
|
|
949
|
+
if (closed) {
|
|
950
|
+
throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
|
|
951
|
+
}
|
|
952
|
+
if (source.indexState(indexName) === null) {
|
|
953
|
+
throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
954
|
+
}
|
|
955
|
+
if (connections >= maxConnections) {
|
|
956
|
+
throw new NarsilError(
|
|
957
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
958
|
+
`The server holds ${connections} feed connections, the most that it accepts`,
|
|
959
|
+
{ maximum: maxConnections }
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
connections += 1;
|
|
963
|
+
let settled = false;
|
|
964
|
+
return {
|
|
965
|
+
open(socket) {
|
|
966
|
+
if (settled) throw new Error("A feed admission opens one connection");
|
|
967
|
+
settled = true;
|
|
968
|
+
return open(indexName, expiresAt, socket);
|
|
969
|
+
},
|
|
970
|
+
cancel() {
|
|
971
|
+
if (settled) return;
|
|
972
|
+
settled = true;
|
|
973
|
+
release();
|
|
974
|
+
}
|
|
975
|
+
};
|
|
976
|
+
},
|
|
977
|
+
connectionCount: () => connections,
|
|
978
|
+
close() {
|
|
979
|
+
closed = true;
|
|
980
|
+
for (const indexName of [...sessionsByIndex.keys()]) {
|
|
981
|
+
each(indexName, (session) => session.close(SYNC_CLOSE_GOING_AWAY, "server closing"));
|
|
982
|
+
}
|
|
983
|
+
stopListening?.();
|
|
984
|
+
stopListening = null;
|
|
985
|
+
}
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// src/sync/feed/engine-source.ts
|
|
990
|
+
function syncSettingsOf(entry) {
|
|
991
|
+
const { config } = entry;
|
|
992
|
+
const synonyms = entry.synonyms === null ? [] : rulesOfStoredSynonyms(entry.synonyms);
|
|
993
|
+
return {
|
|
994
|
+
language: entry.language.name,
|
|
995
|
+
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
996
|
+
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
997
|
+
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
998
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
999
|
+
...config.bm25 !== void 0 ? { bm25: config.bm25 } : {},
|
|
1000
|
+
...synonyms.length > 0 ? { synonyms } : {}
|
|
1001
|
+
};
|
|
1002
|
+
}
|
|
1003
|
+
function storageOf(core) {
|
|
1004
|
+
return core.durability?.manager.syncStorage?.() ?? null;
|
|
1005
|
+
}
|
|
1006
|
+
function engineFeedSource(engine, core) {
|
|
1007
|
+
const requireStorage = () => {
|
|
1008
|
+
const storage = storageOf(core);
|
|
1009
|
+
if (storage === null) {
|
|
1010
|
+
throw new NarsilError(ErrorCodes.SYNC_DURABILITY_REQUIRED, "Device sync needs write-ahead log durability");
|
|
1011
|
+
}
|
|
1012
|
+
return storage;
|
|
1013
|
+
};
|
|
1014
|
+
return {
|
|
1015
|
+
indexState(indexName) {
|
|
1016
|
+
const entry = core.indexRegistry.get(indexName);
|
|
1017
|
+
if (entry === void 0 || entry.sync === null || storageOf(core) === null) return null;
|
|
1018
|
+
return { epoch: entry.sync.epoch, partitionCount: entry.partitionCount, settings: syncSettingsOf(entry) };
|
|
1019
|
+
},
|
|
1020
|
+
async bounds(indexName, partitionId) {
|
|
1021
|
+
const log = await requireStorage().openPartitionLog(indexName, partitionId);
|
|
1022
|
+
const applied = core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0;
|
|
1023
|
+
return { oldestSeqNo: log.oldestSeqNo, head: log.head, highestSeqNo: Math.max(log.head, applied) };
|
|
1024
|
+
},
|
|
1025
|
+
async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
1026
|
+
const read = await requireStorage().readPartitionLog(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes);
|
|
1027
|
+
return { frames: read.frames, gone: read.gone };
|
|
1028
|
+
},
|
|
1029
|
+
async bytesBehind(indexName, seqNos) {
|
|
1030
|
+
const storage = requireStorage();
|
|
1031
|
+
let bytes = 0;
|
|
1032
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
1033
|
+
const log = await storage.openPartitionLog(indexName, partitionId);
|
|
1034
|
+
bytes += await storage.logBytesAfter(log, seqNos[partitionId]);
|
|
1035
|
+
}
|
|
1036
|
+
return bytes;
|
|
1037
|
+
},
|
|
1038
|
+
async snapshotBytes(indexName) {
|
|
1039
|
+
const storage = requireStorage();
|
|
1040
|
+
const published = await storage.published(indexName);
|
|
1041
|
+
if (published !== null) return publishedArchiveLayout(indexName, published, storage).byteLength;
|
|
1042
|
+
const manifest = await storage.currentManifest(indexName);
|
|
1043
|
+
if (manifest === null) return 0;
|
|
1044
|
+
return estimatedArchiveBytes(manifestFileRefs(manifest));
|
|
1045
|
+
},
|
|
1046
|
+
pendingBeyondHead(indexName, partitionId, head) {
|
|
1047
|
+
return (core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0) > head;
|
|
1048
|
+
},
|
|
1049
|
+
listen(onChange) {
|
|
1050
|
+
const onWrite = (payload) => onChange({ kind: "advanced", indexName: payload.indexName });
|
|
1051
|
+
engine.on("write", onWrite);
|
|
1052
|
+
const stopSyncChanges = listenForSyncChanges(core.indexRegistry, onChange);
|
|
1053
|
+
return () => {
|
|
1054
|
+
engine.off("write", onWrite);
|
|
1055
|
+
stopSyncChanges();
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
};
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// src/server/sync/engine-serving.ts
|
|
1062
|
+
function engineSyncServing(engine, core) {
|
|
1063
|
+
startSyncRetentionSweep(core);
|
|
1064
|
+
return {
|
|
1065
|
+
async snapshot(indexName) {
|
|
1066
|
+
const published = await publishedSyncSnapshot(core, indexName);
|
|
1067
|
+
try {
|
|
1068
|
+
return {
|
|
1069
|
+
entityTag: published.snapshot.entityTag,
|
|
1070
|
+
layout: publishedArchiveLayout(indexName, published.snapshot, published.storage),
|
|
1071
|
+
release: published.unpin
|
|
1072
|
+
};
|
|
1073
|
+
} catch (error) {
|
|
1074
|
+
published.unpin();
|
|
1075
|
+
throw error;
|
|
1076
|
+
}
|
|
1077
|
+
},
|
|
1078
|
+
feed: engineFeedSource(engine, core),
|
|
1079
|
+
admitFeed: async () => void 0
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// src/server/sync/feed-host.ts
|
|
1084
|
+
var RelayedFeedSocket = class {
|
|
1085
|
+
constructor(port, feedId) {
|
|
1086
|
+
this.port = port;
|
|
1087
|
+
this.feedId = feedId;
|
|
1088
|
+
}
|
|
1089
|
+
inTransit = 0;
|
|
1090
|
+
threadBuffered = 0;
|
|
1091
|
+
closed = false;
|
|
1092
|
+
send(bytes) {
|
|
1093
|
+
if (this.closed) return;
|
|
1094
|
+
const copy = bytes.slice();
|
|
1095
|
+
this.inTransit += copy.byteLength;
|
|
1096
|
+
this.post({ type: "feed-send", feedId: this.feedId, bytes: copy }, [copy.buffer]);
|
|
1097
|
+
}
|
|
1098
|
+
bufferedAmount() {
|
|
1099
|
+
return this.inTransit + this.threadBuffered;
|
|
1100
|
+
}
|
|
1101
|
+
close(code, reason) {
|
|
1102
|
+
if (this.closed) return;
|
|
1103
|
+
this.closed = true;
|
|
1104
|
+
this.post({ type: "feed-close", feedId: this.feedId, code, reason }, []);
|
|
1105
|
+
}
|
|
1106
|
+
sent(bytes, buffered) {
|
|
1107
|
+
this.inTransit = Math.max(0, this.inTransit - bytes);
|
|
1108
|
+
this.threadBuffered = buffered;
|
|
1109
|
+
}
|
|
1110
|
+
markClosed() {
|
|
1111
|
+
this.closed = true;
|
|
1112
|
+
}
|
|
1113
|
+
post(note, transfer) {
|
|
1114
|
+
try {
|
|
1115
|
+
this.port.postMessage(note, transfer);
|
|
1116
|
+
} catch {
|
|
1117
|
+
this.closed = true;
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
};
|
|
1121
|
+
function routeMissing() {
|
|
1122
|
+
return new NarsilError(ServerErrorCodes.NOT_FOUND, "Route not found");
|
|
1123
|
+
}
|
|
1124
|
+
function createFeedHost(feeds, authorize) {
|
|
1125
|
+
const authorizeContext = (context) => authorize === void 0 ? Promise.resolve({ allowed: true, expiresAt: null }) : authorize(context);
|
|
1126
|
+
function admit(feed, indexName, expiresAt) {
|
|
1127
|
+
const hosted = feeds[feed];
|
|
1128
|
+
if (hosted === void 0) return Promise.reject(routeMissing());
|
|
1129
|
+
return hosted.admit(indexName, expiresAt);
|
|
1130
|
+
}
|
|
1131
|
+
const gateways = {};
|
|
1132
|
+
for (const feed of Object.keys(feeds)) {
|
|
1133
|
+
gateways[feed] = {
|
|
1134
|
+
authorize: authorizeContext,
|
|
1135
|
+
async admit(indexName, expiresAt) {
|
|
1136
|
+
const admission = await admit(feed, indexName, expiresAt);
|
|
1137
|
+
return { open: (socket) => admission.open(socket), cancel: () => admission.cancel() };
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
function forThread(port) {
|
|
1142
|
+
const relayed = /* @__PURE__ */ new Map();
|
|
1143
|
+
let nextFeedId = 0;
|
|
1144
|
+
let closed = false;
|
|
1145
|
+
function closeFeed(feedId) {
|
|
1146
|
+
const feed = relayed.get(feedId);
|
|
1147
|
+
if (feed === void 0) return;
|
|
1148
|
+
relayed.delete(feedId);
|
|
1149
|
+
feed.socket.markClosed();
|
|
1150
|
+
if (feed.endpoint === null) feed.admission.cancel();
|
|
1151
|
+
else feed.endpoint.closedByPeer();
|
|
1152
|
+
}
|
|
1153
|
+
return {
|
|
1154
|
+
async admit(request) {
|
|
1155
|
+
try {
|
|
1156
|
+
const admission = await admit(request.feed, request.indexName, request.expiresAt);
|
|
1157
|
+
if (closed) {
|
|
1158
|
+
admission.cancel();
|
|
1159
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: null };
|
|
1160
|
+
}
|
|
1161
|
+
nextFeedId += 1;
|
|
1162
|
+
const feedId = nextFeedId;
|
|
1163
|
+
relayed.set(feedId, { admission, socket: new RelayedFeedSocket(port, feedId), endpoint: null });
|
|
1164
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId, error: null };
|
|
1165
|
+
} catch (err) {
|
|
1166
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: toHttpError(err).body };
|
|
1167
|
+
}
|
|
1168
|
+
},
|
|
1169
|
+
handle(note) {
|
|
1170
|
+
const feed = relayed.get(note.feedId);
|
|
1171
|
+
if (feed === void 0) return;
|
|
1172
|
+
switch (note.type) {
|
|
1173
|
+
case "feed-open":
|
|
1174
|
+
if (feed.endpoint === null) feed.endpoint = feed.admission.open(feed.socket);
|
|
1175
|
+
return;
|
|
1176
|
+
case "feed-message":
|
|
1177
|
+
feed.endpoint?.receive(note.bytes, note.binary);
|
|
1178
|
+
return;
|
|
1179
|
+
case "feed-sent":
|
|
1180
|
+
feed.socket.sent(note.bytes, note.buffered);
|
|
1181
|
+
feed.endpoint?.drained();
|
|
1182
|
+
return;
|
|
1183
|
+
case "feed-cancel":
|
|
1184
|
+
case "feed-closed":
|
|
1185
|
+
closeFeed(note.feedId);
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
},
|
|
1189
|
+
closeAll() {
|
|
1190
|
+
closed = true;
|
|
1191
|
+
for (const feedId of [...relayed.keys()]) closeFeed(feedId);
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
return {
|
|
1196
|
+
gateways,
|
|
1197
|
+
forThread,
|
|
1198
|
+
close() {
|
|
1199
|
+
for (const hosted of Object.values(feeds)) hosted?.close();
|
|
1200
|
+
}
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
// src/server/feeds.ts
|
|
1205
|
+
function syncServingOf(engine, core) {
|
|
1206
|
+
const ofNode = syncServingSourceOf(engine);
|
|
1207
|
+
if (ofNode !== void 0) return ofNode();
|
|
1208
|
+
return core === void 0 ? null : engineSyncServing(engine, core);
|
|
1209
|
+
}
|
|
1210
|
+
function changesServingOf(engine, core) {
|
|
1211
|
+
const ofNode = changesServingSourceOf(engine);
|
|
1212
|
+
if (ofNode !== void 0) return ofNode();
|
|
1213
|
+
if (core === void 0) return null;
|
|
1214
|
+
return { source: engineChangeSource(core, listenForSyncChanges), admitFeed: async () => void 0 };
|
|
1215
|
+
}
|
|
1216
|
+
function hostedSyncFeed(serving, options) {
|
|
1217
|
+
const hub = createFeedHub(serving.feed, {
|
|
1218
|
+
...options,
|
|
1219
|
+
onError: (error) => console.warn("A device sync feed failed to read its index:", error)
|
|
1220
|
+
});
|
|
1221
|
+
return {
|
|
1222
|
+
async admit(indexName, expiresAt) {
|
|
1223
|
+
await serving.admitFeed(indexName);
|
|
1224
|
+
return hub.admit(indexName, expiresAt);
|
|
1225
|
+
},
|
|
1226
|
+
close: () => hub.close()
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
function hostedChangesFeed(serving, options, allowRegex) {
|
|
1230
|
+
const hub = createChangesHub(serving.source, {
|
|
1231
|
+
...options,
|
|
1232
|
+
refuseFilter: (filter) => refusedRegex(filter, allowRegex),
|
|
1233
|
+
onError: (error) => console.warn("A change feed failed to read its index:", error)
|
|
1234
|
+
});
|
|
1235
|
+
return {
|
|
1236
|
+
async admit(indexName, expiresAt) {
|
|
1237
|
+
await serving.admitFeed(indexName);
|
|
1238
|
+
return hub.admit(indexName, expiresAt);
|
|
1239
|
+
},
|
|
1240
|
+
close: () => hub.close()
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
function startFeeds(engine, core, settings) {
|
|
1244
|
+
const hosted = {};
|
|
1245
|
+
let syncServing = null;
|
|
1246
|
+
if (settings.sync !== null) {
|
|
1247
|
+
syncServing = syncServingOf(engine, core);
|
|
1248
|
+
if (syncServing !== null) hosted.sync = hostedSyncFeed(syncServing, settings.sync);
|
|
1249
|
+
}
|
|
1250
|
+
if (settings.changes !== null) {
|
|
1251
|
+
const serving = changesServingOf(engine, core);
|
|
1252
|
+
if (serving !== null) hosted.changes = hostedChangesFeed(serving, settings.changes, settings.allowRegex);
|
|
1253
|
+
}
|
|
1254
|
+
if (hosted.sync === void 0 && hosted.changes === void 0) return null;
|
|
1255
|
+
return { host: createFeedHost(hosted, settings.authorize), syncServing };
|
|
1256
|
+
}
|
|
1257
|
+
function snapshotHistoryOf(raw) {
|
|
1258
|
+
if (raw === null || raw === "false") return false;
|
|
1259
|
+
return raw === "true" ? true : null;
|
|
1260
|
+
}
|
|
33
1261
|
function createAdminHandlers(deps) {
|
|
34
1262
|
const { engine, tasks, limits } = deps;
|
|
35
1263
|
async function checkpoint(ctx) {
|
|
@@ -41,8 +1269,15 @@ function createAdminHandlers(deps) {
|
|
|
41
1269
|
}
|
|
42
1270
|
}
|
|
43
1271
|
async function snapshot(ctx) {
|
|
1272
|
+
const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
|
|
1273
|
+
if (queryHistory === null) {
|
|
1274
|
+
badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
|
|
1275
|
+
value: ctx.query.get("queryHistory")
|
|
1276
|
+
});
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
44
1279
|
try {
|
|
45
|
-
const archive = await engine.snapshot(ctx.params[0]);
|
|
1280
|
+
const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
|
|
46
1281
|
if (ctx.abort.aborted) {
|
|
47
1282
|
await archive.cancel();
|
|
48
1283
|
return;
|
|
@@ -100,6 +1335,16 @@ function createAdminHandlers(deps) {
|
|
|
100
1335
|
respondError(ctx, err);
|
|
101
1336
|
}
|
|
102
1337
|
}
|
|
1338
|
+
async function updateBm25(ctx) {
|
|
1339
|
+
const body = parseJson(ctx);
|
|
1340
|
+
if (body === void 0) return;
|
|
1341
|
+
try {
|
|
1342
|
+
await engine.updateBM25(ctx.params[0], body);
|
|
1343
|
+
respondJson(ctx, { ok: true });
|
|
1344
|
+
} catch (err) {
|
|
1345
|
+
respondError(ctx, err);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
103
1348
|
async function memory(ctx) {
|
|
104
1349
|
try {
|
|
105
1350
|
const stats = await engine.getMemoryStats();
|
|
@@ -109,15 +1354,15 @@ function createAdminHandlers(deps) {
|
|
|
109
1354
|
respondError(ctx, err);
|
|
110
1355
|
}
|
|
111
1356
|
}
|
|
112
|
-
function requireIndexExists(name) {
|
|
113
|
-
engine.getStats(name);
|
|
1357
|
+
async function requireIndexExists(name) {
|
|
1358
|
+
await indexReadsOf(engine).getStats(name);
|
|
114
1359
|
}
|
|
115
1360
|
async function optimize(ctx) {
|
|
116
1361
|
const body = parseJsonOptional(ctx);
|
|
117
1362
|
if (!body) return;
|
|
118
1363
|
const name = ctx.params[0];
|
|
119
1364
|
try {
|
|
120
|
-
requireIndexExists(name);
|
|
1365
|
+
await requireIndexExists(name);
|
|
121
1366
|
} catch (err) {
|
|
122
1367
|
respondError(ctx, err);
|
|
123
1368
|
return;
|
|
@@ -128,7 +1373,7 @@ function createAdminHandlers(deps) {
|
|
|
128
1373
|
async function rebuildAnalysis(ctx) {
|
|
129
1374
|
const name = ctx.params[0];
|
|
130
1375
|
try {
|
|
131
|
-
requireIndexExists(name);
|
|
1376
|
+
await requireIndexExists(name);
|
|
132
1377
|
} catch (err) {
|
|
133
1378
|
respondError(ctx, err);
|
|
134
1379
|
return;
|
|
@@ -136,6 +1381,24 @@ function createAdminHandlers(deps) {
|
|
|
136
1381
|
const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
|
|
137
1382
|
respondJson(ctx, record, 202);
|
|
138
1383
|
}
|
|
1384
|
+
async function changeAnalysis(ctx) {
|
|
1385
|
+
const body = parseJsonOptional(ctx);
|
|
1386
|
+
if (!body) return;
|
|
1387
|
+
const failure = validateChangeAnalysis(body);
|
|
1388
|
+
if (failure) {
|
|
1389
|
+
rejectInvalid(ctx, failure);
|
|
1390
|
+
return;
|
|
1391
|
+
}
|
|
1392
|
+
const name = ctx.params[0];
|
|
1393
|
+
try {
|
|
1394
|
+
await requireIndexExists(name);
|
|
1395
|
+
} catch (err) {
|
|
1396
|
+
respondError(ctx, err);
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
|
|
1400
|
+
respondJson(ctx, record, 202);
|
|
1401
|
+
}
|
|
139
1402
|
async function rebalance(ctx) {
|
|
140
1403
|
const body = parseJsonOptional(ctx);
|
|
141
1404
|
if (!body) return;
|
|
@@ -147,7 +1410,7 @@ function createAdminHandlers(deps) {
|
|
|
147
1410
|
const target = body.targetPartitionCount;
|
|
148
1411
|
const name = ctx.params[0];
|
|
149
1412
|
try {
|
|
150
|
-
requireIndexExists(name);
|
|
1413
|
+
await requireIndexExists(name);
|
|
151
1414
|
} catch (err) {
|
|
152
1415
|
respondError(ctx, err);
|
|
153
1416
|
return;
|
|
@@ -230,10 +1493,12 @@ function createAdminHandlers(deps) {
|
|
|
230
1493
|
partitionConfig,
|
|
231
1494
|
readSynonyms,
|
|
232
1495
|
replaceSynonyms,
|
|
1496
|
+
updateBm25,
|
|
233
1497
|
memory,
|
|
234
1498
|
optimize,
|
|
235
1499
|
rebalance,
|
|
236
1500
|
rebuildAnalysis,
|
|
1501
|
+
changeAnalysis,
|
|
237
1502
|
restore,
|
|
238
1503
|
getTask,
|
|
239
1504
|
listTasks,
|
|
@@ -241,14 +1506,6 @@ function createAdminHandlers(deps) {
|
|
|
241
1506
|
};
|
|
242
1507
|
}
|
|
243
1508
|
|
|
244
|
-
// src/server/handlers/capabilities.ts
|
|
245
|
-
function createCapabilitiesHandler() {
|
|
246
|
-
function report(ctx) {
|
|
247
|
-
respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
|
|
248
|
-
}
|
|
249
|
-
return { report };
|
|
250
|
-
}
|
|
251
|
-
|
|
252
1509
|
// src/server/handlers/cluster.ts
|
|
253
1510
|
function requireCluster(deps) {
|
|
254
1511
|
if (deps.cluster === void 0) {
|
|
@@ -301,7 +1558,12 @@ function createClusterHandlers(deps) {
|
|
|
301
1558
|
}
|
|
302
1559
|
const table = await cluster.getAllocation(indexName);
|
|
303
1560
|
if (table === null) {
|
|
304
|
-
respondJson(ctx, {
|
|
1561
|
+
respondJson(ctx, {
|
|
1562
|
+
indexName,
|
|
1563
|
+
allocated: false,
|
|
1564
|
+
version: null,
|
|
1565
|
+
partitions: []
|
|
1566
|
+
});
|
|
305
1567
|
return;
|
|
306
1568
|
}
|
|
307
1569
|
const partitions = [];
|
|
@@ -319,12 +1581,158 @@ function createClusterHandlers(deps) {
|
|
|
319
1581
|
});
|
|
320
1582
|
}
|
|
321
1583
|
partitions.sort((left, right) => left.partitionId - right.partitionId);
|
|
322
|
-
respondJson(ctx, {
|
|
1584
|
+
respondJson(ctx, {
|
|
1585
|
+
indexName,
|
|
1586
|
+
allocated: partitions.length > 0,
|
|
1587
|
+
version: table.version,
|
|
1588
|
+
partitions
|
|
1589
|
+
});
|
|
323
1590
|
} catch (err) {
|
|
324
1591
|
respondError(ctx, err);
|
|
325
1592
|
}
|
|
326
1593
|
}
|
|
327
|
-
|
|
1594
|
+
async function replicationFactor(ctx) {
|
|
1595
|
+
try {
|
|
1596
|
+
const cluster = requireCluster(deps);
|
|
1597
|
+
const indexName = ctx.params[0] ?? "";
|
|
1598
|
+
const malformed = malformedIndexName(indexName);
|
|
1599
|
+
if (malformed !== null) {
|
|
1600
|
+
badRequest(ctx.res, malformed, { indexName });
|
|
1601
|
+
return;
|
|
1602
|
+
}
|
|
1603
|
+
const body = parseJson(ctx);
|
|
1604
|
+
if (body === void 0) return;
|
|
1605
|
+
const factor = isRecord(body) ? body.replicationFactor : void 0;
|
|
1606
|
+
if (typeof factor !== "number") {
|
|
1607
|
+
badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
await cluster.setReplicationFactor(indexName, factor);
|
|
1611
|
+
respondJson(ctx, { indexName, replicationFactor: factor });
|
|
1612
|
+
} catch (err) {
|
|
1613
|
+
respondError(ctx, err);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
async function health(ctx) {
|
|
1617
|
+
try {
|
|
1618
|
+
respondJson(ctx, await requireCluster(deps).getHealth());
|
|
1619
|
+
} catch (err) {
|
|
1620
|
+
respondError(ctx, err);
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
return { topology, allocation, replicationFactor, health };
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
// src/server/handlers/backups.ts
|
|
1627
|
+
function stepsOf(context) {
|
|
1628
|
+
return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
|
|
1629
|
+
}
|
|
1630
|
+
function restoreOptionsOf(body) {
|
|
1631
|
+
const { indexName } = body;
|
|
1632
|
+
if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
|
|
1633
|
+
const placement = readRestorePlacement(body);
|
|
1634
|
+
return typeof placement === "string" ? placement : { indexName, ...placement };
|
|
1635
|
+
}
|
|
1636
|
+
function readBody(ctx) {
|
|
1637
|
+
const body = parseJson(ctx);
|
|
1638
|
+
if (body === void 0) return void 0;
|
|
1639
|
+
if (!isRecord(body)) {
|
|
1640
|
+
badRequest(ctx.res, "The body must be a JSON object");
|
|
1641
|
+
return void 0;
|
|
1642
|
+
}
|
|
1643
|
+
return body;
|
|
1644
|
+
}
|
|
1645
|
+
function createBackupHandlers(deps) {
|
|
1646
|
+
async function create(ctx) {
|
|
1647
|
+
try {
|
|
1648
|
+
const cluster = requireCluster(deps);
|
|
1649
|
+
const indexName = ctx.params[0] ?? "";
|
|
1650
|
+
const body = readBody(ctx);
|
|
1651
|
+
if (body === void 0) return;
|
|
1652
|
+
if (typeof body.backup !== "string") {
|
|
1653
|
+
badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
|
|
1654
|
+
return;
|
|
1655
|
+
}
|
|
1656
|
+
if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
|
|
1657
|
+
badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
|
|
1658
|
+
return;
|
|
1659
|
+
}
|
|
1660
|
+
const backupName = validateBackupName(body.backup);
|
|
1661
|
+
const queryHistory = body.queryHistory === true;
|
|
1662
|
+
const record = await deps.tasks.start(
|
|
1663
|
+
"backup",
|
|
1664
|
+
indexName,
|
|
1665
|
+
async (context) => {
|
|
1666
|
+
await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
|
|
1667
|
+
},
|
|
1668
|
+
void 0,
|
|
1669
|
+
backupName
|
|
1670
|
+
);
|
|
1671
|
+
respondJson(ctx, record, 202);
|
|
1672
|
+
} catch (err) {
|
|
1673
|
+
respondError(ctx, err);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
async function list(ctx) {
|
|
1677
|
+
try {
|
|
1678
|
+
const cluster = requireCluster(deps);
|
|
1679
|
+
respondJson(ctx, { backups: await cluster.listBackups() });
|
|
1680
|
+
} catch (err) {
|
|
1681
|
+
respondError(ctx, err);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
async function restore(ctx) {
|
|
1685
|
+
try {
|
|
1686
|
+
const cluster = requireCluster(deps);
|
|
1687
|
+
const backupName = validateBackupName(ctx.params[0]);
|
|
1688
|
+
const body = readBody(ctx);
|
|
1689
|
+
if (body === void 0) return;
|
|
1690
|
+
const options = restoreOptionsOf(body);
|
|
1691
|
+
if (typeof options === "string") {
|
|
1692
|
+
badRequest(ctx.res, options, { backupName });
|
|
1693
|
+
return;
|
|
1694
|
+
}
|
|
1695
|
+
const record = await deps.tasks.start(
|
|
1696
|
+
"backupRestore",
|
|
1697
|
+
options.indexName,
|
|
1698
|
+
async (context) => {
|
|
1699
|
+
await cluster.restoreBackup(backupName, { ...options, ...stepsOf(context) });
|
|
1700
|
+
},
|
|
1701
|
+
void 0,
|
|
1702
|
+
backupName
|
|
1703
|
+
);
|
|
1704
|
+
respondJson(ctx, record, 202);
|
|
1705
|
+
} catch (err) {
|
|
1706
|
+
respondError(ctx, err);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
async function remove(ctx) {
|
|
1710
|
+
try {
|
|
1711
|
+
const cluster = requireCluster(deps);
|
|
1712
|
+
const backupName = validateBackupName(ctx.params[0]);
|
|
1713
|
+
const record = await deps.tasks.start(
|
|
1714
|
+
"backupDelete",
|
|
1715
|
+
"",
|
|
1716
|
+
async (context) => {
|
|
1717
|
+
await cluster.deleteBackup(backupName, stepsOf(context));
|
|
1718
|
+
},
|
|
1719
|
+
void 0,
|
|
1720
|
+
backupName
|
|
1721
|
+
);
|
|
1722
|
+
respondJson(ctx, record, 202);
|
|
1723
|
+
} catch (err) {
|
|
1724
|
+
respondError(ctx, err);
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
return { create, list, restore, remove };
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
// src/server/handlers/capabilities.ts
|
|
1731
|
+
function createCapabilitiesHandler() {
|
|
1732
|
+
function report(ctx) {
|
|
1733
|
+
respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
|
|
1734
|
+
}
|
|
1735
|
+
return { report };
|
|
328
1736
|
}
|
|
329
1737
|
function importResultOf(result, maxErrors) {
|
|
330
1738
|
const errors = result.refused.slice(0, maxErrors);
|
|
@@ -346,7 +1754,7 @@ function createDocumentExportHandlers(deps) {
|
|
|
346
1754
|
}
|
|
347
1755
|
await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
|
|
348
1756
|
} catch (err) {
|
|
349
|
-
|
|
1757
|
+
respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
|
|
350
1758
|
}
|
|
351
1759
|
}
|
|
352
1760
|
async function importExport(ctx) {
|
|
@@ -378,9 +1786,10 @@ function createDocumentExportHandlers(deps) {
|
|
|
378
1786
|
// src/server/handlers/documents.ts
|
|
379
1787
|
function createDocumentHandlers(deps) {
|
|
380
1788
|
const { engine } = deps;
|
|
1789
|
+
const writeBlock = writeBlockLookupOf(engine);
|
|
381
1790
|
const reads = createDocumentReadHandlers(deps);
|
|
382
1791
|
async function insert(ctx) {
|
|
383
|
-
const body =
|
|
1792
|
+
const body = await parseWriteJson(ctx, writeBlock);
|
|
384
1793
|
if (!body) return;
|
|
385
1794
|
const failure = validateDocumentBody(body);
|
|
386
1795
|
if (failure) {
|
|
@@ -395,7 +1804,7 @@ function createDocumentHandlers(deps) {
|
|
|
395
1804
|
}
|
|
396
1805
|
}
|
|
397
1806
|
async function put(ctx) {
|
|
398
|
-
const body =
|
|
1807
|
+
const body = await parseWriteJson(ctx, writeBlock);
|
|
399
1808
|
if (!body) return;
|
|
400
1809
|
const failure = validateDocumentBody(body);
|
|
401
1810
|
if (failure) {
|
|
@@ -416,7 +1825,7 @@ function createDocumentHandlers(deps) {
|
|
|
416
1825
|
}
|
|
417
1826
|
}
|
|
418
1827
|
async function patch(ctx) {
|
|
419
|
-
const body =
|
|
1828
|
+
const body = await parseWriteJson(ctx, writeBlock);
|
|
420
1829
|
if (!body) return;
|
|
421
1830
|
const failure = validateDocumentBody(body);
|
|
422
1831
|
if (failure) {
|
|
@@ -447,7 +1856,7 @@ function createDocumentHandlers(deps) {
|
|
|
447
1856
|
}
|
|
448
1857
|
}
|
|
449
1858
|
async function batch(ctx) {
|
|
450
|
-
const body = parseJson(ctx);
|
|
1859
|
+
const body = parseJson(ctx, ctx.rawBody && isRemoveOnlyBatch(ctx.rawBody) ? "read" : "write");
|
|
451
1860
|
if (!body) return;
|
|
452
1861
|
const failure = validateBatch(body);
|
|
453
1862
|
if (failure) {
|
|
@@ -528,7 +1937,7 @@ function* iterateNdjson(buffer, maxLineBytes) {
|
|
|
528
1937
|
if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
|
|
529
1938
|
if (contentEnd > start) {
|
|
530
1939
|
lineNumber++;
|
|
531
|
-
yield { lineNumber,
|
|
1940
|
+
yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
|
|
532
1941
|
}
|
|
533
1942
|
start = end + 1;
|
|
534
1943
|
}
|
|
@@ -544,6 +1953,7 @@ function isPlainObject(value) {
|
|
|
544
1953
|
async function runImport(engine, options) {
|
|
545
1954
|
const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
|
|
546
1955
|
const bytesTotal = body.length;
|
|
1956
|
+
const writeBlock = writeBlockLookupOf(engine);
|
|
547
1957
|
const errors = [];
|
|
548
1958
|
let indexed = 0;
|
|
549
1959
|
let failed = 0;
|
|
@@ -558,7 +1968,12 @@ async function runImport(engine, options) {
|
|
|
558
1968
|
indexed += result.succeeded.length;
|
|
559
1969
|
for (const failure of result.failed) {
|
|
560
1970
|
const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
|
|
561
|
-
recordFailure({
|
|
1971
|
+
recordFailure({
|
|
1972
|
+
docId: failure.docId,
|
|
1973
|
+
code: serialized.code,
|
|
1974
|
+
message: serialized.message,
|
|
1975
|
+
...serialized.details === void 0 ? {} : { details: serialized.details }
|
|
1976
|
+
});
|
|
562
1977
|
}
|
|
563
1978
|
};
|
|
564
1979
|
const inFlight = /* @__PURE__ */ new Set();
|
|
@@ -584,9 +1999,17 @@ async function runImport(engine, options) {
|
|
|
584
1999
|
try {
|
|
585
2000
|
for (const line of iterateNdjson(body, maxLineBytes)) {
|
|
586
2001
|
bytesProcessed = line.bytesConsumed;
|
|
2002
|
+
const refusal = parseRefusalOf(line.bytes);
|
|
2003
|
+
if (refusal !== null) {
|
|
2004
|
+
const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
|
|
2005
|
+
recordFailure(
|
|
2006
|
+
block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
|
|
2007
|
+
);
|
|
2008
|
+
continue;
|
|
2009
|
+
}
|
|
587
2010
|
let parsed;
|
|
588
2011
|
try {
|
|
589
|
-
parsed = JSON.parse(line.
|
|
2012
|
+
parsed = JSON.parse(line.bytes.toString("utf8"));
|
|
590
2013
|
} catch {
|
|
591
2014
|
recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
|
|
592
2015
|
continue;
|
|
@@ -697,15 +2120,27 @@ function mapHttpIndexConfig(http) {
|
|
|
697
2120
|
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
698
2121
|
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
699
2122
|
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
2123
|
+
if (http.textStorage !== void 0) config.textStorage = http.textStorage;
|
|
700
2124
|
if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
|
|
701
2125
|
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
702
2126
|
if (http.strict !== void 0) config.strict = http.strict;
|
|
703
2127
|
if (http.required !== void 0) config.required = http.required;
|
|
704
2128
|
if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
|
|
705
2129
|
if (http.synonyms !== void 0) config.synonyms = http.synonyms;
|
|
706
|
-
if (http.
|
|
2130
|
+
if (http.tokenizer !== void 0) {
|
|
2131
|
+
if (typeof http.tokenizer !== "string") {
|
|
2132
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
|
|
2133
|
+
}
|
|
2134
|
+
config.tokenizer = http.tokenizer;
|
|
2135
|
+
}
|
|
2136
|
+
if (typeof http.stopWords === "string") {
|
|
2137
|
+
config.stopWords = http.stopWords;
|
|
2138
|
+
} else if (http.stopWords !== void 0) {
|
|
707
2139
|
if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
|
|
708
|
-
throw new NarsilError(
|
|
2140
|
+
throw new NarsilError(
|
|
2141
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2142
|
+
'Field "stopWords" must be a registered name or an array of strings'
|
|
2143
|
+
);
|
|
709
2144
|
}
|
|
710
2145
|
config.stopWords = new Set(http.stopWords);
|
|
711
2146
|
}
|
|
@@ -734,86 +2169,385 @@ function mapEmbedding(embedding) {
|
|
|
734
2169
|
return mapped;
|
|
735
2170
|
}
|
|
736
2171
|
|
|
737
|
-
// src/server/handlers/indexes.ts
|
|
738
|
-
function createIndexHandlers(deps) {
|
|
739
|
-
const { engine } = deps;
|
|
740
|
-
async function create(ctx) {
|
|
2172
|
+
// src/server/handlers/indexes.ts
|
|
2173
|
+
function createIndexHandlers(deps) {
|
|
2174
|
+
const { engine } = deps;
|
|
2175
|
+
async function create(ctx) {
|
|
2176
|
+
const body = parseJson(ctx);
|
|
2177
|
+
if (!body) return;
|
|
2178
|
+
const failure = validateCreateIndex(body);
|
|
2179
|
+
if (failure) {
|
|
2180
|
+
rejectInvalid(ctx, failure);
|
|
2181
|
+
return;
|
|
2182
|
+
}
|
|
2183
|
+
try {
|
|
2184
|
+
const config = mapHttpIndexConfig(body.config);
|
|
2185
|
+
await engine.createIndex(body.name, config);
|
|
2186
|
+
respondJson(ctx, { name: body.name }, 201);
|
|
2187
|
+
} catch (err) {
|
|
2188
|
+
respondError(ctx, err);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
async function list(ctx) {
|
|
2192
|
+
try {
|
|
2193
|
+
respondJson(ctx, { indexes: await indexReadsOf(engine).listIndexes() });
|
|
2194
|
+
} catch (err) {
|
|
2195
|
+
respondError(ctx, err);
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
async function drop(ctx) {
|
|
2199
|
+
try {
|
|
2200
|
+
await engine.dropIndex(ctx.params[0]);
|
|
2201
|
+
respondJson(ctx, { name: ctx.params[0], dropped: true });
|
|
2202
|
+
} catch (err) {
|
|
2203
|
+
respondError(ctx, err);
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
async function stats(ctx) {
|
|
2207
|
+
try {
|
|
2208
|
+
respondJson(ctx, await indexReadsOf(engine).getStats(ctx.params[0]));
|
|
2209
|
+
} catch (err) {
|
|
2210
|
+
respondError(ctx, err);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
async function partitions(ctx) {
|
|
2214
|
+
try {
|
|
2215
|
+
respondJson(ctx, {
|
|
2216
|
+
partitions: await indexReadsOf(engine).getPartitionStats(ctx.params[0])
|
|
2217
|
+
});
|
|
2218
|
+
} catch (err) {
|
|
2219
|
+
respondError(ctx, err);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
async function clear(ctx) {
|
|
2223
|
+
try {
|
|
2224
|
+
await engine.clear(ctx.params[0]);
|
|
2225
|
+
respondJson(ctx, { name: ctx.params[0], cleared: true });
|
|
2226
|
+
} catch (err) {
|
|
2227
|
+
respondError(ctx, err);
|
|
2228
|
+
}
|
|
2229
|
+
}
|
|
2230
|
+
async function open(ctx) {
|
|
2231
|
+
try {
|
|
2232
|
+
await engine.open(ctx.params[0]);
|
|
2233
|
+
respondJson(ctx, { name: ctx.params[0], state: "open" });
|
|
2234
|
+
} catch (err) {
|
|
2235
|
+
respondError(ctx, err);
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
async function close(ctx) {
|
|
2239
|
+
try {
|
|
2240
|
+
await engine.close(ctx.params[0]);
|
|
2241
|
+
respondJson(ctx, { name: ctx.params[0], state: "closed" });
|
|
2242
|
+
} catch (err) {
|
|
2243
|
+
respondError(ctx, err);
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
return { create, list, drop, stats, partitions, clear, open, close };
|
|
2247
|
+
}
|
|
2248
|
+
|
|
2249
|
+
// src/server/handlers/metrics.ts
|
|
2250
|
+
function labelValue(value) {
|
|
2251
|
+
return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
|
|
2252
|
+
}
|
|
2253
|
+
function labels(pairs) {
|
|
2254
|
+
return `{${pairs.map(([name, value]) => `${name}="${labelValue(value)}"`).join(",")}}`;
|
|
2255
|
+
}
|
|
2256
|
+
function family(name, type, help, samples) {
|
|
2257
|
+
return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
|
|
2258
|
+
}
|
|
2259
|
+
function prometheusText(metrics) {
|
|
2260
|
+
const lines = [
|
|
2261
|
+
...family(
|
|
2262
|
+
"narsil_replication_lag",
|
|
2263
|
+
"gauge",
|
|
2264
|
+
"Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
|
|
2265
|
+
metrics.replicationLag.map(
|
|
2266
|
+
(figure) => `narsil_replication_lag${labels([
|
|
2267
|
+
["index", figure.indexName],
|
|
2268
|
+
["partition", String(figure.partitionId)],
|
|
2269
|
+
["replica", figure.replica]
|
|
2270
|
+
])} ${figure.lag}`
|
|
2271
|
+
)
|
|
2272
|
+
),
|
|
2273
|
+
...family(
|
|
2274
|
+
"narsil_commit_point",
|
|
2275
|
+
"gauge",
|
|
2276
|
+
"The highest sequence number that the leader of the partition has acknowledged to a client.",
|
|
2277
|
+
metrics.commitPoints.map(
|
|
2278
|
+
(figure) => `narsil_commit_point${labels([
|
|
2279
|
+
["index", figure.indexName],
|
|
2280
|
+
["partition", String(figure.partitionId)]
|
|
2281
|
+
])} ${figure.commitPoint}`
|
|
2282
|
+
)
|
|
2283
|
+
),
|
|
2284
|
+
...family(
|
|
2285
|
+
"narsil_commit_point_floor",
|
|
2286
|
+
"gauge",
|
|
2287
|
+
"The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
|
|
2288
|
+
metrics.partitions.map(
|
|
2289
|
+
(figure) => `narsil_commit_point_floor${labels([
|
|
2290
|
+
["index", figure.indexName],
|
|
2291
|
+
["partition", String(figure.partitionId)]
|
|
2292
|
+
])} ${figure.commitPointFloor}`
|
|
2293
|
+
)
|
|
2294
|
+
),
|
|
2295
|
+
...family(
|
|
2296
|
+
"narsil_in_sync_replicas",
|
|
2297
|
+
"gauge",
|
|
2298
|
+
"The replicas besides the primary that are in the in-sync set of the partition.",
|
|
2299
|
+
metrics.partitions.map(
|
|
2300
|
+
(figure) => `narsil_in_sync_replicas${labels([
|
|
2301
|
+
["index", figure.indexName],
|
|
2302
|
+
["partition", String(figure.partitionId)]
|
|
2303
|
+
])} ${figure.inSyncReplicas}`
|
|
2304
|
+
)
|
|
2305
|
+
),
|
|
2306
|
+
...family(
|
|
2307
|
+
"narsil_vector_rebuild_waiting_for_memory",
|
|
2308
|
+
"gauge",
|
|
2309
|
+
"A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
|
|
2310
|
+
metrics.vectorRebuildsWaitingForMemory.map(
|
|
2311
|
+
(figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
|
|
2312
|
+
["index", figure.indexName],
|
|
2313
|
+
["field", figure.fieldName]
|
|
2314
|
+
])} 1`
|
|
2315
|
+
)
|
|
2316
|
+
),
|
|
2317
|
+
...family(
|
|
2318
|
+
"narsil_coordinator_request_seconds",
|
|
2319
|
+
"histogram",
|
|
2320
|
+
"How long the calls from this node to the cluster coordinator took.",
|
|
2321
|
+
metrics.coordinatorRequests.flatMap((figure) => {
|
|
2322
|
+
const operation = ["operation", figure.operation];
|
|
2323
|
+
return [
|
|
2324
|
+
...figure.bucketBoundsSeconds.map(
|
|
2325
|
+
(bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
|
|
2326
|
+
),
|
|
2327
|
+
`narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
|
|
2328
|
+
`narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
|
|
2329
|
+
`narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
|
|
2330
|
+
];
|
|
2331
|
+
})
|
|
2332
|
+
)
|
|
2333
|
+
];
|
|
2334
|
+
return `${lines.join("\n")}
|
|
2335
|
+
`;
|
|
2336
|
+
}
|
|
2337
|
+
function createMetricsHandler(deps) {
|
|
2338
|
+
return async function metrics(ctx) {
|
|
2339
|
+
try {
|
|
2340
|
+
if (deps.cluster === void 0) {
|
|
2341
|
+
throw new NarsilError(
|
|
2342
|
+
ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
|
|
2343
|
+
"This server fronts a single engine, so it reports no cluster metrics",
|
|
2344
|
+
{}
|
|
2345
|
+
);
|
|
2346
|
+
}
|
|
2347
|
+
const text = prometheusText(await deps.cluster.getMetrics());
|
|
2348
|
+
if (ctx.abort.aborted) return;
|
|
2349
|
+
sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
|
|
2350
|
+
} catch (err) {
|
|
2351
|
+
respondError(ctx, err);
|
|
2352
|
+
}
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
// src/server/handlers/query-suggestions.ts
|
|
2357
|
+
var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
|
|
2358
|
+
function readObject(ctx) {
|
|
2359
|
+
const body = parseJson(ctx);
|
|
2360
|
+
if (body === void 0) return void 0;
|
|
2361
|
+
if (!isRecord(body)) {
|
|
2362
|
+
badRequest(ctx.res, "The body must be a JSON object");
|
|
2363
|
+
return void 0;
|
|
2364
|
+
}
|
|
2365
|
+
return body;
|
|
2366
|
+
}
|
|
2367
|
+
function readQuery(ctx, body) {
|
|
2368
|
+
if (typeof body.query === "string") return body.query;
|
|
2369
|
+
badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
|
|
2370
|
+
return void 0;
|
|
2371
|
+
}
|
|
2372
|
+
function isStringList(value) {
|
|
2373
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
2374
|
+
}
|
|
2375
|
+
function isIdentity(value) {
|
|
2376
|
+
return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
|
|
2377
|
+
}
|
|
2378
|
+
function previewLimitOf(raw) {
|
|
2379
|
+
if (raw === null) return void 0;
|
|
2380
|
+
return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
|
|
2381
|
+
}
|
|
2382
|
+
async function identityOf(options, ctx) {
|
|
2383
|
+
if (ctx.hookContext === null) return void 0;
|
|
2384
|
+
try {
|
|
2385
|
+
const identity = await options.identify(ctx.hookContext);
|
|
2386
|
+
if (identity === null) return null;
|
|
2387
|
+
return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
|
|
2388
|
+
} catch {
|
|
2389
|
+
return void 0;
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
function noIdentify(indexName) {
|
|
2393
|
+
return new NarsilError(
|
|
2394
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
2395
|
+
"This server has no identify function for popular query suggestions, so it counts no search",
|
|
2396
|
+
{ indexName }
|
|
2397
|
+
);
|
|
2398
|
+
}
|
|
2399
|
+
function createQuerySuggestionHandlers(deps, options) {
|
|
2400
|
+
const { engine, tasks } = deps;
|
|
2401
|
+
const suggestions = () => querySuggestions(engine);
|
|
2402
|
+
async function update(ctx) {
|
|
741
2403
|
const body = parseJson(ctx);
|
|
742
|
-
if (
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
2404
|
+
if (body === void 0) return;
|
|
2405
|
+
try {
|
|
2406
|
+
await suggestions().updateSetting(ctx.params[0], body);
|
|
2407
|
+
respondJson(ctx, { ok: true });
|
|
2408
|
+
} catch (err) {
|
|
2409
|
+
respondError(ctx, err);
|
|
747
2410
|
}
|
|
2411
|
+
}
|
|
2412
|
+
async function status(ctx) {
|
|
748
2413
|
try {
|
|
749
|
-
|
|
750
|
-
await engine.createIndex(body.name, config);
|
|
751
|
-
respondJson(ctx, { name: body.name }, 201);
|
|
2414
|
+
respondJson(ctx, await suggestions().status(ctx.params[0]));
|
|
752
2415
|
} catch (err) {
|
|
753
2416
|
respondError(ctx, err);
|
|
754
2417
|
}
|
|
755
2418
|
}
|
|
756
|
-
function
|
|
2419
|
+
async function record(ctx) {
|
|
2420
|
+
const body = readObject(ctx);
|
|
2421
|
+
if (body === void 0) return;
|
|
2422
|
+
const extra = Object.keys(body).filter((key) => key !== "query");
|
|
2423
|
+
if (extra.length > 0) {
|
|
2424
|
+
badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
|
|
2425
|
+
fields: extra
|
|
2426
|
+
});
|
|
2427
|
+
return;
|
|
2428
|
+
}
|
|
2429
|
+
const query = readQuery(ctx, body);
|
|
2430
|
+
if (query === void 0) return;
|
|
2431
|
+
const indexName = ctx.params[0];
|
|
757
2432
|
try {
|
|
758
|
-
|
|
2433
|
+
if (options === void 0) throw noIdentify(indexName);
|
|
2434
|
+
const identity = await identityOf(options, ctx);
|
|
2435
|
+
if (identity === void 0) {
|
|
2436
|
+
if (!ctx.abort.aborted) {
|
|
2437
|
+
sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
|
|
2438
|
+
}
|
|
2439
|
+
return;
|
|
2440
|
+
}
|
|
2441
|
+
if (identity === null) {
|
|
2442
|
+
respondJson(ctx, { recorded: false, outcome: "unidentified" });
|
|
2443
|
+
return;
|
|
2444
|
+
}
|
|
2445
|
+
respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
|
|
759
2446
|
} catch (err) {
|
|
760
2447
|
respondError(ctx, err);
|
|
761
2448
|
}
|
|
762
2449
|
}
|
|
763
|
-
async function
|
|
2450
|
+
async function build(ctx) {
|
|
2451
|
+
const indexName = ctx.params[0];
|
|
764
2452
|
try {
|
|
765
|
-
|
|
766
|
-
|
|
2453
|
+
const operations = suggestions();
|
|
2454
|
+
const current = await operations.status(indexName);
|
|
2455
|
+
if (current.setting?.enabled !== true) {
|
|
2456
|
+
throw new NarsilError(
|
|
2457
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
2458
|
+
`Index "${indexName}" has popular query suggestions off`,
|
|
2459
|
+
{ indexName }
|
|
2460
|
+
);
|
|
2461
|
+
}
|
|
2462
|
+
const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
|
|
2463
|
+
const finished = await operations.build(indexName);
|
|
2464
|
+
context.reportBuild(finished);
|
|
2465
|
+
if (finished.status === "failed") {
|
|
2466
|
+
throw new NarsilError(
|
|
2467
|
+
finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
|
|
2468
|
+
"The build of popular query suggestions failed",
|
|
2469
|
+
{ indexName, buildId: finished.buildId }
|
|
2470
|
+
);
|
|
2471
|
+
}
|
|
2472
|
+
});
|
|
2473
|
+
respondJson(ctx, task, 202);
|
|
767
2474
|
} catch (err) {
|
|
768
2475
|
respondError(ctx, err);
|
|
769
2476
|
}
|
|
770
2477
|
}
|
|
771
|
-
function
|
|
2478
|
+
async function preview(ctx) {
|
|
2479
|
+
const limit = previewLimitOf(ctx.query.get("limit"));
|
|
2480
|
+
if (limit === null) {
|
|
2481
|
+
badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
772
2484
|
try {
|
|
773
|
-
respondJson(ctx,
|
|
2485
|
+
respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
|
|
774
2486
|
} catch (err) {
|
|
775
2487
|
respondError(ctx, err);
|
|
776
2488
|
}
|
|
777
2489
|
}
|
|
778
|
-
function
|
|
2490
|
+
async function readBans(ctx) {
|
|
779
2491
|
try {
|
|
780
|
-
respondJson(ctx, {
|
|
2492
|
+
respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
|
|
781
2493
|
} catch (err) {
|
|
782
2494
|
respondError(ctx, err);
|
|
783
2495
|
}
|
|
784
2496
|
}
|
|
785
|
-
async function
|
|
2497
|
+
async function replaceBans(ctx) {
|
|
2498
|
+
const body = readObject(ctx);
|
|
2499
|
+
if (body === void 0) return;
|
|
2500
|
+
if (!isStringList(body.bans)) {
|
|
2501
|
+
badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
|
|
2502
|
+
return;
|
|
2503
|
+
}
|
|
786
2504
|
try {
|
|
787
|
-
await
|
|
788
|
-
respondJson(ctx, { name: ctx.params[0], cleared: true });
|
|
2505
|
+
respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
|
|
789
2506
|
} catch (err) {
|
|
790
2507
|
respondError(ctx, err);
|
|
791
2508
|
}
|
|
792
2509
|
}
|
|
793
|
-
async function
|
|
2510
|
+
async function remove(ctx) {
|
|
2511
|
+
const body = readObject(ctx);
|
|
2512
|
+
if (body === void 0) return;
|
|
2513
|
+
const query = readQuery(ctx, body);
|
|
2514
|
+
if (query === void 0) return;
|
|
794
2515
|
try {
|
|
795
|
-
await
|
|
796
|
-
respondJson(ctx, {
|
|
2516
|
+
const held = await suggestions().removeQuery(ctx.params[0], query);
|
|
2517
|
+
respondJson(ctx, { suggestionIndexes: held });
|
|
797
2518
|
} catch (err) {
|
|
798
2519
|
respondError(ctx, err);
|
|
799
2520
|
}
|
|
800
2521
|
}
|
|
801
|
-
async function
|
|
2522
|
+
async function manual(ctx) {
|
|
2523
|
+
const body = readObject(ctx);
|
|
2524
|
+
if (body === void 0) return;
|
|
2525
|
+
const query = readQuery(ctx, body);
|
|
2526
|
+
if (query === void 0) return;
|
|
2527
|
+
const { group, count } = body;
|
|
2528
|
+
if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
|
|
2529
|
+
badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
|
|
2530
|
+
return;
|
|
2531
|
+
}
|
|
802
2532
|
try {
|
|
803
|
-
await
|
|
804
|
-
|
|
2533
|
+
const name = await suggestions().addSuggestion(ctx.params[0], {
|
|
2534
|
+
group,
|
|
2535
|
+
query,
|
|
2536
|
+
...count === void 0 ? {} : { count }
|
|
2537
|
+
});
|
|
2538
|
+
respondJson(ctx, { indexName: name });
|
|
805
2539
|
} catch (err) {
|
|
806
2540
|
respondError(ctx, err);
|
|
807
2541
|
}
|
|
808
2542
|
}
|
|
809
|
-
return {
|
|
2543
|
+
return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
|
|
810
2544
|
}
|
|
811
2545
|
|
|
812
2546
|
// src/server/handlers/search.ts
|
|
813
2547
|
function createSearchHandlers(deps) {
|
|
814
2548
|
const { engine, limits, allowRegex } = deps;
|
|
815
2549
|
async function search(ctx) {
|
|
816
|
-
const params = parseJson(ctx);
|
|
2550
|
+
const params = parseJson(ctx, "read");
|
|
817
2551
|
if (!params) return;
|
|
818
2552
|
const failure = validateQuery(params, limits);
|
|
819
2553
|
if (failure) {
|
|
@@ -822,13 +2556,23 @@ function createSearchHandlers(deps) {
|
|
|
822
2556
|
}
|
|
823
2557
|
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
824
2558
|
try {
|
|
825
|
-
|
|
2559
|
+
const query = withServerRerankWindow(params, limits);
|
|
2560
|
+
const result = await searchWithSignal(ctx, query);
|
|
2561
|
+
await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
|
|
826
2562
|
} catch (err) {
|
|
827
2563
|
respondError(ctx, err);
|
|
828
2564
|
}
|
|
829
2565
|
}
|
|
2566
|
+
async function searchWithSignal(ctx, params) {
|
|
2567
|
+
bindQuerySignal(params, requestSignal(ctx.abort));
|
|
2568
|
+
try {
|
|
2569
|
+
return await engine.query(ctx.params[0], params);
|
|
2570
|
+
} finally {
|
|
2571
|
+
clearQuerySignal(params);
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
830
2574
|
async function preflight(ctx) {
|
|
831
|
-
const params = parseJson(ctx);
|
|
2575
|
+
const params = parseJson(ctx, "read");
|
|
832
2576
|
if (!params) return;
|
|
833
2577
|
const failure = validateQuery(params, limits);
|
|
834
2578
|
if (failure) {
|
|
@@ -837,13 +2581,18 @@ function createSearchHandlers(deps) {
|
|
|
837
2581
|
}
|
|
838
2582
|
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
839
2583
|
try {
|
|
840
|
-
|
|
2584
|
+
bindQuerySignal(params, requestSignal(ctx.abort));
|
|
2585
|
+
try {
|
|
2586
|
+
respondJson(ctx, await engine.preflight(ctx.params[0], params));
|
|
2587
|
+
} finally {
|
|
2588
|
+
clearQuerySignal(params);
|
|
2589
|
+
}
|
|
841
2590
|
} catch (err) {
|
|
842
2591
|
respondError(ctx, err);
|
|
843
2592
|
}
|
|
844
2593
|
}
|
|
845
2594
|
async function suggest(ctx) {
|
|
846
|
-
const params = parseJson(ctx);
|
|
2595
|
+
const params = parseJson(ctx, "read");
|
|
847
2596
|
if (!params) return;
|
|
848
2597
|
const failure = validateSuggest(params);
|
|
849
2598
|
if (failure) {
|
|
@@ -904,6 +2653,15 @@ function assertLimit(field, value, minimum) {
|
|
|
904
2653
|
{ field, value, minimum }
|
|
905
2654
|
);
|
|
906
2655
|
}
|
|
2656
|
+
function assertFraction(field, value) {
|
|
2657
|
+
if (value === void 0) return;
|
|
2658
|
+
if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
|
|
2659
|
+
throw new NarsilError(
|
|
2660
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2661
|
+
`limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
|
|
2662
|
+
{ field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
|
|
2663
|
+
);
|
|
2664
|
+
}
|
|
907
2665
|
function assertRerankWindowCap(value, engineCap) {
|
|
908
2666
|
if (value === void 0) return;
|
|
909
2667
|
const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
|
|
@@ -918,8 +2676,19 @@ function assertServerLimits(limits, engineRerankWindow) {
|
|
|
918
2676
|
if (limits === void 0) return;
|
|
919
2677
|
for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
|
|
920
2678
|
for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
|
|
2679
|
+
assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
|
|
921
2680
|
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
922
2681
|
}
|
|
2682
|
+
function assertQuerySuggestionsOptions(options) {
|
|
2683
|
+
if (options === void 0) return;
|
|
2684
|
+
const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
|
|
2685
|
+
if (typeof identify === "function") return;
|
|
2686
|
+
throw new NarsilError(
|
|
2687
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2688
|
+
"querySuggestions must carry an identify function that returns the person and the group of a request",
|
|
2689
|
+
{ field: "querySuggestions.identify" }
|
|
2690
|
+
);
|
|
2691
|
+
}
|
|
923
2692
|
function assertAllowRegex(allowRegex) {
|
|
924
2693
|
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
925
2694
|
throw new NarsilError(
|
|
@@ -929,231 +2698,36 @@ function assertAllowRegex(allowRegex) {
|
|
|
929
2698
|
);
|
|
930
2699
|
}
|
|
931
2700
|
|
|
932
|
-
// src/server/request-threads/
|
|
933
|
-
var DEFAULT_STATUS = 200;
|
|
934
|
-
var NO_RESPONSE_STATUS = 500;
|
|
935
|
-
function statusOf(line) {
|
|
936
|
-
const parsed = Number.parseInt(line, 10);
|
|
937
|
-
return Number.isFinite(parsed) ? parsed : DEFAULT_STATUS;
|
|
938
|
-
}
|
|
939
|
-
function bodyBytes(body) {
|
|
940
|
-
if (body === void 0) return new Uint8Array(0);
|
|
941
|
-
if (typeof body === "string") return Buffer.from(body, "utf8");
|
|
942
|
-
return body;
|
|
943
|
-
}
|
|
944
|
-
function createRelayAbort() {
|
|
945
|
-
const listeners = [];
|
|
946
|
-
let aborted = false;
|
|
947
|
-
return {
|
|
948
|
-
get aborted() {
|
|
949
|
-
return aborted;
|
|
950
|
-
},
|
|
951
|
-
onAbort(fn) {
|
|
952
|
-
if (aborted) fn();
|
|
953
|
-
else listeners.push(fn);
|
|
954
|
-
},
|
|
955
|
-
trigger() {
|
|
956
|
-
if (aborted) return;
|
|
957
|
-
aborted = true;
|
|
958
|
-
for (const fn of listeners) fn();
|
|
959
|
-
}
|
|
960
|
-
};
|
|
961
|
-
}
|
|
962
|
-
function createCapturingResponse(remoteAddress, requestId, sendPart) {
|
|
963
|
-
let status = DEFAULT_STATUS;
|
|
964
|
-
const headers = [];
|
|
965
|
-
let finished = false;
|
|
966
|
-
let streaming = false;
|
|
967
|
-
let inFlight = 0;
|
|
968
|
-
let writable = null;
|
|
969
|
-
const abort = createRelayAbort();
|
|
970
|
-
let resolveDone = () => void 0;
|
|
971
|
-
const done = new Promise((resolve) => {
|
|
972
|
-
resolveDone = resolve;
|
|
973
|
-
});
|
|
974
|
-
function finish(body) {
|
|
975
|
-
if (finished) return;
|
|
976
|
-
finished = true;
|
|
977
|
-
if (streaming) {
|
|
978
|
-
sendPart({ type: "stream-end", requestId, failed: false }, []);
|
|
979
|
-
resolveDone(null);
|
|
980
|
-
return;
|
|
981
|
-
}
|
|
982
|
-
resolveDone({ status, headers, body });
|
|
983
|
-
}
|
|
984
|
-
function startStreaming() {
|
|
985
|
-
if (streaming) return;
|
|
986
|
-
streaming = true;
|
|
987
|
-
sendPart({ type: "stream-start", requestId, status, headers }, []);
|
|
988
|
-
}
|
|
989
|
-
const sink = {
|
|
990
|
-
cork(callback) {
|
|
991
|
-
callback();
|
|
992
|
-
return sink;
|
|
993
|
-
},
|
|
994
|
-
writeStatus(line) {
|
|
995
|
-
if (!finished && !streaming) status = statusOf(line);
|
|
996
|
-
return sink;
|
|
997
|
-
},
|
|
998
|
-
writeHeader(key, value) {
|
|
999
|
-
if (!finished && !streaming) headers.push([key, value]);
|
|
1000
|
-
return sink;
|
|
1001
|
-
},
|
|
1002
|
-
end(body) {
|
|
1003
|
-
finish(bodyBytes(body));
|
|
1004
|
-
return sink;
|
|
1005
|
-
},
|
|
1006
|
-
write(chunk) {
|
|
1007
|
-
if (finished || abort.aborted) return true;
|
|
1008
|
-
startStreaming();
|
|
1009
|
-
const copy = chunk.slice();
|
|
1010
|
-
inFlight += 1;
|
|
1011
|
-
sendPart({ type: "stream-chunk", requestId, chunk: copy }, [copy.buffer]);
|
|
1012
|
-
return inFlight < RELAY_STREAM_CHUNKS_IN_FLIGHT;
|
|
1013
|
-
},
|
|
1014
|
-
pause: () => void 0,
|
|
1015
|
-
resume: () => void 0,
|
|
1016
|
-
close() {
|
|
1017
|
-
if (finished) return sink;
|
|
1018
|
-
finished = true;
|
|
1019
|
-
if (streaming) {
|
|
1020
|
-
sendPart({ type: "stream-end", requestId, failed: true }, []);
|
|
1021
|
-
resolveDone(null);
|
|
1022
|
-
} else {
|
|
1023
|
-
status = NO_RESPONSE_STATUS;
|
|
1024
|
-
resolveDone({ status, headers, body: new Uint8Array(0) });
|
|
1025
|
-
}
|
|
1026
|
-
return sink;
|
|
1027
|
-
},
|
|
1028
|
-
endWithoutBody() {
|
|
1029
|
-
finish(new Uint8Array(0));
|
|
1030
|
-
return sink;
|
|
1031
|
-
},
|
|
1032
|
-
tryEnd(body) {
|
|
1033
|
-
finish(body);
|
|
1034
|
-
return [true, true];
|
|
1035
|
-
},
|
|
1036
|
-
getWriteOffset: () => 0,
|
|
1037
|
-
onWritable(handler) {
|
|
1038
|
-
writable = handler;
|
|
1039
|
-
return sink;
|
|
1040
|
-
},
|
|
1041
|
-
onAborted: () => sink,
|
|
1042
|
-
onData: () => sink,
|
|
1043
|
-
getRemoteAddressAsText: () => new TextEncoder().encode(remoteAddress).buffer
|
|
1044
|
-
};
|
|
1045
|
-
return {
|
|
1046
|
-
sink,
|
|
1047
|
-
abort,
|
|
1048
|
-
done,
|
|
1049
|
-
credit() {
|
|
1050
|
-
inFlight = Math.max(0, inFlight - 1);
|
|
1051
|
-
if (inFlight >= RELAY_STREAM_CHUNKS_IN_FLIGHT || writable === null) return;
|
|
1052
|
-
const handler = writable;
|
|
1053
|
-
writable = null;
|
|
1054
|
-
handler(0);
|
|
1055
|
-
},
|
|
1056
|
-
cancel() {
|
|
1057
|
-
abort.trigger();
|
|
1058
|
-
if (writable !== null) {
|
|
1059
|
-
const handler = writable;
|
|
1060
|
-
writable = null;
|
|
1061
|
-
handler(0);
|
|
1062
|
-
}
|
|
1063
|
-
if (!finished) {
|
|
1064
|
-
finished = true;
|
|
1065
|
-
resolveDone(null);
|
|
1066
|
-
}
|
|
1067
|
-
},
|
|
1068
|
-
finishUnanswered() {
|
|
1069
|
-
if (finished) return;
|
|
1070
|
-
if (streaming) {
|
|
1071
|
-
finished = true;
|
|
1072
|
-
sendPart({ type: "stream-end", requestId, failed: true }, []);
|
|
1073
|
-
resolveDone(null);
|
|
1074
|
-
return;
|
|
1075
|
-
}
|
|
1076
|
-
status = NO_RESPONSE_STATUS;
|
|
1077
|
-
finish(new Uint8Array(0));
|
|
1078
|
-
}
|
|
1079
|
-
};
|
|
1080
|
-
}
|
|
1081
|
-
function toRelayedResponse(requestId, captured) {
|
|
1082
|
-
return { type: "response", requestId, status: captured.status, headers: captured.headers, body: captured.body };
|
|
1083
|
-
}
|
|
1084
|
-
|
|
1085
|
-
// src/server/request-threads/host.ts
|
|
1086
|
-
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1087
|
-
"request",
|
|
1088
|
-
"authorize",
|
|
1089
|
-
"embed",
|
|
1090
|
-
"rerank",
|
|
1091
|
-
"stream-credit",
|
|
1092
|
-
"stream-cancel"
|
|
1093
|
-
]);
|
|
1094
|
-
function isRelayRequest(message) {
|
|
1095
|
-
if (typeof message !== "object" || message === null) return false;
|
|
1096
|
-
const { type, requestId } = message;
|
|
1097
|
-
if (type === "touch" || type === "leaving") return true;
|
|
1098
|
-
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
1099
|
-
}
|
|
1100
|
-
function spooledBodyOf(relayed) {
|
|
1101
|
-
if (relayed === null) return null;
|
|
1102
|
-
return {
|
|
1103
|
-
path: relayed.path,
|
|
1104
|
-
bytes: relayed.bytes,
|
|
1105
|
-
remove: () => rm(path.dirname(relayed.path), { recursive: true, force: true })
|
|
1106
|
-
};
|
|
1107
|
-
}
|
|
1108
|
-
async function answerRequest(handlers, request, port, streams) {
|
|
1109
|
-
const captured = createCapturingResponse(
|
|
1110
|
-
request.hookContext?.remoteAddress ?? "",
|
|
1111
|
-
request.requestId,
|
|
1112
|
-
(part, moved) => port.postMessage(part, moved)
|
|
1113
|
-
);
|
|
1114
|
-
streams.set(request.requestId, captured);
|
|
1115
|
-
const ctx = {
|
|
1116
|
-
res: captured.sink,
|
|
1117
|
-
params: request.params,
|
|
1118
|
-
query: new URLSearchParams(request.query),
|
|
1119
|
-
contentType: request.contentType,
|
|
1120
|
-
rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody.buffer, request.rawBody.byteOffset, request.rawBody.byteLength),
|
|
1121
|
-
spooledBody: spooledBodyOf(request.spooledBody),
|
|
1122
|
-
abort: captured.abort,
|
|
1123
|
-
hookContext: request.hookContext
|
|
1124
|
-
};
|
|
1125
|
-
try {
|
|
1126
|
-
await handlers[request.route](ctx);
|
|
1127
|
-
} catch {
|
|
1128
|
-
sendError(ctx.res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
|
|
1129
|
-
}
|
|
1130
|
-
captured.finishUnanswered();
|
|
1131
|
-
const answered = await captured.done;
|
|
1132
|
-
streams.delete(request.requestId);
|
|
1133
|
-
return answered === null ? null : toRelayedResponse(request.requestId, answered);
|
|
1134
|
-
}
|
|
2701
|
+
// src/server/request-threads/relay-answers.ts
|
|
1135
2702
|
async function answerAuthorize(authorize, request) {
|
|
1136
2703
|
if (authorize === void 0) {
|
|
1137
|
-
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
|
|
2704
|
+
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
|
|
1138
2705
|
}
|
|
1139
2706
|
const authorization = await authorize(request.context);
|
|
1140
2707
|
if (authorization.allowed) {
|
|
1141
|
-
return {
|
|
2708
|
+
return {
|
|
2709
|
+
type: "authorized",
|
|
2710
|
+
requestId: request.requestId,
|
|
2711
|
+
denial: null,
|
|
2712
|
+
hookError: false,
|
|
2713
|
+
expiresAt: authorization.expiresAt
|
|
2714
|
+
};
|
|
1142
2715
|
}
|
|
1143
2716
|
return {
|
|
1144
2717
|
type: "authorized",
|
|
1145
2718
|
requestId: request.requestId,
|
|
1146
2719
|
denial: authorization.denial,
|
|
1147
|
-
hookError: authorization.denial === null
|
|
2720
|
+
hookError: authorization.denial === null,
|
|
2721
|
+
expiresAt: null
|
|
1148
2722
|
};
|
|
1149
2723
|
}
|
|
1150
|
-
async function answerEmbed(core, request) {
|
|
2724
|
+
async function answerEmbed(core, request, signal) {
|
|
1151
2725
|
try {
|
|
1152
2726
|
const entry = core.requireIndex(request.indexName);
|
|
1153
2727
|
const resolved = await resolveVectorText(
|
|
1154
2728
|
{ vector: request.vector },
|
|
1155
2729
|
entry.embeddingAdapter,
|
|
1156
|
-
core.abortController.signal,
|
|
2730
|
+
signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
|
|
1157
2731
|
entry.embeddingAdapterName
|
|
1158
2732
|
);
|
|
1159
2733
|
const value = resolved.vector?.value;
|
|
@@ -1168,15 +2742,56 @@ async function answerEmbed(core, request) {
|
|
|
1168
2742
|
return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
|
|
1169
2743
|
}
|
|
1170
2744
|
}
|
|
1171
|
-
async function answerRerank(core, request) {
|
|
2745
|
+
async function answerRerank(core, request, signal) {
|
|
1172
2746
|
try {
|
|
1173
|
-
const
|
|
2747
|
+
const { rerankOnHost } = await loadRerankRuntime();
|
|
2748
|
+
const scores = await rerankOnHost(core, request.adapter, request.query, request.documents, signal);
|
|
1174
2749
|
return { type: "reranked", requestId: request.requestId, scores, error: null };
|
|
1175
2750
|
} catch (err) {
|
|
1176
2751
|
if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
|
|
1177
2752
|
return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
|
|
1178
2753
|
}
|
|
1179
2754
|
}
|
|
2755
|
+
async function answerAwaitWrites(core, request) {
|
|
2756
|
+
await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
|
|
2757
|
+
return { type: "writes-applied", requestId: request.requestId };
|
|
2758
|
+
}
|
|
2759
|
+
async function answerFeedAdmit(feeds, request) {
|
|
2760
|
+
if (feeds !== null) return feeds.admit(request);
|
|
2761
|
+
return {
|
|
2762
|
+
type: "feed-admitted",
|
|
2763
|
+
requestId: request.requestId,
|
|
2764
|
+
feedId: null,
|
|
2765
|
+
error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
// src/server/request-threads/host.ts
|
|
2770
|
+
function threadClosed(port, executor, clearSlots) {
|
|
2771
|
+
return new Promise((resolve) => {
|
|
2772
|
+
port.once("close", () => {
|
|
2773
|
+
clearSlots();
|
|
2774
|
+
resolve();
|
|
2775
|
+
});
|
|
2776
|
+
void executor.whenAbandoned?.().then(resolve);
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
2780
|
+
"request",
|
|
2781
|
+
"authorize",
|
|
2782
|
+
"embed",
|
|
2783
|
+
"rerank",
|
|
2784
|
+
"await-writes",
|
|
2785
|
+
"stream-credit",
|
|
2786
|
+
"stream-cancel",
|
|
2787
|
+
"feed-admit"
|
|
2788
|
+
]);
|
|
2789
|
+
function isRelayRequest(message) {
|
|
2790
|
+
if (typeof message !== "object" || message === null) return false;
|
|
2791
|
+
const { type, requestId } = message;
|
|
2792
|
+
if (type === "touch" || type === "leaving") return true;
|
|
2793
|
+
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
2794
|
+
}
|
|
1180
2795
|
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
1181
2796
|
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
1182
2797
|
}
|
|
@@ -1188,20 +2803,42 @@ async function startRequestThreads(options) {
|
|
|
1188
2803
|
const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
|
|
1189
2804
|
const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
|
|
1190
2805
|
const threads = /* @__PURE__ */ new Map();
|
|
2806
|
+
const slotUsers = /* @__PURE__ */ new Map();
|
|
2807
|
+
const longQueriesInFlight = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
|
|
2808
|
+
let longQueryThread = null;
|
|
2809
|
+
const closeAtExit = /* @__PURE__ */ new Set();
|
|
2810
|
+
const retiring = /* @__PURE__ */ new Set();
|
|
1191
2811
|
let stopped = false;
|
|
2812
|
+
function closeThreadsAtExit() {
|
|
2813
|
+
closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
|
|
2814
|
+
}
|
|
1192
2815
|
function releaseDescriptor(thread) {
|
|
2816
|
+
thread.feeds?.closeAll();
|
|
2817
|
+
closeAtExit.delete(thread.closeAtExit);
|
|
1193
2818
|
if (thread.descriptor !== null) {
|
|
1194
2819
|
try {
|
|
1195
2820
|
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
1196
2821
|
} catch {
|
|
1197
2822
|
}
|
|
2823
|
+
thread.descriptor = null;
|
|
1198
2824
|
}
|
|
1199
2825
|
confirmDeparture(thread.departure);
|
|
1200
2826
|
}
|
|
1201
2827
|
function relayFrom(thread) {
|
|
1202
2828
|
const { port } = thread;
|
|
1203
2829
|
const streams = /* @__PURE__ */ new Map();
|
|
2830
|
+
const calls = /* @__PURE__ */ new Map();
|
|
2831
|
+
port.on("close", () => {
|
|
2832
|
+
thread.feeds?.closeAll();
|
|
2833
|
+
for (const stream of streams.values()) stream.cancel();
|
|
2834
|
+
for (const call of calls.values()) call.abort();
|
|
2835
|
+
calls.clear();
|
|
2836
|
+
});
|
|
1204
2837
|
port.on("message", (message) => {
|
|
2838
|
+
if (isThreadFeedNote(message)) {
|
|
2839
|
+
thread.feeds?.handle(message);
|
|
2840
|
+
return;
|
|
2841
|
+
}
|
|
1205
2842
|
if (!isRelayRequest(message)) return;
|
|
1206
2843
|
if (message.type === "leaving") {
|
|
1207
2844
|
releaseDescriptor(thread);
|
|
@@ -1217,31 +2854,116 @@ async function startRequestThreads(options) {
|
|
|
1217
2854
|
}
|
|
1218
2855
|
if (message.type === "stream-cancel") {
|
|
1219
2856
|
streams.get(message.requestId)?.cancel();
|
|
2857
|
+
calls.get(message.requestId)?.abort();
|
|
2858
|
+
calls.delete(message.requestId);
|
|
1220
2859
|
return;
|
|
1221
2860
|
}
|
|
1222
|
-
const
|
|
1223
|
-
void
|
|
1224
|
-
|
|
1225
|
-
|
|
2861
|
+
const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
|
|
2862
|
+
if (call !== void 0) calls.set(message.requestId, call);
|
|
2863
|
+
const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
|
|
2864
|
+
void reply.then(
|
|
2865
|
+
(answer) => {
|
|
2866
|
+
calls.delete(message.requestId);
|
|
2867
|
+
if (answer !== null) port.postMessage(answer);
|
|
2868
|
+
},
|
|
2869
|
+
() => {
|
|
2870
|
+
calls.delete(message.requestId);
|
|
2871
|
+
}
|
|
2872
|
+
);
|
|
1226
2873
|
});
|
|
1227
2874
|
}
|
|
1228
|
-
|
|
2875
|
+
function connectToLongQueries(thread) {
|
|
2876
|
+
if (longQueryThread === null) return;
|
|
2877
|
+
const channel = new MessageChannel();
|
|
2878
|
+
thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
|
|
2879
|
+
longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
|
|
2880
|
+
}
|
|
2881
|
+
function dropLongQueryThread() {
|
|
2882
|
+
if (longQueryThread === null) return;
|
|
2883
|
+
longQueryThread.port.close();
|
|
2884
|
+
longQueryThread = null;
|
|
2885
|
+
}
|
|
2886
|
+
function retireThread(workerId, thread) {
|
|
1229
2887
|
releaseDescriptor(thread);
|
|
1230
|
-
thread.port.
|
|
1231
|
-
|
|
1232
|
-
|
|
2888
|
+
thread.port.unref();
|
|
2889
|
+
retiring.add(thread.closed);
|
|
2890
|
+
void thread.closed.then(() => retiring.delete(thread.closed));
|
|
2891
|
+
void executeUntilAnswered(thread.executor, { type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(
|
|
2892
|
+
() => void 0
|
|
2893
|
+
);
|
|
2894
|
+
return thread.closed;
|
|
1233
2895
|
}
|
|
1234
2896
|
const listener = {
|
|
2897
|
+
async onLongQueryWorkerReady(workerId, executor) {
|
|
2898
|
+
if (stopped) return;
|
|
2899
|
+
dropLongQueryThread();
|
|
2900
|
+
const channel = new MessageChannel();
|
|
2901
|
+
const relaying = {
|
|
2902
|
+
feeds: null,
|
|
2903
|
+
descriptor: null,
|
|
2904
|
+
port: channel.port2,
|
|
2905
|
+
executor,
|
|
2906
|
+
departure: createDepartureWord(),
|
|
2907
|
+
closeAtExit: createCloseAtExitWords(),
|
|
2908
|
+
closed: new Promise((resolve) => channel.port2.once("close", resolve))
|
|
2909
|
+
};
|
|
2910
|
+
relayFrom(relaying);
|
|
2911
|
+
try {
|
|
2912
|
+
await executor.execute(
|
|
2913
|
+
{
|
|
2914
|
+
type: "serveLongQueries",
|
|
2915
|
+
requestId: `serve-long-queries-${workerId}`,
|
|
2916
|
+
settings: {
|
|
2917
|
+
port: channel.port1,
|
|
2918
|
+
touchIntervalMs,
|
|
2919
|
+
limits: options.limits,
|
|
2920
|
+
patternWorkCap: options.core.patternWorkCap,
|
|
2921
|
+
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
2922
|
+
allowRegex: options.allowRegex
|
|
2923
|
+
}
|
|
2924
|
+
},
|
|
2925
|
+
[channel.port1]
|
|
2926
|
+
);
|
|
2927
|
+
} catch (err) {
|
|
2928
|
+
channel.port2.close();
|
|
2929
|
+
console.warn(`Worker ${workerId} could not start answering long queries:`, err);
|
|
2930
|
+
return;
|
|
2931
|
+
}
|
|
2932
|
+
if (stopped) {
|
|
2933
|
+
channel.port2.close();
|
|
2934
|
+
return;
|
|
2935
|
+
}
|
|
2936
|
+
longQueryThread = { workerId, port: channel.port2, executor };
|
|
2937
|
+
for (const thread of threads.values()) connectToLongQueries(thread);
|
|
2938
|
+
},
|
|
1235
2939
|
async onWorkerReady(workerId, executor) {
|
|
1236
2940
|
if (stopped) return;
|
|
1237
2941
|
const channel = new MessageChannel();
|
|
2942
|
+
const slot = gateSlotOfWorker(workerId);
|
|
2943
|
+
slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
|
|
1238
2944
|
const thread = {
|
|
2945
|
+
feeds: options.feeds?.forThread(channel.port2) ?? null,
|
|
1239
2946
|
descriptor: null,
|
|
1240
2947
|
port: channel.port2,
|
|
1241
2948
|
executor,
|
|
1242
|
-
departure: createDepartureWord()
|
|
2949
|
+
departure: createDepartureWord(),
|
|
2950
|
+
closeAtExit: createCloseAtExitWords(),
|
|
2951
|
+
closed: threadClosed(channel.port2, executor, () => {
|
|
2952
|
+
const remaining = (slotUsers.get(slot) ?? 1) - 1;
|
|
2953
|
+
if (remaining > 0) {
|
|
2954
|
+
slotUsers.set(slot, remaining);
|
|
2955
|
+
return;
|
|
2956
|
+
}
|
|
2957
|
+
slotUsers.delete(slot);
|
|
2958
|
+
clearGateSlot(options.gate, slot);
|
|
2959
|
+
clearBodyBudgetSlot(options.bodyBudget, slot);
|
|
2960
|
+
})
|
|
1243
2961
|
};
|
|
1244
2962
|
relayFrom(thread);
|
|
2963
|
+
closeAtExit.add(thread.closeAtExit);
|
|
2964
|
+
const longQueries = longQueryThread;
|
|
2965
|
+
const longQueryChannel = longQueries === null ? null : new MessageChannel();
|
|
2966
|
+
const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
|
|
1245
2967
|
let result;
|
|
1246
2968
|
try {
|
|
1247
2969
|
result = await executor.execute(
|
|
@@ -1251,9 +2973,11 @@ async function startRequestThreads(options) {
|
|
|
1251
2973
|
settings: {
|
|
1252
2974
|
port: channel.port1,
|
|
1253
2975
|
departure: thread.departure.buffer,
|
|
2976
|
+
closeAtExit: thread.closeAtExit.buffer,
|
|
1254
2977
|
gate: options.gate,
|
|
1255
2978
|
gateSlot: gateSlotOfWorker(workerId),
|
|
1256
2979
|
maxConcurrentRequests: options.limits.maxConcurrentRequests,
|
|
2980
|
+
bodyBudget: options.bodyBudget,
|
|
1257
2981
|
touchIntervalMs,
|
|
1258
2982
|
cors: options.cors,
|
|
1259
2983
|
limits: options.limits,
|
|
@@ -1262,33 +2986,50 @@ async function startRequestThreads(options) {
|
|
|
1262
2986
|
searchHooks,
|
|
1263
2987
|
patternWorkCap: options.core.patternWorkCap,
|
|
1264
2988
|
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1265
|
-
allowRegex: options.allowRegex
|
|
2989
|
+
allowRegex: options.allowRegex,
|
|
2990
|
+
feeds: Object.keys(options.feeds?.gateways ?? {}),
|
|
2991
|
+
longQueriesInFlight,
|
|
2992
|
+
longQueryPort: longQueryChannel?.port1 ?? null
|
|
1266
2993
|
}
|
|
1267
2994
|
},
|
|
1268
|
-
|
|
2995
|
+
transfer
|
|
1269
2996
|
);
|
|
1270
2997
|
} catch (err) {
|
|
2998
|
+
closeAtExit.delete(thread.closeAtExit);
|
|
1271
2999
|
channel.port2.close();
|
|
3000
|
+
longQueryChannel?.port2.close();
|
|
1272
3001
|
console.warn(`Worker ${workerId} could not start taking requests:`, err);
|
|
1273
3002
|
return;
|
|
1274
3003
|
}
|
|
1275
3004
|
thread.descriptor = result.descriptor;
|
|
1276
3005
|
if (stopped || departureConfirmed(thread.departure)) {
|
|
1277
|
-
|
|
3006
|
+
longQueryChannel?.port2.close();
|
|
3007
|
+
await retireThread(workerId, thread);
|
|
1278
3008
|
return;
|
|
1279
3009
|
}
|
|
3010
|
+
if (longQueryChannel !== null && longQueries !== null && longQueryThread === longQueries) {
|
|
3011
|
+
longQueries.port.postMessage({ type: "long-query-port", port: longQueryChannel.port2 }, [
|
|
3012
|
+
longQueryChannel.port2
|
|
3013
|
+
]);
|
|
3014
|
+
} else {
|
|
3015
|
+
longQueryChannel?.port2.close();
|
|
3016
|
+
connectToLongQueries(thread);
|
|
3017
|
+
}
|
|
1280
3018
|
acceptor.addChildAppDescriptor(result.descriptor);
|
|
1281
3019
|
threads.set(workerId, thread);
|
|
1282
3020
|
},
|
|
1283
3021
|
onWorkerGone(workerId) {
|
|
3022
|
+
if (longQueryThread?.workerId === workerId) {
|
|
3023
|
+
dropLongQueryThread();
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
1284
3026
|
const thread = threads.get(workerId);
|
|
1285
3027
|
if (thread === void 0) return;
|
|
1286
3028
|
threads.delete(workerId);
|
|
1287
|
-
|
|
1288
|
-
thread.port.close();
|
|
1289
|
-
clearGateSlot(options.gate, gateSlotOfWorker(workerId));
|
|
3029
|
+
void retireThread(workerId, thread);
|
|
1290
3030
|
}
|
|
1291
3031
|
};
|
|
3032
|
+
process.on("exit", closeThreadsAtExit);
|
|
1292
3033
|
await core.orchestrator.serveRequestsOnWorkers(listener);
|
|
1293
3034
|
return {
|
|
1294
3035
|
get threadCount() {
|
|
@@ -1296,16 +3037,87 @@ async function startRequestThreads(options) {
|
|
|
1296
3037
|
},
|
|
1297
3038
|
async stop() {
|
|
1298
3039
|
stopped = true;
|
|
1299
|
-
|
|
3040
|
+
process.off("exit", closeThreadsAtExit);
|
|
1300
3041
|
const stopping = [...threads.entries()].map(([workerId, thread]) => {
|
|
1301
3042
|
threads.delete(workerId);
|
|
1302
|
-
return
|
|
3043
|
+
return retireThread(workerId, thread);
|
|
1303
3044
|
});
|
|
1304
|
-
|
|
3045
|
+
core.orchestrator.stopRequestThreads();
|
|
3046
|
+
const longQueries = longQueryThread;
|
|
3047
|
+
dropLongQueryThread();
|
|
3048
|
+
if (longQueries !== null) {
|
|
3049
|
+
stopping.push(
|
|
3050
|
+
longQueries.executor.execute({ type: "stopServing", requestId: `stop-long-queries-${longQueries.workerId}` }).then(
|
|
3051
|
+
() => void 0,
|
|
3052
|
+
() => void 0
|
|
3053
|
+
)
|
|
3054
|
+
);
|
|
3055
|
+
}
|
|
3056
|
+
await Promise.all([...stopping, ...retiring]);
|
|
1305
3057
|
}
|
|
1306
3058
|
};
|
|
1307
3059
|
}
|
|
1308
3060
|
|
|
3061
|
+
// src/server/sync/options.ts
|
|
3062
|
+
var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
|
|
3063
|
+
var FEED_REFUSALS = {
|
|
3064
|
+
sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
|
|
3065
|
+
changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
|
|
3066
|
+
};
|
|
3067
|
+
function assertCount(feed, field, value) {
|
|
3068
|
+
if (value === void 0) return;
|
|
3069
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
|
|
3070
|
+
throw new NarsilError(
|
|
3071
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3072
|
+
`${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
|
|
3073
|
+
{ field, value: String(value) }
|
|
3074
|
+
);
|
|
3075
|
+
}
|
|
3076
|
+
function resolveFeedOptions(feed, value) {
|
|
3077
|
+
if (value === void 0 || value === false) return null;
|
|
3078
|
+
if (value !== true && (typeof value !== "object" || value === null)) {
|
|
3079
|
+
throw new NarsilError(
|
|
3080
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3081
|
+
`${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
|
|
3082
|
+
{ field: feed, value: String(value) }
|
|
3083
|
+
);
|
|
3084
|
+
}
|
|
3085
|
+
const settings = value === true ? {} : value;
|
|
3086
|
+
for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
|
|
3087
|
+
return {
|
|
3088
|
+
windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
|
|
3089
|
+
maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
|
|
3090
|
+
maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
|
|
3091
|
+
maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
|
|
3092
|
+
};
|
|
3093
|
+
}
|
|
3094
|
+
function assertFeedHasAuthentication(feed, resolved, options) {
|
|
3095
|
+
if (resolved === null || options.onRequest !== void 0) return;
|
|
3096
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
|
|
3097
|
+
}
|
|
3098
|
+
|
|
3099
|
+
// src/server/sync/setting-handlers.ts
|
|
3100
|
+
function createSyncSettingHandlers(engine) {
|
|
3101
|
+
async function getSync(ctx) {
|
|
3102
|
+
try {
|
|
3103
|
+
respondJson(ctx, await engine.getSync(ctx.params[0]));
|
|
3104
|
+
} catch (err) {
|
|
3105
|
+
respondError(ctx, err);
|
|
3106
|
+
}
|
|
3107
|
+
}
|
|
3108
|
+
async function updateSync(ctx) {
|
|
3109
|
+
const body = parseJson(ctx);
|
|
3110
|
+
if (body === void 0) return;
|
|
3111
|
+
try {
|
|
3112
|
+
await engine.updateSync(ctx.params[0], body);
|
|
3113
|
+
respondJson(ctx, { ok: true });
|
|
3114
|
+
} catch (err) {
|
|
3115
|
+
respondError(ctx, err);
|
|
3116
|
+
}
|
|
3117
|
+
}
|
|
3118
|
+
return { getSync, updateSync };
|
|
3119
|
+
}
|
|
3120
|
+
|
|
1309
3121
|
// src/server/task-status.ts
|
|
1310
3122
|
var TERMINAL_TASK_STATUSES = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled"]);
|
|
1311
3123
|
function isTerminalTaskStatus(status) {
|
|
@@ -1390,7 +3202,7 @@ var TaskRegistry = class {
|
|
|
1390
3202
|
* @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
|
|
1391
3203
|
* already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
|
|
1392
3204
|
* one holds its own working set until it finishes. */
|
|
1393
|
-
async start(type, indexName, op, progress) {
|
|
3205
|
+
async start(type, indexName, op, progress, backupName) {
|
|
1394
3206
|
if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
|
|
1395
3207
|
throw new NarsilError(
|
|
1396
3208
|
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
@@ -1403,6 +3215,7 @@ var TaskRegistry = class {
|
|
|
1403
3215
|
id: randomUUID(),
|
|
1404
3216
|
type,
|
|
1405
3217
|
indexName,
|
|
3218
|
+
...backupName === void 0 ? {} : { backupName },
|
|
1406
3219
|
owner: this.instanceId,
|
|
1407
3220
|
status: "running",
|
|
1408
3221
|
createdAt: now,
|
|
@@ -1491,9 +3304,9 @@ var TaskRegistry = class {
|
|
|
1491
3304
|
return task.writes.catch(() => {
|
|
1492
3305
|
});
|
|
1493
3306
|
}
|
|
1494
|
-
reportProgress(task,
|
|
3307
|
+
reportProgress(task, change) {
|
|
1495
3308
|
if (task.settled) return;
|
|
1496
|
-
task.record = { ...task.record,
|
|
3309
|
+
task.record = { ...task.record, ...change };
|
|
1497
3310
|
const now = Date.now();
|
|
1498
3311
|
if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
|
|
1499
3312
|
task.lastProgressWriteAt = now;
|
|
@@ -1502,9 +3315,13 @@ var TaskRegistry = class {
|
|
|
1502
3315
|
async drive(task, op) {
|
|
1503
3316
|
const context = {
|
|
1504
3317
|
signal: task.controller.signal,
|
|
1505
|
-
reportProgress: (progress) => this.reportProgress(task, progress),
|
|
3318
|
+
reportProgress: (progress) => this.reportProgress(task, { progress }),
|
|
3319
|
+
reportSteps: (steps) => this.reportProgress(task, { steps }),
|
|
1506
3320
|
reportResult: (result) => {
|
|
1507
3321
|
task.record = { ...task.record, result };
|
|
3322
|
+
},
|
|
3323
|
+
reportBuild: (build) => {
|
|
3324
|
+
task.record = { ...task.record, build };
|
|
1508
3325
|
}
|
|
1509
3326
|
};
|
|
1510
3327
|
let settled;
|
|
@@ -1526,8 +3343,16 @@ var TaskRegistry = class {
|
|
|
1526
3343
|
};
|
|
1527
3344
|
|
|
1528
3345
|
// src/server/create-server.ts
|
|
3346
|
+
function bodyBytesInFlightOf(fraction) {
|
|
3347
|
+
const processLimitBytes = readMemoryFigures()?.processLimitBytes;
|
|
3348
|
+
if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
|
|
3349
|
+
return Math.floor(fraction * processLimitBytes);
|
|
3350
|
+
}
|
|
1529
3351
|
function resolveLimits(limits, engineRerankWindow) {
|
|
1530
3352
|
return {
|
|
3353
|
+
maxBodyBytesInFlight: bodyBytesInFlightOf(
|
|
3354
|
+
limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
|
|
3355
|
+
),
|
|
1531
3356
|
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
1532
3357
|
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
1533
3358
|
maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
|
|
@@ -1560,7 +3385,7 @@ function assertSecureBinding(host, options) {
|
|
|
1560
3385
|
{ host }
|
|
1561
3386
|
);
|
|
1562
3387
|
}
|
|
1563
|
-
function mainThreadHandlers(deps) {
|
|
3388
|
+
function mainThreadHandlers(deps, syncServing) {
|
|
1564
3389
|
const idx = createIndexHandlers(deps);
|
|
1565
3390
|
const doc = createDocumentHandlers(deps);
|
|
1566
3391
|
const search = createSearchHandlers(deps);
|
|
@@ -1568,8 +3393,12 @@ function mainThreadHandlers(deps) {
|
|
|
1568
3393
|
const documentExport = createDocumentExportHandlers(deps);
|
|
1569
3394
|
const health = createHealthHandlers(deps);
|
|
1570
3395
|
const cluster = createClusterHandlers(deps);
|
|
3396
|
+
const backups = createBackupHandlers(deps);
|
|
1571
3397
|
const version = createVersionHandler(deps);
|
|
1572
3398
|
const capabilities = createCapabilitiesHandler();
|
|
3399
|
+
const syncSetting = createSyncSettingHandlers(deps.engine);
|
|
3400
|
+
const changesSetting = createChangesSettingHandlers(deps.engine);
|
|
3401
|
+
const suggestions = createQuerySuggestionHandlers(deps, deps.querySuggestions);
|
|
1573
3402
|
return {
|
|
1574
3403
|
livez: health.livez,
|
|
1575
3404
|
readyz: health.readyz,
|
|
@@ -1577,6 +3406,13 @@ function mainThreadHandlers(deps) {
|
|
|
1577
3406
|
capabilities: capabilities.report,
|
|
1578
3407
|
clusterTopology: cluster.topology,
|
|
1579
3408
|
clusterAllocation: cluster.allocation,
|
|
3409
|
+
clusterReplicationFactor: cluster.replicationFactor,
|
|
3410
|
+
clusterHealth: cluster.health,
|
|
3411
|
+
createBackup: backups.create,
|
|
3412
|
+
listBackups: backups.list,
|
|
3413
|
+
restoreBackup: backups.restore,
|
|
3414
|
+
deleteBackup: backups.remove,
|
|
3415
|
+
metrics: createMetricsHandler(deps),
|
|
1580
3416
|
memory: admin.memory,
|
|
1581
3417
|
listTasks: admin.listTasks,
|
|
1582
3418
|
getTask: admin.getTask,
|
|
@@ -1606,6 +3442,7 @@ function mainThreadHandlers(deps) {
|
|
|
1606
3442
|
multiGet: doc.multiGet,
|
|
1607
3443
|
checkpoint: admin.checkpoint,
|
|
1608
3444
|
rebuildAnalysis: admin.rebuildAnalysis,
|
|
3445
|
+
changeAnalysis: admin.changeAnalysis,
|
|
1609
3446
|
snapshot: admin.snapshot,
|
|
1610
3447
|
vectorMaintenance: admin.vectorMaintenance,
|
|
1611
3448
|
compact: admin.compact,
|
|
@@ -1614,9 +3451,24 @@ function mainThreadHandlers(deps) {
|
|
|
1614
3451
|
partitionConfig: admin.partitionConfig,
|
|
1615
3452
|
readSynonyms: admin.readSynonyms,
|
|
1616
3453
|
replaceSynonyms: admin.replaceSynonyms,
|
|
3454
|
+
updateBm25: admin.updateBm25,
|
|
1617
3455
|
restore: admin.restore,
|
|
1618
3456
|
exportDocuments: documentExport.exportDocuments,
|
|
1619
|
-
importExport: documentExport.importExport
|
|
3457
|
+
importExport: documentExport.importExport,
|
|
3458
|
+
getSync: syncSetting.getSync,
|
|
3459
|
+
updateSync: syncSetting.updateSync,
|
|
3460
|
+
syncSnapshot: createSyncSnapshotHandler(syncServing),
|
|
3461
|
+
getChanges: changesSetting.getChanges,
|
|
3462
|
+
updateChanges: changesSetting.updateChanges,
|
|
3463
|
+
updateQuerySuggestions: suggestions.update,
|
|
3464
|
+
querySuggestionsStatus: suggestions.status,
|
|
3465
|
+
recordQuery: suggestions.record,
|
|
3466
|
+
buildQuerySuggestions: suggestions.build,
|
|
3467
|
+
previewQuerySuggestions: suggestions.preview,
|
|
3468
|
+
readQueryBans: suggestions.readBans,
|
|
3469
|
+
replaceQueryBans: suggestions.replaceBans,
|
|
3470
|
+
removeQuery: suggestions.remove,
|
|
3471
|
+
addManualSuggestion: suggestions.manual
|
|
1620
3472
|
};
|
|
1621
3473
|
}
|
|
1622
3474
|
var NarsilHttpServer = class {
|
|
@@ -1629,12 +3481,19 @@ var NarsilHttpServer = class {
|
|
|
1629
3481
|
listenSocket = null;
|
|
1630
3482
|
ready = false;
|
|
1631
3483
|
threads = null;
|
|
3484
|
+
sync;
|
|
3485
|
+
changes;
|
|
3486
|
+
syncServing = null;
|
|
3487
|
+
feeds = null;
|
|
1632
3488
|
stopIdleHeapCleanup = () => void 0;
|
|
1633
3489
|
constructor(engine, options = {}) {
|
|
1634
3490
|
assertServerEngine(engine);
|
|
1635
3491
|
const engineRerankWindow = rerankWindowCapOf(engine);
|
|
1636
3492
|
assertServerLimits(options.limits, engineRerankWindow);
|
|
1637
3493
|
assertAllowRegex(options.allowRegex);
|
|
3494
|
+
assertQuerySuggestionsOptions(options.querySuggestions);
|
|
3495
|
+
this.sync = resolveFeedOptions("sync", options.sync);
|
|
3496
|
+
this.changes = resolveFeedOptions("changes", options.changes);
|
|
1638
3497
|
this.host = options.host ?? "127.0.0.1";
|
|
1639
3498
|
this.port = options.port ?? 9876;
|
|
1640
3499
|
this.options = options;
|
|
@@ -1657,10 +3516,13 @@ var NarsilHttpServer = class {
|
|
|
1657
3516
|
isReady: () => this.ready,
|
|
1658
3517
|
build: resolveBuild(options.build),
|
|
1659
3518
|
cluster: options.cluster,
|
|
1660
|
-
requestThreadCount: () => this.requestThreadCount
|
|
3519
|
+
requestThreadCount: () => this.requestThreadCount,
|
|
3520
|
+
querySuggestions: options.querySuggestions
|
|
1661
3521
|
};
|
|
1662
3522
|
}
|
|
1663
3523
|
async listen() {
|
|
3524
|
+
assertFeedHasAuthentication("sync", this.sync, this.options);
|
|
3525
|
+
assertFeedHasAuthentication("changes", this.changes, this.options);
|
|
1664
3526
|
assertSecureBinding(this.host, this.options);
|
|
1665
3527
|
const uws = await loadUWebSockets();
|
|
1666
3528
|
this.uws = uws;
|
|
@@ -1688,6 +3550,9 @@ var NarsilHttpServer = class {
|
|
|
1688
3550
|
this.uws.us_listen_socket_close(this.listenSocket);
|
|
1689
3551
|
this.listenSocket = null;
|
|
1690
3552
|
}
|
|
3553
|
+
this.feeds?.close();
|
|
3554
|
+
this.feeds = null;
|
|
3555
|
+
this.syncServing = null;
|
|
1691
3556
|
if (this.threads !== null) {
|
|
1692
3557
|
const threads = this.threads;
|
|
1693
3558
|
this.threads = null;
|
|
@@ -1705,17 +3570,34 @@ var NarsilHttpServer = class {
|
|
|
1705
3570
|
get requestThreadCount() {
|
|
1706
3571
|
return this.threads?.threadCount ?? 0;
|
|
1707
3572
|
}
|
|
3573
|
+
startFeeds(app, authorize, core) {
|
|
3574
|
+
const started = startFeeds(this.engine, core, {
|
|
3575
|
+
sync: this.sync,
|
|
3576
|
+
changes: this.changes,
|
|
3577
|
+
allowRegex: this.deps.allowRegex,
|
|
3578
|
+
authorize
|
|
3579
|
+
});
|
|
3580
|
+
if (started === null) return;
|
|
3581
|
+
this.syncServing = started.syncServing;
|
|
3582
|
+
this.feeds = started.host;
|
|
3583
|
+
const { sync, changes } = started.host.gateways;
|
|
3584
|
+
if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE);
|
|
3585
|
+
if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE);
|
|
3586
|
+
}
|
|
1708
3587
|
async registerRoutes(app) {
|
|
1709
3588
|
const cors = resolveCors(this.options.cors);
|
|
1710
3589
|
const authorize = this.options.onRequest ? authorizerFor(this.options.onRequest) : void 0;
|
|
1711
3590
|
const core = engineCoreOf(this.engine);
|
|
1712
3591
|
const threadCount = core === void 0 ? 0 : core.orchestrator.requestThreadCount();
|
|
1713
3592
|
const gate = createSharedGateBuffer(gateSlotOfWorker(threadCount));
|
|
3593
|
+
const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
|
|
1714
3594
|
const { limits, build, allowRegex } = this.deps;
|
|
1715
|
-
const handlers = mainThreadHandlers(this.deps);
|
|
3595
|
+
const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
|
|
3596
|
+
this.startFeeds(app, authorize, core);
|
|
1716
3597
|
const run = createRouteRunner({
|
|
1717
3598
|
authorize,
|
|
1718
3599
|
gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
|
|
3600
|
+
bodyBudget: createSharedBodyBudget(bodyBudget, limits.maxBodyBytesInFlight, MAIN_THREAD_GATE_SLOT),
|
|
1719
3601
|
corsHeaders: cors ? corsHeaderReader(cors) : void 0
|
|
1720
3602
|
});
|
|
1721
3603
|
registerServerRoutes(app, run, handlers, limits, cors);
|
|
@@ -1727,10 +3609,12 @@ var NarsilHttpServer = class {
|
|
|
1727
3609
|
handlers,
|
|
1728
3610
|
authorize,
|
|
1729
3611
|
gate,
|
|
3612
|
+
bodyBudget,
|
|
1730
3613
|
limits,
|
|
1731
3614
|
allowRegex,
|
|
1732
3615
|
build,
|
|
1733
|
-
cors
|
|
3616
|
+
cors,
|
|
3617
|
+
feeds: this.feeds
|
|
1734
3618
|
});
|
|
1735
3619
|
} catch (err) {
|
|
1736
3620
|
console.warn("The server could not start request threads, so the main thread answers every request:", err);
|