@delali/narsil 0.3.1-next.99 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +27 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +406 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -1,14 +1,70 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import '../../chunk-
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import
|
|
1
|
+
import { INDEX_METADATA_KEY_PREFIX, indexNameOfMetadataSuffix } from '../../chunk-Q2AP2FSN.mjs';
|
|
2
|
+
import { MAX_NODE_ID_LENGTH, CONTROLLER_LEASE_KEY, MAX_WATCHERS } from '../../chunk-44W3CBNG.mjs';
|
|
3
|
+
import { decode } from '../../chunk-GXCDWLQI.mjs';
|
|
4
|
+
import '../../chunk-7P5YWDRQ.mjs';
|
|
5
|
+
import '../../chunk-ZXX5GZEX.mjs';
|
|
6
|
+
import '../../chunk-4PY4JYMY.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes, describeError } from '../../chunk-ZKOQR7EA.mjs';
|
|
8
|
+
import { compareCodePoints } from '../../chunk-TGPKS2UM.mjs';
|
|
9
|
+
import { encode } from '@msgpack/msgpack';
|
|
6
10
|
|
|
7
11
|
// src/distribution/coordinator/etcd/constants.ts
|
|
8
12
|
var DEFAULT_ETCD_MAX_REQUEST_BYTES = 1572864;
|
|
9
13
|
var MAX_ETCD_REQUEST_BYTES = 2147483647;
|
|
10
14
|
var ETCD_REQUEST_FRAMING_BYTES = 1024;
|
|
11
15
|
var ETCD_MAX_RESPONSE_BYTES = 2147483647;
|
|
16
|
+
var RECONNECT_FIRST_DELAY_MS = 100;
|
|
17
|
+
var RECONNECT_LONGEST_DELAY_MS = 5e3;
|
|
18
|
+
var ETCD_START_PROBE_TIMEOUT_MS = 1e4;
|
|
19
|
+
var ETCD_CALL_TIMEOUT_MS = 5e3;
|
|
20
|
+
var ETCD_KEEPALIVE_TIME_MS = 1e4;
|
|
21
|
+
var ETCD_KEEPALIVE_TIMEOUT_MS = 5e3;
|
|
22
|
+
var ETCD_WATCH_ATTACH_TIMEOUT_MS = 1e4;
|
|
23
|
+
var ETCD_ENDPOINT_FAILURE_THRESHOLD = 3;
|
|
24
|
+
var ETCD_ENDPOINT_SKIP_MS = 5e3;
|
|
25
|
+
|
|
26
|
+
// src/distribution/coordinator/etcd/endpoint-breaker.ts
|
|
27
|
+
var NEVER_ABORTED = new AbortController().signal;
|
|
28
|
+
var noListeners = () => ({ dispose: () => {
|
|
29
|
+
} });
|
|
30
|
+
var EndpointSkippedError = class extends NarsilError {
|
|
31
|
+
isBrokenCircuitError = true;
|
|
32
|
+
};
|
|
33
|
+
var SkippingEndpointBreaker = class {
|
|
34
|
+
constructor(endpoint, counts) {
|
|
35
|
+
this.endpoint = endpoint;
|
|
36
|
+
this.counts = counts;
|
|
37
|
+
}
|
|
38
|
+
onSuccess = noListeners;
|
|
39
|
+
onFailure = noListeners;
|
|
40
|
+
consecutiveFailures = 0;
|
|
41
|
+
skippedUntil = 0;
|
|
42
|
+
async execute(fn, signal = NEVER_ABORTED) {
|
|
43
|
+
if (Date.now() < this.skippedUntil) {
|
|
44
|
+
throw new EndpointSkippedError(
|
|
45
|
+
ErrorCodes.COORDINATOR_UNAVAILABLE,
|
|
46
|
+
`etcd endpoint ${this.endpoint} failed ${ETCD_ENDPOINT_FAILURE_THRESHOLD} calls in a row, so this client skips it for ${ETCD_ENDPOINT_SKIP_MS} ms`,
|
|
47
|
+
{ endpoint: this.endpoint }
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
const result = await fn({ signal });
|
|
52
|
+
this.consecutiveFailures = 0;
|
|
53
|
+
return result;
|
|
54
|
+
} catch (error) {
|
|
55
|
+
if (error instanceof Error && this.counts(error)) {
|
|
56
|
+
this.consecutiveFailures += 1;
|
|
57
|
+
if (this.consecutiveFailures >= ETCD_ENDPOINT_FAILURE_THRESHOLD) {
|
|
58
|
+
this.skippedUntil = Date.now() + ETCD_ENDPOINT_SKIP_MS;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
function createEndpointBreaker(endpoint, counts) {
|
|
66
|
+
return new SkippingEndpointBreaker(endpoint, counts);
|
|
67
|
+
}
|
|
12
68
|
|
|
13
69
|
// src/distribution/coordinator/etcd/types.ts
|
|
14
70
|
var DEFAULT_ETCD_CONFIG = {
|
|
@@ -28,10 +84,12 @@ function buildKey(prefix, ...segments) {
|
|
|
28
84
|
// src/distribution/coordinator/etcd/keys.ts
|
|
29
85
|
var ETCD_KEY_LEASE = "lease";
|
|
30
86
|
var ETCD_KEY_GENERIC = "kv";
|
|
87
|
+
var ETCD_KEY_NODE_STARTED_AT = "node-started-at";
|
|
31
88
|
function coordinatorKeys(config) {
|
|
32
89
|
const prefix = config.keyPrefix;
|
|
33
90
|
return {
|
|
34
91
|
node: (nodeId) => buildKey(prefix, ETCD_KEY_NODES, nodeId),
|
|
92
|
+
nodeStartedAt: (nodeId) => buildKey(prefix, ETCD_KEY_NODE_STARTED_AT, nodeId),
|
|
35
93
|
nodePrefix: () => buildKey(prefix, ETCD_KEY_NODES),
|
|
36
94
|
allocation: (indexName) => buildKey(prefix, ETCD_KEY_ALLOCATION, indexName),
|
|
37
95
|
allocationPrefix: () => buildKey(prefix, ETCD_KEY_ALLOCATION),
|
|
@@ -46,6 +104,42 @@ function extractSuffix(fullKey, prefix) {
|
|
|
46
104
|
return fullKey.slice(prefix.length + 1);
|
|
47
105
|
}
|
|
48
106
|
|
|
107
|
+
// src/distribution/coordinator/etcd/lease-validity.ts
|
|
108
|
+
var MILLISECONDS_PER_SECOND = 1e3;
|
|
109
|
+
var validities = /* @__PURE__ */ new WeakMap();
|
|
110
|
+
function recordLeaseRenewal(lease, sentAt, ttlSeconds) {
|
|
111
|
+
if (!Number.isFinite(ttlSeconds) || ttlSeconds <= 0) return;
|
|
112
|
+
const ttlMs = ttlSeconds * MILLISECONDS_PER_SECOND;
|
|
113
|
+
const heldUntil = sentAt + ttlMs;
|
|
114
|
+
const current = validities.get(lease);
|
|
115
|
+
if (current !== void 0 && current.heldUntil >= heldUntil) return;
|
|
116
|
+
validities.set(lease, { heldUntil, ttlMs });
|
|
117
|
+
}
|
|
118
|
+
function followLeaseRenewals(lease) {
|
|
119
|
+
const unanswered = [];
|
|
120
|
+
lease.on("keepaliveFired", () => {
|
|
121
|
+
unanswered.push(performance.now());
|
|
122
|
+
});
|
|
123
|
+
lease.on("keepaliveSucceeded", (response) => {
|
|
124
|
+
const sentAt = unanswered.shift();
|
|
125
|
+
if (sentAt !== void 0) recordLeaseRenewal(lease, sentAt, Number(response.TTL));
|
|
126
|
+
});
|
|
127
|
+
lease.on("keepaliveEstablished", () => {
|
|
128
|
+
unanswered.length = 0;
|
|
129
|
+
});
|
|
130
|
+
lease.on("keepaliveFailed", () => {
|
|
131
|
+
unanswered.length = 0;
|
|
132
|
+
});
|
|
133
|
+
lease.once("lost", () => {
|
|
134
|
+
validities.delete(lease);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
function leaseHeldFor(lease) {
|
|
138
|
+
const validity = validities.get(lease);
|
|
139
|
+
if (validity === void 0 || lease.revoked()) return null;
|
|
140
|
+
return { remainingMs: validity.heldUntil - performance.now(), ttlMs: validity.ttlMs };
|
|
141
|
+
}
|
|
142
|
+
|
|
49
143
|
// src/distribution/coordinator/etcd/leases.ts
|
|
50
144
|
function payloadsMatch(left, right) {
|
|
51
145
|
if (left === null || left.byteLength !== right.byteLength) {
|
|
@@ -60,9 +154,17 @@ function payloadsMatch(left, right) {
|
|
|
60
154
|
}
|
|
61
155
|
var LeaseManager = class {
|
|
62
156
|
leases = /* @__PURE__ */ new Map();
|
|
157
|
+
holders = /* @__PURE__ */ new Map();
|
|
63
158
|
keyLocks = /* @__PURE__ */ new Map();
|
|
64
159
|
track(key, lease, nodeId, payload = null) {
|
|
65
160
|
this.leases.set(key, { lease, nodeId, key, payload });
|
|
161
|
+
this.holders.set(key, lease);
|
|
162
|
+
}
|
|
163
|
+
holder(key) {
|
|
164
|
+
return this.holders.get(key);
|
|
165
|
+
}
|
|
166
|
+
forgetHolder(key) {
|
|
167
|
+
this.holders.delete(key);
|
|
66
168
|
}
|
|
67
169
|
get(key) {
|
|
68
170
|
return this.leases.get(key);
|
|
@@ -109,6 +211,7 @@ var LeaseManager = class {
|
|
|
109
211
|
}));
|
|
110
212
|
}
|
|
111
213
|
this.leases.clear();
|
|
214
|
+
this.holders.clear();
|
|
112
215
|
await Promise.all(revokePromises);
|
|
113
216
|
}
|
|
114
217
|
entries() {
|
|
@@ -128,6 +231,310 @@ async function loadEtcd3Module() {
|
|
|
128
231
|
);
|
|
129
232
|
}
|
|
130
233
|
}
|
|
234
|
+
|
|
235
|
+
// src/distribution/coordinator/etcd/watch-stream.ts
|
|
236
|
+
var WatchAttachError = class extends NarsilError {
|
|
237
|
+
};
|
|
238
|
+
function shutDownError() {
|
|
239
|
+
return new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
|
|
240
|
+
}
|
|
241
|
+
function attachFailure(message) {
|
|
242
|
+
return new WatchAttachError(ErrorCodes.COORDINATOR_UNAVAILABLE, message);
|
|
243
|
+
}
|
|
244
|
+
function prefixRangeEnd(prefix) {
|
|
245
|
+
for (let position = prefix.length - 1; position >= 0; position--) {
|
|
246
|
+
const byte = prefix[position];
|
|
247
|
+
if (byte !== void 0 && byte < 255) {
|
|
248
|
+
const end = Buffer.from(prefix.subarray(0, position + 1));
|
|
249
|
+
end[position] = byte + 1;
|
|
250
|
+
return end;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return Buffer.from([0]);
|
|
254
|
+
}
|
|
255
|
+
async function openOne(client, prefix, startRevision, listener, registry) {
|
|
256
|
+
const stream = await client.watchClient.watch();
|
|
257
|
+
let attached = false;
|
|
258
|
+
let closing = false;
|
|
259
|
+
let finished = false;
|
|
260
|
+
let settleAttach = () => {
|
|
261
|
+
};
|
|
262
|
+
let markFinished = () => {
|
|
263
|
+
};
|
|
264
|
+
const done = new Promise((resolve) => {
|
|
265
|
+
markFinished = resolve;
|
|
266
|
+
});
|
|
267
|
+
const attachedOrFailed = new Promise((resolve) => {
|
|
268
|
+
settleAttach = resolve;
|
|
269
|
+
});
|
|
270
|
+
const watch = {
|
|
271
|
+
close() {
|
|
272
|
+
if (!closing) {
|
|
273
|
+
closing = true;
|
|
274
|
+
try {
|
|
275
|
+
stream.end();
|
|
276
|
+
stream.cancel();
|
|
277
|
+
} catch (_) {
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return done;
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
function finish() {
|
|
284
|
+
if (finished) return;
|
|
285
|
+
finished = true;
|
|
286
|
+
registry.forget(watch);
|
|
287
|
+
markFinished();
|
|
288
|
+
if (!attached) settleAttach(attachFailure("the etcd watch stream ended before etcd attached the watch"));
|
|
289
|
+
else if (!closing) listener.lost();
|
|
290
|
+
}
|
|
291
|
+
function receive(response) {
|
|
292
|
+
if (closing || finished) return;
|
|
293
|
+
if (!attached) {
|
|
294
|
+
if (!response.created) return;
|
|
295
|
+
if (response.canceled) {
|
|
296
|
+
settleAttach(attachFailure(`etcd refused the watch: ${response.cancel_reason || "no reason given"}`));
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
attached = true;
|
|
300
|
+
settleAttach(null);
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
for (const event of response.events ?? []) {
|
|
304
|
+
try {
|
|
305
|
+
if (event.type === "Put") listener.put(event.kv);
|
|
306
|
+
else if (event.type === "Delete") listener.delete(event.kv);
|
|
307
|
+
} catch (handlerError) {
|
|
308
|
+
void watch.close();
|
|
309
|
+
listener.handlerFailed(handlerError);
|
|
310
|
+
listener.lost();
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
if (response.canceled) {
|
|
315
|
+
void watch.close();
|
|
316
|
+
listener.lost();
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
stream.on("data", (response) => {
|
|
320
|
+
try {
|
|
321
|
+
receive(response);
|
|
322
|
+
} catch (_) {
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
stream.on("error", () => {
|
|
326
|
+
});
|
|
327
|
+
stream.on("status", () => {
|
|
328
|
+
try {
|
|
329
|
+
finish();
|
|
330
|
+
} catch (_) {
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
registry.remember(watch);
|
|
334
|
+
if (registry.isClosed()) {
|
|
335
|
+
await watch.close();
|
|
336
|
+
throw shutDownError();
|
|
337
|
+
}
|
|
338
|
+
const timer = setTimeout(
|
|
339
|
+
() => settleAttach(attachFailure(`etcd did not attach the watch within ${ETCD_WATCH_ATTACH_TIMEOUT_MS} ms`)),
|
|
340
|
+
ETCD_WATCH_ATTACH_TIMEOUT_MS
|
|
341
|
+
);
|
|
342
|
+
timer.unref?.();
|
|
343
|
+
const key = Buffer.from(prefix);
|
|
344
|
+
try {
|
|
345
|
+
stream.write({ create_request: { key, range_end: prefixRangeEnd(key), start_revision: startRevision } });
|
|
346
|
+
} catch (_) {
|
|
347
|
+
settleAttach(attachFailure("the etcd watch stream refused the watch request"));
|
|
348
|
+
}
|
|
349
|
+
const failure = await attachedOrFailed;
|
|
350
|
+
clearTimeout(timer);
|
|
351
|
+
if (failure !== null) {
|
|
352
|
+
await watch.close();
|
|
353
|
+
throw registry.isClosed() ? shutDownError() : failure;
|
|
354
|
+
}
|
|
355
|
+
return watch;
|
|
356
|
+
}
|
|
357
|
+
function createWatchStreams() {
|
|
358
|
+
const open = /* @__PURE__ */ new Set();
|
|
359
|
+
const pending = /* @__PURE__ */ new Set();
|
|
360
|
+
let opening = 0;
|
|
361
|
+
let closed = false;
|
|
362
|
+
function track(operation) {
|
|
363
|
+
const followed = operation.then(
|
|
364
|
+
() => {
|
|
365
|
+
},
|
|
366
|
+
() => {
|
|
367
|
+
}
|
|
368
|
+
);
|
|
369
|
+
pending.add(followed);
|
|
370
|
+
void followed.then(() => pending.delete(followed));
|
|
371
|
+
return operation;
|
|
372
|
+
}
|
|
373
|
+
async function openTracked(client, prefix, startRevision, listener) {
|
|
374
|
+
if (closed) throw shutDownError();
|
|
375
|
+
if (open.size + opening >= MAX_WATCHERS) {
|
|
376
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Maximum watcher limit (${MAX_WATCHERS}) reached`, {
|
|
377
|
+
currentCount: open.size + opening
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
opening += 1;
|
|
381
|
+
let reserved = true;
|
|
382
|
+
const releaseReservation = () => {
|
|
383
|
+
if (!reserved) return;
|
|
384
|
+
reserved = false;
|
|
385
|
+
opening -= 1;
|
|
386
|
+
};
|
|
387
|
+
const registry = {
|
|
388
|
+
remember(watch) {
|
|
389
|
+
releaseReservation();
|
|
390
|
+
open.add(watch);
|
|
391
|
+
},
|
|
392
|
+
forget: (watch) => open.delete(watch),
|
|
393
|
+
isClosed: () => closed
|
|
394
|
+
};
|
|
395
|
+
try {
|
|
396
|
+
return await openOne(client, prefix, startRevision, listener, registry);
|
|
397
|
+
} finally {
|
|
398
|
+
releaseReservation();
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return {
|
|
402
|
+
open: (client, prefix, startRevision, listener) => track(openTracked(client, prefix, startRevision, listener)),
|
|
403
|
+
track,
|
|
404
|
+
isClosed: () => closed,
|
|
405
|
+
async closeAll() {
|
|
406
|
+
closed = true;
|
|
407
|
+
while (open.size > 0 || pending.size > 0) {
|
|
408
|
+
await Promise.all([...[...open].map((watch) => watch.close()), ...pending]);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// src/distribution/coordinator/etcd/prefix-watch.ts
|
|
415
|
+
function blamesTheConnection(error) {
|
|
416
|
+
return error instanceof WatchAttachError || !(error instanceof NarsilError);
|
|
417
|
+
}
|
|
418
|
+
function watchPrefix(context, prefix, handlers) {
|
|
419
|
+
const { streams } = context;
|
|
420
|
+
if (streams.isClosed()) {
|
|
421
|
+
return Promise.reject(new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down"));
|
|
422
|
+
}
|
|
423
|
+
const modRevisionByKey = /* @__PURE__ */ new Map();
|
|
424
|
+
let current = null;
|
|
425
|
+
let stopped = false;
|
|
426
|
+
let reopenTimer = null;
|
|
427
|
+
let reopenDelayMs = RECONNECT_FIRST_DELAY_MS;
|
|
428
|
+
function inactive() {
|
|
429
|
+
return stopped || streams.isClosed();
|
|
430
|
+
}
|
|
431
|
+
function recordPut(kv) {
|
|
432
|
+
modRevisionByKey.set(kv.key.toString(), kv.mod_revision);
|
|
433
|
+
handlers.put(kv);
|
|
434
|
+
}
|
|
435
|
+
function recordDelete(fullKey) {
|
|
436
|
+
modRevisionByKey.delete(fullKey);
|
|
437
|
+
handlers.delete(fullKey);
|
|
438
|
+
}
|
|
439
|
+
function reportingHandlerErrors(record) {
|
|
440
|
+
try {
|
|
441
|
+
record();
|
|
442
|
+
} catch (error) {
|
|
443
|
+
context.reportHandlerError(error);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
async function listMissedChanges(reportMissedChanges) {
|
|
447
|
+
const listing = await context.client.getAll().prefix(prefix).exec();
|
|
448
|
+
const listedKeys = /* @__PURE__ */ new Set();
|
|
449
|
+
for (const kv of listing.kvs) {
|
|
450
|
+
const fullKey = kv.key.toString();
|
|
451
|
+
listedKeys.add(fullKey);
|
|
452
|
+
if (!reportMissedChanges) {
|
|
453
|
+
modRevisionByKey.set(fullKey, kv.mod_revision);
|
|
454
|
+
} else if (modRevisionByKey.get(fullKey) !== kv.mod_revision) {
|
|
455
|
+
reportingHandlerErrors(() => recordPut(kv));
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
if (reportMissedChanges) {
|
|
459
|
+
for (const fullKey of [...modRevisionByKey.keys()]) {
|
|
460
|
+
if (!listedKeys.has(fullKey)) reportingHandlerErrors(() => recordDelete(fullKey));
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return String(BigInt(listing.header.revision) + 1n);
|
|
464
|
+
}
|
|
465
|
+
async function watchFrom(startRevision) {
|
|
466
|
+
const client = context.watchClient.current();
|
|
467
|
+
let opened = null;
|
|
468
|
+
let lostWhileOpening = false;
|
|
469
|
+
let watch;
|
|
470
|
+
try {
|
|
471
|
+
watch = await streams.open(client, prefix, startRevision, {
|
|
472
|
+
put: recordPut,
|
|
473
|
+
delete: (kv) => recordDelete(kv.key.toString()),
|
|
474
|
+
handlerFailed: context.reportHandlerError,
|
|
475
|
+
lost() {
|
|
476
|
+
if (opened === null) lostWhileOpening = true;
|
|
477
|
+
else reopenAfterFailure(opened);
|
|
478
|
+
}
|
|
479
|
+
});
|
|
480
|
+
} catch (error) {
|
|
481
|
+
if (!streams.isClosed() && blamesTheConnection(error)) context.watchClient.replace(client);
|
|
482
|
+
throw error;
|
|
483
|
+
}
|
|
484
|
+
if (inactive()) {
|
|
485
|
+
await watch.close();
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
opened = watch;
|
|
489
|
+
current = watch;
|
|
490
|
+
if (lostWhileOpening) reopenAfterFailure(watch);
|
|
491
|
+
}
|
|
492
|
+
async function listThenWatch(reportMissedChanges) {
|
|
493
|
+
await watchFrom(await listMissedChanges(reportMissedChanges));
|
|
494
|
+
}
|
|
495
|
+
function scheduleReopen() {
|
|
496
|
+
if (inactive()) return;
|
|
497
|
+
const delayMs = reopenDelayMs;
|
|
498
|
+
reopenDelayMs = Math.min(reopenDelayMs * 2, RECONNECT_LONGEST_DELAY_MS);
|
|
499
|
+
reopenTimer = setTimeout(() => {
|
|
500
|
+
reopenTimer = null;
|
|
501
|
+
if (inactive()) return;
|
|
502
|
+
streams.track(listThenWatch(true)).then(
|
|
503
|
+
() => {
|
|
504
|
+
reopenDelayMs = RECONNECT_FIRST_DELAY_MS;
|
|
505
|
+
},
|
|
506
|
+
() => scheduleReopen()
|
|
507
|
+
);
|
|
508
|
+
}, delayMs);
|
|
509
|
+
reopenTimer.unref?.();
|
|
510
|
+
}
|
|
511
|
+
function reopenAfterFailure(failed) {
|
|
512
|
+
if (current !== failed) return;
|
|
513
|
+
current = null;
|
|
514
|
+
void failed.close();
|
|
515
|
+
scheduleReopen();
|
|
516
|
+
}
|
|
517
|
+
async function start() {
|
|
518
|
+
const startRevision = await listMissedChanges(false);
|
|
519
|
+
try {
|
|
520
|
+
await watchFrom(startRevision);
|
|
521
|
+
} catch (_) {
|
|
522
|
+
scheduleReopen();
|
|
523
|
+
}
|
|
524
|
+
return () => {
|
|
525
|
+
stopped = true;
|
|
526
|
+
if (reopenTimer !== null) {
|
|
527
|
+
clearTimeout(reopenTimer);
|
|
528
|
+
reopenTimer = null;
|
|
529
|
+
}
|
|
530
|
+
if (current !== null) {
|
|
531
|
+
void current.close();
|
|
532
|
+
current = null;
|
|
533
|
+
}
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
return streams.track(start());
|
|
537
|
+
}
|
|
131
538
|
function serializeAllocationTable(table) {
|
|
132
539
|
const serializable = {
|
|
133
540
|
indexName: table.indexName,
|
|
@@ -135,6 +542,8 @@ function serializeAllocationTable(table) {
|
|
|
135
542
|
replicationFactor: table.replicationFactor,
|
|
136
543
|
assignments: Array.from(table.assignments.entries())
|
|
137
544
|
};
|
|
545
|
+
if (table.generation !== void 0) serializable.generation = table.generation;
|
|
546
|
+
if (table.indexUuid !== void 0) serializable.indexUuid = table.indexUuid;
|
|
138
547
|
return new Uint8Array(encode(serializable));
|
|
139
548
|
}
|
|
140
549
|
function deserializeAllocationTable(data) {
|
|
@@ -148,12 +557,15 @@ function deserializeAllocationTable(data) {
|
|
|
148
557
|
if (!Array.isArray(raw.assignments)) {
|
|
149
558
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable missing or invalid assignments field");
|
|
150
559
|
}
|
|
151
|
-
|
|
560
|
+
const table = {
|
|
152
561
|
indexName: raw.indexName,
|
|
153
562
|
version: raw.version,
|
|
154
563
|
replicationFactor: typeof raw.replicationFactor === "number" ? raw.replicationFactor : 0,
|
|
155
564
|
assignments: normaliseAssignments(raw.assignments)
|
|
156
565
|
};
|
|
566
|
+
if (typeof raw.generation === "string" && raw.generation.length > 0) table.generation = raw.generation;
|
|
567
|
+
if (typeof raw.indexUuid === "string" && raw.indexUuid.length > 0) table.indexUuid = raw.indexUuid;
|
|
568
|
+
return table;
|
|
157
569
|
}
|
|
158
570
|
function stringsOf(value) {
|
|
159
571
|
if (!Array.isArray(value)) {
|
|
@@ -188,6 +600,12 @@ function normaliseAssignments(entries) {
|
|
|
188
600
|
} else {
|
|
189
601
|
delete normalised.lastHolders;
|
|
190
602
|
}
|
|
603
|
+
const removingReplicas = stringsOf(candidate.removingReplicas);
|
|
604
|
+
if (removingReplicas.length > 0) {
|
|
605
|
+
normalised.removingReplicas = removingReplicas;
|
|
606
|
+
} else {
|
|
607
|
+
delete normalised.removingReplicas;
|
|
608
|
+
}
|
|
191
609
|
assignments.set(partitionId, normalised);
|
|
192
610
|
}
|
|
193
611
|
return assignments;
|
|
@@ -224,32 +642,76 @@ async function writeUnderLease(context, registration) {
|
|
|
224
642
|
const tracked = context.leaseManager.getByNodeId(context.key, registration.nodeId);
|
|
225
643
|
if (tracked !== void 0) {
|
|
226
644
|
try {
|
|
227
|
-
|
|
645
|
+
const sentAt = performance.now();
|
|
646
|
+
const renewal = await tracked.lease.keepaliveOnce();
|
|
647
|
+
recordLeaseRenewal(tracked.lease, sentAt, Number(renewal.TTL));
|
|
228
648
|
if (context.leaseManager.holdsPayload(context.key, payload)) {
|
|
229
649
|
return;
|
|
230
650
|
}
|
|
231
|
-
await tracked.lease.
|
|
651
|
+
await putUnderLease(context, tracked.lease, registration.startedAt, data);
|
|
232
652
|
context.leaseManager.track(context.key, tracked.lease, registration.nodeId, payload);
|
|
233
653
|
return;
|
|
234
654
|
} catch (_) {
|
|
235
655
|
context.leaseManager.remove(context.key);
|
|
236
656
|
}
|
|
237
657
|
}
|
|
238
|
-
const lease =
|
|
239
|
-
await lease.grant();
|
|
658
|
+
const lease = await grantFollowedLease(context);
|
|
240
659
|
try {
|
|
241
|
-
await context.guard.send(write, () =>
|
|
660
|
+
await context.guard.send(write, () => putUnderLease(context, lease, registration.startedAt, data));
|
|
242
661
|
} catch (error) {
|
|
243
662
|
await lease.revoke().catch(() => {
|
|
244
663
|
});
|
|
245
664
|
throw error;
|
|
246
665
|
}
|
|
247
666
|
const existing = context.leaseManager.get(context.key);
|
|
248
|
-
|
|
667
|
+
context.leaseManager.track(context.key, lease, registration.nodeId, payload);
|
|
668
|
+
registerAgainOnLoss(context, lease, registration.nodeId);
|
|
669
|
+
if (existing !== void 0 && existing.lease !== lease) {
|
|
249
670
|
await existing.lease.revoke().catch(() => {
|
|
250
671
|
});
|
|
251
672
|
}
|
|
252
|
-
|
|
673
|
+
}
|
|
674
|
+
async function putUnderLease(context, lease, startedAt, data) {
|
|
675
|
+
await context.client.if(context.key, "Create", ">", -1).then(lease.put(context.startedAtKey).value(startedAt), lease.put(context.key).value(data)).commit();
|
|
676
|
+
}
|
|
677
|
+
async function grantFollowedLease(context) {
|
|
678
|
+
const sentAt = performance.now();
|
|
679
|
+
const lease = context.client.lease(context.ttlSeconds);
|
|
680
|
+
followLeaseRenewals(lease);
|
|
681
|
+
await lease.grant();
|
|
682
|
+
recordLeaseRenewal(lease, sentAt, context.ttlSeconds);
|
|
683
|
+
return lease;
|
|
684
|
+
}
|
|
685
|
+
function registerAgainOnLoss(context, lease, nodeId) {
|
|
686
|
+
lease.once("lost", () => {
|
|
687
|
+
void restoreLostRegistration(context, lease, nodeId, RECONNECT_FIRST_DELAY_MS);
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
async function restoreLostRegistration(context, lost, nodeId, retryDelayMs) {
|
|
691
|
+
const settled = await context.leaseManager.runExclusively(context.key, async () => {
|
|
692
|
+
const tracked = context.leaseManager.get(context.key);
|
|
693
|
+
if (context.isShutdown() || tracked === void 0 || tracked.lease !== lost || tracked.payload === null) {
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
let lease = null;
|
|
697
|
+
try {
|
|
698
|
+
const { startedAt } = deserializeNodeRegistration(Buffer.from(tracked.payload));
|
|
699
|
+
lease = await grantFollowedLease(context);
|
|
700
|
+
await putUnderLease(context, lease, startedAt, Buffer.from(tracked.payload));
|
|
701
|
+
} catch (_) {
|
|
702
|
+
await lease?.revoke().catch(() => {
|
|
703
|
+
});
|
|
704
|
+
return false;
|
|
705
|
+
}
|
|
706
|
+
context.leaseManager.track(context.key, lease, nodeId, tracked.payload);
|
|
707
|
+
registerAgainOnLoss(context, lease, nodeId);
|
|
708
|
+
return true;
|
|
709
|
+
});
|
|
710
|
+
if (settled) return;
|
|
711
|
+
const timer = setTimeout(() => {
|
|
712
|
+
void restoreLostRegistration(context, lost, nodeId, Math.min(retryDelayMs * 2, RECONNECT_LONGEST_DELAY_MS));
|
|
713
|
+
}, retryDelayMs);
|
|
714
|
+
timer.unref?.();
|
|
253
715
|
}
|
|
254
716
|
|
|
255
717
|
// src/distribution/coordinator/etcd/request-size.ts
|
|
@@ -302,9 +764,126 @@ function createEtcdWriteGuard(etcd3, maxRequestBytes) {
|
|
|
302
764
|
};
|
|
303
765
|
}
|
|
304
766
|
|
|
767
|
+
// src/distribution/coordinator/etcd/require-leader.ts
|
|
768
|
+
var ETCD_WATCH_METHOD_PATH = "/etcdserverpb.Watch/Watch";
|
|
769
|
+
var ETCD_REQUIRE_LEADER_METADATA_KEY = "hasleader";
|
|
770
|
+
var ETCD_REQUIRE_LEADER_METADATA_VALUE = "true";
|
|
771
|
+
function requireLeaderOnWatchStreams(properties) {
|
|
772
|
+
if (properties.methodDefinition.path !== ETCD_WATCH_METHOD_PATH) return properties;
|
|
773
|
+
const metadata = properties.metadata.clone();
|
|
774
|
+
metadata.set(ETCD_REQUIRE_LEADER_METADATA_KEY, ETCD_REQUIRE_LEADER_METADATA_VALUE);
|
|
775
|
+
return { ...properties, metadata };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// src/distribution/coordinator/etcd/security.ts
|
|
779
|
+
var SECURE_SCHEME = "https://";
|
|
780
|
+
var LOOPBACK_IPV4 = /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
|
|
781
|
+
var LOOPBACK_NAMES = /* @__PURE__ */ new Set(["localhost", "[::1]", "::1"]);
|
|
782
|
+
function hostnameOf(endpoint) {
|
|
783
|
+
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(endpoint) ? endpoint : `http://${endpoint}`;
|
|
784
|
+
try {
|
|
785
|
+
return new URL(withScheme).hostname.toLowerCase();
|
|
786
|
+
} catch (_) {
|
|
787
|
+
return null;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
function isLoopback(endpoint) {
|
|
791
|
+
const hostname = hostnameOf(endpoint);
|
|
792
|
+
return hostname !== null && (LOOPBACK_NAMES.has(hostname) || LOOPBACK_IPV4.test(hostname));
|
|
793
|
+
}
|
|
794
|
+
function refuse(message, details = {}) {
|
|
795
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
796
|
+
}
|
|
797
|
+
function validateTls(config) {
|
|
798
|
+
const tls = config.tls;
|
|
799
|
+
if (tls === void 0) {
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
if (tls.cert === void 0 !== (tls.key === void 0)) {
|
|
803
|
+
refuse("etcd tls needs both cert and key to present a client certificate, or neither");
|
|
804
|
+
}
|
|
805
|
+
const plain = config.endpoints.filter((endpoint) => !endpoint.startsWith(SECURE_SCHEME));
|
|
806
|
+
if (plain.length > 0) {
|
|
807
|
+
refuse("Every etcd endpoint must start with https:// when tls is set", { endpoints: plain });
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
function validateAuth(config) {
|
|
811
|
+
const auth = config.auth;
|
|
812
|
+
if (auth === void 0) {
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
if (typeof auth.username !== "string" || typeof auth.password !== "string" || auth.username.length === 0 || auth.password.length === 0) {
|
|
816
|
+
refuse("etcd auth needs a username and a password that are not empty");
|
|
817
|
+
}
|
|
818
|
+
const exposed = config.endpoints.filter((endpoint) => !endpoint.startsWith(SECURE_SCHEME) && !isLoopback(endpoint));
|
|
819
|
+
if (exposed.length > 0) {
|
|
820
|
+
refuse("etcd auth sends its password over https://, or over http:// to a loopback address alone", {
|
|
821
|
+
endpoints: exposed
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
function etcdSecurityOptions(config) {
|
|
826
|
+
validateTls(config);
|
|
827
|
+
validateAuth(config);
|
|
828
|
+
const options = {};
|
|
829
|
+
if (config.tls !== void 0) {
|
|
830
|
+
options.credentials = {
|
|
831
|
+
rootCertificate: Buffer.from(config.tls.ca),
|
|
832
|
+
...config.tls.key === void 0 ? {} : { privateKey: Buffer.from(config.tls.key) },
|
|
833
|
+
...config.tls.cert === void 0 ? {} : { certChain: Buffer.from(config.tls.cert) }
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
if (config.auth !== void 0) {
|
|
837
|
+
options.auth = { username: config.auth.username, password: config.auth.password };
|
|
838
|
+
}
|
|
839
|
+
return options;
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
// src/distribution/coordinator/etcd/start-probe.ts
|
|
843
|
+
function refusedCredentials(etcd3, error) {
|
|
844
|
+
return error instanceof etcd3.EtcdAuthenticationFailedError || error instanceof etcd3.EtcdInvalidAuthTokenError || error instanceof etcd3.EtcdPermissionDeniedError || error instanceof etcd3.EtcdUserNotFoundError;
|
|
845
|
+
}
|
|
846
|
+
function probeFailure(etcd3, error, endpoints) {
|
|
847
|
+
if (error instanceof NarsilError) {
|
|
848
|
+
return error;
|
|
849
|
+
}
|
|
850
|
+
if (refusedCredentials(etcd3, error)) {
|
|
851
|
+
return new NarsilError(ErrorCodes.COORDINATOR_AUTH_FAILED, "etcd refused the coordinator credentials", {
|
|
852
|
+
endpoints,
|
|
853
|
+
cause: describeError(error)
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
return new NarsilError(
|
|
857
|
+
ErrorCodes.COORDINATOR_UNAVAILABLE,
|
|
858
|
+
"The coordinator could not reach any etcd endpoint or complete a TLS handshake with one",
|
|
859
|
+
{ endpoints, cause: describeError(error) }
|
|
860
|
+
);
|
|
861
|
+
}
|
|
862
|
+
async function probeEtcd(client, etcd3, probeKey, endpoints) {
|
|
863
|
+
let timer;
|
|
864
|
+
const timedOut = new Promise((_, reject) => {
|
|
865
|
+
timer = setTimeout(() => {
|
|
866
|
+
reject(
|
|
867
|
+
new NarsilError(
|
|
868
|
+
ErrorCodes.COORDINATOR_UNAVAILABLE,
|
|
869
|
+
`No etcd endpoint answered within ${ETCD_START_PROBE_TIMEOUT_MS} ms`,
|
|
870
|
+
{ endpoints, timeoutMs: ETCD_START_PROBE_TIMEOUT_MS }
|
|
871
|
+
)
|
|
872
|
+
);
|
|
873
|
+
}, ETCD_START_PROBE_TIMEOUT_MS);
|
|
874
|
+
});
|
|
875
|
+
try {
|
|
876
|
+
await Promise.race([client.get(probeKey).exec(), timedOut]);
|
|
877
|
+
} catch (error) {
|
|
878
|
+
throw probeFailure(etcd3, error, endpoints);
|
|
879
|
+
} finally {
|
|
880
|
+
clearTimeout(timer);
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
|
|
305
884
|
// src/distribution/coordinator/etcd/stored-values.ts
|
|
306
885
|
function storedValueMethods(context) {
|
|
307
|
-
const { client, keys, guard, assertNotShutdown,
|
|
886
|
+
const { client, keys, guard, assertNotShutdown, watchPrefix: watchPrefix2 } = context;
|
|
308
887
|
return {
|
|
309
888
|
async get(key) {
|
|
310
889
|
assertNotShutdown();
|
|
@@ -361,23 +940,31 @@ function storedValueMethods(context) {
|
|
|
361
940
|
async watchSchemas(handler) {
|
|
362
941
|
assertNotShutdown();
|
|
363
942
|
const prefix = keys.schemaPrefix();
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
watcher.on("put", (kv) => {
|
|
367
|
-
try {
|
|
943
|
+
return watchPrefix2(prefix, {
|
|
944
|
+
put(kv) {
|
|
368
945
|
const schema = deserializeSchema(kv.value);
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
946
|
+
handler({ type: "schema_created", indexName: extractSuffix(kv.key.toString(), prefix), schema });
|
|
947
|
+
},
|
|
948
|
+
delete(fullKey) {
|
|
949
|
+
handler({ type: "schema_dropped", indexName: extractSuffix(fullKey, prefix), schema: null });
|
|
373
950
|
}
|
|
374
951
|
});
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
952
|
+
},
|
|
953
|
+
async watchIndexMetadata(handler) {
|
|
954
|
+
assertNotShutdown();
|
|
955
|
+
const prefix = keys.generic(INDEX_METADATA_KEY_PREFIX);
|
|
956
|
+
const nameOf = (fullKey) => indexNameOfMetadataSuffix(fullKey.slice(prefix.length));
|
|
957
|
+
return watchPrefix2(prefix, {
|
|
958
|
+
put(kv) {
|
|
959
|
+
const indexName = nameOf(kv.key.toString());
|
|
960
|
+
if (indexName === null) return;
|
|
961
|
+
handler({ indexName, metadata: kv.value.byteLength === 0 ? null : new Uint8Array(kv.value) });
|
|
962
|
+
},
|
|
963
|
+
delete(fullKey) {
|
|
964
|
+
const indexName = nameOf(fullKey);
|
|
965
|
+
if (indexName !== null) handler({ indexName, metadata: null });
|
|
966
|
+
}
|
|
379
967
|
});
|
|
380
|
-
return () => removeWatcher(watcher);
|
|
381
968
|
}
|
|
382
969
|
};
|
|
383
970
|
}
|
|
@@ -404,6 +991,14 @@ function validateNodeId(nodeId) {
|
|
|
404
991
|
);
|
|
405
992
|
}
|
|
406
993
|
}
|
|
994
|
+
function isValidNodeId(nodeId) {
|
|
995
|
+
try {
|
|
996
|
+
validateNodeId(nodeId);
|
|
997
|
+
return true;
|
|
998
|
+
} catch (_) {
|
|
999
|
+
return false;
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
407
1002
|
function validatePartitionState(value) {
|
|
408
1003
|
if (!VALID_PARTITION_STATES.has(value)) {
|
|
409
1004
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Unknown partition state '${value}'`, {
|
|
@@ -414,6 +1009,24 @@ function validatePartitionState(value) {
|
|
|
414
1009
|
return value;
|
|
415
1010
|
}
|
|
416
1011
|
|
|
1012
|
+
// src/distribution/coordinator/etcd/watch-client.ts
|
|
1013
|
+
function createReplaceableWatchClient(open) {
|
|
1014
|
+
let client = open();
|
|
1015
|
+
let closed = false;
|
|
1016
|
+
return {
|
|
1017
|
+
current: () => client,
|
|
1018
|
+
replace(stale) {
|
|
1019
|
+
if (closed || stale !== client) return;
|
|
1020
|
+
client = open();
|
|
1021
|
+
stale.close();
|
|
1022
|
+
},
|
|
1023
|
+
close() {
|
|
1024
|
+
closed = true;
|
|
1025
|
+
client.close();
|
|
1026
|
+
}
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
|
|
417
1030
|
// src/distribution/coordinator/etcd/index.ts
|
|
418
1031
|
async function createEtcdCoordinator(config) {
|
|
419
1032
|
const resolvedConfig = {
|
|
@@ -421,36 +1034,47 @@ async function createEtcdCoordinator(config) {
|
|
|
421
1034
|
...config
|
|
422
1035
|
};
|
|
423
1036
|
requireEtcdMaxRequestBytes(resolvedConfig.maxRequestBytes);
|
|
1037
|
+
const securityOptions = etcdSecurityOptions(resolvedConfig);
|
|
424
1038
|
const etcd3Module = await loadEtcd3Module();
|
|
425
|
-
const
|
|
1039
|
+
const openClient = () => new etcd3Module.Etcd3({
|
|
426
1040
|
hosts: resolvedConfig.endpoints,
|
|
427
|
-
grpcOptions: {
|
|
1041
|
+
grpcOptions: {
|
|
1042
|
+
"grpc.max_receive_message_length": ETCD_MAX_RESPONSE_BYTES,
|
|
1043
|
+
"grpc.keepalive_time_ms": ETCD_KEEPALIVE_TIME_MS,
|
|
1044
|
+
"grpc.keepalive_timeout_ms": ETCD_KEEPALIVE_TIMEOUT_MS,
|
|
1045
|
+
callInvocationTransformer: requireLeaderOnWatchStreams
|
|
1046
|
+
},
|
|
1047
|
+
defaultCallOptions: (context) => context.isStream ? {} : { deadline: Date.now() + ETCD_CALL_TIMEOUT_MS },
|
|
1048
|
+
faultHandling: { host: (endpoint) => createEndpointBreaker(endpoint, etcd3Module.isRecoverableError) },
|
|
1049
|
+
...securityOptions
|
|
428
1050
|
});
|
|
1051
|
+
const client = openClient();
|
|
1052
|
+
const keys = coordinatorKeys(resolvedConfig);
|
|
1053
|
+
try {
|
|
1054
|
+
await probeEtcd(client, etcd3Module, keys.nodePrefix(), resolvedConfig.endpoints);
|
|
1055
|
+
} catch (error) {
|
|
1056
|
+
client.close();
|
|
1057
|
+
throw error;
|
|
1058
|
+
}
|
|
429
1059
|
const guard = createEtcdWriteGuard(etcd3Module, resolvedConfig.maxRequestBytes);
|
|
430
1060
|
const leaseManager = new LeaseManager();
|
|
431
|
-
const watchers = /* @__PURE__ */ new Set();
|
|
432
1061
|
let isShutdown = false;
|
|
433
1062
|
function assertNotShutdown() {
|
|
434
1063
|
if (isShutdown) {
|
|
435
1064
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
|
|
436
1065
|
}
|
|
437
1066
|
}
|
|
438
|
-
const
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
1067
|
+
const watchClient = createReplaceableWatchClient(openClient);
|
|
1068
|
+
const streams = createWatchStreams();
|
|
1069
|
+
const { onWatchError } = resolvedConfig;
|
|
1070
|
+
function reportHandlerError(error) {
|
|
1071
|
+
if (onWatchError === void 0) return;
|
|
1072
|
+
try {
|
|
1073
|
+
onWatchError(error instanceof Error ? error : new Error(String(error)));
|
|
1074
|
+
} catch (_) {
|
|
446
1075
|
}
|
|
447
|
-
watchers.add(watcher);
|
|
448
|
-
}
|
|
449
|
-
function removeWatcher(watcher) {
|
|
450
|
-
watcher.cancel().catch(() => {
|
|
451
|
-
});
|
|
452
|
-
watchers.delete(watcher);
|
|
453
1076
|
}
|
|
1077
|
+
const watchContext = { client, watchClient, streams, reportHandlerError };
|
|
454
1078
|
const coordinator = {
|
|
455
1079
|
async registerNode(registration) {
|
|
456
1080
|
assertNotShutdown();
|
|
@@ -461,7 +1085,9 @@ async function createEtcdCoordinator(config) {
|
|
|
461
1085
|
leaseManager,
|
|
462
1086
|
guard,
|
|
463
1087
|
key: keys.node(registration.nodeId),
|
|
464
|
-
|
|
1088
|
+
startedAtKey: keys.nodeStartedAt(registration.nodeId),
|
|
1089
|
+
ttlSeconds: resolvedConfig.nodeHeartbeatTtlSeconds,
|
|
1090
|
+
isShutdown: () => isShutdown
|
|
465
1091
|
},
|
|
466
1092
|
registration
|
|
467
1093
|
);
|
|
@@ -471,11 +1097,28 @@ async function createEtcdCoordinator(config) {
|
|
|
471
1097
|
validateNodeId(nodeId);
|
|
472
1098
|
const key = keys.node(nodeId);
|
|
473
1099
|
const entry = leaseManager.remove(key);
|
|
1100
|
+
leaseManager.forgetHolder(key);
|
|
474
1101
|
if (entry !== void 0) {
|
|
475
1102
|
await entry.lease.revoke().catch(() => {
|
|
476
1103
|
});
|
|
477
1104
|
}
|
|
478
|
-
await client.delete().key(key).
|
|
1105
|
+
await client.if(key, "Create", ">", -1).then(client.delete().key(keys.nodeStartedAt(nodeId)), client.delete().key(key)).commit();
|
|
1106
|
+
},
|
|
1107
|
+
registrationLease(nodeId) {
|
|
1108
|
+
if (isShutdown) return null;
|
|
1109
|
+
const holder = leaseManager.holder(keys.node(nodeId));
|
|
1110
|
+
return holder === void 0 ? null : leaseHeldFor(holder);
|
|
1111
|
+
},
|
|
1112
|
+
async confirmRegistrationLease(nodeId) {
|
|
1113
|
+
if (isShutdown) return null;
|
|
1114
|
+
const holder = leaseManager.holder(keys.node(nodeId));
|
|
1115
|
+
if (holder === void 0) return null;
|
|
1116
|
+
const sentAt = performance.now();
|
|
1117
|
+
try {
|
|
1118
|
+
recordLeaseRenewal(holder, sentAt, Number((await holder.keepaliveOnce()).TTL));
|
|
1119
|
+
} catch (_) {
|
|
1120
|
+
}
|
|
1121
|
+
return leaseHeldFor(holder);
|
|
479
1122
|
},
|
|
480
1123
|
async listNodes() {
|
|
481
1124
|
assertNotShutdown();
|
|
@@ -493,21 +1136,15 @@ async function createEtcdCoordinator(config) {
|
|
|
493
1136
|
async watchNodes(handler) {
|
|
494
1137
|
assertNotShutdown();
|
|
495
1138
|
const prefix = keys.nodePrefix();
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
watcher.on("put", (kv) => {
|
|
499
|
-
try {
|
|
1139
|
+
return watchPrefix(watchContext, prefix, {
|
|
1140
|
+
put(kv) {
|
|
500
1141
|
const registration = deserializeNodeRegistration(kv.value);
|
|
501
1142
|
handler({ type: "node_joined", nodeId: registration.nodeId, registration });
|
|
502
|
-
}
|
|
1143
|
+
},
|
|
1144
|
+
delete(fullKey) {
|
|
1145
|
+
handler({ type: "node_left", nodeId: extractSuffix(fullKey, prefix), registration: null });
|
|
503
1146
|
}
|
|
504
1147
|
});
|
|
505
|
-
watcher.on("delete", (kv) => {
|
|
506
|
-
const fullKey = kv.key.toString();
|
|
507
|
-
const nodeId = extractSuffix(fullKey, prefix);
|
|
508
|
-
handler({ type: "node_left", nodeId, registration: null });
|
|
509
|
-
});
|
|
510
|
-
return () => removeWatcher(watcher);
|
|
511
1148
|
},
|
|
512
1149
|
async getAllocation(indexName) {
|
|
513
1150
|
assertNotShutdown();
|
|
@@ -518,7 +1155,7 @@ async function createEtcdCoordinator(config) {
|
|
|
518
1155
|
}
|
|
519
1156
|
return deserializeAllocationTable(buf);
|
|
520
1157
|
},
|
|
521
|
-
async putAllocation(indexName, table, expectedVersion) {
|
|
1158
|
+
async putAllocation(indexName, table, expectedVersion, writerNodeId, replicaRegistration) {
|
|
522
1159
|
assertNotShutdown();
|
|
523
1160
|
const key = keys.allocation(indexName);
|
|
524
1161
|
const data = Buffer.from(serializeAllocationTable(table));
|
|
@@ -527,11 +1164,23 @@ async function createEtcdCoordinator(config) {
|
|
|
527
1164
|
await guard.send(write, () => client.put(key).value(data).exec());
|
|
528
1165
|
return true;
|
|
529
1166
|
}
|
|
1167
|
+
if (writerNodeId === void 0 || writerNodeId.length === 0) {
|
|
1168
|
+
return false;
|
|
1169
|
+
}
|
|
1170
|
+
if (replicaRegistration !== void 0 && !isValidNodeId(replicaRegistration.nodeId)) {
|
|
1171
|
+
return false;
|
|
1172
|
+
}
|
|
1173
|
+
const controllerLeaseKey = keys.lease(CONTROLLER_LEASE_KEY);
|
|
1174
|
+
const commitGuarded = async (txn) => {
|
|
1175
|
+
txn.and(controllerLeaseKey, "Value", "==", writerNodeId);
|
|
1176
|
+
if (replicaRegistration !== void 0) {
|
|
1177
|
+
txn.and(keys.node(replicaRegistration.nodeId), "Create", ">", 0);
|
|
1178
|
+
txn.and(keys.nodeStartedAt(replicaRegistration.nodeId), "Value", "==", replicaRegistration.startedAt);
|
|
1179
|
+
}
|
|
1180
|
+
return (await txn.then(client.put(key).value(data)).commit()).succeeded;
|
|
1181
|
+
};
|
|
530
1182
|
if (expectedVersion === null) {
|
|
531
|
-
return guard.send(write,
|
|
532
|
-
const txn = await client.if(key, "Create", "==", 0).then(client.put(key).value(data)).commit();
|
|
533
|
-
return txn.succeeded;
|
|
534
|
-
});
|
|
1183
|
+
return guard.send(write, () => commitGuarded(client.if(key, "Create", "==", 0)));
|
|
535
1184
|
}
|
|
536
1185
|
const response = await client.get(key).exec();
|
|
537
1186
|
if (response.kvs.length === 0) {
|
|
@@ -542,10 +1191,7 @@ async function createEtcdCoordinator(config) {
|
|
|
542
1191
|
if (currentTable.version !== expectedVersion) {
|
|
543
1192
|
return false;
|
|
544
1193
|
}
|
|
545
|
-
return guard.send(write,
|
|
546
|
-
const txn = await client.if(key, "Mod", "==", kv.mod_revision).then(client.put(key).value(data)).commit();
|
|
547
|
-
return txn.succeeded;
|
|
548
|
-
});
|
|
1194
|
+
return guard.send(write, () => commitGuarded(client.if(key, "Mod", "==", kv.mod_revision)));
|
|
549
1195
|
},
|
|
550
1196
|
async deleteAllocation(indexName) {
|
|
551
1197
|
assertNotShutdown();
|
|
@@ -553,17 +1199,14 @@ async function createEtcdCoordinator(config) {
|
|
|
553
1199
|
},
|
|
554
1200
|
async watchAllocation(handler) {
|
|
555
1201
|
assertNotShutdown();
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
addWatcher(watcher);
|
|
559
|
-
watcher.on("put", (kv) => {
|
|
560
|
-
try {
|
|
1202
|
+
return watchPrefix(watchContext, keys.allocationPrefix(), {
|
|
1203
|
+
put(kv) {
|
|
561
1204
|
const table = deserializeAllocationTable(kv.value);
|
|
562
1205
|
handler({ indexName: table.indexName, table });
|
|
563
|
-
}
|
|
1206
|
+
},
|
|
1207
|
+
delete() {
|
|
564
1208
|
}
|
|
565
1209
|
});
|
|
566
|
-
return () => removeWatcher(watcher);
|
|
567
1210
|
},
|
|
568
1211
|
async getPartitionState(indexName, partitionId) {
|
|
569
1212
|
assertNotShutdown();
|
|
@@ -629,7 +1272,13 @@ async function createEtcdCoordinator(config) {
|
|
|
629
1272
|
await entry.lease.revoke().catch(() => {
|
|
630
1273
|
});
|
|
631
1274
|
},
|
|
632
|
-
...storedValueMethods({
|
|
1275
|
+
...storedValueMethods({
|
|
1276
|
+
client,
|
|
1277
|
+
keys,
|
|
1278
|
+
guard,
|
|
1279
|
+
assertNotShutdown,
|
|
1280
|
+
watchPrefix: (prefix, handlers) => watchPrefix(watchContext, prefix, handlers)
|
|
1281
|
+
}),
|
|
633
1282
|
async getLeaseHolder(key) {
|
|
634
1283
|
assertNotShutdown();
|
|
635
1284
|
const etcdKey = keys.lease(key);
|
|
@@ -641,14 +1290,9 @@ async function createEtcdCoordinator(config) {
|
|
|
641
1290
|
return;
|
|
642
1291
|
}
|
|
643
1292
|
isShutdown = true;
|
|
644
|
-
|
|
645
|
-
for (const watcher of watchers) {
|
|
646
|
-
cancelPromises.push(watcher.cancel().catch(() => {
|
|
647
|
-
}));
|
|
648
|
-
}
|
|
649
|
-
await Promise.all(cancelPromises);
|
|
650
|
-
watchers.clear();
|
|
1293
|
+
await streams.closeAll();
|
|
651
1294
|
await leaseManager.revokeAll();
|
|
1295
|
+
watchClient.close();
|
|
652
1296
|
client.close();
|
|
653
1297
|
}
|
|
654
1298
|
};
|