@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -0,0 +1,2951 @@
|
|
|
1
|
+
// src/engine/orchestration/worker-copies-host.browser.ts
|
|
2
|
+
var COMPACTION_SEGMENT_TRIGGER;
|
|
3
|
+
var DEFAULT_BACKPRESSURE_LIMIT;
|
|
4
|
+
var DEFAULT_COPY_IDLE_TIMEOUT_MS;
|
|
5
|
+
var DEFAULT_COPY_THRESHOLD;
|
|
6
|
+
var DEFAULT_REQUEST_TIMEOUT_MS;
|
|
7
|
+
var ErrorCodes;
|
|
8
|
+
var IDLE_MERGE_DELAY_MS;
|
|
9
|
+
var LIVE_TAIL_FLUSH_DOCUMENTS;
|
|
10
|
+
var LIVE_TAIL_FREEZE_FLOOR;
|
|
11
|
+
var LOAD_BUFFER_YIELD_INTERVAL;
|
|
12
|
+
var LONG_QUERY_THREADS;
|
|
13
|
+
var MAIN_COPY_LONE_QUERY_DOCUMENTS;
|
|
14
|
+
var MATCHER_STATE_BUDGET_BYTES;
|
|
15
|
+
var MAX_PENDING_REPLICATION_DOCUMENTS;
|
|
16
|
+
var MAX_SWEEP_INTERVAL_MS;
|
|
17
|
+
var MIN_SWEEP_INTERVAL_MS;
|
|
18
|
+
var NO_REMOVED_SPELLINGS;
|
|
19
|
+
var NarsilError;
|
|
20
|
+
var POOL_RESTART_DELAY_MAX_MS;
|
|
21
|
+
var POOL_RESTART_DELAY_MS;
|
|
22
|
+
var REPLICATION_WINDOW;
|
|
23
|
+
var RETIRED_THREADS_EXIT_WAIT_MS;
|
|
24
|
+
var SEGMENTS_PER_MERGE;
|
|
25
|
+
var SHUTDOWN_TIMEOUT_MS;
|
|
26
|
+
var SegmentRefusal;
|
|
27
|
+
var afterCurrentTurn;
|
|
28
|
+
var availablePatternSearch;
|
|
29
|
+
var createWorkerFactory;
|
|
30
|
+
var encodeSegmentState;
|
|
31
|
+
var extractVectorFieldsFromSchema;
|
|
32
|
+
var fnv1a;
|
|
33
|
+
var generateId;
|
|
34
|
+
var growPartitionsTo;
|
|
35
|
+
var isCompositePartition;
|
|
36
|
+
var kWayMerge;
|
|
37
|
+
var loadStorageCode;
|
|
38
|
+
var mergeFacets;
|
|
39
|
+
var noteThreadSlotGone;
|
|
40
|
+
var oversampledFacetConfig;
|
|
41
|
+
var patternHost;
|
|
42
|
+
var prepareDocumentVectors;
|
|
43
|
+
var requireLoadedVectorCode;
|
|
44
|
+
var resolvePartitionInsertOptions;
|
|
45
|
+
var resolveRequestThreadCount;
|
|
46
|
+
var resolveWorkerCount;
|
|
47
|
+
var returnsEveryMatch;
|
|
48
|
+
var schemaFieldsOf;
|
|
49
|
+
var splitWorkerBudget;
|
|
50
|
+
var threadSlotOfWorker;
|
|
51
|
+
var workersEnabledByDefault;
|
|
52
|
+
var yieldToEventLoop;
|
|
53
|
+
var createSharedFrozenSegment;
|
|
54
|
+
var alreadyPresentOnWorker;
|
|
55
|
+
var buildSegmentPayloads;
|
|
56
|
+
var createRequestId;
|
|
57
|
+
var eligibleIndexNames;
|
|
58
|
+
var freezeEncodedSegmentShared;
|
|
59
|
+
var freezeSegmentShared;
|
|
60
|
+
var isDeterministicFailure;
|
|
61
|
+
var isValidWorkerResponse;
|
|
62
|
+
var mergeSegmentContents;
|
|
63
|
+
var reportIneligible;
|
|
64
|
+
var selectionKeeps;
|
|
65
|
+
var toError;
|
|
66
|
+
var workerIneligibility;
|
|
67
|
+
function bindWorkerCopiesHost(host, segmentCodec, segmentMerge) {
|
|
68
|
+
({
|
|
69
|
+
COMPACTION_SEGMENT_TRIGGER,
|
|
70
|
+
DEFAULT_BACKPRESSURE_LIMIT,
|
|
71
|
+
DEFAULT_COPY_IDLE_TIMEOUT_MS,
|
|
72
|
+
DEFAULT_COPY_THRESHOLD,
|
|
73
|
+
DEFAULT_REQUEST_TIMEOUT_MS,
|
|
74
|
+
ErrorCodes,
|
|
75
|
+
IDLE_MERGE_DELAY_MS,
|
|
76
|
+
LIVE_TAIL_FLUSH_DOCUMENTS,
|
|
77
|
+
LIVE_TAIL_FREEZE_FLOOR,
|
|
78
|
+
LOAD_BUFFER_YIELD_INTERVAL,
|
|
79
|
+
LONG_QUERY_THREADS,
|
|
80
|
+
MAIN_COPY_LONE_QUERY_DOCUMENTS,
|
|
81
|
+
MATCHER_STATE_BUDGET_BYTES,
|
|
82
|
+
MAX_PENDING_REPLICATION_DOCUMENTS,
|
|
83
|
+
MAX_SWEEP_INTERVAL_MS,
|
|
84
|
+
MIN_SWEEP_INTERVAL_MS,
|
|
85
|
+
NO_REMOVED_SPELLINGS,
|
|
86
|
+
NarsilError,
|
|
87
|
+
POOL_RESTART_DELAY_MAX_MS,
|
|
88
|
+
POOL_RESTART_DELAY_MS,
|
|
89
|
+
REPLICATION_WINDOW,
|
|
90
|
+
RETIRED_THREADS_EXIT_WAIT_MS,
|
|
91
|
+
SEGMENTS_PER_MERGE,
|
|
92
|
+
SHUTDOWN_TIMEOUT_MS,
|
|
93
|
+
SegmentRefusal,
|
|
94
|
+
afterCurrentTurn,
|
|
95
|
+
availablePatternSearch,
|
|
96
|
+
createWorkerFactory,
|
|
97
|
+
encodeSegmentState,
|
|
98
|
+
extractVectorFieldsFromSchema,
|
|
99
|
+
fnv1a,
|
|
100
|
+
generateId,
|
|
101
|
+
growPartitionsTo,
|
|
102
|
+
isCompositePartition,
|
|
103
|
+
kWayMerge,
|
|
104
|
+
loadStorageCode,
|
|
105
|
+
mergeFacets,
|
|
106
|
+
noteThreadSlotGone,
|
|
107
|
+
oversampledFacetConfig,
|
|
108
|
+
patternHost,
|
|
109
|
+
prepareDocumentVectors,
|
|
110
|
+
requireLoadedVectorCode,
|
|
111
|
+
resolvePartitionInsertOptions,
|
|
112
|
+
resolveRequestThreadCount,
|
|
113
|
+
resolveWorkerCount,
|
|
114
|
+
returnsEveryMatch,
|
|
115
|
+
schemaFieldsOf,
|
|
116
|
+
splitWorkerBudget,
|
|
117
|
+
threadSlotOfWorker,
|
|
118
|
+
workersEnabledByDefault,
|
|
119
|
+
yieldToEventLoop
|
|
120
|
+
} = host);
|
|
121
|
+
({ createSharedFrozenSegment } = segmentCodec);
|
|
122
|
+
({
|
|
123
|
+
alreadyPresentOnWorker,
|
|
124
|
+
buildSegmentPayloads,
|
|
125
|
+
createRequestId,
|
|
126
|
+
eligibleIndexNames,
|
|
127
|
+
freezeEncodedSegmentShared,
|
|
128
|
+
freezeSegmentShared,
|
|
129
|
+
isDeterministicFailure,
|
|
130
|
+
isValidWorkerResponse,
|
|
131
|
+
mergeSegmentContents,
|
|
132
|
+
reportIneligible,
|
|
133
|
+
selectionKeeps,
|
|
134
|
+
toError,
|
|
135
|
+
workerIneligibility
|
|
136
|
+
} = segmentMerge);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// src/workers/liveness/constants.ts
|
|
140
|
+
var DEFAULT_WORKER_SILENCE_BEFORE_PROBE_MS = 1e4;
|
|
141
|
+
var DEFAULT_WORKER_SILENCE_BEFORE_RETIREMENT_MS = 9e4;
|
|
142
|
+
var DEFAULT_MAX_THREADS_LEFT_RUNNING = 1;
|
|
143
|
+
|
|
144
|
+
// src/workers/liveness/config.ts
|
|
145
|
+
function requireWholeNumber(field, value, minimum) {
|
|
146
|
+
if (value === void 0 || Number.isSafeInteger(value) && value >= minimum) return;
|
|
147
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `workers.${field} must be a whole number of at least ${minimum}`, {
|
|
148
|
+
[field]: value
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
function validateWorkerLiveness(workers) {
|
|
152
|
+
requireWholeNumber("silenceBeforeProbeMs", workers.silenceBeforeProbeMs, 1);
|
|
153
|
+
requireWholeNumber("silenceBeforeRetirementMs", workers.silenceBeforeRetirementMs, 1);
|
|
154
|
+
requireWholeNumber("maxThreadsLeftRunning", workers.maxThreadsLeftRunning, 0);
|
|
155
|
+
const beforeProbe = workers.silenceBeforeProbeMs ?? DEFAULT_WORKER_SILENCE_BEFORE_PROBE_MS;
|
|
156
|
+
const beforeRetirement = workers.silenceBeforeRetirementMs ?? DEFAULT_WORKER_SILENCE_BEFORE_RETIREMENT_MS;
|
|
157
|
+
if (beforeProbe < beforeRetirement) return;
|
|
158
|
+
throw new NarsilError(
|
|
159
|
+
ErrorCodes.CONFIG_INVALID,
|
|
160
|
+
"workers.silenceBeforeProbeMs must be less than workers.silenceBeforeRetirementMs, so that a worker gets a probe before the engine retires it",
|
|
161
|
+
{ silenceBeforeProbeMs: beforeProbe, silenceBeforeRetirementMs: beforeRetirement }
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// src/workers/pool-memory.ts
|
|
166
|
+
function toFinite(value) {
|
|
167
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
168
|
+
}
|
|
169
|
+
function toHeapLimit(value) {
|
|
170
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
|
|
171
|
+
}
|
|
172
|
+
async function memoryStatsOf(slots) {
|
|
173
|
+
const indices = [];
|
|
174
|
+
const pending = [];
|
|
175
|
+
for (const [slotIndex, slot] of slots) {
|
|
176
|
+
indices.push(slotIndex);
|
|
177
|
+
pending.push(
|
|
178
|
+
slot.executor.execute({ type: "memoryReport", requestId: createRequestId() }).catch(() => null)
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
const reports = await Promise.all(pending);
|
|
182
|
+
const stats = [];
|
|
183
|
+
for (let n = 0; n < indices.length; n++) {
|
|
184
|
+
const report = reports[n];
|
|
185
|
+
stats.push({
|
|
186
|
+
workerId: indices[n],
|
|
187
|
+
heapUsed: toFinite(report?.heapUsed),
|
|
188
|
+
heapTotal: toFinite(report?.heapTotal),
|
|
189
|
+
heapLimit: toHeapLimit(report?.heapLimit),
|
|
190
|
+
external: toFinite(report?.external)
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return stats;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// src/workers/pool.ts
|
|
197
|
+
function createWorkerPool(config) {
|
|
198
|
+
const workerCount = resolveWorkerCount(config.count);
|
|
199
|
+
const queryWorkerCount = workerCount - Math.min(config.longQueryWorkers ?? 0, workerCount - 1);
|
|
200
|
+
const workers = /* @__PURE__ */ new Map();
|
|
201
|
+
const deadSlots = /* @__PURE__ */ new Set();
|
|
202
|
+
const goneSlots = /* @__PURE__ */ new Set();
|
|
203
|
+
const threadsStillRunning = /* @__PURE__ */ new Map();
|
|
204
|
+
const waitingForEveryThread = [];
|
|
205
|
+
const indexAssignment = /* @__PURE__ */ new Map();
|
|
206
|
+
let isShutdown = false;
|
|
207
|
+
function handleWorkerDeath(slotIndex, error) {
|
|
208
|
+
if (isShutdown) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const slot = workers.get(slotIndex);
|
|
212
|
+
if (slot !== void 0 && !slot.serving) {
|
|
213
|
+
workers.delete(slotIndex);
|
|
214
|
+
goneSlots.delete(slotIndex);
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
if (deadSlots.has(slotIndex)) {
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
deadSlots.add(slotIndex);
|
|
221
|
+
const indexNames = slot ? [...slot.indexes].sort() : [];
|
|
222
|
+
workers.delete(slotIndex);
|
|
223
|
+
config.onWorkerCrash?.(slotIndex, indexNames, error);
|
|
224
|
+
}
|
|
225
|
+
function handleWorkerGone(slotIndex) {
|
|
226
|
+
if (!deadSlots.has(slotIndex) || workers.has(slotIndex)) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
goneSlots.add(slotIndex);
|
|
230
|
+
config.onWorkerGone?.(slotIndex);
|
|
231
|
+
}
|
|
232
|
+
function spawnSlot(slotIndex, serving) {
|
|
233
|
+
const slot = {
|
|
234
|
+
executor: config.workerFactory(slotIndex, {
|
|
235
|
+
...config.silence,
|
|
236
|
+
onDeath: (error) => handleWorkerDeath(slotIndex, error),
|
|
237
|
+
onGone: () => {
|
|
238
|
+
noteThreadGone(slot.executor);
|
|
239
|
+
handleWorkerGone(slotIndex);
|
|
240
|
+
}
|
|
241
|
+
}),
|
|
242
|
+
indexes: /* @__PURE__ */ new Set(),
|
|
243
|
+
inFlight: 0,
|
|
244
|
+
serving
|
|
245
|
+
};
|
|
246
|
+
let finish = () => void 0;
|
|
247
|
+
const gone = new Promise((resolve) => {
|
|
248
|
+
finish = resolve;
|
|
249
|
+
});
|
|
250
|
+
threadsStillRunning.set(slot.executor, { indexes: slot.indexes, gone, finish });
|
|
251
|
+
workers.set(slotIndex, slot);
|
|
252
|
+
watchForThreadLeftRunning(slotIndex, slot.executor);
|
|
253
|
+
return slot;
|
|
254
|
+
}
|
|
255
|
+
function watchForThreadLeftRunning(slotIndex, executor) {
|
|
256
|
+
const exited = executor.whenTheThreadExits?.();
|
|
257
|
+
if (exited === void 0) return;
|
|
258
|
+
void executor.whenAbandoned?.().then(() => config.onThreadLeftRunning?.(slotIndex, exited));
|
|
259
|
+
}
|
|
260
|
+
function noteThreadGone(executor) {
|
|
261
|
+
threadsStillRunning.get(executor)?.finish();
|
|
262
|
+
threadsStillRunning.delete(executor);
|
|
263
|
+
if (threadsStillRunning.size > 0) return;
|
|
264
|
+
for (const resume of waitingForEveryThread.splice(0)) resume();
|
|
265
|
+
}
|
|
266
|
+
function whenEveryThreadIsGone() {
|
|
267
|
+
if (threadsStillRunning.size === 0) return Promise.resolve();
|
|
268
|
+
return new Promise((resolve) => {
|
|
269
|
+
waitingForEveryThread.push(resolve);
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
async function whenRetiredCopiesAreGone(indexName) {
|
|
273
|
+
const active = new Set(getAllExecutors());
|
|
274
|
+
const pending = [];
|
|
275
|
+
for (const [executor, lifetime] of threadsStillRunning) {
|
|
276
|
+
if (!active.has(executor) && lifetime.indexes.has(indexName)) pending.push(lifetime.gone);
|
|
277
|
+
}
|
|
278
|
+
await Promise.all(pending);
|
|
279
|
+
}
|
|
280
|
+
function shutDownUntilTheThreadIsGone(executor) {
|
|
281
|
+
return executor.shutdown().then(() => noteThreadGone(executor));
|
|
282
|
+
}
|
|
283
|
+
function ensureWorker(slotIndex) {
|
|
284
|
+
const slot = workers.get(slotIndex);
|
|
285
|
+
if (slot !== void 0) return slot;
|
|
286
|
+
if (deadSlots.has(slotIndex)) return void 0;
|
|
287
|
+
return spawnSlot(slotIndex, true);
|
|
288
|
+
}
|
|
289
|
+
function deadWorkerIds() {
|
|
290
|
+
return [...deadSlots].filter((slotIndex) => goneSlots.has(slotIndex) && !workers.has(slotIndex)).sort((a, b) => a - b);
|
|
291
|
+
}
|
|
292
|
+
function spawnReplacement(slotIndex) {
|
|
293
|
+
if (isShutdown || !deadSlots.has(slotIndex) || !goneSlots.has(slotIndex) || workers.has(slotIndex)) return null;
|
|
294
|
+
const slot = spawnSlot(slotIndex, false);
|
|
295
|
+
return {
|
|
296
|
+
workerId: slotIndex,
|
|
297
|
+
executor: slot.executor,
|
|
298
|
+
hold(indexName) {
|
|
299
|
+
slot.indexes.add(indexName);
|
|
300
|
+
},
|
|
301
|
+
admit() {
|
|
302
|
+
if (workers.get(slotIndex) !== slot) return;
|
|
303
|
+
deadSlots.delete(slotIndex);
|
|
304
|
+
goneSlots.delete(slotIndex);
|
|
305
|
+
slot.serving = true;
|
|
306
|
+
},
|
|
307
|
+
abandon() {
|
|
308
|
+
if (workers.get(slotIndex) === slot) workers.delete(slotIndex);
|
|
309
|
+
void shutDownUntilTheThreadIsGone(slot.executor).catch(() => void 0);
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
function isLongQueryWorker(workerId) {
|
|
314
|
+
return workerId >= queryWorkerCount;
|
|
315
|
+
}
|
|
316
|
+
function assignSlot(indexName) {
|
|
317
|
+
const preferred = fnv1a(indexName) % queryWorkerCount;
|
|
318
|
+
for (let probe = 0; probe < queryWorkerCount; probe++) {
|
|
319
|
+
const candidate = (preferred + probe) % queryWorkerCount;
|
|
320
|
+
if (!deadSlots.has(candidate)) {
|
|
321
|
+
return candidate;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker in the pool has crashed`);
|
|
325
|
+
}
|
|
326
|
+
function addIndex(indexName) {
|
|
327
|
+
if (isShutdown) {
|
|
328
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
329
|
+
}
|
|
330
|
+
if (indexAssignment.has(indexName)) {
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
const slotIndex = assignSlot(indexName);
|
|
334
|
+
const slot = ensureWorker(slotIndex);
|
|
335
|
+
if (!slot) {
|
|
336
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker ${slotIndex} for index "${indexName}" has crashed`);
|
|
337
|
+
}
|
|
338
|
+
slot.indexes.add(indexName);
|
|
339
|
+
indexAssignment.set(indexName, slotIndex);
|
|
340
|
+
}
|
|
341
|
+
function removeIndex(indexName) {
|
|
342
|
+
for (const slot of workers.values()) slot.indexes.delete(indexName);
|
|
343
|
+
indexAssignment.delete(indexName);
|
|
344
|
+
}
|
|
345
|
+
function getExecutor(indexName) {
|
|
346
|
+
if (isShutdown) {
|
|
347
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
348
|
+
}
|
|
349
|
+
const slotIndex = indexAssignment.get(indexName);
|
|
350
|
+
if (slotIndex === void 0) {
|
|
351
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" is not registered in the worker pool`);
|
|
352
|
+
}
|
|
353
|
+
const slot = workers.get(slotIndex);
|
|
354
|
+
if (slot?.serving) {
|
|
355
|
+
return slot.executor;
|
|
356
|
+
}
|
|
357
|
+
for (const survivor of workers.values()) {
|
|
358
|
+
if (survivor.serving && survivor.indexes.has(indexName)) {
|
|
359
|
+
return survivor.executor;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `Every worker holding index "${indexName}" has crashed`);
|
|
363
|
+
}
|
|
364
|
+
async function shutdown() {
|
|
365
|
+
if (isShutdown) {
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
isShutdown = true;
|
|
369
|
+
const shutdownPromises = [...threadsStillRunning.keys()].map((executor) => {
|
|
370
|
+
let timeoutId;
|
|
371
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
372
|
+
timeoutId = setTimeout(resolve, SHUTDOWN_TIMEOUT_MS);
|
|
373
|
+
});
|
|
374
|
+
return Promise.race([shutDownUntilTheThreadIsGone(executor), timeoutPromise]).finally(
|
|
375
|
+
() => clearTimeout(timeoutId)
|
|
376
|
+
);
|
|
377
|
+
});
|
|
378
|
+
await Promise.allSettled(shutdownPromises);
|
|
379
|
+
indexAssignment.clear();
|
|
380
|
+
}
|
|
381
|
+
function getAllExecutors() {
|
|
382
|
+
return [...workers.values()].map((slot) => slot.executor);
|
|
383
|
+
}
|
|
384
|
+
function executorEntries() {
|
|
385
|
+
return [...workers].map(([workerId, slot]) => ({ workerId, executor: slot.executor }));
|
|
386
|
+
}
|
|
387
|
+
function executorsHolding(indexName) {
|
|
388
|
+
const holders = [];
|
|
389
|
+
for (const slot of workers.values()) {
|
|
390
|
+
if (slot.indexes.has(indexName)) holders.push(slot.executor);
|
|
391
|
+
}
|
|
392
|
+
return holders;
|
|
393
|
+
}
|
|
394
|
+
function lease(workerId, slot) {
|
|
395
|
+
slot.inFlight += 1;
|
|
396
|
+
let released = false;
|
|
397
|
+
return {
|
|
398
|
+
workerId,
|
|
399
|
+
executor: slot.executor,
|
|
400
|
+
release() {
|
|
401
|
+
if (released) return;
|
|
402
|
+
released = true;
|
|
403
|
+
slot.inFlight -= 1;
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function leaseLeastBusy(indexName) {
|
|
408
|
+
let chosenId = -1;
|
|
409
|
+
let chosen = null;
|
|
410
|
+
for (const [workerId, slot] of workers) {
|
|
411
|
+
if (!slot.serving || isLongQueryWorker(workerId)) continue;
|
|
412
|
+
if (indexName !== void 0 && !slot.indexes.has(indexName)) continue;
|
|
413
|
+
if (chosen === null || slot.inFlight < chosen.inFlight) {
|
|
414
|
+
chosenId = workerId;
|
|
415
|
+
chosen = slot;
|
|
416
|
+
}
|
|
417
|
+
if (chosen.inFlight === 0) break;
|
|
418
|
+
}
|
|
419
|
+
return chosen === null ? null : lease(chosenId, chosen);
|
|
420
|
+
}
|
|
421
|
+
function queriesInFlight() {
|
|
422
|
+
let total = 0;
|
|
423
|
+
for (const [workerId, slot] of workers) {
|
|
424
|
+
if (slot.serving && !isLongQueryWorker(workerId)) total += slot.inFlight;
|
|
425
|
+
}
|
|
426
|
+
return total;
|
|
427
|
+
}
|
|
428
|
+
function leaseIdle(limit) {
|
|
429
|
+
const leases = [];
|
|
430
|
+
for (const [workerId, slot] of workers) {
|
|
431
|
+
if (leases.length >= limit) break;
|
|
432
|
+
if (slot.serving && slot.inFlight === 0 && !isLongQueryWorker(workerId)) leases.push(lease(workerId, slot));
|
|
433
|
+
}
|
|
434
|
+
return leases;
|
|
435
|
+
}
|
|
436
|
+
function addIndexToAll(indexName) {
|
|
437
|
+
if (isShutdown) {
|
|
438
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
439
|
+
}
|
|
440
|
+
for (let i = 0; i < workerCount; i++) {
|
|
441
|
+
ensureWorker(i)?.indexes.add(indexName);
|
|
442
|
+
}
|
|
443
|
+
indexAssignment.set(indexName, assignSlot(indexName));
|
|
444
|
+
}
|
|
445
|
+
function spawnAll() {
|
|
446
|
+
if (isShutdown) {
|
|
447
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker pool has been shut down");
|
|
448
|
+
}
|
|
449
|
+
for (let i = 0; i < workerCount; i++) ensureWorker(i);
|
|
450
|
+
}
|
|
451
|
+
function retire(executor, error) {
|
|
452
|
+
for (const [workerId, slot] of workers) {
|
|
453
|
+
if (slot.executor !== executor) continue;
|
|
454
|
+
handleWorkerDeath(workerId, error);
|
|
455
|
+
void shutDownUntilTheThreadIsGone(executor).catch(() => void 0);
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return {
|
|
460
|
+
getExecutor,
|
|
461
|
+
getAllExecutors,
|
|
462
|
+
executorEntries,
|
|
463
|
+
executorsHolding,
|
|
464
|
+
leaseLeastBusy,
|
|
465
|
+
leaseIdle,
|
|
466
|
+
isLongQueryWorker,
|
|
467
|
+
queriesInFlight,
|
|
468
|
+
spawnAll,
|
|
469
|
+
deadWorkerIds,
|
|
470
|
+
spawnReplacement,
|
|
471
|
+
retire,
|
|
472
|
+
get workerCount() {
|
|
473
|
+
return workerCount;
|
|
474
|
+
},
|
|
475
|
+
addIndex,
|
|
476
|
+
addIndexToAll,
|
|
477
|
+
removeIndex,
|
|
478
|
+
getMemoryStats: () => memoryStatsOf(workers.entries()),
|
|
479
|
+
shutdown,
|
|
480
|
+
whenEveryThreadIsGone,
|
|
481
|
+
whenRetiredCopiesAreGone,
|
|
482
|
+
whenExecutorIsGone: (executor) => threadsStillRunning.get(executor)?.gone ?? Promise.resolve()
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// src/core/partition/frozen/merge-split.ts
|
|
487
|
+
function splitMergeSelections(selections) {
|
|
488
|
+
let count = 0;
|
|
489
|
+
for (const selection of selections) {
|
|
490
|
+
for (let ordinal = 0; ordinal < selection.segment.arrays.documentCount; ordinal++) {
|
|
491
|
+
if (selectionKeeps(selection, ordinal)) count++;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
if (count < 2) throw new SegmentRefusal(8);
|
|
495
|
+
const middle = Math.ceil(count / 2);
|
|
496
|
+
const halves = [[], []];
|
|
497
|
+
let at = 0;
|
|
498
|
+
for (const selection of selections) {
|
|
499
|
+
const marks = new Uint8Array(selection.segment.arrays.documentCount);
|
|
500
|
+
let first = false;
|
|
501
|
+
let second = false;
|
|
502
|
+
for (let ordinal = 0; ordinal < marks.length; ordinal++) {
|
|
503
|
+
if (!selectionKeeps(selection, ordinal)) continue;
|
|
504
|
+
if (at++ < middle) {
|
|
505
|
+
marks[ordinal] = 1;
|
|
506
|
+
first = true;
|
|
507
|
+
} else {
|
|
508
|
+
marks[ordinal] = 2;
|
|
509
|
+
second = true;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
if (first) halves[0].push({ segment: selection.segment, keep: (ordinal) => marks[ordinal] === 1 });
|
|
513
|
+
if (second) halves[1].push({ segment: selection.segment, keep: (ordinal) => marks[ordinal] === 2 });
|
|
514
|
+
}
|
|
515
|
+
return halves;
|
|
516
|
+
}
|
|
517
|
+
function mergeFrozenSegmentOutputs(segments, analysis, textAnalysis) {
|
|
518
|
+
if (typeof SharedArrayBuffer !== "function") return null;
|
|
519
|
+
const outputs = [];
|
|
520
|
+
const pending = [segments.map((segment) => ({ segment }))];
|
|
521
|
+
while (pending.length > 0) {
|
|
522
|
+
const selections = pending.pop();
|
|
523
|
+
if (selections === void 0) break;
|
|
524
|
+
try {
|
|
525
|
+
const merged = mergeSegmentContents(selections, analysis, textAnalysis);
|
|
526
|
+
const snapshot = freezeEncodedSegmentShared(
|
|
527
|
+
merged.payload,
|
|
528
|
+
merged.documents,
|
|
529
|
+
void 0,
|
|
530
|
+
analysis,
|
|
531
|
+
merged.reusedPartialWords
|
|
532
|
+
);
|
|
533
|
+
if (snapshot === null) return null;
|
|
534
|
+
outputs.push(snapshot);
|
|
535
|
+
} catch (error) {
|
|
536
|
+
if (!(error instanceof SegmentRefusal) || error.status !== 8) throw error;
|
|
537
|
+
const [first, second] = splitMergeSelections(selections);
|
|
538
|
+
pending.push(second, first);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
return outputs;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// src/workers/segment-actions.ts
|
|
545
|
+
function isPagedSnapshot(snapshot) {
|
|
546
|
+
return "path" in snapshot;
|
|
547
|
+
}
|
|
548
|
+
async function attachSegments(entry, action) {
|
|
549
|
+
for (const segment of action.segments) {
|
|
550
|
+
growPartitionsTo(entry.manager, segment.partitionId);
|
|
551
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
552
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
553
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await loadStorageCode()).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
554
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
555
|
+
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
function requireComposite(entry, indexName, partitionId, verb) {
|
|
559
|
+
const partition = entry.manager.getPartition(partitionId);
|
|
560
|
+
if (!isCompositePartition(partition)) {
|
|
561
|
+
throw new NarsilError(
|
|
562
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
563
|
+
`Partition ${partitionId} of "${indexName}" holds no frozen segments to ${verb}`,
|
|
564
|
+
{ indexName, partitionId }
|
|
565
|
+
);
|
|
566
|
+
}
|
|
567
|
+
return partition;
|
|
568
|
+
}
|
|
569
|
+
function runSegmentAction(entry, action) {
|
|
570
|
+
switch (action.type) {
|
|
571
|
+
case "buildSegment": {
|
|
572
|
+
const options = resolvePartitionInsertOptions(entry.config, entry.manager.analysis, action.options);
|
|
573
|
+
const outputs = buildSegmentPayloads(
|
|
574
|
+
action.documents,
|
|
575
|
+
entry.config.schema,
|
|
576
|
+
entry.language,
|
|
577
|
+
options,
|
|
578
|
+
entry.config.trackPositions ?? true
|
|
579
|
+
);
|
|
580
|
+
return outputs.map(({ payload, documents }, index) => {
|
|
581
|
+
const snapshot = freezeSegmentShared(
|
|
582
|
+
payload,
|
|
583
|
+
documents.map((doc) => doc.document),
|
|
584
|
+
index === 0 ? action.segmentId : void 0
|
|
585
|
+
);
|
|
586
|
+
return snapshot === null ? { kind: "payload", payload } : { kind: "shared", snapshot };
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
case "mergeSegments": {
|
|
590
|
+
for (const segment of action.segments) {
|
|
591
|
+
if (segment.payload.docIds.some((docId) => entry.manager.has(docId))) {
|
|
592
|
+
console.warn(
|
|
593
|
+
`Skipping replicated segment for index "${action.indexName}": its documents already exist on this copy`
|
|
594
|
+
);
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
entry.manager.mergeSegment(segment.partitionId, segment.payload, segment.documents);
|
|
598
|
+
}
|
|
599
|
+
return void 0;
|
|
600
|
+
}
|
|
601
|
+
case "attachSegments":
|
|
602
|
+
return attachSegments(entry, action);
|
|
603
|
+
case "adoptDiskSegments":
|
|
604
|
+
return loadStorageCode().then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
605
|
+
case "compactSegments": {
|
|
606
|
+
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
607
|
+
return mergeFrozenSegmentOutputs(
|
|
608
|
+
partition.frozenSegmentsById(action.segmentIds),
|
|
609
|
+
entry.manager.partialWordAnalysis,
|
|
610
|
+
entry.manager.textAnalysis
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
case "swapSegments": {
|
|
614
|
+
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
615
|
+
entry.manager.swapFrozenSegments(
|
|
616
|
+
action.partitionId,
|
|
617
|
+
action.dropSegmentIds,
|
|
618
|
+
action.snapshots.map((snapshot) => createSharedFrozenSegment(snapshot)),
|
|
619
|
+
action.removedSpellings
|
|
620
|
+
);
|
|
621
|
+
return void 0;
|
|
622
|
+
}
|
|
623
|
+
case "freezeLiveTail": {
|
|
624
|
+
growPartitionsTo(entry.manager, action.partitionId);
|
|
625
|
+
const tail = entry.manager.getPartition(action.partitionId);
|
|
626
|
+
const held = isCompositePartition(tail) ? tail.live.count() : tail.count();
|
|
627
|
+
if (held !== action.snapshot.documentCount) {
|
|
628
|
+
console.warn(
|
|
629
|
+
`Partition ${action.partitionId} of "${action.indexName}" held ${held} live documents where the frozen tail holds ${action.snapshot.documentCount}`
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
entry.manager.replaceLiveTail(action.partitionId, createSharedFrozenSegment(action.snapshot));
|
|
633
|
+
return void 0;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// src/workers/worker-executor.ts
|
|
639
|
+
function errorFromEventLike(event) {
|
|
640
|
+
if (event instanceof Error) {
|
|
641
|
+
return event;
|
|
642
|
+
}
|
|
643
|
+
if (typeof event === "object" && event !== null) {
|
|
644
|
+
const detail = event;
|
|
645
|
+
if (detail.error instanceof Error) {
|
|
646
|
+
return detail.error;
|
|
647
|
+
}
|
|
648
|
+
if (typeof detail.message === "string" && detail.message.length > 0) {
|
|
649
|
+
return new Error(detail.message);
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
return new Error("The worker reported an error event carrying no detail");
|
|
653
|
+
}
|
|
654
|
+
function createWorkerExecutor(worker, config) {
|
|
655
|
+
const backpressureLimit = config?.backpressureLimit ?? DEFAULT_BACKPRESSURE_LIMIT;
|
|
656
|
+
const requestTimeout = config?.requestTimeout ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
657
|
+
const silenceBeforeProbe = config?.silenceBeforeProbeMs ?? DEFAULT_WORKER_SILENCE_BEFORE_PROBE_MS;
|
|
658
|
+
const silenceBeforeRetirement = config?.silenceBeforeRetirementMs ?? DEFAULT_WORKER_SILENCE_BEFORE_RETIREMENT_MS;
|
|
659
|
+
const pending = /* @__PURE__ */ new Map();
|
|
660
|
+
const retained = /* @__PURE__ */ new Map();
|
|
661
|
+
const unanswered = /* @__PURE__ */ new Set();
|
|
662
|
+
let deathError = null;
|
|
663
|
+
let shutdownRequested = false;
|
|
664
|
+
let shutdownRequestId = null;
|
|
665
|
+
let gone = false;
|
|
666
|
+
let lastSignOfLife = performance.now();
|
|
667
|
+
let probeId = null;
|
|
668
|
+
let silenceCheck;
|
|
669
|
+
let askedToLeave = false;
|
|
670
|
+
let answeredSinceAskedToLeave = false;
|
|
671
|
+
let servesConnections = false;
|
|
672
|
+
const stopServingIds = /* @__PURE__ */ new Set();
|
|
673
|
+
let abandon = () => void 0;
|
|
674
|
+
const abandoned = new Promise((resolve) => {
|
|
675
|
+
abandon = resolve;
|
|
676
|
+
});
|
|
677
|
+
let noteExit = () => void 0;
|
|
678
|
+
const exited = new Promise((resolve) => {
|
|
679
|
+
noteExit = resolve;
|
|
680
|
+
});
|
|
681
|
+
const waitingForTheThreadToGo = [];
|
|
682
|
+
const reportsItsExit = typeof worker.on === "function";
|
|
683
|
+
function watchingSilence() {
|
|
684
|
+
if (gone) return false;
|
|
685
|
+
if (askedToLeave) return true;
|
|
686
|
+
if (deathError !== null || shutdownRequested) return false;
|
|
687
|
+
return reportsItsExit || unanswered.size > 0;
|
|
688
|
+
}
|
|
689
|
+
function stopWatchingSilence() {
|
|
690
|
+
clearTimeout(silenceCheck);
|
|
691
|
+
silenceCheck = void 0;
|
|
692
|
+
}
|
|
693
|
+
function silenceBeforeStopping() {
|
|
694
|
+
return askedToLeave && deathError !== null && !answeredSinceAskedToLeave ? silenceBeforeProbe : silenceBeforeRetirement;
|
|
695
|
+
}
|
|
696
|
+
function watchSilence() {
|
|
697
|
+
if (silenceCheck !== void 0 || !watchingSilence()) return;
|
|
698
|
+
const allowedSilence = probeId === null ? silenceBeforeProbe : silenceBeforeStopping();
|
|
699
|
+
const timer = setTimeout(
|
|
700
|
+
() => {
|
|
701
|
+
const afterQueuedAnswers = setTimeout(checkSilence, 0);
|
|
702
|
+
if (typeof afterQueuedAnswers.unref === "function") afterQueuedAnswers.unref();
|
|
703
|
+
silenceCheck = afterQueuedAnswers;
|
|
704
|
+
},
|
|
705
|
+
Math.max(0, lastSignOfLife + allowedSilence - performance.now())
|
|
706
|
+
);
|
|
707
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
708
|
+
silenceCheck = timer;
|
|
709
|
+
}
|
|
710
|
+
function probe() {
|
|
711
|
+
const requestId = createRequestId();
|
|
712
|
+
probeId = requestId;
|
|
713
|
+
try {
|
|
714
|
+
worker.postMessage({ type: "probe", requestId });
|
|
715
|
+
} catch {
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
function checkSilence() {
|
|
719
|
+
silenceCheck = void 0;
|
|
720
|
+
if (!watchingSilence()) return;
|
|
721
|
+
const silentFor = performance.now() - lastSignOfLife;
|
|
722
|
+
if (silentFor >= silenceBeforeStopping()) {
|
|
723
|
+
if (askedToLeave) stopOrLeaveTheThread();
|
|
724
|
+
else retireUnansweringWorker(new Error(`no answer to any request or probe for ${Math.round(silentFor)}ms`));
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
if (probeId === null && silentFor >= silenceBeforeProbe) probe();
|
|
728
|
+
watchSilence();
|
|
729
|
+
}
|
|
730
|
+
function noteAnswer(requestId) {
|
|
731
|
+
lastSignOfLife = performance.now();
|
|
732
|
+
unanswered.delete(requestId);
|
|
733
|
+
const probeAnswered = requestId === probeId;
|
|
734
|
+
if (probeAnswered) probeId = null;
|
|
735
|
+
const firstAnswerSinceAskedToLeave = askedToLeave && !answeredSinceAskedToLeave;
|
|
736
|
+
if (askedToLeave) answeredSinceAskedToLeave = true;
|
|
737
|
+
if (!watchingSilence()) {
|
|
738
|
+
stopWatchingSilence();
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
if (!probeAnswered && !firstAnswerSinceAskedToLeave) return;
|
|
742
|
+
stopWatchingSilence();
|
|
743
|
+
watchSilence();
|
|
744
|
+
}
|
|
745
|
+
function processResponse(msg) {
|
|
746
|
+
if (!isValidWorkerResponse(msg)) {
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
const response = msg;
|
|
750
|
+
noteAnswer(response.requestId);
|
|
751
|
+
if (stopServingIds.delete(response.requestId) && response.type === "success") servesConnections = false;
|
|
752
|
+
if (response.requestId === shutdownRequestId && response.type === "success") {
|
|
753
|
+
shutdownRequestId = null;
|
|
754
|
+
servesConnections = false;
|
|
755
|
+
if (reportsItsExit) worker.unref?.();
|
|
756
|
+
else stopAWorkerThatReportsNoExit();
|
|
757
|
+
}
|
|
758
|
+
retained.get(response.requestId)?.();
|
|
759
|
+
retained.delete(response.requestId);
|
|
760
|
+
const entry = pending.get(response.requestId);
|
|
761
|
+
if (!entry) {
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
clearTimeout(entry.timeoutId);
|
|
765
|
+
pending.delete(response.requestId);
|
|
766
|
+
if (response.type === "error") {
|
|
767
|
+
entry.reject(new NarsilError(response.code, response.message, response.details));
|
|
768
|
+
} else {
|
|
769
|
+
entry.resolve(response.data);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
function handleDeath(cause) {
|
|
773
|
+
if (deathError !== null) {
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
777
|
+
deathError = new NarsilError(ErrorCodes.WORKER_CRASHED, `Worker died: ${message}`);
|
|
778
|
+
stopWatchingSilence();
|
|
779
|
+
unanswered.clear();
|
|
780
|
+
for (const [id, entry] of pending) {
|
|
781
|
+
clearTimeout(entry.timeoutId);
|
|
782
|
+
entry.reject(deathError);
|
|
783
|
+
pending.delete(id);
|
|
784
|
+
}
|
|
785
|
+
if (!shutdownRequested) {
|
|
786
|
+
config?.onDeath?.(cause instanceof Error ? cause : new Error(message));
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
function reportGone() {
|
|
790
|
+
if (gone) {
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
gone = true;
|
|
794
|
+
stopWatchingSilence();
|
|
795
|
+
for (const release of retained.values()) release();
|
|
796
|
+
retained.clear();
|
|
797
|
+
for (const resume of waitingForTheThreadToGo.splice(0)) resume();
|
|
798
|
+
config?.onGone?.();
|
|
799
|
+
}
|
|
800
|
+
function threadGoneWithin(timeoutMs) {
|
|
801
|
+
if (gone) return Promise.resolve(true);
|
|
802
|
+
return new Promise((resolve) => {
|
|
803
|
+
const timeoutId = setTimeout(() => resolve(false), timeoutMs);
|
|
804
|
+
waitingForTheThreadToGo.push(() => {
|
|
805
|
+
clearTimeout(timeoutId);
|
|
806
|
+
resolve(true);
|
|
807
|
+
});
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
function stopTheThread() {
|
|
811
|
+
try {
|
|
812
|
+
const stopping = worker.terminate?.();
|
|
813
|
+
if (stopping instanceof Promise) stopping.catch(() => void 0);
|
|
814
|
+
} catch {
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
function stopOrLeaveTheThread() {
|
|
818
|
+
if (!servesConnections) {
|
|
819
|
+
stopTheThread();
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
worker.unref?.();
|
|
823
|
+
abandon();
|
|
824
|
+
reportGone();
|
|
825
|
+
}
|
|
826
|
+
function stopAWorkerThatReportsNoExit() {
|
|
827
|
+
stopTheThread();
|
|
828
|
+
reportGone();
|
|
829
|
+
}
|
|
830
|
+
function askTheThreadToLeave(requestId) {
|
|
831
|
+
try {
|
|
832
|
+
worker.postMessage({ type: "shutdown", requestId });
|
|
833
|
+
} catch {
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
function watchTheThreadLeave() {
|
|
837
|
+
if (!reportsItsExit || gone || askedToLeave) return;
|
|
838
|
+
askedToLeave = true;
|
|
839
|
+
lastSignOfLife = performance.now();
|
|
840
|
+
stopWatchingSilence();
|
|
841
|
+
probe();
|
|
842
|
+
watchSilence();
|
|
843
|
+
}
|
|
844
|
+
function retireUnansweringWorker(cause) {
|
|
845
|
+
handleDeath(cause);
|
|
846
|
+
if (!reportsItsExit) {
|
|
847
|
+
stopAWorkerThatReportsNoExit();
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
askTheThreadToLeave(createRequestId());
|
|
851
|
+
watchTheThreadLeave();
|
|
852
|
+
}
|
|
853
|
+
if (typeof worker.on === "function") {
|
|
854
|
+
worker.on("message", (msg) => processResponse(msg));
|
|
855
|
+
worker.on("error", (cause) => {
|
|
856
|
+
handleDeath(cause);
|
|
857
|
+
});
|
|
858
|
+
worker.on("exit", (code) => {
|
|
859
|
+
handleDeath(new Error(`Worker exited with code ${String(code)}`));
|
|
860
|
+
reportGone();
|
|
861
|
+
noteExit();
|
|
862
|
+
});
|
|
863
|
+
} else if (typeof worker.addEventListener === "function") {
|
|
864
|
+
worker.addEventListener("message", (event) => {
|
|
865
|
+
const msg = event.data;
|
|
866
|
+
processResponse(msg);
|
|
867
|
+
});
|
|
868
|
+
worker.addEventListener("error", (event) => {
|
|
869
|
+
handleDeath(errorFromEventLike(event));
|
|
870
|
+
stopAWorkerThatReportsNoExit();
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
function declareTimedOut(requestId, entry) {
|
|
874
|
+
if (pending.get(requestId) !== entry) return;
|
|
875
|
+
pending.delete(requestId);
|
|
876
|
+
entry.reject(new NarsilError(ErrorCodes.WORKER_TIMEOUT, `Request ${requestId} timed out after ${requestTimeout}ms`));
|
|
877
|
+
}
|
|
878
|
+
function send(action, transfer, settled2, bounded) {
|
|
879
|
+
if (deathError !== null) {
|
|
880
|
+
settled2?.();
|
|
881
|
+
return Promise.reject(deathError);
|
|
882
|
+
}
|
|
883
|
+
if (bounded && pending.size >= backpressureLimit) {
|
|
884
|
+
settled2?.();
|
|
885
|
+
return Promise.reject(
|
|
886
|
+
new NarsilError(ErrorCodes.WORKER_BUSY, `Backpressure limit of ${backpressureLimit} pending requests reached`)
|
|
887
|
+
);
|
|
888
|
+
}
|
|
889
|
+
const requestId = createRequestId();
|
|
890
|
+
const taggedAction = { ...action, requestId };
|
|
891
|
+
if (action.type === "serveRequests") servesConnections = true;
|
|
892
|
+
if (action.type === "stopServing") stopServingIds.add(requestId);
|
|
893
|
+
return new Promise((resolve, reject) => {
|
|
894
|
+
const entry = {
|
|
895
|
+
resolve,
|
|
896
|
+
reject,
|
|
897
|
+
timeoutId: bounded ? setTimeout(() => {
|
|
898
|
+
entry.timeoutId = setTimeout(() => declareTimedOut(requestId, entry), 0);
|
|
899
|
+
}, requestTimeout) : void 0
|
|
900
|
+
};
|
|
901
|
+
pending.set(requestId, entry);
|
|
902
|
+
if (settled2 !== void 0) retained.set(requestId, settled2);
|
|
903
|
+
try {
|
|
904
|
+
if (transfer === void 0) worker.postMessage(taggedAction);
|
|
905
|
+
else worker.postMessage(taggedAction, transfer);
|
|
906
|
+
} catch (error) {
|
|
907
|
+
clearTimeout(entry.timeoutId);
|
|
908
|
+
pending.delete(requestId);
|
|
909
|
+
retained.delete(requestId);
|
|
910
|
+
stopServingIds.delete(requestId);
|
|
911
|
+
settled2?.();
|
|
912
|
+
reject(error);
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
if (unanswered.size === 0 && probeId === null) lastSignOfLife = performance.now();
|
|
916
|
+
unanswered.add(requestId);
|
|
917
|
+
watchSilence();
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
function stopOnceTheWorkerAcknowledges() {
|
|
921
|
+
const requestId = createRequestId();
|
|
922
|
+
shutdownRequestId = requestId;
|
|
923
|
+
askTheThreadToLeave(requestId);
|
|
924
|
+
}
|
|
925
|
+
async function shutdown() {
|
|
926
|
+
const firstShutdown = !shutdownRequested;
|
|
927
|
+
shutdownRequested = true;
|
|
928
|
+
if (firstShutdown && deathError === null && !gone) stopOnceTheWorkerAcknowledges();
|
|
929
|
+
if (reportsItsExit) watchTheThreadLeave();
|
|
930
|
+
else stopWatchingSilence();
|
|
931
|
+
try {
|
|
932
|
+
if (await threadGoneWithin(SHUTDOWN_TIMEOUT_MS)) return;
|
|
933
|
+
if (!reportsItsExit) {
|
|
934
|
+
stopAWorkerThatReportsNoExit();
|
|
935
|
+
return;
|
|
936
|
+
}
|
|
937
|
+
throw new NarsilError(ErrorCodes.WORKER_TIMEOUT, "Shutdown timed out");
|
|
938
|
+
} finally {
|
|
939
|
+
for (const [id, entry] of pending) {
|
|
940
|
+
clearTimeout(entry.timeoutId);
|
|
941
|
+
entry.reject(new NarsilError(ErrorCodes.WORKER_CRASHED, "Worker is shutting down"));
|
|
942
|
+
pending.delete(id);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
watchSilence();
|
|
947
|
+
return {
|
|
948
|
+
execute: (action, transfer) => send(action, transfer, void 0, true),
|
|
949
|
+
executeUntilAnswered: (action, settled2) => send(action, void 0, settled2, false),
|
|
950
|
+
shutdown,
|
|
951
|
+
whenAbandoned: () => abandoned,
|
|
952
|
+
whenTheThreadExits: () => exited
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// src/core/partition/frozen/share.ts
|
|
957
|
+
function shareFrozenSegment(segment, analysis) {
|
|
958
|
+
segment.settleRemovals();
|
|
959
|
+
if (segment.paged !== null) {
|
|
960
|
+
return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
961
|
+
}
|
|
962
|
+
if (segment.sharedSnapshot !== null) {
|
|
963
|
+
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
964
|
+
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
965
|
+
return { snapshot: segment.sharedSnapshot, tombstonedDocIds };
|
|
966
|
+
}
|
|
967
|
+
return {
|
|
968
|
+
snapshot: segment.sharedSnapshot,
|
|
969
|
+
tombstonedDocIds,
|
|
970
|
+
survivingSurfaceForms: segment.surfaces.reader.serialize()
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
const payload = encodeSegmentState(segment);
|
|
974
|
+
const documents = payload.docIds.map((docId) => {
|
|
975
|
+
const stored = segment.docStore.get(docId);
|
|
976
|
+
if (stored === void 0) {
|
|
977
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while sharing a segment`, {
|
|
978
|
+
docId
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
return stored.fields;
|
|
982
|
+
});
|
|
983
|
+
const snapshot = freezeSegmentShared(payload, documents, segment.segmentId, analysis);
|
|
984
|
+
return snapshot === null ? null : { snapshot, tombstonedDocIds: [] };
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// src/workers/copy-generation.ts
|
|
988
|
+
var generations = /* @__PURE__ */ new WeakMap();
|
|
989
|
+
function copyGenerationOf(manager) {
|
|
990
|
+
return generations.get(manager) ?? 0;
|
|
991
|
+
}
|
|
992
|
+
function advanceCopyGeneration(manager) {
|
|
993
|
+
const generation = copyGenerationOf(manager) + 1;
|
|
994
|
+
generations.set(manager, generation);
|
|
995
|
+
return generation;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
// src/workers/copy-publication.ts
|
|
999
|
+
function holdCopyPublication(manager, advances) {
|
|
1000
|
+
const held = [];
|
|
1001
|
+
try {
|
|
1002
|
+
const fields = [];
|
|
1003
|
+
for (const [fieldName, vector] of manager.getVectorIndexes()) {
|
|
1004
|
+
const pin = vector.holdCopyPublication();
|
|
1005
|
+
held.push(pin);
|
|
1006
|
+
fields.push({ fieldName, vector: pin.field });
|
|
1007
|
+
}
|
|
1008
|
+
const generation = advances ? advanceCopyGeneration(manager) : copyGenerationOf(manager);
|
|
1009
|
+
let references = 0;
|
|
1010
|
+
const retain = () => {
|
|
1011
|
+
references++;
|
|
1012
|
+
let released = false;
|
|
1013
|
+
return () => {
|
|
1014
|
+
if (released) return;
|
|
1015
|
+
released = true;
|
|
1016
|
+
if (--references === 0) for (const pin of held.splice(0)) pin.release();
|
|
1017
|
+
};
|
|
1018
|
+
};
|
|
1019
|
+
return {
|
|
1020
|
+
publication: { generation, fields },
|
|
1021
|
+
release: retain(),
|
|
1022
|
+
retain
|
|
1023
|
+
};
|
|
1024
|
+
} catch (error) {
|
|
1025
|
+
for (const pin of held) pin.release();
|
|
1026
|
+
throw error;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// src/workers/executor.ts
|
|
1031
|
+
function executeUntilAnswered(executor, action, settled2) {
|
|
1032
|
+
try {
|
|
1033
|
+
if (executor.executeUntilAnswered !== void 0) return executor.executeUntilAnswered(action, settled2);
|
|
1034
|
+
const answered = executor.execute(action);
|
|
1035
|
+
return settled2 === void 0 ? answered : answered.finally(settled2);
|
|
1036
|
+
} catch (error) {
|
|
1037
|
+
settled2?.();
|
|
1038
|
+
return Promise.reject(error);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
// src/engine/worker-resync.ts
|
|
1043
|
+
var copyLoadSequence = 0;
|
|
1044
|
+
function emptySerializablePartition(manager, partitionId) {
|
|
1045
|
+
return {
|
|
1046
|
+
indexName: manager.indexName,
|
|
1047
|
+
partitionId,
|
|
1048
|
+
totalPartitions: manager.partitionCount,
|
|
1049
|
+
language: manager.language.name,
|
|
1050
|
+
schema: {},
|
|
1051
|
+
docCount: 0,
|
|
1052
|
+
avgDocLength: 0,
|
|
1053
|
+
documents: {},
|
|
1054
|
+
invertedIndex: {},
|
|
1055
|
+
fieldIndexes: { numeric: {}, boolean: {}, enum: {}, geopoint: {} },
|
|
1056
|
+
statistics: { totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} }
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
function captureLiveTail(partition) {
|
|
1060
|
+
if (partition.count() === 0) return null;
|
|
1061
|
+
const payload = partition.encodeSegment();
|
|
1062
|
+
const documents = payload.docIds.map((docId) => {
|
|
1063
|
+
const document = partition.getRef(docId);
|
|
1064
|
+
if (document === void 0) {
|
|
1065
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while copying a partition`, {
|
|
1066
|
+
docId
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
return document;
|
|
1070
|
+
});
|
|
1071
|
+
return { payload, documents };
|
|
1072
|
+
}
|
|
1073
|
+
function captureComposite(manager, partitionId) {
|
|
1074
|
+
const partition = manager.getPartition(partitionId);
|
|
1075
|
+
if (!isCompositePartition(partition)) return null;
|
|
1076
|
+
const segmentIds = partition.frozenSegmentSizes().map((size) => size.segmentId);
|
|
1077
|
+
const frozen = [];
|
|
1078
|
+
for (const segment of partition.frozenSegmentsById(segmentIds)) {
|
|
1079
|
+
const shared = shareFrozenSegment(segment, manager.partialWordAnalysis);
|
|
1080
|
+
if (shared === null) return null;
|
|
1081
|
+
frozen.push(shared);
|
|
1082
|
+
}
|
|
1083
|
+
return {
|
|
1084
|
+
partitionId,
|
|
1085
|
+
empty: emptySerializablePartition(manager, partitionId),
|
|
1086
|
+
live: captureLiveTail(partition.live),
|
|
1087
|
+
frozen
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
function capturePartitions(manager) {
|
|
1091
|
+
const captured = [];
|
|
1092
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
1093
|
+
captured.push(
|
|
1094
|
+
captureComposite(manager, partitionId) ?? {
|
|
1095
|
+
partitionId,
|
|
1096
|
+
empty: emptySerializablePartition(manager, partitionId),
|
|
1097
|
+
live: captureLiveTail(manager.getPartition(partitionId)),
|
|
1098
|
+
frozen: []
|
|
1099
|
+
}
|
|
1100
|
+
);
|
|
1101
|
+
}
|
|
1102
|
+
return captured;
|
|
1103
|
+
}
|
|
1104
|
+
async function captureCopy(manager) {
|
|
1105
|
+
for (; ; ) {
|
|
1106
|
+
const vectors = [...manager.getVectorIndexes()];
|
|
1107
|
+
await Promise.all(vectors.map(([, vector]) => vector.prepareCopyPublication()));
|
|
1108
|
+
const captured = await manager.publication.write(() => {
|
|
1109
|
+
const current = manager.getVectorIndexes();
|
|
1110
|
+
if (current.size !== vectors.length || vectors.some(([name, vector]) => current.get(name) !== vector)) return null;
|
|
1111
|
+
if (vectors.some(([, vector]) => vector.maintenanceStatus().loadingFromFiles)) return null;
|
|
1112
|
+
const partitions = capturePartitions(manager);
|
|
1113
|
+
const { embedding, ...settings } = manager.config;
|
|
1114
|
+
const config = structuredClone({
|
|
1115
|
+
...settings,
|
|
1116
|
+
...embedding === void 0 ? {} : {
|
|
1117
|
+
embedding: {
|
|
1118
|
+
fields: embedding.fields,
|
|
1119
|
+
...typeof embedding.adapter === "string" ? { adapter: embedding.adapter } : {}
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
});
|
|
1123
|
+
const synonyms = manager.synonyms?.encodedRules.slice() ?? null;
|
|
1124
|
+
return { captured: partitions, config, synonyms, held: holdCopyPublication(manager, false) };
|
|
1125
|
+
});
|
|
1126
|
+
if (captured !== null) return captured;
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
async function sendPartition(manager, executors, captured) {
|
|
1130
|
+
const { indexName } = manager;
|
|
1131
|
+
await Promise.all(
|
|
1132
|
+
executors.map(
|
|
1133
|
+
(workerExecutor) => workerExecutor.execute({
|
|
1134
|
+
type: "deserialize",
|
|
1135
|
+
indexName,
|
|
1136
|
+
partitionId: captured.partitionId,
|
|
1137
|
+
data: captured.empty,
|
|
1138
|
+
requestId: `resync-${indexName}-${captured.partitionId}`
|
|
1139
|
+
})
|
|
1140
|
+
)
|
|
1141
|
+
);
|
|
1142
|
+
const live = captured.live;
|
|
1143
|
+
if (live !== null) {
|
|
1144
|
+
await Promise.all(
|
|
1145
|
+
executors.map(
|
|
1146
|
+
(workerExecutor) => workerExecutor.execute({
|
|
1147
|
+
type: "mergeSegments",
|
|
1148
|
+
indexName,
|
|
1149
|
+
segments: [{ partitionId: captured.partitionId, payload: live.payload, documents: live.documents }],
|
|
1150
|
+
requestId: `resync-live-${indexName}-${captured.partitionId}`
|
|
1151
|
+
})
|
|
1152
|
+
)
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
if (captured.frozen.length === 0) return;
|
|
1156
|
+
const segments = captured.frozen.map((shared) => ({
|
|
1157
|
+
partitionId: captured.partitionId,
|
|
1158
|
+
snapshot: shared.snapshot,
|
|
1159
|
+
tombstonedDocIds: shared.tombstonedDocIds,
|
|
1160
|
+
survivingSurfaceForms: shared.survivingSurfaceForms
|
|
1161
|
+
}));
|
|
1162
|
+
await Promise.all(
|
|
1163
|
+
executors.map(
|
|
1164
|
+
(workerExecutor) => workerExecutor.execute({
|
|
1165
|
+
type: "attachSegments",
|
|
1166
|
+
indexName,
|
|
1167
|
+
segments,
|
|
1168
|
+
requestId: `resync-attach-${indexName}-${captured.partitionId}`
|
|
1169
|
+
})
|
|
1170
|
+
)
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
function transferIndexToPool(indexName, pool, source, manager, analysisStale = false) {
|
|
1174
|
+
return loadIndexOntoWorkers(
|
|
1175
|
+
indexName,
|
|
1176
|
+
pool.getAllExecutors(),
|
|
1177
|
+
source,
|
|
1178
|
+
manager,
|
|
1179
|
+
() => pool.addIndexToAll(indexName),
|
|
1180
|
+
analysisStale
|
|
1181
|
+
);
|
|
1182
|
+
}
|
|
1183
|
+
async function loadIndexOntoWorkers(indexName, executors, source, manager, onDropped, analysisStale = false) {
|
|
1184
|
+
copyLoadSequence += 1;
|
|
1185
|
+
const copyLoad = `${indexName}#${copyLoadSequence}`;
|
|
1186
|
+
const { captured, held, config, synonyms } = await captureCopy(manager);
|
|
1187
|
+
try {
|
|
1188
|
+
source.onPublication?.(held.publication);
|
|
1189
|
+
await Promise.allSettled(
|
|
1190
|
+
executors.map(
|
|
1191
|
+
(workerExecutor) => workerExecutor.execute({ type: "dropIndex", indexName, requestId: `resync-drop-${indexName}` })
|
|
1192
|
+
)
|
|
1193
|
+
);
|
|
1194
|
+
onDropped?.();
|
|
1195
|
+
const created = await Promise.allSettled(
|
|
1196
|
+
executors.map(
|
|
1197
|
+
(workerExecutor) => executeUntilAnswered(
|
|
1198
|
+
workerExecutor,
|
|
1199
|
+
{
|
|
1200
|
+
type: "createIndex",
|
|
1201
|
+
indexName,
|
|
1202
|
+
config,
|
|
1203
|
+
requestId: `resync-create-${indexName}`,
|
|
1204
|
+
analysisStale,
|
|
1205
|
+
copyLoad,
|
|
1206
|
+
publication: held.publication,
|
|
1207
|
+
partitionCount: captured.length,
|
|
1208
|
+
...synonyms !== null ? { synonyms } : {}
|
|
1209
|
+
},
|
|
1210
|
+
held.retain()
|
|
1211
|
+
)
|
|
1212
|
+
)
|
|
1213
|
+
);
|
|
1214
|
+
const failure = created.find((result) => result.status === "rejected");
|
|
1215
|
+
if (failure?.status === "rejected") throw failure.reason;
|
|
1216
|
+
for (const partition of captured) {
|
|
1217
|
+
await sendPartition(manager, executors, partition);
|
|
1218
|
+
}
|
|
1219
|
+
return copyLoad;
|
|
1220
|
+
} catch (error) {
|
|
1221
|
+
await Promise.allSettled(
|
|
1222
|
+
executors.map(
|
|
1223
|
+
(executor) => executor.execute({
|
|
1224
|
+
type: "dropIndex",
|
|
1225
|
+
indexName,
|
|
1226
|
+
requestId: `failed-load-${copyLoad}`
|
|
1227
|
+
})
|
|
1228
|
+
)
|
|
1229
|
+
);
|
|
1230
|
+
throw error;
|
|
1231
|
+
} finally {
|
|
1232
|
+
held.release();
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
// src/engine/orchestration/publication-stores.ts
|
|
1237
|
+
function trackPublicationStores(state, publication) {
|
|
1238
|
+
for (const { vector } of publication.fields) {
|
|
1239
|
+
state.sharedVectorFields.set(vector.handle, vector.handles);
|
|
1240
|
+
state.publishedVectorStores.add(vector.handle);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
function prunePublicationStores(state) {
|
|
1244
|
+
for (const handle of state.publishedVectorStores) {
|
|
1245
|
+
const field = state.sharedVectorFields.get(handle);
|
|
1246
|
+
if (field !== void 0) {
|
|
1247
|
+
let pinned = false;
|
|
1248
|
+
for (let slot = 0; slot < field.store.pins.length; slot++) {
|
|
1249
|
+
if (Atomics.load(field.store.pins, slot) !== 0n) {
|
|
1250
|
+
pinned = true;
|
|
1251
|
+
break;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
if (pinned) continue;
|
|
1255
|
+
}
|
|
1256
|
+
state.sharedVectorFields.delete(handle);
|
|
1257
|
+
state.publishedVectorStores.delete(handle);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
// src/engine/orchestration/replication.ts
|
|
1262
|
+
async function dispatchToWorkers(state, action) {
|
|
1263
|
+
const pool = state.workerPool;
|
|
1264
|
+
if (!pool) return;
|
|
1265
|
+
const executors = "indexName" in action ? pool.executorsHolding(action.indexName) : pool.getAllExecutors();
|
|
1266
|
+
if ("indexName" in action && !state.scaledOutIndexes.has(action.indexName)) return;
|
|
1267
|
+
const results = await Promise.allSettled(
|
|
1268
|
+
executors.map((workerExecutor) => executeUntilAnswered(workerExecutor, action, retainPublicationOf(action)))
|
|
1269
|
+
);
|
|
1270
|
+
for (let i = 0; i < results.length; i++) {
|
|
1271
|
+
const result = results[i];
|
|
1272
|
+
if (result.status === "rejected") {
|
|
1273
|
+
if (action.publication === void 0 && action.type === "insert" && alreadyPresentOnWorker(result.reason)) {
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
if (action.publication !== void 0) pool.retire(executors[i], toError(result.reason));
|
|
1277
|
+
console.warn("Worker replication failed:", result.reason);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
function documentWeight(action) {
|
|
1282
|
+
if (action.type === "mergeSegments") {
|
|
1283
|
+
let total = 0;
|
|
1284
|
+
for (const segment of action.segments) {
|
|
1285
|
+
total += segment.documents.length;
|
|
1286
|
+
}
|
|
1287
|
+
return total;
|
|
1288
|
+
}
|
|
1289
|
+
if (action.type === "attachSegments") {
|
|
1290
|
+
let total = 0;
|
|
1291
|
+
for (const segment of action.segments) {
|
|
1292
|
+
total += segment.snapshot.documentCount;
|
|
1293
|
+
}
|
|
1294
|
+
return total;
|
|
1295
|
+
}
|
|
1296
|
+
if (action.type === "swapSegments") {
|
|
1297
|
+
let total = 0;
|
|
1298
|
+
for (const snapshot of action.snapshots) total += snapshot.documentCount;
|
|
1299
|
+
return total;
|
|
1300
|
+
}
|
|
1301
|
+
return 1;
|
|
1302
|
+
}
|
|
1303
|
+
var NO_VECTOR_FIELDS = /* @__PURE__ */ new Set();
|
|
1304
|
+
function vectorFieldPathsOf(state, indexName) {
|
|
1305
|
+
return state.indexRegistry.get(indexName)?.vectorFieldPaths ?? NO_VECTOR_FIELDS;
|
|
1306
|
+
}
|
|
1307
|
+
function withoutVectors(document, fieldPaths) {
|
|
1308
|
+
if (fieldPaths.size === 0) return document;
|
|
1309
|
+
return prepareDocumentVectors(document, fieldPaths).partitionDoc;
|
|
1310
|
+
}
|
|
1311
|
+
function detachCallerDocuments(state, action) {
|
|
1312
|
+
if (action.type !== "insert" && action.type !== "update" && action.type !== "mergeSegments") return action;
|
|
1313
|
+
const fieldPaths = vectorFieldPathsOf(state, action.indexName);
|
|
1314
|
+
if (action.type === "insert") {
|
|
1315
|
+
const stripped = withoutVectors(action.document, fieldPaths);
|
|
1316
|
+
const document = action.skipClone === true || stripped !== action.document ? stripped : structuredClone(stripped);
|
|
1317
|
+
return { ...action, document };
|
|
1318
|
+
}
|
|
1319
|
+
if (action.type === "update") {
|
|
1320
|
+
const stripped = withoutVectors(action.document, fieldPaths);
|
|
1321
|
+
return { ...action, document: stripped !== action.document ? stripped : structuredClone(stripped) };
|
|
1322
|
+
}
|
|
1323
|
+
return {
|
|
1324
|
+
...action,
|
|
1325
|
+
segments: action.segments.map((segment) => {
|
|
1326
|
+
const documents = segment.documents.map((document) => withoutVectors(document, fieldPaths));
|
|
1327
|
+
const cloned = action.skipClone === true || fieldPaths.size > 0 ? documents : structuredClone(documents);
|
|
1328
|
+
return { ...segment, documents: cloned };
|
|
1329
|
+
})
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
function partitionLedger(state, indexName, partitionId) {
|
|
1333
|
+
let byPartition = state.segmentLedger.get(indexName);
|
|
1334
|
+
if (byPartition === void 0) {
|
|
1335
|
+
byPartition = /* @__PURE__ */ new Map();
|
|
1336
|
+
state.segmentLedger.set(indexName, byPartition);
|
|
1337
|
+
}
|
|
1338
|
+
let entries = byPartition.get(partitionId);
|
|
1339
|
+
if (entries === void 0) {
|
|
1340
|
+
entries = [];
|
|
1341
|
+
byPartition.set(partitionId, entries);
|
|
1342
|
+
}
|
|
1343
|
+
return entries;
|
|
1344
|
+
}
|
|
1345
|
+
function recordSegmentBroadcast(state, action) {
|
|
1346
|
+
if (action.type === "clear") {
|
|
1347
|
+
state.segmentLedger.delete(action.indexName);
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
if (action.type === "attachSegments") {
|
|
1351
|
+
for (const segment of action.segments) {
|
|
1352
|
+
partitionLedger(state, action.indexName, segment.partitionId).push({
|
|
1353
|
+
segmentId: segment.snapshot.segmentId,
|
|
1354
|
+
documentCount: segment.snapshot.documentCount
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
if (action.type === "freezeLiveTail") {
|
|
1360
|
+
partitionLedger(state, action.indexName, action.partitionId).push({
|
|
1361
|
+
segmentId: action.snapshot.segmentId,
|
|
1362
|
+
documentCount: action.snapshot.documentCount
|
|
1363
|
+
});
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
if (action.type === "swapSegments") {
|
|
1367
|
+
const entries = partitionLedger(state, action.indexName, action.partitionId);
|
|
1368
|
+
const dropped = new Set(action.dropSegmentIds);
|
|
1369
|
+
const kept = entries.filter((entry) => !dropped.has(entry.segmentId));
|
|
1370
|
+
for (const snapshot of action.snapshots) {
|
|
1371
|
+
kept.push({ segmentId: snapshot.segmentId, documentCount: snapshot.documentCount });
|
|
1372
|
+
}
|
|
1373
|
+
entries.length = 0;
|
|
1374
|
+
entries.push(...kept);
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
function queueFor(state, indexName) {
|
|
1378
|
+
let queue = state.replicationQueues.get(indexName);
|
|
1379
|
+
if (queue === void 0) {
|
|
1380
|
+
queue = {
|
|
1381
|
+
tail: Promise.resolve(),
|
|
1382
|
+
oldestRun: void 0,
|
|
1383
|
+
newestRun: void 0,
|
|
1384
|
+
runsInFlight: 0,
|
|
1385
|
+
pendingActions: 0,
|
|
1386
|
+
pendingDocuments: 0,
|
|
1387
|
+
drainWaiters: []
|
|
1388
|
+
};
|
|
1389
|
+
state.replicationQueues.set(indexName, queue);
|
|
1390
|
+
}
|
|
1391
|
+
return queue;
|
|
1392
|
+
}
|
|
1393
|
+
function dispatchGate(queue) {
|
|
1394
|
+
if (queue.runsInFlight < REPLICATION_WINDOW) return Promise.resolve();
|
|
1395
|
+
let run = queue.newestRun;
|
|
1396
|
+
for (let step = 1; step < REPLICATION_WINDOW && run !== void 0; step++) run = run.older;
|
|
1397
|
+
return run?.done ?? Promise.resolve();
|
|
1398
|
+
}
|
|
1399
|
+
function appendRun(queue, done) {
|
|
1400
|
+
const run = { done, older: queue.newestRun, newer: void 0 };
|
|
1401
|
+
if (queue.newestRun === void 0) queue.oldestRun = run;
|
|
1402
|
+
else queue.newestRun.newer = run;
|
|
1403
|
+
queue.newestRun = run;
|
|
1404
|
+
queue.runsInFlight += 1;
|
|
1405
|
+
return run;
|
|
1406
|
+
}
|
|
1407
|
+
function removeRun(queue, run) {
|
|
1408
|
+
if (run.older === void 0) queue.oldestRun = run.newer;
|
|
1409
|
+
else run.older.newer = run.newer;
|
|
1410
|
+
if (run.newer === void 0) queue.newestRun = run.older;
|
|
1411
|
+
else run.newer.older = run.older;
|
|
1412
|
+
run.older = void 0;
|
|
1413
|
+
run.newer = void 0;
|
|
1414
|
+
queue.runsInFlight -= 1;
|
|
1415
|
+
}
|
|
1416
|
+
function enqueueReplication(state, indexName, action) {
|
|
1417
|
+
const weight = documentWeight(action);
|
|
1418
|
+
recordSegmentBroadcast(state, action);
|
|
1419
|
+
const queue = queueFor(state, indexName);
|
|
1420
|
+
queue.pendingActions += 1;
|
|
1421
|
+
queue.pendingDocuments += weight;
|
|
1422
|
+
const gate = dispatchGate(queue);
|
|
1423
|
+
const entry = appendRun(queue, gate);
|
|
1424
|
+
const run = gate.then(async () => {
|
|
1425
|
+
try {
|
|
1426
|
+
if (action.publication !== void 0) {
|
|
1427
|
+
const manager = state.executor.getManager(indexName);
|
|
1428
|
+
await manager?.publication.afterQueuedWrites();
|
|
1429
|
+
if (state.executor.getManager(indexName) !== manager) return;
|
|
1430
|
+
}
|
|
1431
|
+
await dispatchToWorkers(state, action);
|
|
1432
|
+
} catch (err) {
|
|
1433
|
+
console.warn("Worker replication failed:", err);
|
|
1434
|
+
} finally {
|
|
1435
|
+
releasePublicationOf(action);
|
|
1436
|
+
removeRun(queue, entry);
|
|
1437
|
+
queue.pendingActions -= 1;
|
|
1438
|
+
queue.pendingDocuments -= weight;
|
|
1439
|
+
if (queue.pendingDocuments < MAX_PENDING_REPLICATION_DOCUMENTS && queue.drainWaiters.length > 0) {
|
|
1440
|
+
for (const resume of queue.drainWaiters.splice(0)) resume();
|
|
1441
|
+
}
|
|
1442
|
+
if (queue.pendingActions === 0 && state.replicationQueues.get(indexName) === queue) {
|
|
1443
|
+
state.replicationQueues.delete(indexName);
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
});
|
|
1447
|
+
entry.done = run;
|
|
1448
|
+
queue.tail = run;
|
|
1449
|
+
}
|
|
1450
|
+
function copyLoadBufferHoldOf(state, buffer) {
|
|
1451
|
+
let hold = state.copyLoadBufferHolds.get(buffer);
|
|
1452
|
+
if (hold === void 0) {
|
|
1453
|
+
hold = { yieldedAt: 0, documents: 0, waiting: [] };
|
|
1454
|
+
state.copyLoadBufferHolds.set(buffer, hold);
|
|
1455
|
+
}
|
|
1456
|
+
return hold;
|
|
1457
|
+
}
|
|
1458
|
+
function endCopyLoadBuffer(state, indexName, buffer) {
|
|
1459
|
+
if (state.copyLoadBuffers.get(indexName) === buffer) state.copyLoadBuffers.delete(indexName);
|
|
1460
|
+
const hold = state.copyLoadBufferHolds.get(buffer);
|
|
1461
|
+
if (hold === void 0) return;
|
|
1462
|
+
state.copyLoadBufferHolds.delete(buffer);
|
|
1463
|
+
for (const resume of hold.waiting.splice(0)) resume();
|
|
1464
|
+
}
|
|
1465
|
+
function markCopiesLoaded(state, indexName, copyLoad) {
|
|
1466
|
+
enqueueReplication(state, indexName, {
|
|
1467
|
+
type: "copyLoaded",
|
|
1468
|
+
indexName,
|
|
1469
|
+
copyLoad,
|
|
1470
|
+
requestId: `copy-loaded-${copyLoad}`
|
|
1471
|
+
});
|
|
1472
|
+
}
|
|
1473
|
+
function copiesWillReceive(state, indexName) {
|
|
1474
|
+
if (state.copyLoadBuffers.has(indexName) || state.replicationQueues.has(indexName)) return true;
|
|
1475
|
+
return state.workerPool !== null && state.scaledOutIndexes.has(indexName);
|
|
1476
|
+
}
|
|
1477
|
+
async function replicateToWorkers(state, action) {
|
|
1478
|
+
if (!("indexName" in action)) {
|
|
1479
|
+
if (state.workerPool === null) return;
|
|
1480
|
+
await awaitReplicationIdle(state);
|
|
1481
|
+
await dispatchToWorkers(state, action);
|
|
1482
|
+
return;
|
|
1483
|
+
}
|
|
1484
|
+
recordMainWrite(state, action);
|
|
1485
|
+
const waiting = replicationRoomWait(state, action.indexName);
|
|
1486
|
+
if (waiting !== null) await waiting;
|
|
1487
|
+
}
|
|
1488
|
+
function queueForCopies(state, action) {
|
|
1489
|
+
if (!("indexName" in action)) return;
|
|
1490
|
+
const loading = state.copyLoadBuffers.get(action.indexName);
|
|
1491
|
+
if (loading !== void 0) {
|
|
1492
|
+
loading.push(action);
|
|
1493
|
+
copyLoadBufferHoldOf(state, loading).documents += documentWeight(action);
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) {
|
|
1497
|
+
releasePublicationOf(action);
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
enqueueReplication(state, action.indexName, action);
|
|
1501
|
+
}
|
|
1502
|
+
function recordMainWrite(state, action) {
|
|
1503
|
+
if (!("indexName" in action) || !copiesWillReceive(state, action.indexName)) return;
|
|
1504
|
+
queueForCopies(state, detachCallerDocuments(state, action));
|
|
1505
|
+
}
|
|
1506
|
+
function replicationRoomWait(state, indexName) {
|
|
1507
|
+
const loading = state.copyLoadBuffers.get(indexName);
|
|
1508
|
+
if (loading !== void 0) {
|
|
1509
|
+
const hold = copyLoadBufferHoldOf(state, loading);
|
|
1510
|
+
if (hold.documents >= MAX_PENDING_REPLICATION_DOCUMENTS) {
|
|
1511
|
+
return new Promise((resume) => hold.waiting.push(resume));
|
|
1512
|
+
}
|
|
1513
|
+
if (loading.length - hold.yieldedAt < LOAD_BUFFER_YIELD_INTERVAL) return null;
|
|
1514
|
+
hold.yieldedAt = loading.length;
|
|
1515
|
+
return yieldToEventLoop();
|
|
1516
|
+
}
|
|
1517
|
+
const queue = state.replicationQueues.get(indexName);
|
|
1518
|
+
if (queue === void 0 || queue.pendingDocuments < MAX_PENDING_REPLICATION_DOCUMENTS) return null;
|
|
1519
|
+
return new Promise((resume) => queue.drainWaiters.push(resume));
|
|
1520
|
+
}
|
|
1521
|
+
function settled(queue) {
|
|
1522
|
+
const runs = [queue.tail];
|
|
1523
|
+
for (let run = queue.oldestRun; run !== void 0; run = run.newer) runs.push(run.done);
|
|
1524
|
+
return Promise.all(runs).then(() => void 0);
|
|
1525
|
+
}
|
|
1526
|
+
function publicationCapacityWait(state, indexName) {
|
|
1527
|
+
const waiting = replicationRoomWait(state, indexName);
|
|
1528
|
+
return waiting === null ? null : awaitPublicationCapacity(state, indexName, waiting);
|
|
1529
|
+
}
|
|
1530
|
+
async function awaitPublicationCapacity(state, indexName, first) {
|
|
1531
|
+
await first;
|
|
1532
|
+
for (; ; ) {
|
|
1533
|
+
const waiting = replicationRoomWait(state, indexName);
|
|
1534
|
+
if (waiting === null) return;
|
|
1535
|
+
await waiting;
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
async function awaitReplicationIdle(state, indexName) {
|
|
1539
|
+
if (indexName !== void 0) {
|
|
1540
|
+
const queue = state.replicationQueues.get(indexName);
|
|
1541
|
+
if (queue !== void 0) await settled(queue);
|
|
1542
|
+
return;
|
|
1543
|
+
}
|
|
1544
|
+
while (state.replicationQueues.size > 0) {
|
|
1545
|
+
await Promise.all(Array.from(state.replicationQueues.values(), settled));
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
async function awaitWritesApplied(state, indexName) {
|
|
1549
|
+
let awaitedTransition;
|
|
1550
|
+
let awaitedRepair = null;
|
|
1551
|
+
for (; ; ) {
|
|
1552
|
+
const transition = state.copyTransitions.get(indexName);
|
|
1553
|
+
if (transition !== void 0 && transition !== awaitedTransition) {
|
|
1554
|
+
awaitedTransition = transition;
|
|
1555
|
+
await transition.done;
|
|
1556
|
+
continue;
|
|
1557
|
+
}
|
|
1558
|
+
const repair = state.poolRepair;
|
|
1559
|
+
if (repair !== null && repair !== awaitedRepair) {
|
|
1560
|
+
awaitedRepair = repair;
|
|
1561
|
+
await repair;
|
|
1562
|
+
continue;
|
|
1563
|
+
}
|
|
1564
|
+
await awaitReplicationIdle(state, indexName);
|
|
1565
|
+
return;
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// src/engine/orchestration/publication.ts
|
|
1570
|
+
var heldPublications = /* @__PURE__ */ new WeakMap();
|
|
1571
|
+
function retainPublicationOf2(action) {
|
|
1572
|
+
return heldPublications.get(action)?.retain();
|
|
1573
|
+
}
|
|
1574
|
+
function releasePublicationOf2(action) {
|
|
1575
|
+
heldPublications.get(action)?.release();
|
|
1576
|
+
heldPublications.delete(action);
|
|
1577
|
+
}
|
|
1578
|
+
function publishToWorkers(state, action) {
|
|
1579
|
+
if (!("indexName" in action)) return;
|
|
1580
|
+
if (!state.copyLoadBuffers.has(action.indexName) && !state.scaledOutIndexes.has(action.indexName)) return;
|
|
1581
|
+
const manager = state.executor.getManager(action.indexName);
|
|
1582
|
+
if (manager === void 0) return;
|
|
1583
|
+
if (!state.scaledOutIndexes.has(action.indexName)) {
|
|
1584
|
+
for (const vector of manager.getVectorIndexes().values()) {
|
|
1585
|
+
if (vector.maintenanceStatus().loadingFromFiles) return;
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
const detached = detachCallerDocuments(state, action);
|
|
1589
|
+
const held = holdCopyPublication(manager, true);
|
|
1590
|
+
const publication = { ...detached, publication: held.publication };
|
|
1591
|
+
heldPublications.set(publication, held);
|
|
1592
|
+
trackPublicationStores(state, held.publication);
|
|
1593
|
+
try {
|
|
1594
|
+
queueForCopies(state, publication);
|
|
1595
|
+
} catch (error) {
|
|
1596
|
+
releasePublicationOf2(publication);
|
|
1597
|
+
throw error;
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
// src/engine/orchestration/publication-code.ts
|
|
1602
|
+
var publicationLoaded = false;
|
|
1603
|
+
async function loadPublicationCode() {
|
|
1604
|
+
publicationLoaded = true;
|
|
1605
|
+
}
|
|
1606
|
+
function publishToWorkers2(state, action) {
|
|
1607
|
+
return publicationLoaded ? publishToWorkers(state, action) : void 0;
|
|
1608
|
+
}
|
|
1609
|
+
function releasePublicationOf(action) {
|
|
1610
|
+
if (publicationLoaded) releasePublicationOf2(action);
|
|
1611
|
+
}
|
|
1612
|
+
function retainPublicationOf(action) {
|
|
1613
|
+
return publicationLoaded ? retainPublicationOf2(action) : void 0;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
// src/engine/orchestration/live-tail.ts
|
|
1617
|
+
function liveTailCount(manager, partitionId) {
|
|
1618
|
+
if (partitionId >= manager.partitionCount) return 0;
|
|
1619
|
+
const partition = manager.getPartition(partitionId);
|
|
1620
|
+
return isCompositePartition(partition) ? partition.live.count() : partition.count();
|
|
1621
|
+
}
|
|
1622
|
+
function freezeIntoSharedSegment(manager, partitionId) {
|
|
1623
|
+
return manager.freezeLiveTail(partitionId, (payload, documents) => {
|
|
1624
|
+
const snapshot = freezeSegmentShared(payload, documents, void 0, manager.partialWordAnalysis);
|
|
1625
|
+
return snapshot === null ? null : createSharedFrozenSegment(snapshot);
|
|
1626
|
+
});
|
|
1627
|
+
}
|
|
1628
|
+
async function freezeLiveTailsBeforeCopiesLoad(manager) {
|
|
1629
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
1630
|
+
try {
|
|
1631
|
+
await manager.publication.write(() => {
|
|
1632
|
+
if (liveTailCount(manager, partitionId) >= LIVE_TAIL_FREEZE_FLOOR) freezeIntoSharedSegment(manager, partitionId);
|
|
1633
|
+
});
|
|
1634
|
+
} catch (err) {
|
|
1635
|
+
console.warn(`Freezing the live tail of partition ${partitionId} of "${manager.indexName}" failed:`, err);
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
function freezeTail(state, indexName, manager, partitionId) {
|
|
1640
|
+
const segment = freezeIntoSharedSegment(manager, partitionId);
|
|
1641
|
+
if (segment === null || segment.sharedSnapshot === null) return;
|
|
1642
|
+
publishToWorkers2(state, {
|
|
1643
|
+
type: "freezeLiveTail",
|
|
1644
|
+
indexName,
|
|
1645
|
+
partitionId,
|
|
1646
|
+
snapshot: segment.sharedSnapshot,
|
|
1647
|
+
requestId: createRequestId()
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
async function freezeLiveTail(state, indexName, manager, partitionId, floor) {
|
|
1651
|
+
if (!state.scaledOutIndexes.has(indexName)) return;
|
|
1652
|
+
if (liveTailCount(manager, partitionId) < floor) return;
|
|
1653
|
+
try {
|
|
1654
|
+
await manager.publication.write(() => {
|
|
1655
|
+
if (state.executor.getManager(indexName) !== manager || liveTailCount(manager, partitionId) < floor) return;
|
|
1656
|
+
freezeTail(state, indexName, manager, partitionId);
|
|
1657
|
+
});
|
|
1658
|
+
} catch (err) {
|
|
1659
|
+
console.warn(`Freezing the live tail of partition ${partitionId} of "${indexName}" failed:`, err);
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
async function flushGrownTails(state, indexName) {
|
|
1663
|
+
if (!state.scaledOutIndexes.has(indexName)) return;
|
|
1664
|
+
const manager = state.executor.getManager(indexName);
|
|
1665
|
+
if (manager === void 0) return;
|
|
1666
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
1667
|
+
await freezeLiveTail(state, indexName, manager, partitionId, LIVE_TAIL_FLUSH_DOCUMENTS);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
// src/engine/orchestration/compaction.ts
|
|
1672
|
+
function smallestSegmentIds(entries, limit) {
|
|
1673
|
+
return [...entries].sort((a, b) => a.documentCount - b.documentCount).slice(0, limit).map((entry) => entry.segmentId);
|
|
1674
|
+
}
|
|
1675
|
+
function pickSegmentIds(entries, policy) {
|
|
1676
|
+
if (policy === "idle") {
|
|
1677
|
+
return entries.length >= 2 ? smallestSegmentIds(entries, SEGMENTS_PER_MERGE) : null;
|
|
1678
|
+
}
|
|
1679
|
+
return entries.length >= COMPACTION_SEGMENT_TRIGGER ? smallestSegmentIds(entries, COMPACTION_SEGMENT_TRIGGER) : null;
|
|
1680
|
+
}
|
|
1681
|
+
function mainCompositeOf(manager, partitionId) {
|
|
1682
|
+
if (partitionId >= manager.partitionCount) return null;
|
|
1683
|
+
const partition = manager.getPartition(partitionId);
|
|
1684
|
+
return isCompositePartition(partition) ? partition : null;
|
|
1685
|
+
}
|
|
1686
|
+
function candidateSegmentIds(state, indexName, manager, partitionId, policy) {
|
|
1687
|
+
const composite = mainCompositeOf(manager, partitionId);
|
|
1688
|
+
if (composite !== null) {
|
|
1689
|
+
const sizes = composite.frozenSegmentSizes().map((size) => ({ segmentId: size.segmentId, documentCount: size.liveDocumentCount }));
|
|
1690
|
+
const picks = pickSegmentIds(sizes, policy);
|
|
1691
|
+
if (picks !== null) return picks;
|
|
1692
|
+
}
|
|
1693
|
+
const ledger = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
1694
|
+
return ledger === void 0 ? null : pickSegmentIds(ledger, policy);
|
|
1695
|
+
}
|
|
1696
|
+
function mainHoldsAll(composite, segmentIds) {
|
|
1697
|
+
if (composite === null) return false;
|
|
1698
|
+
const held = new Set(composite.frozenSegmentSizes().map((size) => size.segmentId));
|
|
1699
|
+
return segmentIds.every((segmentId) => held.has(segmentId));
|
|
1700
|
+
}
|
|
1701
|
+
async function compactOnWorker(state, indexName, partitionId, segmentIds) {
|
|
1702
|
+
const pool = state.workerPool;
|
|
1703
|
+
if (pool === null || !state.scaledOutIndexes.has(indexName)) return null;
|
|
1704
|
+
const lease = pool.leaseLeastBusy(indexName);
|
|
1705
|
+
if (lease === null) return null;
|
|
1706
|
+
try {
|
|
1707
|
+
return await lease.executor.execute({
|
|
1708
|
+
type: "compactSegments",
|
|
1709
|
+
indexName,
|
|
1710
|
+
partitionId,
|
|
1711
|
+
segmentIds: [...segmentIds],
|
|
1712
|
+
requestId: createRequestId()
|
|
1713
|
+
});
|
|
1714
|
+
} catch (err) {
|
|
1715
|
+
if (!(err instanceof NarsilError) || err.code !== ErrorCodes.PARTITION_CORRUPTED) {
|
|
1716
|
+
console.warn("Worker segment compaction failed:", err);
|
|
1717
|
+
}
|
|
1718
|
+
return null;
|
|
1719
|
+
} finally {
|
|
1720
|
+
lease.release();
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
function ledgerHoldsAll(state, indexName, partitionId, segmentIds) {
|
|
1724
|
+
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
1725
|
+
if (entries === void 0) return false;
|
|
1726
|
+
const held = new Set(entries.map((entry) => entry.segmentId));
|
|
1727
|
+
return segmentIds.every((segmentId) => held.has(segmentId));
|
|
1728
|
+
}
|
|
1729
|
+
function segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain) {
|
|
1730
|
+
if (state.executor.getManager(indexName) !== manager) return false;
|
|
1731
|
+
if (mergedOnMain === null) return ledgerHoldsAll(state, indexName, partitionId, segmentIds);
|
|
1732
|
+
return mainCompositeOf(manager, partitionId) === mergedOnMain && mainHoldsAll(mergedOnMain, segmentIds);
|
|
1733
|
+
}
|
|
1734
|
+
function forgetLedgerSegments(state, indexName, partitionId, segmentIds) {
|
|
1735
|
+
const entries = state.segmentLedger.get(indexName)?.get(partitionId);
|
|
1736
|
+
if (entries === void 0) return;
|
|
1737
|
+
const stale = new Set(segmentIds);
|
|
1738
|
+
const kept = entries.filter((entry) => !stale.has(entry.segmentId));
|
|
1739
|
+
entries.length = 0;
|
|
1740
|
+
entries.push(...kept);
|
|
1741
|
+
}
|
|
1742
|
+
async function compactPartitionSegments(state, indexName, manager, partitionId, segmentIds, policy) {
|
|
1743
|
+
await awaitReplicationIdle(state, indexName);
|
|
1744
|
+
if (policy === "idle" && !state.scaledOutIndexes.has(indexName)) return false;
|
|
1745
|
+
const composite = mainCompositeOf(manager, partitionId);
|
|
1746
|
+
const mainCanMerge = composite !== null && mainHoldsAll(composite, segmentIds);
|
|
1747
|
+
let snapshots = await compactOnWorker(state, indexName, partitionId, segmentIds);
|
|
1748
|
+
if (snapshots === null && mainCanMerge && composite !== null) {
|
|
1749
|
+
snapshots = mergeFrozenSegmentOutputs(
|
|
1750
|
+
composite.frozenSegmentsById(segmentIds),
|
|
1751
|
+
manager.partialWordAnalysis,
|
|
1752
|
+
manager.textAnalysis
|
|
1753
|
+
);
|
|
1754
|
+
}
|
|
1755
|
+
if (snapshots === null) {
|
|
1756
|
+
if (!mainCanMerge) forgetLedgerSegments(state, indexName, partitionId, segmentIds);
|
|
1757
|
+
return false;
|
|
1758
|
+
}
|
|
1759
|
+
const mergedOnMain = mainCanMerge ? composite : null;
|
|
1760
|
+
const replacement = snapshots.map((snapshot) => createSharedFrozenSegment(snapshot));
|
|
1761
|
+
const publication = await manager.publication.write(() => {
|
|
1762
|
+
if (!segmentsStillHeld(state, indexName, manager, partitionId, segmentIds, mergedOnMain)) return null;
|
|
1763
|
+
const removedSpellings = mainCanMerge ? manager.swapFrozenSegments(partitionId, segmentIds, replacement) : void 0;
|
|
1764
|
+
publishToWorkers2(state, {
|
|
1765
|
+
type: "swapSegments",
|
|
1766
|
+
indexName,
|
|
1767
|
+
partitionId,
|
|
1768
|
+
dropSegmentIds: [...segmentIds],
|
|
1769
|
+
snapshots,
|
|
1770
|
+
removedSpellings,
|
|
1771
|
+
requestId: createRequestId()
|
|
1772
|
+
});
|
|
1773
|
+
return true;
|
|
1774
|
+
});
|
|
1775
|
+
if (publication === null) return false;
|
|
1776
|
+
return snapshots.length < segmentIds.length;
|
|
1777
|
+
}
|
|
1778
|
+
async function compactIndexSegments(state, indexName, policy) {
|
|
1779
|
+
const manager = state.executor.getManager(indexName);
|
|
1780
|
+
if (!manager || manager.config.textStorage === "disk") return;
|
|
1781
|
+
const partitionIds = /* @__PURE__ */ new Set();
|
|
1782
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
|
|
1783
|
+
for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
|
|
1784
|
+
for (const partitionId of partitionIds) {
|
|
1785
|
+
if (policy === "idle") {
|
|
1786
|
+
await awaitReplicationIdle(state, indexName);
|
|
1787
|
+
await freezeLiveTail(state, indexName, manager, partitionId, LIVE_TAIL_FREEZE_FLOOR);
|
|
1788
|
+
}
|
|
1789
|
+
let picks = candidateSegmentIds(state, indexName, manager, partitionId, policy);
|
|
1790
|
+
while (picks !== null) {
|
|
1791
|
+
const compacted = await compactPartitionSegments(state, indexName, manager, partitionId, picks, policy);
|
|
1792
|
+
if (!compacted) break;
|
|
1793
|
+
picks = candidateSegmentIds(state, indexName, manager, partitionId, policy);
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
function maybeCompactSegments(state, indexName, policy = "ingest") {
|
|
1798
|
+
if (state.compactionsInFlight.has(indexName)) {
|
|
1799
|
+
if (policy === "idle") scheduleIdleMerge(state, indexName);
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
const run = compactIndexSegments(state, indexName, policy).catch((err) => {
|
|
1803
|
+
console.warn("Segment compaction failed:", err);
|
|
1804
|
+
}).finally(() => {
|
|
1805
|
+
state.compactionsInFlight.delete(indexName);
|
|
1806
|
+
});
|
|
1807
|
+
state.compactionsInFlight.set(indexName, run);
|
|
1808
|
+
}
|
|
1809
|
+
function armIdleMerge(state, indexName, dueAt, delayMs) {
|
|
1810
|
+
const timer = setTimeout(() => {
|
|
1811
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
1812
|
+
if (pending === void 0 || pending.timer !== timer) return;
|
|
1813
|
+
const remainingMs = pending.dueAt - Date.now();
|
|
1814
|
+
if (remainingMs > 0) {
|
|
1815
|
+
armIdleMerge(state, indexName, pending.dueAt, remainingMs);
|
|
1816
|
+
return;
|
|
1817
|
+
}
|
|
1818
|
+
state.idleMergeTimers.delete(indexName);
|
|
1819
|
+
maybeCompactSegments(state, indexName, "idle");
|
|
1820
|
+
}, delayMs);
|
|
1821
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
1822
|
+
state.idleMergeTimers.set(indexName, { timer, dueAt });
|
|
1823
|
+
}
|
|
1824
|
+
function scheduleIdleMerge(state, indexName) {
|
|
1825
|
+
const dueAt = Date.now() + IDLE_MERGE_DELAY_MS;
|
|
1826
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
1827
|
+
if (pending !== void 0) {
|
|
1828
|
+
pending.dueAt = dueAt;
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
armIdleMerge(state, indexName, dueAt, IDLE_MERGE_DELAY_MS);
|
|
1832
|
+
}
|
|
1833
|
+
function cancelIdleMerge(state, indexName) {
|
|
1834
|
+
const pending = state.idleMergeTimers.get(indexName);
|
|
1835
|
+
if (pending === void 0) return;
|
|
1836
|
+
clearTimeout(pending.timer);
|
|
1837
|
+
state.idleMergeTimers.delete(indexName);
|
|
1838
|
+
}
|
|
1839
|
+
async function awaitCompactions(state) {
|
|
1840
|
+
while (state.compactionsInFlight.size > 0) {
|
|
1841
|
+
await Promise.all([...state.compactionsInFlight.values()]);
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
// src/engine/orchestration/vector-copies.ts
|
|
1846
|
+
function holdsIndex(state, indexName) {
|
|
1847
|
+
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName);
|
|
1848
|
+
}
|
|
1849
|
+
async function insertOnLease(lease, indexName, fieldName, handle, ordinals) {
|
|
1850
|
+
try {
|
|
1851
|
+
const outcome = await lease.executor.execute({
|
|
1852
|
+
type: "insertVectorOrdinals",
|
|
1853
|
+
indexName,
|
|
1854
|
+
fieldName,
|
|
1855
|
+
handle,
|
|
1856
|
+
ordinals,
|
|
1857
|
+
requestId: createRequestId()
|
|
1858
|
+
});
|
|
1859
|
+
return outcome ?? null;
|
|
1860
|
+
} catch {
|
|
1861
|
+
return null;
|
|
1862
|
+
} finally {
|
|
1863
|
+
lease.release();
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
function sharedCopyHostOf(state) {
|
|
1867
|
+
return {
|
|
1868
|
+
get workerCount() {
|
|
1869
|
+
return state.workerPool?.workerCount ?? state.keywordWorkerCount + state.longQueryWorkerCount;
|
|
1870
|
+
},
|
|
1871
|
+
holdsIndex: (indexName) => holdsIndex(state, indexName),
|
|
1872
|
+
resolvePartition: (indexName, docId) => state.executor.getManager(indexName)?.partitionIdOf(docId),
|
|
1873
|
+
async loadShared(indexName, fieldName, handle, handles) {
|
|
1874
|
+
if (!holdsIndex(state, indexName)) return false;
|
|
1875
|
+
const publication = state.executor.getManager(indexName)?.getVectorIndexes().get(fieldName)?.holdCopyPublication();
|
|
1876
|
+
if (publication === void 0) return false;
|
|
1877
|
+
state.sharedVectorFields.set(handle, handles);
|
|
1878
|
+
try {
|
|
1879
|
+
await replicateToWorkers(state, {
|
|
1880
|
+
type: "loadVectorCopy",
|
|
1881
|
+
indexName,
|
|
1882
|
+
fieldName,
|
|
1883
|
+
handle,
|
|
1884
|
+
storeHandle: publication.field.handle,
|
|
1885
|
+
handles,
|
|
1886
|
+
requestId: createRequestId()
|
|
1887
|
+
});
|
|
1888
|
+
await awaitWritesApplied(state, indexName);
|
|
1889
|
+
return state.scaledOutIndexes.has(indexName);
|
|
1890
|
+
} finally {
|
|
1891
|
+
publication.release();
|
|
1892
|
+
}
|
|
1893
|
+
},
|
|
1894
|
+
async drop(indexName, fieldName, handle) {
|
|
1895
|
+
state.sharedVectorFields.delete(handle);
|
|
1896
|
+
if (!holdsIndex(state, indexName)) return;
|
|
1897
|
+
await replicateToWorkers(state, {
|
|
1898
|
+
type: "dropVectorCopy",
|
|
1899
|
+
indexName,
|
|
1900
|
+
fieldName,
|
|
1901
|
+
handle,
|
|
1902
|
+
requestId: createRequestId()
|
|
1903
|
+
});
|
|
1904
|
+
},
|
|
1905
|
+
async insertOrdinals(indexName, fieldName, handle, ordinals) {
|
|
1906
|
+
const pool = state.workerPool;
|
|
1907
|
+
if (pool === null || !state.scaledOutIndexes.has(indexName)) return null;
|
|
1908
|
+
for (let attempt = 0; attempt < pool.workerCount; attempt++) {
|
|
1909
|
+
const lease = pool.leaseLeastBusy();
|
|
1910
|
+
if (lease === null) return null;
|
|
1911
|
+
const outcome = await insertOnLease(lease, indexName, fieldName, handle, ordinals);
|
|
1912
|
+
if (outcome !== null) return outcome;
|
|
1913
|
+
if (!state.scaledOutIndexes.has(indexName)) return null;
|
|
1914
|
+
}
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
};
|
|
1918
|
+
}
|
|
1919
|
+
function releaseVectorLocksOf(state, workerId) {
|
|
1920
|
+
noteThreadSlotGone(threadSlotOfWorker(workerId));
|
|
1921
|
+
if (state.sharedVectorFields.size === 0) return;
|
|
1922
|
+
const { releaseLocksHeldBy, releasePinsHeldBy } = requireLoadedVectorCode();
|
|
1923
|
+
const threadSlot = threadSlotOfWorker(workerId);
|
|
1924
|
+
for (const handles of state.sharedVectorFields.values()) {
|
|
1925
|
+
releasePinsHeldBy(handles.store.pins, threadSlot);
|
|
1926
|
+
if (handles.graph !== null) releaseLocksHeldBy(handles.graph, threadSlot);
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
function refreshVectorCopies(state, indexName) {
|
|
1930
|
+
if (state.vectorCopyPolicy?.host === void 0) return;
|
|
1931
|
+
const manager = state.executor.getManager(indexName);
|
|
1932
|
+
if (manager === void 0) return;
|
|
1933
|
+
for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.refreshWorkerCopies();
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
// src/engine/orchestration/request-threads.ts
|
|
1937
|
+
async function announceRequestThreads(state, pool) {
|
|
1938
|
+
const listener = state.requestThreads;
|
|
1939
|
+
if (listener === null) return;
|
|
1940
|
+
const entries = pool.executorEntries();
|
|
1941
|
+
for (const { workerId, executor } of entries) {
|
|
1942
|
+
if (pool.isLongQueryWorker(workerId)) await listener.onLongQueryWorkerReady(workerId, executor);
|
|
1943
|
+
}
|
|
1944
|
+
for (const { workerId, executor } of entries) {
|
|
1945
|
+
if (!pool.isLongQueryWorker(workerId)) await listener.onWorkerReady(workerId, executor);
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
function scheduleRequestThreadPoolRestart(state) {
|
|
1949
|
+
if (state.requestThreads === null) return;
|
|
1950
|
+
const timer = setTimeout(
|
|
1951
|
+
() => {
|
|
1952
|
+
if (state.requestThreads === null || state.workerPool !== null || threadsLeftRunningPastLimit(state)) return;
|
|
1953
|
+
void ensurePool(state).then(() => scaleOutReadyIndexes(state)).catch((err) => {
|
|
1954
|
+
console.warn("Restarting the request threads failed:", err);
|
|
1955
|
+
scheduleRequestThreadPoolRestart(state);
|
|
1956
|
+
});
|
|
1957
|
+
},
|
|
1958
|
+
Math.max(0, state.poolRetryAt - Date.now())
|
|
1959
|
+
);
|
|
1960
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
1961
|
+
}
|
|
1962
|
+
async function serveRequestsOnWorkers(state, listener) {
|
|
1963
|
+
if (!state.workersEnabled) return 0;
|
|
1964
|
+
state.requestThreads = listener;
|
|
1965
|
+
state.keywordWorkerCount = resolveRequestThreadCount(state.config?.workers?.count);
|
|
1966
|
+
state.longQueryWorkerCount = LONG_QUERY_THREADS;
|
|
1967
|
+
if (state.vectorCopyPolicy !== void 0) {
|
|
1968
|
+
state.vectorCopyPolicy.host = sharedCopyHostOf(state);
|
|
1969
|
+
state.vectorCopyPolicy.enabled = state.workersEnabled;
|
|
1970
|
+
}
|
|
1971
|
+
if (state.poolStart !== null) await state.poolStart.catch(() => void 0);
|
|
1972
|
+
const running = state.workerPool;
|
|
1973
|
+
if (running !== null && running.workerCount !== resolveWorkerCount(state.keywordWorkerCount + LONG_QUERY_THREADS)) {
|
|
1974
|
+
retirePool(state, running);
|
|
1975
|
+
state.poolRetryAt = 0;
|
|
1976
|
+
}
|
|
1977
|
+
const pool = await ensurePool(state);
|
|
1978
|
+
for (const indexName of state.indexRegistry.keys()) {
|
|
1979
|
+
refreshVectorCopies(state, indexName);
|
|
1980
|
+
if (indexReadyForCopies(state, indexName)) {
|
|
1981
|
+
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
return pool.workerCount;
|
|
1985
|
+
}
|
|
1986
|
+
function requestThreadCountOf(state) {
|
|
1987
|
+
if (!state.workersEnabled) return 0;
|
|
1988
|
+
return resolveRequestThreadCount(state.config?.workers?.count);
|
|
1989
|
+
}
|
|
1990
|
+
function stopRequestThreads(state) {
|
|
1991
|
+
const listener = state.requestThreads;
|
|
1992
|
+
if (listener === null) return;
|
|
1993
|
+
state.requestThreads = null;
|
|
1994
|
+
const pool = state.workerPool;
|
|
1995
|
+
if (pool === null) return;
|
|
1996
|
+
for (const { workerId } of pool.executorEntries()) listener.onWorkerGone(workerId);
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
// src/engine/orchestration/repair.ts
|
|
2000
|
+
var COPY_RESTART_REASON = "A request arrived after every worker crashed and the restart delay passed";
|
|
2001
|
+
function nextRestartDelay(state) {
|
|
2002
|
+
const delay = state.poolRetryDelayMs;
|
|
2003
|
+
state.poolRetryDelayMs = Math.min(delay * 2, POOL_RESTART_DELAY_MAX_MS);
|
|
2004
|
+
return delay;
|
|
2005
|
+
}
|
|
2006
|
+
function deferPoolRestart(state) {
|
|
2007
|
+
state.poolRetryAt = Date.now() + nextRestartDelay(state);
|
|
2008
|
+
}
|
|
2009
|
+
function retirePool(state, pool) {
|
|
2010
|
+
if (state.workerPool !== pool) return;
|
|
2011
|
+
state.workerPool = null;
|
|
2012
|
+
cancelRepair(state);
|
|
2013
|
+
deferPoolRestart(state);
|
|
2014
|
+
for (const indexName of state.scaledOutIndexes) state.droppedCopies.set(indexName, COPY_RESTART_REASON);
|
|
2015
|
+
state.scaledOutIndexes.clear();
|
|
2016
|
+
state.segmentLedger.clear();
|
|
2017
|
+
for (const { workerId } of pool.executorEntries()) state.requestThreads?.onWorkerGone(workerId);
|
|
2018
|
+
const earlierThreadsGone = state.retiredThreadsGone;
|
|
2019
|
+
state.retiredThreadsGone = pool.whenEveryThreadIsGone().then(() => earlierThreadsGone);
|
|
2020
|
+
void pool.shutdown().catch(() => void 0);
|
|
2021
|
+
scheduleRequestThreadPoolRestart(state);
|
|
2022
|
+
}
|
|
2023
|
+
function handleWorkerCrash(state, pool, workerId, indexNames, error) {
|
|
2024
|
+
state.callbacks?.onWorkerCrash?.(workerId, indexNames, error);
|
|
2025
|
+
state.requestThreads?.onWorkerGone(workerId);
|
|
2026
|
+
if (pool.getAllExecutors().length === 0) retirePool(state, pool);
|
|
2027
|
+
}
|
|
2028
|
+
function handleWorkerThreadGone(state, pool, workerId) {
|
|
2029
|
+
releaseVectorLocksOf(state, workerId);
|
|
2030
|
+
scheduleRepair(state, pool);
|
|
2031
|
+
}
|
|
2032
|
+
function threadsLeftRunningPastLimit(state) {
|
|
2033
|
+
return state.threadsLeftRunning > state.maxThreadsLeftRunning;
|
|
2034
|
+
}
|
|
2035
|
+
function noteThreadLeftRunning(state, workerId, exited) {
|
|
2036
|
+
state.threadsLeftRunning += 1;
|
|
2037
|
+
if (threadsLeftRunningPastLimit(state)) {
|
|
2038
|
+
console.warn(
|
|
2039
|
+
`Worker ${workerId} stopped answering while it held HTTP connections, so its thread keeps running in the background. ${state.threadsLeftRunning} such threads pass the limit of ${state.maxThreadsLeftRunning} that workers.maxThreadsLeftRunning sets, so the engine starts no replacement thread until one of them exits, and the main thread and the remaining request threads answer meanwhile.`
|
|
2040
|
+
);
|
|
2041
|
+
}
|
|
2042
|
+
void exited.then(() => {
|
|
2043
|
+
state.threadsLeftRunning -= 1;
|
|
2044
|
+
if (state.shuttingDown) return;
|
|
2045
|
+
const pool = state.workerPool;
|
|
2046
|
+
if (pool !== null) scheduleRepair(state, pool);
|
|
2047
|
+
else scheduleRequestThreadPoolRestart(state);
|
|
2048
|
+
});
|
|
2049
|
+
}
|
|
2050
|
+
function scheduleRepair(state, pool) {
|
|
2051
|
+
if (state.workerPool !== pool || state.repairTimer !== null || state.poolRepair !== null) return;
|
|
2052
|
+
if (threadsLeftRunningPastLimit(state)) return;
|
|
2053
|
+
const timer = setTimeout(() => {
|
|
2054
|
+
state.repairTimer = null;
|
|
2055
|
+
void repairPool(state, pool);
|
|
2056
|
+
}, state.poolRetryDelayMs);
|
|
2057
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
2058
|
+
state.repairTimer = timer;
|
|
2059
|
+
}
|
|
2060
|
+
function cancelRepair(state) {
|
|
2061
|
+
if (state.repairTimer === null) return;
|
|
2062
|
+
clearTimeout(state.repairTimer);
|
|
2063
|
+
state.repairTimer = null;
|
|
2064
|
+
}
|
|
2065
|
+
async function loadOntoReplacement(state, replacement, indexName) {
|
|
2066
|
+
const entry = state.indexRegistry.get(indexName);
|
|
2067
|
+
const manager = state.executor.getManager(indexName);
|
|
2068
|
+
if (entry === void 0 || manager === void 0 || !state.scaledOutIndexes.has(indexName)) return;
|
|
2069
|
+
const buffered = [];
|
|
2070
|
+
state.copyLoadBuffers.set(indexName, buffered);
|
|
2071
|
+
let copyLoad;
|
|
2072
|
+
try {
|
|
2073
|
+
copyLoad = await loadIndexOntoWorkers(
|
|
2074
|
+
indexName,
|
|
2075
|
+
[replacement.executor],
|
|
2076
|
+
{ ...entry, onPublication: (publication) => trackPublicationStores(state, publication) },
|
|
2077
|
+
manager,
|
|
2078
|
+
void 0,
|
|
2079
|
+
state.callbacks?.isAnalysisStale?.(indexName)
|
|
2080
|
+
);
|
|
2081
|
+
replacement.hold(indexName);
|
|
2082
|
+
} finally {
|
|
2083
|
+
endCopyLoadBuffer(state, indexName, buffered);
|
|
2084
|
+
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
2085
|
+
}
|
|
2086
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
2087
|
+
refreshVectorCopies(state, indexName);
|
|
2088
|
+
}
|
|
2089
|
+
async function replaceWorker(state, pool, workerId) {
|
|
2090
|
+
const replacement = pool.spawnReplacement(workerId);
|
|
2091
|
+
if (replacement === null) return;
|
|
2092
|
+
try {
|
|
2093
|
+
if (state.bootstrapModule !== void 0) {
|
|
2094
|
+
await replacement.executor.execute({
|
|
2095
|
+
type: "bootstrap",
|
|
2096
|
+
moduleUrl: state.bootstrapModule,
|
|
2097
|
+
requestId: createRequestId()
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
2100
|
+
for (const indexName of [...state.scaledOutIndexes]) {
|
|
2101
|
+
await loadOntoReplacement(state, replacement, indexName);
|
|
2102
|
+
}
|
|
2103
|
+
replacement.admit();
|
|
2104
|
+
if (pool.isLongQueryWorker(workerId))
|
|
2105
|
+
await state.requestThreads?.onLongQueryWorkerReady(workerId, replacement.executor);
|
|
2106
|
+
else await state.requestThreads?.onWorkerReady(workerId, replacement.executor);
|
|
2107
|
+
} catch (err) {
|
|
2108
|
+
replacement.abandon();
|
|
2109
|
+
throw err;
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async function replaceDeadWorkers(state, pool) {
|
|
2113
|
+
for (const workerId of pool.deadWorkerIds()) {
|
|
2114
|
+
if (state.workerPool !== pool || threadsLeftRunningPastLimit(state)) return;
|
|
2115
|
+
await replaceWorker(state, pool, workerId);
|
|
2116
|
+
}
|
|
2117
|
+
state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
|
|
2118
|
+
}
|
|
2119
|
+
function repairPool(state, pool) {
|
|
2120
|
+
if (state.workerPool !== pool) return Promise.resolve();
|
|
2121
|
+
if (state.poolRepair !== null) return state.poolRepair;
|
|
2122
|
+
const run = replaceDeadWorkers(state, pool).catch((err) => {
|
|
2123
|
+
nextRestartDelay(state);
|
|
2124
|
+
console.warn("Replacing a crashed worker failed:", err);
|
|
2125
|
+
}).finally(() => {
|
|
2126
|
+
state.poolRepair = null;
|
|
2127
|
+
if (pool.deadWorkerIds().length > 0) scheduleRepair(state, pool);
|
|
2128
|
+
});
|
|
2129
|
+
state.poolRepair = run;
|
|
2130
|
+
return run;
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
// src/engine/orchestration/scale-out.ts
|
|
2134
|
+
var COPY_RELOAD_REASON = "A request arrived after an idle spell dropped the worker copies";
|
|
2135
|
+
function copiesAllowed(state) {
|
|
2136
|
+
if (!state.workersEnabled || state.scaleOutBlocked || state.shuttingDown) return false;
|
|
2137
|
+
if (state.workerPool !== null) return true;
|
|
2138
|
+
return Date.now() >= state.poolRetryAt && !threadsLeftRunningPastLimit(state);
|
|
2139
|
+
}
|
|
2140
|
+
function retiredThreadsGoneWithin(state, timeoutMs) {
|
|
2141
|
+
return new Promise((resolve) => {
|
|
2142
|
+
const timeoutId = setTimeout(() => resolve(false), timeoutMs);
|
|
2143
|
+
const settle = () => {
|
|
2144
|
+
clearTimeout(timeoutId);
|
|
2145
|
+
resolve(true);
|
|
2146
|
+
};
|
|
2147
|
+
void state.retiredThreadsGone.then(settle);
|
|
2148
|
+
void state.shutdownStarted.then(settle);
|
|
2149
|
+
});
|
|
2150
|
+
}
|
|
2151
|
+
async function startPool(state) {
|
|
2152
|
+
const everyRetiredThreadIsGone = await retiredThreadsGoneWithin(state, RETIRED_THREADS_EXIT_WAIT_MS);
|
|
2153
|
+
if (state.shuttingDown) {
|
|
2154
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, "The engine is shutting down, so no worker pool starts");
|
|
2155
|
+
}
|
|
2156
|
+
if (!everyRetiredThreadIsGone) {
|
|
2157
|
+
throw new NarsilError(
|
|
2158
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
2159
|
+
`A thread of the retired worker pool still runs after ${RETIRED_THREADS_EXIT_WAIT_MS}ms, so no new pool starts yet`
|
|
2160
|
+
);
|
|
2161
|
+
}
|
|
2162
|
+
if (threadsLeftRunningPastLimit(state)) {
|
|
2163
|
+
throw new NarsilError(
|
|
2164
|
+
ErrorCodes.WORKER_CRASHED,
|
|
2165
|
+
`${state.threadsLeftRunning} worker threads that stopped answering still run in the background, past the limit of ${state.maxThreadsLeftRunning} that workers.maxThreadsLeftRunning sets, so no new pool starts until one of them exits`
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
eligibleIndexNames(state);
|
|
2169
|
+
const factory = await createWorkerFactory();
|
|
2170
|
+
let started = null;
|
|
2171
|
+
const pool = createWorkerPool({
|
|
2172
|
+
count: state.keywordWorkerCount + state.longQueryWorkerCount,
|
|
2173
|
+
longQueryWorkers: state.longQueryWorkerCount,
|
|
2174
|
+
workerFactory: factory,
|
|
2175
|
+
silence: {
|
|
2176
|
+
silenceBeforeProbeMs: state.config?.workers?.silenceBeforeProbeMs,
|
|
2177
|
+
silenceBeforeRetirementMs: state.config?.workers?.silenceBeforeRetirementMs
|
|
2178
|
+
},
|
|
2179
|
+
onWorkerCrash(workerId, indexNames, error) {
|
|
2180
|
+
if (started !== null) handleWorkerCrash(state, started, workerId, indexNames, error);
|
|
2181
|
+
},
|
|
2182
|
+
onWorkerGone(workerId) {
|
|
2183
|
+
if (started !== null) handleWorkerThreadGone(state, started, workerId);
|
|
2184
|
+
},
|
|
2185
|
+
onThreadLeftRunning(workerId, exited) {
|
|
2186
|
+
noteThreadLeftRunning(state, workerId, exited);
|
|
2187
|
+
}
|
|
2188
|
+
});
|
|
2189
|
+
started = pool;
|
|
2190
|
+
try {
|
|
2191
|
+
pool.spawnAll();
|
|
2192
|
+
if (state.bootstrapModule !== void 0) {
|
|
2193
|
+
const moduleUrl = state.bootstrapModule;
|
|
2194
|
+
await Promise.all(
|
|
2195
|
+
pool.getAllExecutors().map(
|
|
2196
|
+
(workerExecutor) => workerExecutor.execute({ type: "bootstrap", moduleUrl, requestId: "copies-bootstrap" })
|
|
2197
|
+
)
|
|
2198
|
+
);
|
|
2199
|
+
}
|
|
2200
|
+
} catch (err) {
|
|
2201
|
+
await pool.shutdown().catch(() => void 0);
|
|
2202
|
+
throw err;
|
|
2203
|
+
}
|
|
2204
|
+
return pool;
|
|
2205
|
+
}
|
|
2206
|
+
async function ensurePool(state) {
|
|
2207
|
+
await loadPublicationCode();
|
|
2208
|
+
if (state.workerPool !== null) return state.workerPool;
|
|
2209
|
+
if (state.poolStart === null) {
|
|
2210
|
+
state.poolStart = startPool(state).then(
|
|
2211
|
+
async (pool) => {
|
|
2212
|
+
state.workerPool = pool;
|
|
2213
|
+
state.poolStart = null;
|
|
2214
|
+
try {
|
|
2215
|
+
await announceRequestThreads(state, pool);
|
|
2216
|
+
} catch (err) {
|
|
2217
|
+
retirePool(state, pool);
|
|
2218
|
+
throw err;
|
|
2219
|
+
}
|
|
2220
|
+
return pool;
|
|
2221
|
+
},
|
|
2222
|
+
(err) => {
|
|
2223
|
+
state.poolStart = null;
|
|
2224
|
+
deferPoolRestart(state);
|
|
2225
|
+
throw err;
|
|
2226
|
+
}
|
|
2227
|
+
);
|
|
2228
|
+
}
|
|
2229
|
+
return state.poolStart;
|
|
2230
|
+
}
|
|
2231
|
+
function documentCountOf(state, indexName) {
|
|
2232
|
+
return state.executor.getManager(indexName)?.countDocuments() ?? 0;
|
|
2233
|
+
}
|
|
2234
|
+
function copyThresholdReason(state, indexName, incomingCount = 0) {
|
|
2235
|
+
const held = documentCountOf(state, indexName);
|
|
2236
|
+
if (incomingCount <= 0) {
|
|
2237
|
+
return `Index "${indexName}" holds ${held} documents, reaching the copy threshold of ${state.copyThreshold}`;
|
|
2238
|
+
}
|
|
2239
|
+
return `Index "${indexName}" holds ${held} documents, and the batch that the engine is writing adds ${incomingCount}, for ${held + incomingCount} against the copy threshold of ${state.copyThreshold}`;
|
|
2240
|
+
}
|
|
2241
|
+
function indexReadyForCopies(state, indexName, incomingCount = 0) {
|
|
2242
|
+
if (!copiesAllowed(state)) return false;
|
|
2243
|
+
if (state.scaledOutIndexes.has(indexName) || state.desyncedIndexes.has(indexName)) return false;
|
|
2244
|
+
if (state.copyTransitions.has(indexName)) return false;
|
|
2245
|
+
if (state.callbacks?.shouldDeferCopies?.()) return false;
|
|
2246
|
+
const entry = state.indexRegistry.get(indexName);
|
|
2247
|
+
if (entry === void 0 || state.executor.getManager(indexName) === void 0) return false;
|
|
2248
|
+
if (documentCountOf(state, indexName) + incomingCount < state.copyThreshold) return false;
|
|
2249
|
+
const ineligibility = workerIneligibility(indexName, entry.config, state.bootstrapModule);
|
|
2250
|
+
if (ineligibility !== null) {
|
|
2251
|
+
reportIneligible(state, indexName, ineligibility);
|
|
2252
|
+
return false;
|
|
2253
|
+
}
|
|
2254
|
+
return true;
|
|
2255
|
+
}
|
|
2256
|
+
function discardLoadedCopies(state, pool, indexName) {
|
|
2257
|
+
const holders = pool.executorsHolding(indexName);
|
|
2258
|
+
pool.removeIndex(indexName);
|
|
2259
|
+
state.segmentLedger.delete(indexName);
|
|
2260
|
+
void Promise.allSettled(
|
|
2261
|
+
holders.map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `stale-load-drop-${indexName}` }))
|
|
2262
|
+
);
|
|
2263
|
+
}
|
|
2264
|
+
async function loadCopies(state, indexName, reason) {
|
|
2265
|
+
const entry = state.indexRegistry.get(indexName);
|
|
2266
|
+
const manager = state.executor.getManager(indexName);
|
|
2267
|
+
if (entry === void 0 || manager === void 0) return;
|
|
2268
|
+
const ineligibility = workerIneligibility(indexName, entry.config, state.bootstrapModule);
|
|
2269
|
+
if (ineligibility !== null) {
|
|
2270
|
+
reportIneligible(state, indexName, ineligibility);
|
|
2271
|
+
return;
|
|
2272
|
+
}
|
|
2273
|
+
const pool = await ensurePool(state);
|
|
2274
|
+
if (state.executor.getManager(indexName) === void 0) return;
|
|
2275
|
+
const dropReason = state.droppedCopies.get(indexName);
|
|
2276
|
+
const reload = state.droppedCopies.delete(indexName);
|
|
2277
|
+
const buffered = [];
|
|
2278
|
+
let dispatched = false;
|
|
2279
|
+
state.copyLoadBuffers.set(indexName, buffered);
|
|
2280
|
+
try {
|
|
2281
|
+
await freezeLiveTailsBeforeCopiesLoad(manager);
|
|
2282
|
+
const copyLoad = await transferIndexToPool(
|
|
2283
|
+
indexName,
|
|
2284
|
+
pool,
|
|
2285
|
+
{ ...entry, onPublication: (publication) => trackPublicationStores(state, publication) },
|
|
2286
|
+
manager,
|
|
2287
|
+
state.callbacks?.isAnalysisStale?.(indexName)
|
|
2288
|
+
);
|
|
2289
|
+
if (state.desyncedIndexes.has(indexName)) {
|
|
2290
|
+
discardLoadedCopies(state, pool, indexName);
|
|
2291
|
+
return;
|
|
2292
|
+
}
|
|
2293
|
+
state.scaledOutIndexes.add(indexName);
|
|
2294
|
+
state.lastAccessAt.set(indexName, Date.now());
|
|
2295
|
+
state.poolRetryDelayMs = POOL_RESTART_DELAY_MS;
|
|
2296
|
+
for (const action of buffered) enqueueReplication(state, indexName, action);
|
|
2297
|
+
dispatched = true;
|
|
2298
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
2299
|
+
} catch (err) {
|
|
2300
|
+
if (dropReason !== void 0) state.droppedCopies.set(indexName, dropReason);
|
|
2301
|
+
throw err;
|
|
2302
|
+
} finally {
|
|
2303
|
+
endCopyLoadBuffer(state, indexName, buffered);
|
|
2304
|
+
if (!dispatched) for (const action of buffered) releasePublicationOf(action);
|
|
2305
|
+
}
|
|
2306
|
+
if (reload) state.copyReloadCounts.set(indexName, (state.copyReloadCounts.get(indexName) ?? 0) + 1);
|
|
2307
|
+
refreshVectorCopies(state, indexName);
|
|
2308
|
+
scheduleIdleMerge(state, indexName);
|
|
2309
|
+
state.callbacks?.onCopiesLoaded?.(pool.workerCount, reason);
|
|
2310
|
+
}
|
|
2311
|
+
async function loadAfter(previous, state, indexName, reason) {
|
|
2312
|
+
if (previous !== void 0) await previous.done;
|
|
2313
|
+
if (state.poolRepair !== null) await state.poolRepair;
|
|
2314
|
+
if (!copiesAllowed(state)) return;
|
|
2315
|
+
if (state.scaledOutIndexes.has(indexName) || state.desyncedIndexes.has(indexName)) return;
|
|
2316
|
+
await loadCopies(state, indexName, reason);
|
|
2317
|
+
}
|
|
2318
|
+
function scaleOutIndex(state, indexName, reason) {
|
|
2319
|
+
const previous = state.copyTransitions.get(indexName);
|
|
2320
|
+
if (previous !== void 0 && previous.kind !== "drop") return previous.done;
|
|
2321
|
+
const kind = state.droppedCopies.has(indexName) ? "reload" : "load";
|
|
2322
|
+
const done = loadAfter(previous, state, indexName, reason).catch((err) => {
|
|
2323
|
+
const error = toError(err);
|
|
2324
|
+
state.scaleOutBlocked = isDeterministicFailure(error);
|
|
2325
|
+
state.callbacks?.onCopyLoadFailure?.(reason, error, !state.scaleOutBlocked);
|
|
2326
|
+
}).finally(() => {
|
|
2327
|
+
if (state.copyTransitions.get(indexName)?.done === done) state.copyTransitions.delete(indexName);
|
|
2328
|
+
});
|
|
2329
|
+
state.copyTransitions.set(indexName, { kind, done });
|
|
2330
|
+
return done;
|
|
2331
|
+
}
|
|
2332
|
+
async function scaleOutReadyIndexes(state) {
|
|
2333
|
+
if (!copiesAllowed(state)) return;
|
|
2334
|
+
for (const indexName of state.indexRegistry.keys()) {
|
|
2335
|
+
if (!indexReadyForCopies(state, indexName)) continue;
|
|
2336
|
+
const reason = copyThresholdReason(state, indexName);
|
|
2337
|
+
setTimeout(() => {
|
|
2338
|
+
void scaleOutIndex(state, indexName, reason);
|
|
2339
|
+
}, 0);
|
|
2340
|
+
}
|
|
2341
|
+
}
|
|
2342
|
+
async function scaleOutBeforeBatch(state, indexName, incomingCount) {
|
|
2343
|
+
if (incomingCount <= 0) return;
|
|
2344
|
+
const pending = state.copyTransitions.get(indexName);
|
|
2345
|
+
if (pending !== void 0) {
|
|
2346
|
+
if (pending.kind === "load") await pending.done;
|
|
2347
|
+
return;
|
|
2348
|
+
}
|
|
2349
|
+
if (!indexReadyForCopies(state, indexName, incomingCount)) return;
|
|
2350
|
+
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName, incomingCount));
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
// src/engine/orchestration/idle.ts
|
|
2354
|
+
function isIndexBusy(state, indexName) {
|
|
2355
|
+
return state.poolRepair !== null || state.copyTransitions.has(indexName) || state.replicationQueues.has(indexName) || state.compactionsInFlight.has(indexName);
|
|
2356
|
+
}
|
|
2357
|
+
function noteAccess(state, indexName) {
|
|
2358
|
+
state.lastAccessAt.set(indexName, Date.now());
|
|
2359
|
+
if (!copiesAllowed(state)) return;
|
|
2360
|
+
const reloadReason = state.droppedCopies.get(indexName);
|
|
2361
|
+
if (reloadReason !== void 0) {
|
|
2362
|
+
const transition = state.copyTransitions.get(indexName);
|
|
2363
|
+
if (transition !== void 0 && transition.kind !== "drop") return;
|
|
2364
|
+
void scaleOutIndex(state, indexName, reloadReason);
|
|
2365
|
+
return;
|
|
2366
|
+
}
|
|
2367
|
+
if (indexReadyForCopies(state, indexName)) {
|
|
2368
|
+
void scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
async function releaseCopies(state, indexName) {
|
|
2372
|
+
const pool = state.workerPool;
|
|
2373
|
+
if (pool === null) return;
|
|
2374
|
+
await awaitReplicationIdle(state, indexName);
|
|
2375
|
+
const cutoff = Date.now() - state.copyIdleTimeoutMs;
|
|
2376
|
+
if ((state.lastAccessAt.get(indexName) ?? Date.now()) > cutoff) return;
|
|
2377
|
+
if (!state.scaledOutIndexes.has(indexName) || state.compactionsInFlight.has(indexName)) return;
|
|
2378
|
+
state.scaledOutIndexes.delete(indexName);
|
|
2379
|
+
state.droppedCopies.set(indexName, COPY_RELOAD_REASON);
|
|
2380
|
+
cancelIdleMerge(state, indexName);
|
|
2381
|
+
const outcomes = await Promise.allSettled(
|
|
2382
|
+
pool.executorsHolding(indexName).map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `idle-drop-${indexName}` }))
|
|
2383
|
+
);
|
|
2384
|
+
for (const outcome of outcomes) {
|
|
2385
|
+
if (outcome.status === "rejected") console.warn("Dropping an idle worker copy failed:", outcome.reason);
|
|
2386
|
+
}
|
|
2387
|
+
pool.removeIndex(indexName);
|
|
2388
|
+
state.segmentLedger.delete(indexName);
|
|
2389
|
+
}
|
|
2390
|
+
async function dropIdleCopies(state, indexName) {
|
|
2391
|
+
const previous = state.copyTransitions.get(indexName);
|
|
2392
|
+
if (previous !== void 0) return;
|
|
2393
|
+
const done = releaseCopies(state, indexName).catch((err) => {
|
|
2394
|
+
console.warn("Dropping idle worker copies failed:", err);
|
|
2395
|
+
}).finally(() => {
|
|
2396
|
+
if (state.copyTransitions.get(indexName)?.done === done) state.copyTransitions.delete(indexName);
|
|
2397
|
+
});
|
|
2398
|
+
state.copyTransitions.set(indexName, { kind: "drop", done });
|
|
2399
|
+
await done;
|
|
2400
|
+
}
|
|
2401
|
+
async function sweepIdleCopies(state) {
|
|
2402
|
+
prunePublicationStores(state);
|
|
2403
|
+
const cutoff = Date.now() - state.copyIdleTimeoutMs;
|
|
2404
|
+
for (const indexName of [...state.scaledOutIndexes]) {
|
|
2405
|
+
if ((state.lastAccessAt.get(indexName) ?? Date.now()) > cutoff) continue;
|
|
2406
|
+
if (isIndexBusy(state, indexName)) continue;
|
|
2407
|
+
await dropIdleCopies(state, indexName);
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
function startIdleSweep(state) {
|
|
2411
|
+
if (!state.workersEnabled || state.idleSweep !== null) return;
|
|
2412
|
+
const interval = Math.min(MAX_SWEEP_INTERVAL_MS, Math.max(MIN_SWEEP_INTERVAL_MS, state.copyIdleTimeoutMs / 2));
|
|
2413
|
+
const timer = setInterval(() => void sweepIdleCopies(state).catch(() => void 0), interval);
|
|
2414
|
+
if (typeof timer.unref === "function") timer.unref();
|
|
2415
|
+
state.idleSweep = timer;
|
|
2416
|
+
}
|
|
2417
|
+
function stopIdleSweep(state) {
|
|
2418
|
+
if (state.idleSweep === null) return;
|
|
2419
|
+
clearInterval(state.idleSweep);
|
|
2420
|
+
state.idleSweep = null;
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
// src/engine/orchestration/release.ts
|
|
2424
|
+
async function releaseIndexCopies(state, indexName, steps) {
|
|
2425
|
+
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
2426
|
+
const loading = state.copyTransitions.get(indexName);
|
|
2427
|
+
if (loading !== void 0) await loading.done.catch(() => void 0);
|
|
2428
|
+
const previousPool = state.workerPool;
|
|
2429
|
+
await steps.awaitReplicationIdle(state, indexName);
|
|
2430
|
+
await previousPool?.whenRetiredCopiesAreGone(indexName);
|
|
2431
|
+
await state.retiredThreadsGone;
|
|
2432
|
+
steps.cancelIdleMerge(state, indexName);
|
|
2433
|
+
const reloads = wasScaledOut || loading !== void 0 && loading.kind !== "drop";
|
|
2434
|
+
const pool = state.workerPool;
|
|
2435
|
+
if (pool === null) return reloads;
|
|
2436
|
+
pool.removeIndex(indexName);
|
|
2437
|
+
state.segmentLedger.delete(indexName);
|
|
2438
|
+
await Promise.all(
|
|
2439
|
+
pool.executorEntries().map(
|
|
2440
|
+
({ executor }) => executor.execute({ type: "dropIndex", indexName, requestId: `restore-release-${indexName}` }).catch(async (error) => {
|
|
2441
|
+
if (steps.heldNowhere(error)) return;
|
|
2442
|
+
if (pool.getAllExecutors().includes(executor)) throw error;
|
|
2443
|
+
await pool.whenExecutorIsGone(executor);
|
|
2444
|
+
})
|
|
2445
|
+
)
|
|
2446
|
+
);
|
|
2447
|
+
return reloads;
|
|
2448
|
+
}
|
|
2449
|
+
async function endCopyRelease(state, indexName, reloads, steps) {
|
|
2450
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
2451
|
+
if (hold === void 0) return steps.resyncIndex(indexName, reloads);
|
|
2452
|
+
hold.holders -= 1;
|
|
2453
|
+
hold.reloads ||= reloads;
|
|
2454
|
+
if (hold.holders > 0) return;
|
|
2455
|
+
state.copyReleaseHolds.delete(indexName);
|
|
2456
|
+
await steps.resyncIndex(indexName, hold.reloads);
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
// src/core/pattern-index/matcher-budget.ts
|
|
2460
|
+
function widestPatternSplit(footprintBytes) {
|
|
2461
|
+
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2464
|
+
// src/core/pattern-index/query-positions.ts
|
|
2465
|
+
function largestPatternFootprint(expression, schema) {
|
|
2466
|
+
if (expression === void 0) return null;
|
|
2467
|
+
const module = availablePatternSearch();
|
|
2468
|
+
if (module === null) return null;
|
|
2469
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
2470
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
2471
|
+
return patternFields.size === 0 ? null : module.largestFootprint(expression, patternFields, patternHost());
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
// src/engine/orchestration/search.ts
|
|
2475
|
+
function partitionsPerLease(scope, leaseCount) {
|
|
2476
|
+
const assignments = Array.from({ length: leaseCount }, () => []);
|
|
2477
|
+
for (let at = 0; at < scope.length; at++) {
|
|
2478
|
+
assignments[at % leaseCount].push(scope[at]);
|
|
2479
|
+
}
|
|
2480
|
+
return assignments;
|
|
2481
|
+
}
|
|
2482
|
+
function mergedDocuments(results) {
|
|
2483
|
+
let documents;
|
|
2484
|
+
for (const result of results) {
|
|
2485
|
+
if (result.documents === void 0) continue;
|
|
2486
|
+
documents ??= /* @__PURE__ */ new Map();
|
|
2487
|
+
for (const [docId, document] of result.documents) documents.set(docId, document);
|
|
2488
|
+
}
|
|
2489
|
+
return documents;
|
|
2490
|
+
}
|
|
2491
|
+
function mergeWorkerResults(results, facetConfig) {
|
|
2492
|
+
const merged = kWayMerge(results.map((result) => result.scored));
|
|
2493
|
+
let totalMatched = 0;
|
|
2494
|
+
let patternWorkSpent = 0;
|
|
2495
|
+
const workerFacets = [];
|
|
2496
|
+
for (const result of results) {
|
|
2497
|
+
totalMatched += result.totalMatched;
|
|
2498
|
+
patternWorkSpent += result.patternWorkSpent ?? 0;
|
|
2499
|
+
if (result.facets !== void 0) workerFacets.push(result.facets);
|
|
2500
|
+
}
|
|
2501
|
+
const documents = mergedDocuments(results);
|
|
2502
|
+
return {
|
|
2503
|
+
scored: merged,
|
|
2504
|
+
totalMatched,
|
|
2505
|
+
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0,
|
|
2506
|
+
patternWorkSpent,
|
|
2507
|
+
...documents !== void 0 ? { documents } : {}
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
function splitReadsOf(params, reads) {
|
|
2511
|
+
if (reads === void 0 || reads.kind === "documents") return reads;
|
|
2512
|
+
return returnsEveryMatch(params) ? null : { kind: "documents" };
|
|
2513
|
+
}
|
|
2514
|
+
function scoresPerPartition(state, indexName, params, globalStats) {
|
|
2515
|
+
const mode = params.scoring ?? state.indexRegistry.get(indexName)?.config.defaultScoring ?? "local";
|
|
2516
|
+
if (mode === "local") return true;
|
|
2517
|
+
return mode === "broadcast" && globalStats !== void 0;
|
|
2518
|
+
}
|
|
2519
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork, bm25, reads, readGeneration) {
|
|
2520
|
+
const workerParams = params.group?.reduce === void 0 ? params : { ...params, group: { ...params.group, reduce: void 0 } };
|
|
2521
|
+
const workerReads = params.group?.reduce === void 0 ? reads : { kind: "documents" };
|
|
2522
|
+
return {
|
|
2523
|
+
type: "query",
|
|
2524
|
+
readGeneration,
|
|
2525
|
+
indexName,
|
|
2526
|
+
params: workerParams,
|
|
2527
|
+
requestId: createRequestId(),
|
|
2528
|
+
patternWork,
|
|
2529
|
+
bm25: bm25 ?? null,
|
|
2530
|
+
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
2531
|
+
...globalStats !== void 0 ? { globalStats } : {},
|
|
2532
|
+
...workerReads !== void 0 ? { reads: workerReads } : {}
|
|
2533
|
+
};
|
|
2534
|
+
}
|
|
2535
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork, bm25, reads, readGeneration) {
|
|
2536
|
+
const assignments = partitionsPerLease(scope, leases.length);
|
|
2537
|
+
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
2538
|
+
const outcomes = await Promise.allSettled(
|
|
2539
|
+
assignments.map(
|
|
2540
|
+
(partitionIds, at) => leases[at].executor.execute(
|
|
2541
|
+
queryAction(indexName, workerParams, globalStats, partitionIds, patternWork, bm25, reads, readGeneration)
|
|
2542
|
+
).finally(() => leases[at].release())
|
|
2543
|
+
)
|
|
2544
|
+
);
|
|
2545
|
+
const results = [];
|
|
2546
|
+
const failures = [];
|
|
2547
|
+
for (const outcome of outcomes) {
|
|
2548
|
+
if (outcome.status === "fulfilled") results.push(outcome.value);
|
|
2549
|
+
else failures.push(outcome.reason);
|
|
2550
|
+
}
|
|
2551
|
+
if (failures.length > 0) throw failures.find(failsTheSearch) ?? failures[0];
|
|
2552
|
+
return mergeWorkerResults(results, params.facets);
|
|
2553
|
+
}
|
|
2554
|
+
function failsTheSearch(err) {
|
|
2555
|
+
if (!(err instanceof NarsilError)) return false;
|
|
2556
|
+
return err.code === ErrorCodes.SEARCH_WORK_CAP_EXCEEDED || err.code === ErrorCodes.SEARCH_STATE_MOVED;
|
|
2557
|
+
}
|
|
2558
|
+
function patternSplitWidth(state, indexName, params) {
|
|
2559
|
+
const schema = state.indexRegistry.get(indexName)?.config.schema;
|
|
2560
|
+
if (schema === void 0) return Number.POSITIVE_INFINITY;
|
|
2561
|
+
const largest = largestPatternFootprint(params.filters, schema);
|
|
2562
|
+
if (largest === null) return Number.POSITIVE_INFINITY;
|
|
2563
|
+
if (typeof SharedArrayBuffer !== "function") return 1;
|
|
2564
|
+
return widestPatternSplit(largest);
|
|
2565
|
+
}
|
|
2566
|
+
function takeMainCopyTurn(state) {
|
|
2567
|
+
if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
|
|
2568
|
+
state.mainCopyTurnTaken = true;
|
|
2569
|
+
afterCurrentTurn(() => {
|
|
2570
|
+
state.mainCopyTurnTaken = false;
|
|
2571
|
+
});
|
|
2572
|
+
return true;
|
|
2573
|
+
}
|
|
2574
|
+
function answersFasterOnMainCopy(state, pool, manager) {
|
|
2575
|
+
if (pool.queriesInFlight() > 0) return false;
|
|
2576
|
+
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
2577
|
+
return takeMainCopyTurn(state);
|
|
2578
|
+
}
|
|
2579
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25, reads) {
|
|
2580
|
+
const pool = state.workerPool;
|
|
2581
|
+
if (!pool) return null;
|
|
2582
|
+
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
2583
|
+
const manager = state.executor.getManager(indexName);
|
|
2584
|
+
if (!manager) return null;
|
|
2585
|
+
if (answersFasterOnMainCopy(state, pool, manager)) return null;
|
|
2586
|
+
const readGeneration = copyGenerationOf(manager);
|
|
2587
|
+
await awaitReplicationIdle(state, indexName);
|
|
2588
|
+
const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
|
|
2589
|
+
const idle = pool.leaseIdle(Math.min(scope.length, patternSplitWidth(state, indexName, params)));
|
|
2590
|
+
const leases = [];
|
|
2591
|
+
let fork = null;
|
|
2592
|
+
const splitReads = splitReadsOf(params, reads);
|
|
2593
|
+
try {
|
|
2594
|
+
if (idle.length >= 2 && splitReads !== null && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
2595
|
+
leases.push(...idle);
|
|
2596
|
+
fork = patternWork.fork(leases.length);
|
|
2597
|
+
const merged = await runSplit(
|
|
2598
|
+
leases,
|
|
2599
|
+
scope,
|
|
2600
|
+
indexName,
|
|
2601
|
+
params,
|
|
2602
|
+
globalStats,
|
|
2603
|
+
fork.shared,
|
|
2604
|
+
bm25,
|
|
2605
|
+
splitReads,
|
|
2606
|
+
readGeneration
|
|
2607
|
+
);
|
|
2608
|
+
fork.settle(merged.patternWorkSpent ?? 0);
|
|
2609
|
+
return merged;
|
|
2610
|
+
}
|
|
2611
|
+
for (const lease2 of idle.slice(1)) lease2.release();
|
|
2612
|
+
const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
|
|
2613
|
+
if (lease === null) return null;
|
|
2614
|
+
leases.push(lease);
|
|
2615
|
+
fork = patternWork.fork(1);
|
|
2616
|
+
const result = await lease.executor.execute(
|
|
2617
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared, bm25, reads, readGeneration)
|
|
2618
|
+
);
|
|
2619
|
+
fork.settle(result.patternWorkSpent ?? 0);
|
|
2620
|
+
return result;
|
|
2621
|
+
} catch (err) {
|
|
2622
|
+
if (failsTheSearch(err)) throw err;
|
|
2623
|
+
fork?.settle(0);
|
|
2624
|
+
console.warn("Worker search failed, falling back to local:", err);
|
|
2625
|
+
return null;
|
|
2626
|
+
} finally {
|
|
2627
|
+
for (const lease of leases) lease.release();
|
|
2628
|
+
}
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
// src/engine/orchestration/segments.ts
|
|
2632
|
+
function segmentBuildConcurrency(state, indexName) {
|
|
2633
|
+
if (!state.scaledOutIndexes.has(indexName)) return 0;
|
|
2634
|
+
return state.workerPool?.workerCount ?? 0;
|
|
2635
|
+
}
|
|
2636
|
+
function builtSegmentsOf(request, results) {
|
|
2637
|
+
let start = 0;
|
|
2638
|
+
return results.map((result) => {
|
|
2639
|
+
const count = result.kind === "shared" ? result.snapshot.documentCount : result.payload.documentCount;
|
|
2640
|
+
const segmentId = result.kind === "shared" ? result.snapshot.segmentId : start === 0 ? request.action.segmentId : generateId();
|
|
2641
|
+
const base = {
|
|
2642
|
+
partitionId: request.partitionId,
|
|
2643
|
+
segmentId,
|
|
2644
|
+
documents: request.documents.slice(start, start + count),
|
|
2645
|
+
docIds: request.action.documents.slice(start, start + count).map((document) => document.docId)
|
|
2646
|
+
};
|
|
2647
|
+
start += count;
|
|
2648
|
+
if (result.kind === "shared") return { ...base, snapshot: result.snapshot, payload: null };
|
|
2649
|
+
return { ...base, snapshot: null, payload: result.payload };
|
|
2650
|
+
});
|
|
2651
|
+
}
|
|
2652
|
+
async function buildSegments(state, requests) {
|
|
2653
|
+
const pool = state.workerPool;
|
|
2654
|
+
if (!pool || requests.length === 0) return null;
|
|
2655
|
+
const leases = requests.map(() => pool.leaseLeastBusy());
|
|
2656
|
+
if (leases.some((lease) => lease === null)) {
|
|
2657
|
+
for (const lease of leases) lease?.release();
|
|
2658
|
+
return null;
|
|
2659
|
+
}
|
|
2660
|
+
try {
|
|
2661
|
+
const outputs = await Promise.all(
|
|
2662
|
+
requests.map((request, i) => {
|
|
2663
|
+
const lease = leases[i];
|
|
2664
|
+
if (lease === null) throw new Error("Segment build lease missing");
|
|
2665
|
+
return lease.executor.execute(request.action).then((result) => builtSegmentsOf(request, result));
|
|
2666
|
+
})
|
|
2667
|
+
);
|
|
2668
|
+
return outputs.flat();
|
|
2669
|
+
} finally {
|
|
2670
|
+
for (const lease of leases) lease?.release();
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
// src/engine/orchestration/index.ts
|
|
2675
|
+
function holdsCopies(state, indexName) {
|
|
2676
|
+
return state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName);
|
|
2677
|
+
}
|
|
2678
|
+
async function settleCopiedPublications(state, indexName, growth, room) {
|
|
2679
|
+
if (room !== null) await room;
|
|
2680
|
+
if (growth === "segments") maybeCompactSegments(state, indexName);
|
|
2681
|
+
if (!holdsCopies(state, indexName)) return;
|
|
2682
|
+
if (growth === "liveTail") await flushGrownTails(state, indexName);
|
|
2683
|
+
scheduleIdleMerge(state, indexName);
|
|
2684
|
+
}
|
|
2685
|
+
function copyIdleTimeoutBeforeClose(closeAfterMs) {
|
|
2686
|
+
return closeAfterMs === void 0 ? DEFAULT_COPY_IDLE_TIMEOUT_MS : Math.min(DEFAULT_COPY_IDLE_TIMEOUT_MS, closeAfterMs);
|
|
2687
|
+
}
|
|
2688
|
+
function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, vectorCopyPolicy) {
|
|
2689
|
+
let announceShutdown = () => void 0;
|
|
2690
|
+
const shutdownStarted = new Promise((resolve) => {
|
|
2691
|
+
announceShutdown = resolve;
|
|
2692
|
+
});
|
|
2693
|
+
const state = {
|
|
2694
|
+
config,
|
|
2695
|
+
executor,
|
|
2696
|
+
indexRegistry,
|
|
2697
|
+
callbacks,
|
|
2698
|
+
vectorCopyPolicy,
|
|
2699
|
+
workersEnabled: config?.workers?.enabled ?? workersEnabledByDefault(),
|
|
2700
|
+
keywordWorkerCount: splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).keyword,
|
|
2701
|
+
longQueryWorkerCount: 0,
|
|
2702
|
+
requestThreads: null,
|
|
2703
|
+
copyThreshold: config?.workers?.promotionThreshold ?? DEFAULT_COPY_THRESHOLD,
|
|
2704
|
+
copyIdleTimeoutMs: config?.workers?.idleTimeoutMs ?? copyIdleTimeoutBeforeClose(config?.lifecycle?.idleTimeoutMs),
|
|
2705
|
+
bootstrapModule: config?.workers?.bootstrapModule,
|
|
2706
|
+
reportedIneligible: /* @__PURE__ */ new Set(),
|
|
2707
|
+
scaledOutIndexes: /* @__PURE__ */ new Set(),
|
|
2708
|
+
desyncedIndexes: /* @__PURE__ */ new Set(),
|
|
2709
|
+
copyReleaseHolds: /* @__PURE__ */ new Map(),
|
|
2710
|
+
copyLoadBuffers: /* @__PURE__ */ new Map(),
|
|
2711
|
+
copyLoadBufferHolds: /* @__PURE__ */ new WeakMap(),
|
|
2712
|
+
copyTransitions: /* @__PURE__ */ new Map(),
|
|
2713
|
+
droppedCopies: /* @__PURE__ */ new Map(),
|
|
2714
|
+
lastAccessAt: /* @__PURE__ */ new Map(),
|
|
2715
|
+
copyReloadCounts: /* @__PURE__ */ new Map(),
|
|
2716
|
+
replicationQueues: /* @__PURE__ */ new Map(),
|
|
2717
|
+
segmentLedger: /* @__PURE__ */ new Map(),
|
|
2718
|
+
compactionsInFlight: /* @__PURE__ */ new Map(),
|
|
2719
|
+
idleMergeTimers: /* @__PURE__ */ new Map(),
|
|
2720
|
+
sharedVectorFields: /* @__PURE__ */ new Map(),
|
|
2721
|
+
publishedVectorStores: /* @__PURE__ */ new Set(),
|
|
2722
|
+
workerPool: null,
|
|
2723
|
+
retiredThreadsGone: Promise.resolve(),
|
|
2724
|
+
shutdownStarted,
|
|
2725
|
+
announceShutdown,
|
|
2726
|
+
poolStart: null,
|
|
2727
|
+
poolRetryAt: 0,
|
|
2728
|
+
poolRetryDelayMs: POOL_RESTART_DELAY_MS,
|
|
2729
|
+
poolRepair: null,
|
|
2730
|
+
threadsLeftRunning: 0,
|
|
2731
|
+
maxThreadsLeftRunning: config?.workers?.maxThreadsLeftRunning ?? DEFAULT_MAX_THREADS_LEFT_RUNNING,
|
|
2732
|
+
mainCopyTurnTaken: false,
|
|
2733
|
+
mainCopyQueries: config?.workers?.mainCopyQueries,
|
|
2734
|
+
repairTimer: null,
|
|
2735
|
+
scaleOutBlocked: false,
|
|
2736
|
+
shuttingDown: false,
|
|
2737
|
+
idleSweep: null
|
|
2738
|
+
};
|
|
2739
|
+
startIdleSweep(state);
|
|
2740
|
+
function settlingCopyWork(indexName) {
|
|
2741
|
+
return state.copyTransitions.get(indexName)?.done ?? state.compactionsInFlight.get(indexName);
|
|
2742
|
+
}
|
|
2743
|
+
async function resyncIndex(indexName, wasScaledOut) {
|
|
2744
|
+
for (let pending = settlingCopyWork(indexName); pending !== void 0; pending = settlingCopyWork(indexName)) {
|
|
2745
|
+
await pending;
|
|
2746
|
+
}
|
|
2747
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
2748
|
+
if (hold !== void 0) {
|
|
2749
|
+
hold.reloads ||= wasScaledOut;
|
|
2750
|
+
return;
|
|
2751
|
+
}
|
|
2752
|
+
state.desyncedIndexes.delete(indexName);
|
|
2753
|
+
if (!wasScaledOut) return;
|
|
2754
|
+
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
2755
|
+
}
|
|
2756
|
+
const copyReleaseSteps = {
|
|
2757
|
+
awaitReplicationIdle,
|
|
2758
|
+
cancelIdleMerge,
|
|
2759
|
+
heldNowhere: (error) => error instanceof NarsilError && error.code === ErrorCodes.INDEX_NOT_FOUND,
|
|
2760
|
+
resyncIndex
|
|
2761
|
+
};
|
|
2762
|
+
async function openIndex(indexName) {
|
|
2763
|
+
state.droppedCopies.delete(indexName);
|
|
2764
|
+
if (!indexReadyForCopies(state, indexName)) return;
|
|
2765
|
+
await scaleOutIndex(state, indexName, copyThresholdReason(state, indexName));
|
|
2766
|
+
}
|
|
2767
|
+
async function getWorkerMemoryStats() {
|
|
2768
|
+
if (!state.workerPool) return [];
|
|
2769
|
+
const reports = await state.workerPool.getMemoryStats();
|
|
2770
|
+
return reports.map((report) => ({
|
|
2771
|
+
workerId: report.workerId,
|
|
2772
|
+
heapUsed: report.heapUsed,
|
|
2773
|
+
heapTotal: report.heapTotal,
|
|
2774
|
+
heapLimit: report.heapLimit,
|
|
2775
|
+
external: report.external
|
|
2776
|
+
}));
|
|
2777
|
+
}
|
|
2778
|
+
function workerCopies() {
|
|
2779
|
+
const reports = [];
|
|
2780
|
+
for (const indexName of indexRegistry.keys()) {
|
|
2781
|
+
reports.push({
|
|
2782
|
+
indexName,
|
|
2783
|
+
scaledOut: state.scaledOutIndexes.has(indexName),
|
|
2784
|
+
reloadCount: state.copyReloadCounts.get(indexName) ?? 0
|
|
2785
|
+
});
|
|
2786
|
+
}
|
|
2787
|
+
return reports;
|
|
2788
|
+
}
|
|
2789
|
+
async function shutdown() {
|
|
2790
|
+
state.shuttingDown = true;
|
|
2791
|
+
state.announceShutdown();
|
|
2792
|
+
stopIdleSweep(state);
|
|
2793
|
+
cancelRepair(state);
|
|
2794
|
+
stopRequestThreads(state);
|
|
2795
|
+
for (const indexName of [...state.idleMergeTimers.keys()]) cancelIdleMerge(state, indexName);
|
|
2796
|
+
await Promise.allSettled([...state.copyTransitions.values()].map((transition) => transition.done));
|
|
2797
|
+
if (state.poolRepair !== null) await state.poolRepair;
|
|
2798
|
+
if (state.poolStart !== null) await state.poolStart.catch(() => void 0);
|
|
2799
|
+
let threadsGone = state.retiredThreadsGone;
|
|
2800
|
+
if (state.workerPool) {
|
|
2801
|
+
await awaitCompactions(state);
|
|
2802
|
+
await awaitReplicationIdle(state);
|
|
2803
|
+
await state.workerPool.shutdown();
|
|
2804
|
+
threadsGone = Promise.all([threadsGone, state.workerPool.whenEveryThreadIsGone()]).then(() => void 0);
|
|
2805
|
+
state.workerPool = null;
|
|
2806
|
+
state.scaledOutIndexes.clear();
|
|
2807
|
+
state.replicationQueues.clear();
|
|
2808
|
+
state.segmentLedger.clear();
|
|
2809
|
+
}
|
|
2810
|
+
void threadsGone.then(() => {
|
|
2811
|
+
state.sharedVectorFields.clear();
|
|
2812
|
+
state.publishedVectorStores.clear();
|
|
2813
|
+
});
|
|
2814
|
+
}
|
|
2815
|
+
async function restoreAfterFailedClose(indexName, failure) {
|
|
2816
|
+
const pool = state.workerPool;
|
|
2817
|
+
const entry = indexRegistry.get(indexName);
|
|
2818
|
+
const manager = executor.getManager(indexName);
|
|
2819
|
+
if (pool !== null && entry !== void 0 && manager !== void 0) {
|
|
2820
|
+
try {
|
|
2821
|
+
const copyLoad = await transferIndexToPool(
|
|
2822
|
+
indexName,
|
|
2823
|
+
pool,
|
|
2824
|
+
{ ...entry, onPublication: (publication) => trackPublicationStores(state, publication) },
|
|
2825
|
+
manager,
|
|
2826
|
+
callbacks?.isAnalysisStale?.(indexName)
|
|
2827
|
+
);
|
|
2828
|
+
state.scaledOutIndexes.add(indexName);
|
|
2829
|
+
markCopiesLoaded(state, indexName, copyLoad);
|
|
2830
|
+
refreshVectorCopies(state, indexName);
|
|
2831
|
+
} catch (repairError) {
|
|
2832
|
+
throw new NarsilError(
|
|
2833
|
+
ErrorCodes.WORKER_CRASHED,
|
|
2834
|
+
`Worker copies for index "${indexName}" could not be restored after close failed`,
|
|
2835
|
+
{
|
|
2836
|
+
indexName,
|
|
2837
|
+
closeError: failure.reason instanceof Error ? failure.reason.message : String(failure.reason),
|
|
2838
|
+
repairError: repairError instanceof Error ? repairError.message : String(repairError)
|
|
2839
|
+
}
|
|
2840
|
+
);
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
if (failure.reason instanceof NarsilError) throw failure.reason;
|
|
2844
|
+
throw new NarsilError(ErrorCodes.WORKER_CRASHED, `A worker could not close index "${indexName}"`, {
|
|
2845
|
+
indexName,
|
|
2846
|
+
cause: failure.reason instanceof Error ? failure.reason.message : String(failure.reason)
|
|
2847
|
+
});
|
|
2848
|
+
}
|
|
2849
|
+
function withdrawCopies(indexName) {
|
|
2850
|
+
const pool = state.workerPool;
|
|
2851
|
+
if (pool === null) return;
|
|
2852
|
+
const holders = pool.executorsHolding(indexName);
|
|
2853
|
+
pool.removeIndex(indexName);
|
|
2854
|
+
state.segmentLedger.delete(indexName);
|
|
2855
|
+
void Promise.allSettled(
|
|
2856
|
+
holders.map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `desync-drop-${indexName}` }))
|
|
2857
|
+
);
|
|
2858
|
+
}
|
|
2859
|
+
async function closeIndex(indexName) {
|
|
2860
|
+
if (state.poolRepair !== null) await state.poolRepair;
|
|
2861
|
+
const transition = state.copyTransitions.get(indexName);
|
|
2862
|
+
if (transition !== void 0) await transition.done;
|
|
2863
|
+
await awaitReplicationIdle(state, indexName);
|
|
2864
|
+
await awaitCompactions(state);
|
|
2865
|
+
cancelIdleMerge(state, indexName);
|
|
2866
|
+
const pool = state.workerPool;
|
|
2867
|
+
if (pool !== null && state.scaledOutIndexes.has(indexName)) {
|
|
2868
|
+
state.scaledOutIndexes.delete(indexName);
|
|
2869
|
+
const outcomes = await Promise.allSettled(
|
|
2870
|
+
pool.executorsHolding(indexName).map((worker) => worker.execute({ type: "dropIndex", indexName, requestId: `close-${indexName}` }))
|
|
2871
|
+
);
|
|
2872
|
+
const failure = outcomes.find((outcome) => outcome.status === "rejected");
|
|
2873
|
+
if (failure?.status === "rejected") await restoreAfterFailedClose(indexName, failure);
|
|
2874
|
+
pool.removeIndex(indexName);
|
|
2875
|
+
}
|
|
2876
|
+
state.droppedCopies.delete(indexName);
|
|
2877
|
+
if (!state.copyReleaseHolds.has(indexName)) state.desyncedIndexes.delete(indexName);
|
|
2878
|
+
state.lastAccessAt.delete(indexName);
|
|
2879
|
+
state.replicationQueues.delete(indexName);
|
|
2880
|
+
state.segmentLedger.delete(indexName);
|
|
2881
|
+
}
|
|
2882
|
+
async function replicate(action) {
|
|
2883
|
+
await replicateToWorkers(state, action);
|
|
2884
|
+
if (!("indexName" in action)) return;
|
|
2885
|
+
if (action.type === "attachSegments") maybeCompactSegments(state, action.indexName);
|
|
2886
|
+
if (action.type === "insert" || action.type === "update") await flushGrownTails(state, action.indexName);
|
|
2887
|
+
if (state.scaledOutIndexes.has(action.indexName) || state.copyLoadBuffers.has(action.indexName)) {
|
|
2888
|
+
scheduleIdleMerge(state, action.indexName);
|
|
2889
|
+
}
|
|
2890
|
+
}
|
|
2891
|
+
return {
|
|
2892
|
+
scaleOutReadyIndexes: () => scaleOutReadyIndexes(state),
|
|
2893
|
+
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
2894
|
+
replicateToWorkers: replicate,
|
|
2895
|
+
publishToWorkers: (action) => publishToWorkers2(state, action),
|
|
2896
|
+
settlePublications(indexName, growth = "liveTail") {
|
|
2897
|
+
const room = publicationCapacityWait(state, indexName);
|
|
2898
|
+
if (room === null && (growth === "none" || !holdsCopies(state, indexName))) {
|
|
2899
|
+
if (growth === "segments") maybeCompactSegments(state, indexName);
|
|
2900
|
+
if (holdsCopies(state, indexName)) scheduleIdleMerge(state, indexName);
|
|
2901
|
+
return void 0;
|
|
2902
|
+
}
|
|
2903
|
+
return settleCopiedPublications(state, indexName, growth, room);
|
|
2904
|
+
},
|
|
2905
|
+
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
2906
|
+
queueDiskAdoption: (action) => {
|
|
2907
|
+
publishToWorkers2(state, action);
|
|
2908
|
+
},
|
|
2909
|
+
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
2910
|
+
awaitCompactions: () => awaitCompactions(state),
|
|
2911
|
+
openIndex,
|
|
2912
|
+
closeIndex,
|
|
2913
|
+
isIndexBusy: (indexName) => isIndexBusy(state, indexName),
|
|
2914
|
+
buildSegments: (requests) => buildSegments(state, requests),
|
|
2915
|
+
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
2916
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25, reads) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25, reads),
|
|
2917
|
+
hasWorkerPool: () => state.workerPool !== null,
|
|
2918
|
+
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
2919
|
+
shareMainThread: () => {
|
|
2920
|
+
if (state.mainCopyQueries === void 0) state.mainCopyQueries = "none";
|
|
2921
|
+
},
|
|
2922
|
+
serveRequestsOnWorkers: (listener) => serveRequestsOnWorkers(state, listener),
|
|
2923
|
+
requestThreadCount: () => requestThreadCountOf(state),
|
|
2924
|
+
copyIdleTimeoutMs: () => state.copyIdleTimeoutMs,
|
|
2925
|
+
stopRequestThreads: () => stopRequestThreads(state),
|
|
2926
|
+
desyncIndex: (indexName) => {
|
|
2927
|
+
state.desyncedIndexes.add(indexName);
|
|
2928
|
+
const loading = state.copyTransitions.get(indexName);
|
|
2929
|
+
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
2930
|
+
if (wasScaledOut) withdrawCopies(indexName);
|
|
2931
|
+
return wasScaledOut || loading !== void 0 && loading.kind !== "drop";
|
|
2932
|
+
},
|
|
2933
|
+
async releaseIndexCopies(indexName) {
|
|
2934
|
+
const hold = state.copyReleaseHolds.get(indexName);
|
|
2935
|
+
if (hold === void 0) state.copyReleaseHolds.set(indexName, { holders: 1, reloads: false });
|
|
2936
|
+
else hold.holders += 1;
|
|
2937
|
+
state.desyncedIndexes.add(indexName);
|
|
2938
|
+
return releaseIndexCopies(state, indexName, copyReleaseSteps);
|
|
2939
|
+
},
|
|
2940
|
+
async endCopyRelease(indexName, reloads) {
|
|
2941
|
+
await endCopyRelease(state, indexName, reloads, copyReleaseSteps);
|
|
2942
|
+
},
|
|
2943
|
+
resyncIndex,
|
|
2944
|
+
noteAccess: (indexName) => noteAccess(state, indexName),
|
|
2945
|
+
workerCopies,
|
|
2946
|
+
getWorkerMemoryStats,
|
|
2947
|
+
shutdown
|
|
2948
|
+
};
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2951
|
+
export { bindWorkerCopiesHost, createWorkerExecutor, createWorkerOrchestrator, createWorkerPool, emptySerializablePartition, endCopyRelease, loadIndexOntoWorkers, publishToWorkers, releaseIndexCopies, releasePublicationOf2 as releasePublicationOf, retainPublicationOf2 as retainPublicationOf, runSegmentAction, transferIndexToPool, validateWorkerLiveness };
|