@delali/narsil 0.3.1-next.157 → 0.3.1-next.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +3 -3
- package/dist/adapters/gcs.mjs +3 -3
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +3 -3
- package/dist/browser/{chunk-N23GL72Q.mjs → chunk-4RNSFVWI.mjs} +20777 -19056
- package/dist/browser/chunk-MMHIIL5B.mjs +564 -0
- package/dist/browser/chunk-MZHQ65VI.mjs +1228 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/{chunk-VLJQW4J6.mjs → core-leaf-TJAGWPZZ.mjs} +4674 -3102
- package/dist/{react-native/chunks/durability-config-QXVZEO7T.mjs → browser/durability-config-SGB5ZP5K.mjs} +8 -3
- package/dist/browser/embed-RF2MPGAO.mjs +222 -0
- package/dist/{react-native/chunks/geo-index-PX3BIHAH.mjs → browser/geo-index-GNHY4PUB.mjs} +190 -93
- package/dist/{react-native/chunks/module-TTEVPRCW.mjs → browser/module-362VSIGR.mjs} +1570 -275
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-WMZX4OEC.mjs +3196 -0
- package/dist/browser/phrase-module-22TKMXWI.mjs +616 -0
- package/dist/browser/resolve-vector-text-KI65IHMD.mjs +60 -0
- package/dist/browser/{restore-hooks-IE3GGYHG.mjs → restore-queue-S4BWOGMU.mjs} +98 -24
- package/dist/{react-native/chunks/runtime-R5E3TB3J.mjs → browser/runtime-NU6VOVH2.mjs} +61 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-X6CNRHJS.mjs +18221 -0
- package/dist/browser/store-leaf-AEMTAVHX.mjs +2069 -0
- package/dist/browser/{chunk-GFAZPSWK.mjs → suggestions-leaf-3Y6MR6OF.mjs} +441 -119
- package/dist/browser/worker-copies-WQCPP3TS.mjs +2750 -0
- package/dist/{capabilities-D3AoBJoF.d.ts → capabilities-BKDrry0z.d.ts} +2 -2
- package/dist/chunk-2JRERUNZ.mjs +217 -0
- package/dist/chunk-2RIS76RJ.mjs +212 -0
- package/dist/chunk-3B775F3C.mjs +252 -0
- package/dist/{chunk-24FNVRKB.mjs → chunk-3D6RZGJH.mjs} +3 -3
- package/dist/{chunk-INJYO3BU.mjs → chunk-4CAMH7FO.mjs} +1 -1
- package/dist/{chunk-U476IXKQ.mjs → chunk-4PY4JYMY.mjs} +2 -1
- package/dist/chunk-6EZW2GQL.mjs +978 -0
- package/dist/chunk-75UNINFA.mjs +271 -0
- package/dist/chunk-77AUVDQ5.mjs +7 -0
- package/dist/{chunk-JTYPJF3T.mjs → chunk-7MWYORQC.mjs} +2 -2
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-XIHUI4KB.mjs → chunk-AB27FWZM.mjs} +8 -3
- package/dist/{chunk-FUD5XSZ5.mjs → chunk-BXM7XM4G.mjs} +7 -3
- package/dist/{chunk-HJJI72X7.mjs → chunk-C7YQHCSO.mjs} +1 -1
- package/dist/{chunk-RVGRFOGN.mjs → chunk-D456ZFPQ.mjs} +2 -1
- package/dist/{chunk-KKQC526K.mjs → chunk-E2F6LZR3.mjs} +19 -21
- package/dist/{chunk-SMOOSIAW.mjs → chunk-FE3MRIBN.mjs} +104 -13
- package/dist/{chunk-KYYABCSW.mjs → chunk-FETZYCH7.mjs} +15 -1
- package/dist/{chunk-XXJZMTZT.mjs → chunk-FVZ5DB6Y.mjs} +3 -2
- package/dist/{chunk-EYD7K3H6.mjs → chunk-GF4K7IUK.mjs} +317 -120
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/{chunk-IIRIG7H5.mjs → chunk-H64YDF5M.mjs} +3 -3
- package/dist/{browser/chunk-VY5KW3HN.mjs → chunk-HOOR4CP5.mjs} +63 -7
- package/dist/chunk-IB4BYLE3.mjs +127 -0
- package/dist/{chunk-3C7MTCCI.mjs → chunk-IE3Q62QD.mjs} +2 -2
- package/dist/chunk-IH3G6SJO.mjs +40 -0
- package/dist/chunk-JV74RKCY.mjs +3041 -0
- package/dist/{chunk-ZPZ3YGXV.mjs → chunk-K67I3RF4.mjs} +261 -101
- package/dist/chunk-KLVMK3R5.mjs +47 -0
- package/dist/{chunk-5VV4R6P7.mjs → chunk-LAWIJRMJ.mjs} +4154 -2577
- package/dist/chunk-LF4GB4PO.mjs +281 -0
- package/dist/chunk-NC4QLCZC.mjs +59 -0
- package/dist/chunk-OOO54OVC.mjs +627 -0
- package/dist/chunk-OUWZKNO3.mjs +207 -0
- package/dist/{chunk-FJKKCV4D.mjs → chunk-P5YLI5ED.mjs} +343 -182
- package/dist/{chunk-4ZTHGTVX.mjs → chunk-PTTMGRG5.mjs} +9 -10
- package/dist/{react-native/chunks/chunk-DZ7H7D2C.mjs → chunk-Q3ZZH5Z7.mjs} +6 -0
- package/dist/{browser/chunk-EKMGCRSR.mjs → chunk-QBVTYY4J.mjs} +2 -9
- package/dist/{chunk-W5EB5XKI.mjs → chunk-R4CE4IVU.mjs} +341 -183
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/{chunk-KJ3RTZDJ.mjs → chunk-S3BSDBF7.mjs} +10 -3
- package/dist/{chunk-4LMSZAK6.mjs → chunk-SHGN3QIF.mjs} +69 -13
- package/dist/chunk-SXIRM4MX.mjs +3549 -0
- package/dist/{chunk-TBC3XUL2.mjs → chunk-TGPKS2UM.mjs} +47 -1
- package/dist/chunk-TINW4BSH.mjs +31301 -0
- package/dist/{cold-module-VOJXMNZB.mjs → chunk-TSLV5P2J.mjs} +397 -80
- package/dist/{chunk-HHBUCIUY.mjs → chunk-TW543F33.mjs} +3 -3
- package/dist/chunk-U6B7DUQW.mjs +788 -0
- package/dist/{chunk-OQLD2BSW.mjs → chunk-V3SGPXWQ.mjs} +8 -9
- package/dist/chunk-W34VJXY7.mjs +4723 -0
- package/dist/chunk-YETKPX5M.mjs +5 -0
- package/dist/chunk-ZQ6QY42L.mjs +59 -0
- package/dist/{client-kqSMevVi.d.ts → client-CTp8RRiK.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +66 -18
- package/dist/{config-B4v_Htpu.d.ts → config-PCNzWjHL.d.ts} +6 -3
- package/dist/{config-CTP01fns.d.ts → config-_WT5grSL.d.ts} +3 -3
- package/dist/core-leaf-KXFGP74A.mjs +7 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-MKTZBR26.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +7 -4
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -2
- package/dist/distribution/transport/grpc.mjs +7 -2
- package/dist/distribution/transport/tcp.mjs +7 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +1199 -1092
- package/dist/{browser/durability-config-TIYKW6XX.mjs → durability-config-XV4XR7Y4.mjs} +7 -3
- package/dist/embed-4H7RTBVL.mjs +221 -0
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CkPn2PWE.d.ts → engine-DBRVR_d-.d.ts} +1252 -1239
- package/dist/{browser/geo-index-OTV7VPIR.mjs → geo-index-2ZMMWWSK.mjs} +190 -93
- package/dist/{index-C4AmUnl1.d.ts → index-Ck9w5WHz.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -16
- package/dist/index.d.ts +1495 -78
- package/dist/index.mjs +14 -50
- package/dist/index.react-native.mjs +11 -28
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/{long-query-thread-QDFQSJD6.mjs → long-query-thread-JMGNGF6V.mjs} +74 -58
- package/dist/{module-N55W4Q5C.mjs → module-ZJ6ARG6V.mjs} +444 -111
- package/dist/msgpack-leaf-5WFRP6JM.mjs +9 -0
- package/dist/optional-reads-BR3BOVZW.mjs +3373 -0
- package/dist/optional-writes-DDFHYUOH.mjs +218 -0
- package/dist/{paged-partition-archive-RLRGOXDX.mjs → paged-partition-archive-G576XGVE.mjs} +29 -69
- package/dist/partial-words.browser.mjs +1649 -0
- package/dist/partial-words.mjs +15 -14
- package/dist/{chunk-CITFCZKF.mjs → pattern.browser.mjs} +153 -186
- package/dist/pattern.mjs +6 -8
- package/dist/persistence/durability/checkpoint-worker.d.ts +32 -24
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -59
- package/dist/phrase-module-XK3T44VU.mjs +634 -0
- package/dist/query-actions-OMXJEC2J.mjs +118 -0
- package/dist/query-suggestions/merge-worker.mjs +18 -54
- package/dist/query-suggestions.browser.mjs +26 -227
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +16 -67
- package/dist/react-native/chunks/chunk-4EXKQ2IB.mjs +565 -0
- package/dist/react-native/chunks/chunk-52QD7CBJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-72OKYHZL.mjs +969 -0
- package/dist/react-native/chunks/chunk-AQOWRH2C.mjs +802 -0
- package/dist/react-native/chunks/{chunk-4COWCEY7.mjs → chunk-FTQ7JNMU.mjs} +341 -180
- package/dist/react-native/chunks/chunk-G3CUQ333.mjs +42 -0
- package/dist/react-native/chunks/{chunk-EJ33LCHV.mjs → chunk-HRDMIQ63.mjs} +53 -1
- package/dist/react-native/chunks/{chunk-4AV5PPH6.mjs → chunk-IY6U6HRA.mjs} +63 -7
- package/dist/react-native/chunks/chunk-J2NQW353.mjs +212 -0
- package/dist/react-native/chunks/chunk-L4R5ITVG.mjs +29047 -0
- package/dist/react-native/chunks/{chunk-OXM66LO6.mjs → chunk-OK3WUA6N.mjs} +2 -10
- package/dist/react-native/chunks/{chunk-MVXELMJX.mjs → chunk-PVMEHBQO.mjs} +1 -1
- package/dist/{browser/chunk-SRMZXJSL.mjs → react-native/chunks/chunk-TQ26BKBH.mjs} +6 -0
- package/dist/{browser/chunk-GPPOG6PT.mjs → react-native/chunks/chunk-VUJCSQG5.mjs} +3 -1
- package/dist/react-native/chunks/{chunk-ENLMYKSX.mjs → chunk-XEGKQKLY.mjs} +5385 -3114
- package/dist/react-native/chunks/chunk-YOTP2FPE.mjs +232 -0
- package/dist/react-native/chunks/chunk-ZIU6TWXG.mjs +127 -0
- package/dist/react-native/chunks/core-leaf-5B4JXCVA.mjs +6 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/{durability-config-GE4U7MCZ.mjs → react-native/chunks/durability-config-3IOXIPYZ.mjs} +7 -3
- package/dist/react-native/chunks/embed-3WQ2CCR5.mjs +221 -0
- package/dist/{geo-index-MRH5JMIR.mjs → react-native/chunks/geo-index-QEYKEVPH.mjs} +190 -93
- package/dist/react-native/chunks/{chunk-CKNRFDTZ.mjs → gzip.react-native-DLO4WW27.mjs} +2 -1
- package/dist/{browser/module-TCT66FWA.mjs → react-native/chunks/module-VUA27HZZ.mjs} +1018 -540
- package/dist/react-native/chunks/msgpack-leaf-TR66ZVAX.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-WDTLLHUT.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-FZV3QW34.mjs +267 -0
- package/dist/react-native/chunks/phrase-module-KCQWVXYZ.mjs +635 -0
- package/dist/react-native/chunks/resolve-vector-text-LXRYXF7M.mjs +59 -0
- package/dist/react-native/chunks/restore-queue-EI2V2I4S.mjs +179 -0
- package/dist/{browser/runtime-OTAGEFQW.mjs → react-native/chunks/runtime-BUCSAK4U.mjs} +61 -25
- package/dist/react-native/chunks/segment-codec-leaf-6ULP7H3I.mjs +12 -0
- package/dist/react-native/chunks/segment-merge-leaf-F4GQZNK6.mjs +13 -0
- package/dist/react-native/chunks/storage-leaf-OAXBU2L6.mjs +18042 -0
- package/dist/react-native/chunks/store-leaf-GY2ZZFXB.mjs +14 -0
- package/dist/react-native/chunks/{chunk-WW2X2AEI.mjs → suggestions-leaf-SQEZ4IBS.mjs} +332 -118
- package/dist/react-native/chunks/worker-copies-EEHSCFJW.mjs +2627 -0
- package/dist/react-native/partial-words.mjs +954 -35
- package/dist/react-native/pattern.mjs +238 -111
- package/dist/react-native/query-suggestions.mjs +33 -239
- package/dist/react-native/sync.mjs +95 -52
- package/dist/react-native/synonyms.mjs +256 -112
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +6 -0
- package/dist/{rerank-MQ37IYRV.mjs → rerank-Y6MWBMTT.mjs} +17 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-4PDRTPNQ.mjs +2 -0
- package/dist/restore-queue-KPYSRJ4A.mjs +184 -0
- package/dist/runtime-4GKUIAKV.mjs +595 -0
- package/dist/{schema-Jwt928nJ.d.ts → schema-C_7yRJLt.d.ts} +1 -1
- package/dist/segment-codec-leaf-6IFBSBPN.mjs +17 -0
- package/dist/segment-merge-leaf-VE3FUSVE.mjs +17 -0
- package/dist/server.d.ts +33 -13
- package/dist/server.mjs +81 -100
- package/dist/{browser/durability-wiring-GVZEE46O.mjs → storage-leaf-HMIIOUYN.mjs} +2334 -931
- package/dist/store-leaf-M7T7SUZJ.mjs +19 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-Q3Q5D6XV.mjs +53 -0
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +27 -11
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +7 -2
- package/dist/sync.browser.mjs +88 -72
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +89 -59
- package/dist/{chunk-I5B7RXU7.mjs → synonyms.browser.mjs} +248 -135
- package/dist/synonyms.mjs +15 -11
- package/dist/{tasks-CoNjxbop.d.ts → tasks-B4tT4vmP.d.ts} +4 -4
- package/dist/thread-SNILJXX3.mjs +30 -0
- package/dist/{types-DPMjfOr_.d.ts → types-CTg0KaMy.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +7 -1
- package/dist/vector/search-worker.mjs +9 -13
- package/dist/worker-copies-ZLZS2CUM.mjs +2152 -0
- package/dist/workers/entry.mjs +60 -30
- package/openapi.json +66 -50
- package/package.json +233 -8
- package/dist/analysis-change-CKEMM2MC.mjs +0 -55
- package/dist/analysis-change-entry-SEDMJ6MW.mjs +0 -1
- package/dist/archive-history-E4R73RWD.mjs +0 -29
- package/dist/browser/analysis-change-ZKGJFENM.mjs +0 -942
- package/dist/browser/analysis-change-entry-5BFUVMHJ.mjs +0 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +0 -29
- package/dist/browser/chunk-25MTVDYD.mjs +0 -432
- package/dist/browser/chunk-3I27KXQD.mjs +0 -34
- package/dist/browser/chunk-3NLMB5CC.mjs +0 -217
- package/dist/browser/chunk-4RD3AFSB.mjs +0 -32
- package/dist/browser/chunk-5W3MMTLG.mjs +0 -19
- package/dist/browser/chunk-5YWB3E2W.mjs +0 -92
- package/dist/browser/chunk-7IGKGWSQ.mjs +0 -79
- package/dist/browser/chunk-AI5UI46D.mjs +0 -12
- package/dist/browser/chunk-ATEBYXP6.mjs +0 -10
- package/dist/browser/chunk-BLT4RHCJ.mjs +0 -241
- package/dist/browser/chunk-C26SPCGQ.mjs +0 -2636
- package/dist/browser/chunk-C4FJ3WDY.mjs +0 -177
- package/dist/browser/chunk-CXNTHHA7.mjs +0 -63
- package/dist/browser/chunk-DSXIBE6N.mjs +0 -78
- package/dist/browser/chunk-DUH6QUVU.mjs +0 -2374
- package/dist/browser/chunk-ELV4OZOI.mjs +0 -29
- package/dist/browser/chunk-EXQZGUQH.mjs +0 -193
- package/dist/browser/chunk-G64Q6OJF.mjs +0 -96
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-HUCSHDTB.mjs +0 -100
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-K3IPRX6D.mjs +0 -2559
- package/dist/browser/chunk-ML7THVLJ.mjs +0 -38
- package/dist/browser/chunk-MY2XKH22.mjs +0 -40
- package/dist/browser/chunk-NLWTBAGB.mjs +0 -111
- package/dist/browser/chunk-NXQG7DRA.mjs +0 -318
- package/dist/browser/chunk-OEDCEZXJ.mjs +0 -92
- package/dist/browser/chunk-OIZS5KIX.mjs +0 -6
- package/dist/browser/chunk-RA6LSBCM.mjs +0 -43
- package/dist/browser/chunk-TK63WFXK.mjs +0 -90
- package/dist/browser/chunk-TY7L6OXO.mjs +0 -69
- package/dist/browser/chunk-U3JOQGWX.mjs +0 -19
- package/dist/browser/chunk-U4YDIBUT.mjs +0 -653
- package/dist/browser/chunk-UATIJ4KU.mjs +0 -21
- package/dist/browser/chunk-UKOWD2K5.mjs +0 -315
- package/dist/browser/chunk-UTMHUHUV.mjs +0 -150
- package/dist/browser/chunk-V43N6TZO.mjs +0 -857
- package/dist/browser/chunk-VI7BQR4U.mjs +0 -21
- package/dist/browser/chunk-WN464PTL.mjs +0 -375
- package/dist/browser/chunk-WXLBZJMT.mjs +0 -25
- package/dist/browser/chunk-X7TXYUKO.mjs +0 -86
- package/dist/browser/chunk-XAWNZ4G2.mjs +0 -151
- package/dist/browser/chunk-Y6IBAUNO.mjs +0 -561
- package/dist/browser/chunk-ZI2V4VD7.mjs +0 -21
- package/dist/browser/cold-module-YNUUDNOM.mjs +0 -761
- package/dist/browser/engine-hooks-HBVS6YM4.mjs +0 -18
- package/dist/browser/gzip-5S7ROJFJ.mjs +0 -1
- package/dist/browser/import-5ESVMFDJ.mjs +0 -20
- package/dist/browser/methods-KTQCSIN4.mjs +0 -30
- package/dist/browser/module-NDSQLPMQ.mjs +0 -25
- package/dist/browser/operations-OJ3BXS55.mjs +0 -5
- package/dist/browser/phrase-module-HAT3TLZN.mjs +0 -328
- package/dist/browser/pool-3JLET5RO.mjs +0 -297
- package/dist/browser/release-HYXPVJVS.mjs +0 -32
- package/dist/browser/restore-queue-LQBBTODR.mjs +0 -55
- package/dist/browser/retention-TY3L3BFN.mjs +0 -211
- package/dist/browser/snapshot-5FHPPCNZ.mjs +0 -965
- package/dist/browser/start-2UZTOBG5.mjs +0 -26
- package/dist/browser/sync-state-P5XG6LTQ.mjs +0 -15
- package/dist/browser/write-ULXAN5B4.mjs +0 -196
- package/dist/chunk-2NE7VCYE.mjs +0 -212
- package/dist/chunk-2ROUFMQX.mjs +0 -543
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-3MXV3VHR.mjs +0 -46
- package/dist/chunk-3OH67G7R.mjs +0 -214
- package/dist/chunk-4B7JIGQD.mjs +0 -106
- package/dist/chunk-5YNRKI2C.mjs +0 -81
- package/dist/chunk-6HTGOXIK.mjs +0 -875
- package/dist/chunk-6OEGXLBX.mjs +0 -57
- package/dist/chunk-6RC4K46J.mjs +0 -86
- package/dist/chunk-7GFE3PEY.mjs +0 -40
- package/dist/chunk-7GXJUJ24.mjs +0 -49
- package/dist/chunk-7V5IUP2T.mjs +0 -63
- package/dist/chunk-A4L7YTKX.mjs +0 -458
- package/dist/chunk-BEZH7VEH.mjs +0 -23
- package/dist/chunk-BNL5OPRV.mjs +0 -634
- package/dist/chunk-BTPGQC6Z.mjs +0 -27
- package/dist/chunk-C7XQYG6U.mjs +0 -50
- package/dist/chunk-CQZXZ75V.mjs +0 -562
- package/dist/chunk-CWS55TEX.mjs +0 -242
- package/dist/chunk-CZQ33DLW.mjs +0 -78
- package/dist/chunk-EEBCDP36.mjs +0 -32
- package/dist/chunk-FB6463IN.mjs +0 -1499
- package/dist/chunk-FCJ553HS.mjs +0 -655
- package/dist/chunk-FEZVDOWX.mjs +0 -6996
- package/dist/chunk-FOCV4CBZ.mjs +0 -87
- package/dist/chunk-FQBFZAWG.mjs +0 -27
- package/dist/chunk-G6HRSGOU.mjs +0 -74
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-GH4WQ22V.mjs +0 -34
- package/dist/chunk-GPWMUPCA.mjs +0 -1126
- package/dist/chunk-GQPTDQCL.mjs +0 -54
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-I54T6CQP.mjs +0 -117
- package/dist/chunk-IPN7S4KX.mjs +0 -130
- package/dist/chunk-J4IJTTEX.mjs +0 -226
- package/dist/chunk-JSV6OQ4C.mjs +0 -74
- package/dist/chunk-K4ZIGL2O.mjs +0 -290
- package/dist/chunk-KAKQ74LS.mjs +0 -53
- package/dist/chunk-KCQYFO4K.mjs +0 -62
- package/dist/chunk-KHKIRS5D.mjs +0 -113
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KP7TYHWT.mjs +0 -92
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LDNCLRZU.mjs +0 -218
- package/dist/chunk-LHKOSI2J.mjs +0 -58
- package/dist/chunk-LQG5G2LY.mjs +0 -39
- package/dist/chunk-MATMVH65.mjs +0 -9
- package/dist/chunk-MP7ABPTD.mjs +0 -189
- package/dist/chunk-MW6BR4QP.mjs +0 -316
- package/dist/chunk-MWD5VPN7.mjs +0 -21
- package/dist/chunk-ODYKO5TO.mjs +0 -101
- package/dist/chunk-OG7UGJ6F.mjs +0 -63
- package/dist/chunk-ORN5BLL7.mjs +0 -11
- package/dist/chunk-OYABFKTW.mjs +0 -1974
- package/dist/chunk-P5LANSH6.mjs +0 -121
- package/dist/chunk-PELHVTO7.mjs +0 -24
- package/dist/chunk-PIH5SJ4N.mjs +0 -7481
- package/dist/chunk-PSZI4O24.mjs +0 -253
- package/dist/chunk-PV2CG7EJ.mjs +0 -886
- package/dist/chunk-PWGYQXML.mjs +0 -470
- package/dist/chunk-Q2UL2TPC.mjs +0 -21
- package/dist/chunk-QHPYS427.mjs +0 -66
- package/dist/chunk-QJVOLKU6.mjs +0 -111
- package/dist/chunk-QLDABJZM.mjs +0 -20
- package/dist/chunk-QYRNLLLI.mjs +0 -99
- package/dist/chunk-RCX4WNLC.mjs +0 -151
- package/dist/chunk-RFBLQLLZ.mjs +0 -16
- package/dist/chunk-RIHIVFIQ.mjs +0 -240
- package/dist/chunk-RLOES5SX.mjs +0 -31
- package/dist/chunk-RWURUDSP.mjs +0 -2639
- package/dist/chunk-RZWWB2XR.mjs +0 -96
- package/dist/chunk-S5LN746F.mjs +0 -10
- package/dist/chunk-SOI6LTHM.mjs +0 -216
- package/dist/chunk-SQJWAIF4.mjs +0 -12
- package/dist/chunk-TDIK6NTV.mjs +0 -10
- package/dist/chunk-TQXHPFWT.mjs +0 -177
- package/dist/chunk-U55XJEIG.mjs +0 -18
- package/dist/chunk-UD5HIS4U.mjs +0 -382
- package/dist/chunk-UHICS3JI.mjs +0 -83
- package/dist/chunk-UJDJB3NE.mjs +0 -913
- package/dist/chunk-VD7J552I.mjs +0 -28
- package/dist/chunk-VSLSGASN.mjs +0 -30
- package/dist/chunk-VZ2ONHEE.mjs +0 -3715
- package/dist/chunk-W7OSSKH5.mjs +0 -1427
- package/dist/chunk-WZ7YV54N.mjs +0 -152
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-Y4WPNHMC.mjs +0 -50
- package/dist/chunk-YEKL4JBK.mjs +0 -96
- package/dist/chunk-ZXNQEW43.mjs +0 -18
- package/dist/chunk-ZYAY26KG.mjs +0 -29
- package/dist/direct-executor-V5DRIECI.mjs +0 -34
- package/dist/durability-wiring-3TD62MBI.mjs +0 -4072
- package/dist/engine-hooks-HA3SBOBR.mjs +0 -54
- package/dist/gzip-F4HMIEEQ.mjs +0 -1
- package/dist/import-GPKLTSKA.mjs +0 -56
- package/dist/methods-7IOINERF.mjs +0 -30
- package/dist/module-JYHFTVPS.mjs +0 -63
- package/dist/operations-G5EP6JPX.mjs +0 -5
- package/dist/phrase-module-XW5MKWP5.mjs +0 -330
- package/dist/pool-Y4A46T36.mjs +0 -298
- package/dist/react-native/chunks/analysis-change-QUVDNABS.mjs +0 -934
- package/dist/react-native/chunks/analysis-change-entry-OC4XASBF.mjs +0 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +0 -29
- package/dist/react-native/chunks/chunk-2PRHILX3.mjs +0 -529
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +0 -1
- package/dist/react-native/chunks/chunk-3WI35N75.mjs +0 -226
- package/dist/react-native/chunks/chunk-4DII3ZSM.mjs +0 -73
- package/dist/react-native/chunks/chunk-4J74WHQM.mjs +0 -177
- package/dist/react-native/chunks/chunk-4NWR3NGS.mjs +0 -151
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +0 -17
- package/dist/react-native/chunks/chunk-5MSX5FRT.mjs +0 -858
- package/dist/react-native/chunks/chunk-6OFDUYVX.mjs +0 -92
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +0 -9
- package/dist/react-native/chunks/chunk-6ZELFCK5.mjs +0 -315
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +0 -27
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +0 -32
- package/dist/react-native/chunks/chunk-BITYFZSZ.mjs +0 -242
- package/dist/react-native/chunks/chunk-C5G6TJ7A.mjs +0 -140
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +0 -43
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +0 -10
- package/dist/react-native/chunks/chunk-DMIFXZ3Q.mjs +0 -432
- package/dist/react-native/chunks/chunk-DVDX4SDR.mjs +0 -375
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +0 -12
- package/dist/react-native/chunks/chunk-F753DA54.mjs +0 -23
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +0 -40
- package/dist/react-native/chunks/chunk-FTZHMRD2.mjs +0 -18
- package/dist/react-native/chunks/chunk-HJBJBH26.mjs +0 -34
- package/dist/react-native/chunks/chunk-IAENKE2D.mjs +0 -21
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +0 -10
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +0 -111
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +0 -38
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +0 -15
- package/dist/react-native/chunks/chunk-KSBYEMCB.mjs +0 -100
- package/dist/react-native/chunks/chunk-KU646XXT.mjs +0 -164
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +0 -68
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +0 -93
- package/dist/react-native/chunks/chunk-LZSLNSUN.mjs +0 -193
- package/dist/react-native/chunks/chunk-MJDQU5MW.mjs +0 -1225
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +0 -8
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +0 -78
- package/dist/react-native/chunks/chunk-OCSLSH27.mjs +0 -22723
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +0 -29
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +0 -79
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +0 -9
- package/dist/react-native/chunks/chunk-RDWG33MM.mjs +0 -79
- package/dist/react-native/chunks/chunk-SK2PWYDH.mjs +0 -654
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +0 -96
- package/dist/react-native/chunks/chunk-TORU2OHQ.mjs +0 -218
- package/dist/react-native/chunks/chunk-U4KSKA4Z.mjs +0 -23
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +0 -68
- package/dist/react-native/chunks/chunk-WBYQRHDY.mjs +0 -86
- package/dist/react-native/chunks/chunk-WVWJNUMN.mjs +0 -2562
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +0 -288
- package/dist/react-native/chunks/chunk-YZW4CHM6.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZNZ6CPAZ.mjs +0 -562
- package/dist/react-native/chunks/chunk-ZVQMBF2H.mjs +0 -63
- package/dist/react-native/chunks/chunk-ZWAIN4GG.mjs +0 -2636
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +0 -92
- package/dist/react-native/chunks/cold-module-N6KFNNZA.mjs +0 -773
- package/dist/react-native/chunks/durability-wiring-RN64S2TL.mjs +0 -4984
- package/dist/react-native/chunks/engine-hooks-CWA5CKCA.mjs +0 -30
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +0 -3
- package/dist/react-native/chunks/import-5UAWBRJP.mjs +0 -32
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +0 -30
- package/dist/react-native/chunks/module-6RP4ZCBW.mjs +0 -37
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +0 -5
- package/dist/react-native/chunks/phrase-module-QK22HSQ2.mjs +0 -329
- package/dist/react-native/chunks/pool-JR3NJCAF.mjs +0 -297
- package/dist/react-native/chunks/release-2SIHZWDO.mjs +0 -32
- package/dist/react-native/chunks/restore-hooks-ENFMDTWC.mjs +0 -127
- package/dist/react-native/chunks/restore-queue-4R6AASE5.mjs +0 -55
- package/dist/react-native/chunks/retention-5GTFT3GR.mjs +0 -223
- package/dist/react-native/chunks/snapshot-5LGPQXFS.mjs +0 -976
- package/dist/react-native/chunks/start-ZQJVZ5DN.mjs +0 -38
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +0 -15
- package/dist/react-native/chunks/write-6BRYIIZ7.mjs +0 -208
- package/dist/release-SMR4WJD4.mjs +0 -32
- package/dist/restore-hooks-HXLIEB4A.mjs +0 -153
- package/dist/restore-queue-LZH7LMLR.mjs +0 -55
- package/dist/retention-3K7ARGAG.mjs +0 -66
- package/dist/runtime-COIUORCN.mjs +0 -177
- package/dist/snapshot-O7ICP7NJ.mjs +0 -783
- package/dist/start-S4PFP6RS.mjs +0 -66
- package/dist/sync-state-ZWHLAFLE.mjs +0 -15
- package/dist/thread-YLJ3ZPQ3.mjs +0 -50
- package/dist/write-2OBJGLFS.mjs +0 -233
- /package/dist/browser/{chunk-M3MQE2X7.mjs → digest-leaf-64IPRPZD.mjs} +0 -0
- /package/dist/browser/{chunk-JNOAKDUM.mjs → gzip-JF3S6PPT.mjs} +0 -0
- /package/dist/{chunk-ZXNNR73R.mjs → gzip-2UCZW2MM.mjs} +0 -0
package/dist/distribution.mjs
CHANGED
|
@@ -1,112 +1,56 @@
|
|
|
1
|
-
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest } from './chunk-
|
|
5
|
-
import { readVectorFile } from './chunk-
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-6OEGXLBX.mjs';
|
|
9
|
-
import { addBackgroundAdmission, workWaitingForMemory } from './chunk-PELHVTO7.mjs';
|
|
10
|
-
import './chunk-FCJ553HS.mjs';
|
|
11
|
-
import './chunk-MATMVH65.mjs';
|
|
12
|
-
import './chunk-VD7J552I.mjs';
|
|
13
|
-
import './chunk-ZXNNR73R.mjs';
|
|
14
|
-
import './chunk-WZ7YV54N.mjs';
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-3D6RZGJH.mjs';
|
|
2
|
+
import { applyStoredIndexBm25 } from './chunk-NC4QLCZC.mjs';
|
|
3
|
+
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-GF4K7IUK.mjs';
|
|
4
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal } from './chunk-SXIRM4MX.mjs';
|
|
5
|
+
import { readVectorFile } from './chunk-TSLV5P2J.mjs';
|
|
6
|
+
import { emptySerializablePartition } from './chunk-LF4GB4PO.mjs';
|
|
7
|
+
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-75UNINFA.mjs';
|
|
15
8
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
16
|
-
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-
|
|
17
|
-
import {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import { requireSyncSetting, requireNoQuerySuggestions } from './chunk-QJVOLKU6.mjs';
|
|
24
|
-
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
25
|
-
import { readSnapshotArchive, sourceIndexNameOf } from './chunk-MW6BR4QP.mjs';
|
|
26
|
-
import { openManifestPayload, listedKeysOf, decodeManifestPayload } from './chunk-CWS55TEX.mjs';
|
|
27
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-K4ZIGL2O.mjs';
|
|
28
|
-
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
9
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-V3SGPXWQ.mjs';
|
|
10
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, suggestionStateOfSettings, indexConfigKey, metadataWithSync, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState } from './chunk-PTTMGRG5.mjs';
|
|
11
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-H64YDF5M.mjs';
|
|
12
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-H64YDF5M.mjs';
|
|
13
|
+
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireSyncSetting, requireNoQuerySuggestions, decodeIndexConfiguration, INDEX_CONFIGURATION_ENVELOPE_VERSION } from './chunk-2JRERUNZ.mjs';
|
|
14
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-4CAMH7FO.mjs';
|
|
15
|
+
import { estimatedArchiveBytes, readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload } from './chunk-OOO54OVC.mjs';
|
|
29
16
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
-
import './chunk-
|
|
32
|
-
import {
|
|
33
|
-
import { encodeSegmentManifest } from './chunk-C7XQYG6U.mjs';
|
|
34
|
-
import './chunk-BNL5OPRV.mjs';
|
|
35
|
-
import './chunk-CQZXZ75V.mjs';
|
|
36
|
-
import { directoryCheckpointStore, readListedFile } from './chunk-UHICS3JI.mjs';
|
|
17
|
+
import './chunk-YETKPX5M.mjs';
|
|
18
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, createInsyncAddMessage, validateAckPayload, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-W34VJXY7.mjs';
|
|
19
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-U6B7DUQW.mjs';
|
|
37
20
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
|
|
38
21
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
39
|
-
import './chunk-
|
|
40
|
-
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-JTYPJF3T.mjs';
|
|
22
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-7MWYORQC.mjs';
|
|
41
23
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
42
24
|
import './chunk-QDZJM7NI.mjs';
|
|
43
|
-
import
|
|
44
|
-
import { keyBytesOf, QuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-EYD7K3H6.mjs';
|
|
25
|
+
import './chunk-77AUVDQ5.mjs';
|
|
45
26
|
import './chunk-LNGE7SLY.mjs';
|
|
46
|
-
import
|
|
47
|
-
import
|
|
48
|
-
import {
|
|
49
|
-
import './chunk-
|
|
27
|
+
import './chunk-QBVTYY4J.mjs';
|
|
28
|
+
import './chunk-IB4BYLE3.mjs';
|
|
29
|
+
import { deserializePartition } from './chunk-2RIS76RJ.mjs';
|
|
30
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_OVERSAMPLE } from './chunk-LAWIJRMJ.mjs';
|
|
31
|
+
import './chunk-BXM7XM4G.mjs';
|
|
32
|
+
import { RestoredHistory } from './chunk-HOOR4CP5.mjs';
|
|
50
33
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
51
|
-
import './chunk-
|
|
52
|
-
import './chunk-
|
|
53
|
-
import './chunk-
|
|
54
|
-
import './chunk-
|
|
55
|
-
import {
|
|
56
|
-
import {
|
|
57
|
-
import {
|
|
58
|
-
import './chunk-
|
|
59
|
-
import './chunk-
|
|
60
|
-
import {
|
|
61
|
-
import './chunk-3MXV3VHR.mjs';
|
|
62
|
-
import { BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-EEBCDP36.mjs';
|
|
63
|
-
import { synonymModule, generateId } from './chunk-VZ2ONHEE.mjs';
|
|
64
|
-
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, mergePartitionStats } from './chunk-PIH5SJ4N.mjs';
|
|
65
|
-
import { decodeIndexConfiguration, INDEX_CONFIGURATION_ENVELOPE_VERSION } from './chunk-GH4WQ22V.mjs';
|
|
66
|
-
import { openChecksummedEnvelope } from './chunk-3OH67G7R.mjs';
|
|
67
|
-
import './chunk-IPN7S4KX.mjs';
|
|
68
|
-
import { fnv1a, fnv1aUint32s } from './chunk-KAKQ74LS.mjs';
|
|
69
|
-
import { crc32Init, crc32Final, crc32Update } from './chunk-7GXJUJ24.mjs';
|
|
70
|
-
import './chunk-Q2UL2TPC.mjs';
|
|
71
|
-
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE } from './chunk-CITFCZKF.mjs';
|
|
72
|
-
import './chunk-RLOES5SX.mjs';
|
|
73
|
-
import './chunk-PV2CG7EJ.mjs';
|
|
74
|
-
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
75
|
-
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf, getNestedValue, isLatitude, isLongitude } from './chunk-W5EB5XKI.mjs';
|
|
76
|
-
import { MIN_POLYGON_POINTS, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES } from './chunk-SQJWAIF4.mjs';
|
|
77
|
-
import './chunk-GES7U2PM.mjs';
|
|
78
|
-
import './chunk-I54T6CQP.mjs';
|
|
79
|
-
import { configOfNamedSettings } from './chunk-I5B7RXU7.mjs';
|
|
80
|
-
import './chunk-Y4WPNHMC.mjs';
|
|
81
|
-
import { getLanguage, resolveIndexAnalysis, hasLanguage } from './chunk-RZWWB2XR.mjs';
|
|
82
|
-
import './chunk-TGP5M5R7.mjs';
|
|
83
|
-
import './chunk-G6HRSGOU.mjs';
|
|
34
|
+
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-6EZW2GQL.mjs';
|
|
35
|
+
import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
|
|
36
|
+
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, visitTokens, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-TINW4BSH.mjs';
|
|
37
|
+
import { encodeDocument, encodeDocumentReply } from './chunk-KLVMK3R5.mjs';
|
|
38
|
+
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-AB27FWZM.mjs';
|
|
39
|
+
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
40
|
+
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
41
|
+
import './chunk-7T4LAYT4.mjs';
|
|
42
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-JV74RKCY.mjs';
|
|
43
|
+
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-R4CE4IVU.mjs';
|
|
84
44
|
import './chunk-U2BKYEC7.mjs';
|
|
85
|
-
import './chunk-
|
|
86
|
-
import './chunk-
|
|
87
|
-
import {
|
|
88
|
-
import './chunk-TDIK6NTV.mjs';
|
|
89
|
-
import { bm25SetsValue, holdsQuotedText } from './chunk-FB6463IN.mjs';
|
|
90
|
-
import './chunk-7GFE3PEY.mjs';
|
|
91
|
-
import './chunk-ZXNQEW43.mjs';
|
|
92
|
-
import './chunk-7V5IUP2T.mjs';
|
|
93
|
-
import { MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
94
|
-
import './chunk-X5CV2E2B.mjs';
|
|
95
|
-
import { visitTokens } from './chunk-A4L7YTKX.mjs';
|
|
96
|
-
import './chunk-U55XJEIG.mjs';
|
|
97
|
-
import './chunk-KOKM4PQN.mjs';
|
|
98
|
-
import { MIN_OVERSAMPLE } from './chunk-ORN5BLL7.mjs';
|
|
99
|
-
import './chunk-BTPGQC6Z.mjs';
|
|
100
|
-
import './chunk-LQG5G2LY.mjs';
|
|
45
|
+
import './chunk-OUWZKNO3.mjs';
|
|
46
|
+
import { collectAndReturnPages, BATCH_CHUNK_SIZE, MAX_SORT_FIELDS, MAX_DOC_ID_LENGTH, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, DEFAULT_PAGE_SIZE, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, HYBRID_RESCORE_WINDOW, DEFAULT_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT } from './chunk-3B775F3C.mjs';
|
|
47
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-E2F6LZR3.mjs';
|
|
101
48
|
import './chunk-ZXX5GZEX.mjs';
|
|
102
|
-
import './chunk-
|
|
103
|
-
import './chunk-
|
|
104
|
-
import {
|
|
105
|
-
import {
|
|
106
|
-
import {
|
|
107
|
-
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-J4IJTTEX.mjs';
|
|
108
|
-
import { compareCodePoints, exceedsCodePoints } from './chunk-TBC3XUL2.mjs';
|
|
109
|
-
import { encode, decode } from '@msgpack/msgpack';
|
|
49
|
+
import { joinChunks, readWhole } from './chunk-IE3Q62QD.mjs';
|
|
50
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-4PY4JYMY.mjs';
|
|
51
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
52
|
+
import { holdsUnpairedSurrogate, compareCodePoints, readSortField, isSortMode, SORT_MODES, exceedsCodePoints, toComparableSortValue } from './chunk-TGPKS2UM.mjs';
|
|
53
|
+
import { encode } from '@msgpack/msgpack';
|
|
110
54
|
import { randomUUID, createHash } from 'crypto';
|
|
111
55
|
import { mkdtemp, rm, mkdir, open } from 'fs/promises';
|
|
112
56
|
import { join } from 'path';
|
|
@@ -2285,8 +2229,8 @@ function validateScoredEntry(value, fieldLabel) {
|
|
|
2285
2229
|
`Invalid SearchResultPayload: "${fieldLabel}.score" is absent only where "sortValues" carries the sort key`
|
|
2286
2230
|
);
|
|
2287
2231
|
}
|
|
2288
|
-
} else if (
|
|
2289
|
-
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}.score" must be a
|
|
2232
|
+
} else if (typeof value.score !== "number") {
|
|
2233
|
+
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}.score" must be a number`);
|
|
2290
2234
|
}
|
|
2291
2235
|
if (value.sortValues !== null && value.sortValues !== void 0) {
|
|
2292
2236
|
if (!Array.isArray(value.sortValues)) {
|
|
@@ -3384,6 +3328,8 @@ async function handleInsyncAdmission(payload, coordinator, controllerNodeId) {
|
|
|
3384
3328
|
};
|
|
3385
3329
|
});
|
|
3386
3330
|
}
|
|
3331
|
+
|
|
3332
|
+
// src/distribution/replication/primary.ts
|
|
3387
3333
|
function previousTermOf(log, seqNo) {
|
|
3388
3334
|
return seqNo <= 1 ? 0 : log.termAt(seqNo - 1);
|
|
3389
3335
|
}
|
|
@@ -3519,6 +3465,23 @@ function forgetReplica(state, indexName, partitionId, replicaNodeId) {
|
|
|
3519
3465
|
}
|
|
3520
3466
|
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
3521
3467
|
}
|
|
3468
|
+
function forgetKeysOfIndex(entries, indexName) {
|
|
3469
|
+
for (const key of [...entries.keys()]) {
|
|
3470
|
+
if (key.slice(0, key.lastIndexOf(":")) === indexName) entries.delete(key);
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
3473
|
+
function forgetCatchUpIndex(state, indexName) {
|
|
3474
|
+
for (const [key, replicas] of [...state.cursors]) {
|
|
3475
|
+
if (key.slice(0, key.lastIndexOf(":")) !== indexName) continue;
|
|
3476
|
+
for (const cursor of replicas.values()) cursor.feed?.cancel();
|
|
3477
|
+
state.cursors.delete(key);
|
|
3478
|
+
}
|
|
3479
|
+
forgetKeysOfIndex(state.pendingAdmissions, indexName);
|
|
3480
|
+
forgetKeysOfIndex(state.admittedReplicas, indexName);
|
|
3481
|
+
forgetKeysOfIndex(state.acknowledgedSeqNos, indexName);
|
|
3482
|
+
forgetKeysOfIndex(state.removedReplicas, indexName);
|
|
3483
|
+
forgetKeysOfIndex(state.startedTerms, indexName);
|
|
3484
|
+
}
|
|
3522
3485
|
function getPendingAdmissions(state, indexName, partitionId) {
|
|
3523
3486
|
const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
|
|
3524
3487
|
return pending === void 0 ? [] : [...pending];
|
|
@@ -6012,6 +5975,8 @@ async function routeUpdateItems(table, indexName, updates, deps, conditions) {
|
|
|
6012
5975
|
}
|
|
6013
5976
|
return { succeeded: succeeded2, failed: failed2 };
|
|
6014
5977
|
}
|
|
5978
|
+
|
|
5979
|
+
// src/distribution/cluster-node/write-routing/forward-batch.ts
|
|
6015
5980
|
function groupIntoRuns(payload, partitionCount) {
|
|
6016
5981
|
const runsByPartition = /* @__PURE__ */ new Map();
|
|
6017
5982
|
for (let index = 0; index < payload.operations.length; index++) {
|
|
@@ -7309,7 +7274,7 @@ async function readPartitionManifest(store, backupName, partition) {
|
|
|
7309
7274
|
try {
|
|
7310
7275
|
const raw = await openManifestPayload(bytes);
|
|
7311
7276
|
sourceIndexName = sourceIndexNameOf(listedKeysOf(raw));
|
|
7312
|
-
decoded = decodeManifestPayload(raw, sourceIndexName);
|
|
7277
|
+
decoded = decodeManifestPayload(raw, sourceIndexName, partition.partitionId);
|
|
7313
7278
|
} catch (error) {
|
|
7314
7279
|
throw damaged(
|
|
7315
7280
|
backupName,
|
|
@@ -8046,6 +8011,8 @@ function createAnalysisView(deps) {
|
|
|
8046
8011
|
}
|
|
8047
8012
|
};
|
|
8048
8013
|
}
|
|
8014
|
+
|
|
8015
|
+
// src/distribution/cluster-node/bm25-view.ts
|
|
8049
8016
|
function settingOfMetadata(bytes, indexName) {
|
|
8050
8017
|
const decoded = decode(bytes);
|
|
8051
8018
|
if (!isRecord2(decoded) || typeof decoded.indexUuid !== "string") return null;
|
|
@@ -8365,6 +8332,15 @@ function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
|
8365
8332
|
inFlight.abortResolve();
|
|
8366
8333
|
state.inFlight.delete(key);
|
|
8367
8334
|
}
|
|
8335
|
+
function clearBootstrapSyncOfIndex(state, indexName) {
|
|
8336
|
+
const keys = /* @__PURE__ */ new Set([...state.inFlight.keys(), ...state.completed, ...state.generations.keys()]);
|
|
8337
|
+
for (const key of keys) {
|
|
8338
|
+
const separator = key.lastIndexOf(":");
|
|
8339
|
+
if (key.slice(0, separator) === indexName) {
|
|
8340
|
+
clearBootstrapSyncIndex(state, indexName, Number(key.slice(separator + 1)));
|
|
8341
|
+
}
|
|
8342
|
+
}
|
|
8343
|
+
}
|
|
8368
8344
|
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
8369
8345
|
return state.completed.has(entryKey(indexName, partitionId));
|
|
8370
8346
|
}
|
|
@@ -8901,6 +8877,8 @@ function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo,
|
|
|
8901
8877
|
}
|
|
8902
8878
|
return null;
|
|
8903
8879
|
}
|
|
8880
|
+
|
|
8881
|
+
// src/distribution/cluster-node/bootstrap-sync/frames.ts
|
|
8904
8882
|
function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
8905
8883
|
return {
|
|
8906
8884
|
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
@@ -13174,6 +13152,8 @@ function createNodeController(config, nodeId, transport, errors) {
|
|
|
13174
13152
|
extraMessages: errors.extraMessages
|
|
13175
13153
|
});
|
|
13176
13154
|
}
|
|
13155
|
+
|
|
13156
|
+
// src/distribution/cluster-node/copy-failure.ts
|
|
13177
13157
|
function delay(ms) {
|
|
13178
13158
|
return new Promise((resolve) => {
|
|
13179
13159
|
const timer = setTimeout(resolve, ms);
|
|
@@ -14444,394 +14424,86 @@ function createAllocationErrorForwarder(forwardOnError) {
|
|
|
14444
14424
|
};
|
|
14445
14425
|
}
|
|
14446
14426
|
|
|
14447
|
-
// src/
|
|
14448
|
-
function
|
|
14449
|
-
|
|
14450
|
-
if (entry === void 0) return null;
|
|
14451
|
-
const manager = core.executor.getManager(indexName);
|
|
14452
|
-
const language = manager?.language ?? getLanguage(entry.config.language ?? "english");
|
|
14453
|
-
const customTokenizer = (manager?.analysis ?? resolveIndexAnalysis(entry.config)).customTokenizer;
|
|
14454
|
-
return {
|
|
14455
|
-
schema: entry.config.schema,
|
|
14456
|
-
language,
|
|
14457
|
-
...customTokenizer === void 0 ? {} : { customTokenizer }
|
|
14458
|
-
};
|
|
14459
|
-
}
|
|
14460
|
-
function vectorGrowthBytesOf(core, indexName, documents) {
|
|
14461
|
-
const manager = core.executor.getManager(indexName);
|
|
14462
|
-
if (manager === void 0 || manager === null) return 0;
|
|
14463
|
-
let bytes = 0;
|
|
14464
|
-
for (const vectorIndex of manager.getVectorIndexes().values()) bytes += vectorIndex.growthBytesFor(documents);
|
|
14465
|
-
return bytes;
|
|
14427
|
+
// src/distribution/replication/log.ts
|
|
14428
|
+
function estimateEntrySize(entry) {
|
|
14429
|
+
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
14466
14430
|
}
|
|
14467
|
-
|
|
14468
|
-
|
|
14469
|
-
|
|
14470
|
-
|
|
14471
|
-
|
|
14431
|
+
var computeChecksum = computeEntryChecksum;
|
|
14432
|
+
function ownedDocument(document) {
|
|
14433
|
+
if (document === null || document.byteOffset === 0 && document.byteLength === document.buffer.byteLength) {
|
|
14434
|
+
return document;
|
|
14435
|
+
}
|
|
14436
|
+
return document.slice();
|
|
14472
14437
|
}
|
|
14473
|
-
|
|
14474
|
-
|
|
14475
|
-
|
|
14476
|
-
|
|
14477
|
-
|
|
14478
|
-
|
|
14479
|
-
|
|
14480
|
-
|
|
14481
|
-
|
|
14482
|
-
|
|
14483
|
-
|
|
14484
|
-
|
|
14485
|
-
|
|
14486
|
-
|
|
14487
|
-
|
|
14438
|
+
function createReplicationLog(partitionId, config) {
|
|
14439
|
+
const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
|
|
14440
|
+
let nextSeqNo = config?.startSeqNo ?? 1;
|
|
14441
|
+
let localLogEnd = nextSeqNo - 1;
|
|
14442
|
+
let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
|
|
14443
|
+
let seedSeqNo = localLogEnd;
|
|
14444
|
+
let seedPrimaryTerm = localLogEndPrimaryTerm;
|
|
14445
|
+
let commitPoint = nextSeqNo - 1;
|
|
14446
|
+
let entries = [];
|
|
14447
|
+
let headIndex = 0;
|
|
14448
|
+
let totalSizeBytes = 0;
|
|
14449
|
+
function liveCount() {
|
|
14450
|
+
return entries.length - headIndex;
|
|
14451
|
+
}
|
|
14452
|
+
function compact() {
|
|
14453
|
+
if (headIndex > 0) {
|
|
14454
|
+
entries = entries.slice(headIndex);
|
|
14455
|
+
headIndex = 0;
|
|
14488
14456
|
}
|
|
14489
|
-
if (next === null) await storage?.release(indexName);
|
|
14490
|
-
return true;
|
|
14491
|
-
});
|
|
14492
|
-
}
|
|
14493
|
-
|
|
14494
|
-
// src/persistence/durability/partition-log.ts
|
|
14495
|
-
function persistedPosition(manager, indexName, partitionId) {
|
|
14496
|
-
const partition = manager.partitionRegistry?.indexes.get(indexName)?.partitions.get(partitionId);
|
|
14497
|
-
if (partition !== void 0) return { seqNo: partition.appliedSeqNo, primaryTerm: partition.seqOwner.primaryTerm };
|
|
14498
|
-
return { seqNo: manager.highestPersistedSeqNo(indexName, partitionId), primaryTerm: 0 };
|
|
14499
|
-
}
|
|
14500
|
-
async function closePrevious(registry, indexName, partitionId) {
|
|
14501
|
-
const state = registry.indexState(indexName);
|
|
14502
|
-
if (state.checkpointInFlight !== null) await state.checkpointInFlight.catch(() => void 0);
|
|
14503
|
-
const previous = state.partitions.get(partitionId);
|
|
14504
|
-
if (previous === void 0) return;
|
|
14505
|
-
if (previous.failed === null) await previous.appendChain;
|
|
14506
|
-
else await previous.appendChain.catch(() => void 0);
|
|
14507
|
-
await closePartitionWriter(previous);
|
|
14508
|
-
}
|
|
14509
|
-
async function reopenPartitionLogAfterReplay(manager, indexName, partitionId, replay) {
|
|
14510
|
-
const registry = manager.partitionRegistry;
|
|
14511
|
-
if (registry === void 0) return;
|
|
14512
|
-
await closePrevious(registry, indexName, partitionId);
|
|
14513
|
-
const position = await replay();
|
|
14514
|
-
registry.indexState(indexName).partitions.set(partitionId, registry.freshPartition(indexName, partitionId, position));
|
|
14515
|
-
}
|
|
14516
|
-
async function replacePartitionLog(registry, indexName, partitionId, position) {
|
|
14517
|
-
await closePrevious(registry, indexName, partitionId);
|
|
14518
|
-
for (const key of await registry.directory.list(`${indexName}/wal/${partitionId}/`)) {
|
|
14519
|
-
await registry.directory.remove(key);
|
|
14520
14457
|
}
|
|
14521
|
-
|
|
14522
|
-
|
|
14523
|
-
|
|
14524
|
-
|
|
14525
|
-
|
|
14526
|
-
|
|
14527
|
-
|
|
14528
|
-
|
|
14529
|
-
}
|
|
14530
|
-
|
|
14531
|
-
|
|
14532
|
-
|
|
14533
|
-
|
|
14534
|
-
ErrorCodes.BACKUP_DURABILITY_REQUIRED,
|
|
14535
|
-
`The primary of partition ${partitionId} of index '${indexName}' runs without write-ahead log durability, so it can't take a backup`,
|
|
14536
|
-
{ indexName, partitionId }
|
|
14537
|
-
);
|
|
14538
|
-
}
|
|
14539
|
-
async function manifestWithoutTermHistory(manifest) {
|
|
14540
|
-
const stripped = {
|
|
14541
|
-
...manifest,
|
|
14542
|
-
checkpoint: manifest.checkpoint.map(({ partitionId, lastSeqNo, primaryTerm }) => ({
|
|
14543
|
-
partitionId,
|
|
14544
|
-
lastSeqNo,
|
|
14545
|
-
primaryTerm
|
|
14546
|
-
}))
|
|
14547
|
-
};
|
|
14548
|
-
const encoded2 = await encodeSegmentManifest(stripped);
|
|
14549
|
-
return { manifest: stripped, manifestBytes: joinChunks(encoded2.chunks, encoded2.byteLength) };
|
|
14550
|
-
}
|
|
14551
|
-
async function openOnce(core, indexName, partitionId) {
|
|
14552
|
-
const durable = core.durability?.manager;
|
|
14553
|
-
if (durable?.openArchiveCheckpoint === void 0) throw durabilityRequired(indexName, partitionId);
|
|
14554
|
-
const release = await core.indexState.acquire(indexName, false);
|
|
14555
|
-
let checkpoint = null;
|
|
14556
|
-
let handedOver = false;
|
|
14557
|
-
try {
|
|
14558
|
-
const entry = core.requireIndex(indexName);
|
|
14559
|
-
const manager = core.requireManager(indexName);
|
|
14560
|
-
checkpoint = await durable.openArchiveCheckpoint(indexName);
|
|
14561
|
-
const { directory } = checkpoint;
|
|
14562
|
-
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
14563
|
-
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
14564
|
-
const config = await encodeIndexConfiguration(entry);
|
|
14565
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-RLRGOXDX.mjs');
|
|
14566
|
-
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
14567
|
-
handedOver = true;
|
|
14568
|
-
try {
|
|
14569
|
-
const narrowed = await manifestWithoutTermHistory(archive.manifest);
|
|
14570
|
-
return {
|
|
14571
|
-
config,
|
|
14572
|
-
...narrowed,
|
|
14573
|
-
lastSeqNo: archive.lastSeqNo,
|
|
14574
|
-
sourceIndexName: indexName,
|
|
14575
|
-
readRange: archive.readRange,
|
|
14576
|
-
release: archive.release
|
|
14577
|
-
};
|
|
14578
|
-
} catch (error) {
|
|
14579
|
-
archive.release();
|
|
14580
|
-
throw error;
|
|
14458
|
+
function findIndexBySeqNo(seqNo) {
|
|
14459
|
+
let low = headIndex;
|
|
14460
|
+
let high = entries.length - 1;
|
|
14461
|
+
while (low <= high) {
|
|
14462
|
+
const mid = low + high >>> 1;
|
|
14463
|
+
const midSeqNo = entries[mid].seqNo;
|
|
14464
|
+
if (midSeqNo < seqNo) {
|
|
14465
|
+
low = mid + 1;
|
|
14466
|
+
} else if (midSeqNo > seqNo) {
|
|
14467
|
+
high = mid - 1;
|
|
14468
|
+
} else {
|
|
14469
|
+
return mid;
|
|
14470
|
+
}
|
|
14581
14471
|
}
|
|
14582
|
-
|
|
14583
|
-
if (!handedOver) checkpoint?.release();
|
|
14584
|
-
release();
|
|
14472
|
+
return low;
|
|
14585
14473
|
}
|
|
14586
|
-
|
|
14587
|
-
|
|
14588
|
-
|
|
14589
|
-
|
|
14590
|
-
|
|
14591
|
-
|
|
14474
|
+
function evictOldEntries() {
|
|
14475
|
+
while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
|
|
14476
|
+
totalSizeBytes -= estimateEntrySize(entries[headIndex]);
|
|
14477
|
+
entries[headIndex] = null;
|
|
14478
|
+
headIndex += 1;
|
|
14479
|
+
}
|
|
14480
|
+
if (totalSizeBytes < 0) {
|
|
14481
|
+
totalSizeBytes = 0;
|
|
14482
|
+
}
|
|
14483
|
+
if (headIndex >= 256 || headIndex > liveCount()) {
|
|
14484
|
+
compact();
|
|
14485
|
+
}
|
|
14592
14486
|
}
|
|
14593
|
-
|
|
14594
|
-
|
|
14595
|
-
|
|
14596
|
-
|
|
14597
|
-
|
|
14598
|
-
}
|
|
14599
|
-
|
|
14600
|
-
|
|
14601
|
-
|
|
14602
|
-
const held = [...manager.getPartition(partitionId).docIds()];
|
|
14603
|
-
for (const vectorIndex of manager.getVectorIndexes().values()) {
|
|
14604
|
-
for (const docId of held) if (vectorIndex.has(docId)) vectorIndex.remove(docId);
|
|
14487
|
+
function assertEntryFitsRetention(entry) {
|
|
14488
|
+
const entrySize = REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
14489
|
+
if (entrySize > retentionBytes) {
|
|
14490
|
+
throw new NarsilError(
|
|
14491
|
+
ErrorCodes.REPLICATION_LOG_FULL,
|
|
14492
|
+
`Replication log entry of ${entrySize} bytes exceeds the log retention limit of ${retentionBytes} bytes and cannot be buffered`,
|
|
14493
|
+
{ partitionId, entrySize, retentionBytes, seqNo: entry.seqNo }
|
|
14494
|
+
);
|
|
14495
|
+
}
|
|
14605
14496
|
}
|
|
14606
|
-
|
|
14607
|
-
}
|
|
14608
|
-
|
|
14609
|
-
|
|
14610
|
-
|
|
14611
|
-
|
|
14612
|
-
try {
|
|
14613
|
-
await work();
|
|
14614
|
-
} finally {
|
|
14615
|
-
await core.orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
14497
|
+
function storeEntry(appended) {
|
|
14498
|
+
const entry = { ...appended, document: ownedDocument(appended.document) };
|
|
14499
|
+
entries.push(entry);
|
|
14500
|
+
if (entry.seqNo >= localLogEnd) {
|
|
14501
|
+
localLogEnd = entry.seqNo;
|
|
14502
|
+
localLogEndPrimaryTerm = entry.primaryTerm;
|
|
14616
14503
|
}
|
|
14617
|
-
|
|
14618
|
-
|
|
14619
|
-
|
|
14620
|
-
}
|
|
14621
|
-
async function emptyPartitionCopy(core, indexName, partitionId, writeRecords) {
|
|
14622
|
-
if (!core.indexRegistry.has(indexName)) return;
|
|
14623
|
-
await withPartitionHeld(core, indexName, async () => {
|
|
14624
|
-
clearPartition(core.requireManager(indexName), partitionId);
|
|
14625
|
-
writeRecords.resetPartition(indexName, partitionId);
|
|
14626
|
-
});
|
|
14627
|
-
const durable = core.durability?.manager;
|
|
14628
|
-
if (durable !== void 0) await restartPartitionLog(durable, indexName, partitionId, { seqNo: 0, primaryTerm: 0 });
|
|
14629
|
-
}
|
|
14630
|
-
async function loadPartitionVectors(store, manifest, manager, partitionId) {
|
|
14631
|
-
const partition = manager.getPartition(partitionId);
|
|
14632
|
-
for (const field of manifest.vectors) {
|
|
14633
|
-
const vectorIndex = manager.getVectorIndexes().get(field.fieldPath);
|
|
14634
|
-
if (vectorIndex === void 0) continue;
|
|
14635
|
-
for (const file of field.files) {
|
|
14636
|
-
const { payload } = await readVectorFile(store, file, field.fieldPath);
|
|
14637
|
-
const vectors = bytesToVectors(payload.vectors);
|
|
14638
|
-
for (let i = 0; i < payload.docIds.length; i++) {
|
|
14639
|
-
if (isDead(file.dead, i) || !partition.has(payload.docIds[i])) continue;
|
|
14640
|
-
vectorIndex.insert(
|
|
14641
|
-
payload.docIds[i],
|
|
14642
|
-
vectors.slice(i * payload.dimension, (i + 1) * payload.dimension),
|
|
14643
|
-
partitionId
|
|
14644
|
-
);
|
|
14645
|
-
}
|
|
14646
|
-
}
|
|
14647
|
-
vectorIndex.scheduleBuild();
|
|
14648
|
-
}
|
|
14649
|
-
}
|
|
14650
|
-
async function reloadPartitionCopy(core, indexName, partitionId, writeRecords) {
|
|
14651
|
-
const durable = core.durability?.manager;
|
|
14652
|
-
const registry = durable?.partitionRegistry;
|
|
14653
|
-
if (durable === void 0 || registry === void 0) {
|
|
14654
|
-
throw new NarsilError(
|
|
14655
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
14656
|
-
`Partition ${partitionId} of index "${indexName}" has no files on disk to reload its copy from`,
|
|
14657
|
-
{ indexName, partitionId }
|
|
14658
|
-
);
|
|
14659
|
-
}
|
|
14660
|
-
await withPartitionHeld(core, indexName, async () => {
|
|
14661
|
-
const manager = core.requireManager(indexName);
|
|
14662
|
-
clearPartition(manager, partitionId);
|
|
14663
|
-
writeRecords.resetPartition(indexName, partitionId);
|
|
14664
|
-
const store = directoryCheckpointStore(registry.directory);
|
|
14665
|
-
const decoded = await readSegmentManifest(store, indexName);
|
|
14666
|
-
if (decoded !== null && decoded.kind !== "current") {
|
|
14667
|
-
throw new NarsilError(
|
|
14668
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
14669
|
-
`Index "${indexName}" holds the checkpoint layout of an earlier release, so partition ${partitionId} cannot reload alone`,
|
|
14670
|
-
{ indexName, partitionId }
|
|
14671
|
-
);
|
|
14672
|
-
}
|
|
14673
|
-
const manifest = decoded?.manifest ?? null;
|
|
14674
|
-
const listed = manifest?.partitions.find((entry) => entry.partitionId === partitionId);
|
|
14675
|
-
if (manifest !== null && listed !== void 0) {
|
|
14676
|
-
await loadPartitionSegments(store, listed, manager, {
|
|
14677
|
-
schema: manifest.schema,
|
|
14678
|
-
decodesEveryDocument: true,
|
|
14679
|
-
writeRecords
|
|
14680
|
-
});
|
|
14681
|
-
await loadPartitionVectors(store, manifest, manager, partitionId);
|
|
14682
|
-
}
|
|
14683
|
-
const checkpoint = manifest?.checkpoint ?? [];
|
|
14684
|
-
writeRecords.loadTermHistory(
|
|
14685
|
-
indexName,
|
|
14686
|
-
partitionId,
|
|
14687
|
-
checkpoint.find((entry) => entry.partitionId === partitionId)
|
|
14688
|
-
);
|
|
14689
|
-
await reopenPartitionLogAfterReplay(durable, indexName, partitionId, async () => {
|
|
14690
|
-
const replayed = await replaySyncedWal(
|
|
14691
|
-
registry.directory,
|
|
14692
|
-
indexName,
|
|
14693
|
-
partitionId,
|
|
14694
|
-
checkpointLastSeqNo(checkpoint, partitionId),
|
|
14695
|
-
{
|
|
14696
|
-
manager,
|
|
14697
|
-
vectorFieldPaths: core.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
14698
|
-
vectorIndexes: manager.getVectorIndexes(),
|
|
14699
|
-
writeRecords
|
|
14700
|
-
}
|
|
14701
|
-
);
|
|
14702
|
-
return {
|
|
14703
|
-
seqNo: replayed.highestSeqNo,
|
|
14704
|
-
primaryTerm: Math.max(checkpointPrimaryTerm(checkpoint, partitionId), replayed.highestPrimaryTerm)
|
|
14705
|
-
};
|
|
14706
|
-
});
|
|
14707
|
-
});
|
|
14708
|
-
}
|
|
14709
|
-
|
|
14710
|
-
// src/distribution/cluster-node/held-partitions.ts
|
|
14711
|
-
function createHeldPartitionRecord(core) {
|
|
14712
|
-
const writesByIndex = /* @__PURE__ */ new Map();
|
|
14713
|
-
const indexesBehindOnDisk = /* @__PURE__ */ new Set();
|
|
14714
|
-
function queueWrite(indexName, next) {
|
|
14715
|
-
const previous = writesByIndex.get(indexName) ?? Promise.resolve();
|
|
14716
|
-
const write = previous.then(() => writeHeldPartitions(core, indexName, next, indexesBehindOnDisk));
|
|
14717
|
-
writesByIndex.set(
|
|
14718
|
-
indexName,
|
|
14719
|
-
write.catch(() => void 0)
|
|
14720
|
-
);
|
|
14721
|
-
return write;
|
|
14722
|
-
}
|
|
14723
|
-
return {
|
|
14724
|
-
record(indexName, partitionId) {
|
|
14725
|
-
return queueWrite(indexName, (held) => held.includes(partitionId) ? null : [...held, partitionId]);
|
|
14726
|
-
},
|
|
14727
|
-
forget(indexName, partitionId) {
|
|
14728
|
-
return queueWrite(indexName, (held) => held.includes(partitionId) ? held.filter((id) => id !== partitionId) : null);
|
|
14729
|
-
},
|
|
14730
|
-
held(indexName) {
|
|
14731
|
-
return core.indexRegistry.get(indexName)?.heldPartitions ?? void 0;
|
|
14732
|
-
}
|
|
14733
|
-
};
|
|
14734
|
-
}
|
|
14735
|
-
async function writeHeldPartitions(core, indexName, next, indexesBehindOnDisk) {
|
|
14736
|
-
const entry = core.indexRegistry.get(indexName);
|
|
14737
|
-
if (entry === void 0) {
|
|
14738
|
-
return;
|
|
14739
|
-
}
|
|
14740
|
-
const updated = next(entry.heldPartitions ?? []);
|
|
14741
|
-
if (updated === null && !indexesBehindOnDisk.has(indexName)) {
|
|
14742
|
-
return;
|
|
14743
|
-
}
|
|
14744
|
-
if (updated !== null) {
|
|
14745
|
-
core.indexRegistry.set(indexName, { ...entry, heldPartitions: [...updated].sort((a, b) => a - b) });
|
|
14746
|
-
}
|
|
14747
|
-
if (core.durability === null) {
|
|
14748
|
-
return;
|
|
14749
|
-
}
|
|
14750
|
-
indexesBehindOnDisk.add(indexName);
|
|
14751
|
-
await core.durability.manager.persistMetadata(indexName);
|
|
14752
|
-
indexesBehindOnDisk.delete(indexName);
|
|
14753
|
-
}
|
|
14754
|
-
|
|
14755
|
-
// src/distribution/replication/log.ts
|
|
14756
|
-
function estimateEntrySize(entry) {
|
|
14757
|
-
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
14758
|
-
}
|
|
14759
|
-
var computeChecksum = computeEntryChecksum;
|
|
14760
|
-
function ownedDocument(document) {
|
|
14761
|
-
if (document === null || document.byteOffset === 0 && document.byteLength === document.buffer.byteLength) {
|
|
14762
|
-
return document;
|
|
14763
|
-
}
|
|
14764
|
-
return document.slice();
|
|
14765
|
-
}
|
|
14766
|
-
function createReplicationLog(partitionId, config) {
|
|
14767
|
-
const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
|
|
14768
|
-
let nextSeqNo = config?.startSeqNo ?? 1;
|
|
14769
|
-
let localLogEnd = nextSeqNo - 1;
|
|
14770
|
-
let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
|
|
14771
|
-
let seedSeqNo = localLogEnd;
|
|
14772
|
-
let seedPrimaryTerm = localLogEndPrimaryTerm;
|
|
14773
|
-
let commitPoint = nextSeqNo - 1;
|
|
14774
|
-
let entries = [];
|
|
14775
|
-
let headIndex = 0;
|
|
14776
|
-
let totalSizeBytes = 0;
|
|
14777
|
-
function liveCount() {
|
|
14778
|
-
return entries.length - headIndex;
|
|
14779
|
-
}
|
|
14780
|
-
function compact() {
|
|
14781
|
-
if (headIndex > 0) {
|
|
14782
|
-
entries = entries.slice(headIndex);
|
|
14783
|
-
headIndex = 0;
|
|
14784
|
-
}
|
|
14785
|
-
}
|
|
14786
|
-
function findIndexBySeqNo(seqNo) {
|
|
14787
|
-
let low = headIndex;
|
|
14788
|
-
let high = entries.length - 1;
|
|
14789
|
-
while (low <= high) {
|
|
14790
|
-
const mid = low + high >>> 1;
|
|
14791
|
-
const midSeqNo = entries[mid].seqNo;
|
|
14792
|
-
if (midSeqNo < seqNo) {
|
|
14793
|
-
low = mid + 1;
|
|
14794
|
-
} else if (midSeqNo > seqNo) {
|
|
14795
|
-
high = mid - 1;
|
|
14796
|
-
} else {
|
|
14797
|
-
return mid;
|
|
14798
|
-
}
|
|
14799
|
-
}
|
|
14800
|
-
return low;
|
|
14801
|
-
}
|
|
14802
|
-
function evictOldEntries() {
|
|
14803
|
-
while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
|
|
14804
|
-
totalSizeBytes -= estimateEntrySize(entries[headIndex]);
|
|
14805
|
-
entries[headIndex] = null;
|
|
14806
|
-
headIndex += 1;
|
|
14807
|
-
}
|
|
14808
|
-
if (totalSizeBytes < 0) {
|
|
14809
|
-
totalSizeBytes = 0;
|
|
14810
|
-
}
|
|
14811
|
-
if (headIndex >= 256 || headIndex > liveCount()) {
|
|
14812
|
-
compact();
|
|
14813
|
-
}
|
|
14814
|
-
}
|
|
14815
|
-
function assertEntryFitsRetention(entry) {
|
|
14816
|
-
const entrySize = REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
14817
|
-
if (entrySize > retentionBytes) {
|
|
14818
|
-
throw new NarsilError(
|
|
14819
|
-
ErrorCodes.REPLICATION_LOG_FULL,
|
|
14820
|
-
`Replication log entry of ${entrySize} bytes exceeds the log retention limit of ${retentionBytes} bytes and cannot be buffered`,
|
|
14821
|
-
{ partitionId, entrySize, retentionBytes, seqNo: entry.seqNo }
|
|
14822
|
-
);
|
|
14823
|
-
}
|
|
14824
|
-
}
|
|
14825
|
-
function storeEntry(appended) {
|
|
14826
|
-
const entry = { ...appended, document: ownedDocument(appended.document) };
|
|
14827
|
-
entries.push(entry);
|
|
14828
|
-
if (entry.seqNo >= localLogEnd) {
|
|
14829
|
-
localLogEnd = entry.seqNo;
|
|
14830
|
-
localLogEndPrimaryTerm = entry.primaryTerm;
|
|
14831
|
-
}
|
|
14832
|
-
totalSizeBytes += estimateEntrySize(entry);
|
|
14833
|
-
evictOldEntries();
|
|
14834
|
-
return { ...entry };
|
|
14504
|
+
totalSizeBytes += estimateEntrySize(entry);
|
|
14505
|
+
evictOldEntries();
|
|
14506
|
+
return { ...entry };
|
|
14835
14507
|
}
|
|
14836
14508
|
function readEntry(seqNo) {
|
|
14837
14509
|
if (liveCount() === 0) return void 0;
|
|
@@ -14996,24 +14668,630 @@ function getReplicationLog(replicationLogs, indexName, partitionId, replicationC
|
|
|
14996
14668
|
noteAcceptedPrimaryTerm(log, persisted.primaryTerm);
|
|
14997
14669
|
replicationLogs.set(key, log);
|
|
14998
14670
|
}
|
|
14999
|
-
return log;
|
|
15000
|
-
}
|
|
15001
|
-
function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm = 0, replicationConfig) {
|
|
15002
|
-
const key = replicationLogKey(indexName, partitionId);
|
|
15003
|
-
const seeded = createReplicationLog(partitionId, {
|
|
15004
|
-
startSeqNo,
|
|
15005
|
-
lastPrimaryTerm,
|
|
15006
|
-
logRetentionBytes: replicationConfig?.logRetentionBytes
|
|
15007
|
-
});
|
|
15008
|
-
carryKnownTerms(replicationLogs.get(key), seeded);
|
|
15009
|
-
noteAcceptedPrimaryTerm(seeded, lastPrimaryTerm);
|
|
15010
|
-
replicationLogs.set(key, seeded);
|
|
14671
|
+
return log;
|
|
14672
|
+
}
|
|
14673
|
+
function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm = 0, replicationConfig) {
|
|
14674
|
+
const key = replicationLogKey(indexName, partitionId);
|
|
14675
|
+
const seeded = createReplicationLog(partitionId, {
|
|
14676
|
+
startSeqNo,
|
|
14677
|
+
lastPrimaryTerm,
|
|
14678
|
+
logRetentionBytes: replicationConfig?.logRetentionBytes
|
|
14679
|
+
});
|
|
14680
|
+
carryKnownTerms(replicationLogs.get(key), seeded);
|
|
14681
|
+
noteAcceptedPrimaryTerm(seeded, lastPrimaryTerm);
|
|
14682
|
+
replicationLogs.set(key, seeded);
|
|
14683
|
+
}
|
|
14684
|
+
function deleteIndexReplicationLogs(replicationLogs, indexName) {
|
|
14685
|
+
const prefix = `${indexName}:`;
|
|
14686
|
+
for (const key of replicationLogs.keys()) {
|
|
14687
|
+
if (key.startsWith(prefix)) replicationLogs.delete(key);
|
|
14688
|
+
}
|
|
14689
|
+
}
|
|
14690
|
+
|
|
14691
|
+
// src/distribution/cluster-node/local-copy-state.ts
|
|
14692
|
+
function forgetClosedCopy(state, indexName) {
|
|
14693
|
+
deleteIndexReplicationLogs(state.replicationLogs, indexName);
|
|
14694
|
+
forgetReshardIndex(state.reshard, indexName);
|
|
14695
|
+
}
|
|
14696
|
+
function forgetDroppedCopy(state, indexName) {
|
|
14697
|
+
forgetClosedCopy(state, indexName);
|
|
14698
|
+
clearBootstrapSyncOfIndex(state.bootstrapSync, indexName);
|
|
14699
|
+
forgetCatchUpIndex(state.catchUp(), indexName);
|
|
14700
|
+
}
|
|
14701
|
+
|
|
14702
|
+
// src/engine/memory/write-analysis.ts
|
|
14703
|
+
function writeAnalysisOf(core, indexName) {
|
|
14704
|
+
const entry = core.indexRegistry.get(indexName);
|
|
14705
|
+
if (entry === void 0) return null;
|
|
14706
|
+
const manager = core.executor.getManager(indexName);
|
|
14707
|
+
const language = manager?.language ?? getLanguage(entry.config.language ?? "english");
|
|
14708
|
+
const customTokenizer = (manager?.analysis ?? resolveIndexAnalysis(entry.config)).customTokenizer;
|
|
14709
|
+
return {
|
|
14710
|
+
schema: entry.config.schema,
|
|
14711
|
+
language,
|
|
14712
|
+
...customTokenizer === void 0 ? {} : { customTokenizer }
|
|
14713
|
+
};
|
|
14714
|
+
}
|
|
14715
|
+
function vectorGrowthBytesOf(core, indexName, documents) {
|
|
14716
|
+
const manager = core.executor.getManager(indexName);
|
|
14717
|
+
if (manager === void 0 || manager === null) return 0;
|
|
14718
|
+
let bytes = 0;
|
|
14719
|
+
for (const vectorIndex of manager.getVectorIndexes().values()) bytes += vectorIndex.growthBytesFor(documents);
|
|
14720
|
+
return bytes;
|
|
14721
|
+
}
|
|
14722
|
+
|
|
14723
|
+
// src/engine/sync/adopt.ts
|
|
14724
|
+
function sameState(held, next) {
|
|
14725
|
+
if (held === null || next === null) return held === next;
|
|
14726
|
+
return held.epoch === next.epoch && held.active === next.active && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
|
|
14727
|
+
}
|
|
14728
|
+
async function adoptIndexSyncState(core, indexName, next) {
|
|
14729
|
+
return oneSyncChangeAtATime(core.indexRegistry, indexName, async () => {
|
|
14730
|
+
const entry = core.indexRegistry.get(indexName);
|
|
14731
|
+
if (entry === void 0) return false;
|
|
14732
|
+
const held = entry.sync;
|
|
14733
|
+
if (sameState(held, next)) return false;
|
|
14734
|
+
const storage = core.durability?.manager.syncStorage?.();
|
|
14735
|
+
const anotherEpoch = held !== null && next !== null && held.epoch !== next.epoch;
|
|
14736
|
+
if (anotherEpoch) await storage?.release(indexName);
|
|
14737
|
+
entry.sync = next;
|
|
14738
|
+
try {
|
|
14739
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
14740
|
+
} catch (error) {
|
|
14741
|
+
entry.sync = held;
|
|
14742
|
+
throw error;
|
|
14743
|
+
}
|
|
14744
|
+
if (next === null) await storage?.release(indexName);
|
|
14745
|
+
return true;
|
|
14746
|
+
});
|
|
14747
|
+
}
|
|
14748
|
+
|
|
14749
|
+
// src/persistence/durability/partition-log.ts
|
|
14750
|
+
function persistedPosition(manager, indexName, partitionId) {
|
|
14751
|
+
const partition = manager.partitionRegistry?.indexes.get(indexName)?.partitions.get(partitionId);
|
|
14752
|
+
if (partition !== void 0) return { seqNo: partition.appliedSeqNo, primaryTerm: partition.seqOwner.primaryTerm };
|
|
14753
|
+
return { seqNo: manager.highestPersistedSeqNo(indexName, partitionId), primaryTerm: 0 };
|
|
14754
|
+
}
|
|
14755
|
+
async function closePrevious(registry, indexName, partitionId) {
|
|
14756
|
+
const state = registry.indexState(indexName);
|
|
14757
|
+
if (state.checkpointInFlight !== null) await state.checkpointInFlight.catch(() => void 0);
|
|
14758
|
+
const previous = state.partitions.get(partitionId);
|
|
14759
|
+
if (previous === void 0) return;
|
|
14760
|
+
if (previous.failed === null) await previous.appendChain;
|
|
14761
|
+
else await previous.appendChain.catch(() => void 0);
|
|
14762
|
+
await closePartitionWriter(previous);
|
|
14763
|
+
}
|
|
14764
|
+
async function reopenPartitionLogAfterReplay(manager, indexName, partitionId, replay) {
|
|
14765
|
+
const registry = manager.partitionRegistry;
|
|
14766
|
+
if (registry === void 0) return;
|
|
14767
|
+
await closePrevious(registry, indexName, partitionId);
|
|
14768
|
+
const position = await replay();
|
|
14769
|
+
registry.indexState(indexName).partitions.set(partitionId, registry.freshPartition(indexName, partitionId, position));
|
|
14770
|
+
}
|
|
14771
|
+
async function replacePartitionLog(registry, indexName, partitionId, position) {
|
|
14772
|
+
await closePrevious(registry, indexName, partitionId);
|
|
14773
|
+
for (const key of await registry.directory.list(`${indexName}/wal/${partitionId}/`)) {
|
|
14774
|
+
await registry.directory.remove(key);
|
|
14775
|
+
}
|
|
14776
|
+
registry.indexState(indexName).partitions.set(partitionId, registry.freshPartition(indexName, partitionId, position));
|
|
14777
|
+
}
|
|
14778
|
+
async function restartPartitionLog(manager, indexName, partitionId, position) {
|
|
14779
|
+
const registry = manager.partitionRegistry;
|
|
14780
|
+
if (registry === void 0 || manager.checkpointFromMemory === void 0) return;
|
|
14781
|
+
await replacePartitionLog(registry, indexName, partitionId, position);
|
|
14782
|
+
await manager.syncStorage?.().removeHistory(indexName, partitionId);
|
|
14783
|
+
await manager.checkpointFromMemory(indexName);
|
|
14784
|
+
}
|
|
14785
|
+
|
|
14786
|
+
// src/distribution/cluster-node/backup-partition-source.ts
|
|
14787
|
+
function durabilityRequired(indexName, partitionId) {
|
|
14788
|
+
return new NarsilError(
|
|
14789
|
+
ErrorCodes.BACKUP_DURABILITY_REQUIRED,
|
|
14790
|
+
`The primary of partition ${partitionId} of index '${indexName}' runs without write-ahead log durability, so it can't take a backup`,
|
|
14791
|
+
{ indexName, partitionId }
|
|
14792
|
+
);
|
|
14793
|
+
}
|
|
14794
|
+
async function manifestWithoutTermHistory(manifest) {
|
|
14795
|
+
const stripped = {
|
|
14796
|
+
...manifest,
|
|
14797
|
+
checkpoint: manifest.checkpoint.map(({ partitionId, lastSeqNo, primaryTerm }) => ({
|
|
14798
|
+
partitionId,
|
|
14799
|
+
lastSeqNo,
|
|
14800
|
+
primaryTerm
|
|
14801
|
+
}))
|
|
14802
|
+
};
|
|
14803
|
+
const encoded2 = await encodeSegmentManifest(stripped);
|
|
14804
|
+
return { manifest: stripped, manifestBytes: joinChunks(encoded2.chunks, encoded2.byteLength) };
|
|
14805
|
+
}
|
|
14806
|
+
async function openOnce(core, indexName, partitionId) {
|
|
14807
|
+
const durable = core.durability?.manager;
|
|
14808
|
+
if (durable?.openArchiveCheckpoint === void 0) throw durabilityRequired(indexName, partitionId);
|
|
14809
|
+
const release = await core.indexState.acquire(indexName, false);
|
|
14810
|
+
let checkpoint = null;
|
|
14811
|
+
let handedOver = false;
|
|
14812
|
+
try {
|
|
14813
|
+
const entry = core.requireIndex(indexName);
|
|
14814
|
+
const manager = core.requireManager(indexName);
|
|
14815
|
+
checkpoint = await durable.openArchiveCheckpoint(indexName);
|
|
14816
|
+
const { directory } = checkpoint;
|
|
14817
|
+
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
14818
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
14819
|
+
const config = await encodeIndexConfiguration(entry);
|
|
14820
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-G576XGVE.mjs');
|
|
14821
|
+
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
14822
|
+
handedOver = true;
|
|
14823
|
+
try {
|
|
14824
|
+
const narrowed = await manifestWithoutTermHistory(archive.manifest);
|
|
14825
|
+
return {
|
|
14826
|
+
config,
|
|
14827
|
+
...narrowed,
|
|
14828
|
+
lastSeqNo: archive.lastSeqNo,
|
|
14829
|
+
sourceIndexName: indexName,
|
|
14830
|
+
readRange: archive.readRange,
|
|
14831
|
+
release: archive.release
|
|
14832
|
+
};
|
|
14833
|
+
} catch (error) {
|
|
14834
|
+
archive.release();
|
|
14835
|
+
throw error;
|
|
14836
|
+
}
|
|
14837
|
+
} finally {
|
|
14838
|
+
if (!handedOver) checkpoint?.release();
|
|
14839
|
+
release();
|
|
14840
|
+
}
|
|
14841
|
+
}
|
|
14842
|
+
async function openBackupPartitionSource(core, indexName, partitionId) {
|
|
14843
|
+
core.guardShutdown();
|
|
14844
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
14845
|
+
const source = await openOnce(core, indexName, partitionId);
|
|
14846
|
+
if (source !== null) return source;
|
|
14847
|
+
}
|
|
14848
|
+
throw new NarsilError(
|
|
14849
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
14850
|
+
`The analysis of index '${indexName}' changed while partition ${partitionId} took its backup checkpoint, so try the backup again once the change has a durable checkpoint`,
|
|
14851
|
+
{ indexName, partitionId }
|
|
14852
|
+
);
|
|
14853
|
+
}
|
|
14854
|
+
|
|
14855
|
+
// src/distribution/cluster-node/write-records/constants.ts
|
|
14856
|
+
var INITIAL_WRITE_RECORD_SLOTS = 64;
|
|
14857
|
+
|
|
14858
|
+
// src/distribution/cluster-node/write-records/table.ts
|
|
14859
|
+
function createWriteRecordTable() {
|
|
14860
|
+
const slots = /* @__PURE__ */ new Map();
|
|
14861
|
+
let seqNos = new Float64Array(0);
|
|
14862
|
+
let primaryTerms = new Float64Array(0);
|
|
14863
|
+
let freeSlots = [];
|
|
14864
|
+
let nextSlot = 0;
|
|
14865
|
+
function grow() {
|
|
14866
|
+
const capacity = Math.max(INITIAL_WRITE_RECORD_SLOTS, seqNos.length * 2);
|
|
14867
|
+
const grownSeqNos = new Float64Array(capacity);
|
|
14868
|
+
const grownTerms = new Float64Array(capacity);
|
|
14869
|
+
grownSeqNos.set(seqNos);
|
|
14870
|
+
grownTerms.set(primaryTerms);
|
|
14871
|
+
seqNos = grownSeqNos;
|
|
14872
|
+
primaryTerms = grownTerms;
|
|
14873
|
+
}
|
|
14874
|
+
function claimSlot() {
|
|
14875
|
+
const reused = freeSlots.pop();
|
|
14876
|
+
if (reused !== void 0) return reused;
|
|
14877
|
+
if (nextSlot === seqNos.length) grow();
|
|
14878
|
+
return nextSlot++;
|
|
14879
|
+
}
|
|
14880
|
+
return {
|
|
14881
|
+
get count() {
|
|
14882
|
+
return slots.size;
|
|
14883
|
+
},
|
|
14884
|
+
set(documentId, seqNo, primaryTerm) {
|
|
14885
|
+
let slot = slots.get(documentId);
|
|
14886
|
+
if (slot === void 0) {
|
|
14887
|
+
slot = claimSlot();
|
|
14888
|
+
slots.set(documentId, slot);
|
|
14889
|
+
}
|
|
14890
|
+
seqNos[slot] = seqNo;
|
|
14891
|
+
primaryTerms[slot] = primaryTerm;
|
|
14892
|
+
},
|
|
14893
|
+
remove(documentId) {
|
|
14894
|
+
const slot = slots.get(documentId);
|
|
14895
|
+
if (slot === void 0) return;
|
|
14896
|
+
slots.delete(documentId);
|
|
14897
|
+
seqNos[slot] = 0;
|
|
14898
|
+
primaryTerms[slot] = 0;
|
|
14899
|
+
freeSlots.push(slot);
|
|
14900
|
+
},
|
|
14901
|
+
recordOf(documentId) {
|
|
14902
|
+
const slot = slots.get(documentId);
|
|
14903
|
+
if (slot === void 0) return null;
|
|
14904
|
+
return { seqNo: seqNos[slot], primaryTerm: primaryTerms[slot] };
|
|
14905
|
+
},
|
|
14906
|
+
seqNoOf(documentId) {
|
|
14907
|
+
const slot = slots.get(documentId);
|
|
14908
|
+
return slot === void 0 ? 0 : seqNos[slot];
|
|
14909
|
+
},
|
|
14910
|
+
primaryTermOf(documentId) {
|
|
14911
|
+
const slot = slots.get(documentId);
|
|
14912
|
+
return slot === void 0 ? 0 : primaryTerms[slot];
|
|
14913
|
+
},
|
|
14914
|
+
clear() {
|
|
14915
|
+
slots.clear();
|
|
14916
|
+
seqNos = new Float64Array(0);
|
|
14917
|
+
primaryTerms = new Float64Array(0);
|
|
14918
|
+
freeSlots = [];
|
|
14919
|
+
nextSlot = 0;
|
|
14920
|
+
}
|
|
14921
|
+
};
|
|
14922
|
+
}
|
|
14923
|
+
|
|
14924
|
+
// src/distribution/cluster-node/write-records/term-history.ts
|
|
14925
|
+
function createTermHistory() {
|
|
14926
|
+
let starts = [];
|
|
14927
|
+
let highestSeqNo = 0;
|
|
14928
|
+
return {
|
|
14929
|
+
get highestSeqNo() {
|
|
14930
|
+
return highestSeqNo;
|
|
14931
|
+
},
|
|
14932
|
+
noteEntry(seqNo, primaryTerm) {
|
|
14933
|
+
const last = starts.at(-1);
|
|
14934
|
+
if (last === void 0 || primaryTerm > last.primaryTerm) starts.push({ primaryTerm, firstSeqNo: seqNo });
|
|
14935
|
+
if (seqNo > highestSeqNo) highestSeqNo = seqNo;
|
|
14936
|
+
},
|
|
14937
|
+
termOf(seqNo) {
|
|
14938
|
+
if (seqNo > highestSeqNo) return null;
|
|
14939
|
+
let term = null;
|
|
14940
|
+
for (const start of starts) {
|
|
14941
|
+
if (start.firstSeqNo > seqNo) break;
|
|
14942
|
+
term = start.primaryTerm;
|
|
14943
|
+
}
|
|
14944
|
+
return term;
|
|
14945
|
+
},
|
|
14946
|
+
starts(upToSeqNo) {
|
|
14947
|
+
return starts.filter((start) => start.firstSeqNo <= upToSeqNo).map((start) => ({ ...start }));
|
|
14948
|
+
},
|
|
14949
|
+
replace(next, nextHighest) {
|
|
14950
|
+
starts = next.map((start) => ({ ...start }));
|
|
14951
|
+
highestSeqNo = nextHighest;
|
|
14952
|
+
},
|
|
14953
|
+
prune(floorSeqNo) {
|
|
14954
|
+
while (starts.length > 1 && starts[1].firstSeqNo <= floorSeqNo) starts.shift();
|
|
14955
|
+
},
|
|
14956
|
+
clear() {
|
|
14957
|
+
starts = [];
|
|
14958
|
+
highestSeqNo = 0;
|
|
14959
|
+
}
|
|
14960
|
+
};
|
|
14961
|
+
}
|
|
14962
|
+
|
|
14963
|
+
// src/distribution/cluster-node/write-records/keeper.ts
|
|
14964
|
+
function createClusterWriteRecords() {
|
|
14965
|
+
const indexes = /* @__PURE__ */ new Map();
|
|
14966
|
+
let oldestRetainedSeqNo = () => void 0;
|
|
14967
|
+
function partition(indexName, partitionId) {
|
|
14968
|
+
let partitions = indexes.get(indexName);
|
|
14969
|
+
if (partitions === void 0) {
|
|
14970
|
+
partitions = /* @__PURE__ */ new Map();
|
|
14971
|
+
indexes.set(indexName, partitions);
|
|
14972
|
+
}
|
|
14973
|
+
let held = partitions.get(partitionId);
|
|
14974
|
+
if (held === void 0) {
|
|
14975
|
+
held = { table: createWriteRecordTable(), history: createTermHistory(), pagedSegments: [] };
|
|
14976
|
+
partitions.set(partitionId, held);
|
|
14977
|
+
}
|
|
14978
|
+
return held;
|
|
14979
|
+
}
|
|
14980
|
+
function applied(entry) {
|
|
14981
|
+
const held = partition(entry.indexName, entry.partitionId);
|
|
14982
|
+
if (entry.operation === "INDEX") held.table.set(entry.documentId, entry.seqNo, entry.primaryTerm);
|
|
14983
|
+
else held.table.remove(entry.documentId);
|
|
14984
|
+
held.history.noteEntry(entry.seqNo, entry.primaryTerm);
|
|
14985
|
+
}
|
|
14986
|
+
function recordOfDocument(indexName, partitionId, documentId) {
|
|
14987
|
+
const held = indexes.get(indexName)?.get(partitionId);
|
|
14988
|
+
if (held === void 0) return null;
|
|
14989
|
+
const record = held.table.recordOf(documentId);
|
|
14990
|
+
if (record !== null) return record;
|
|
14991
|
+
for (let at = held.pagedSegments.length - 1; at >= 0; at--) {
|
|
14992
|
+
const segment = held.pagedSegments[at];
|
|
14993
|
+
if (!segment.hasDocument(documentId) || segment.paged === null) continue;
|
|
14994
|
+
const stored = segment.paged.writeRecordAt(segment.arrays.idTable.ordinalOf(documentId));
|
|
14995
|
+
return stored.seqNo === 0 ? null : stored;
|
|
14996
|
+
}
|
|
14997
|
+
return null;
|
|
14998
|
+
}
|
|
14999
|
+
function captureSegment(indexName, partitionId, segment) {
|
|
15000
|
+
const count2 = segment.arrays.documentCount;
|
|
15001
|
+
const seqNos = new Float64Array(count2);
|
|
15002
|
+
const primaryTerms = new Float64Array(count2);
|
|
15003
|
+
for (let ordinal = 0; ordinal < count2; ordinal++) {
|
|
15004
|
+
if (segment.isTombstoned(ordinal)) continue;
|
|
15005
|
+
const record = recordOfDocument(indexName, partitionId, segment.arrays.idTable.idAt(ordinal));
|
|
15006
|
+
if (record === null) continue;
|
|
15007
|
+
seqNos[ordinal] = record.seqNo;
|
|
15008
|
+
primaryTerms[ordinal] = record.primaryTerm;
|
|
15009
|
+
}
|
|
15010
|
+
return { seqNos, primaryTerms };
|
|
15011
|
+
}
|
|
15012
|
+
return {
|
|
15013
|
+
readRetainedLogWith(read) {
|
|
15014
|
+
oldestRetainedSeqNo = read;
|
|
15015
|
+
},
|
|
15016
|
+
applied,
|
|
15017
|
+
replayed: applied,
|
|
15018
|
+
partition,
|
|
15019
|
+
adoptPartition(indexName, partitionId, records) {
|
|
15020
|
+
Object.assign(partition(indexName, partitionId), records);
|
|
15021
|
+
},
|
|
15022
|
+
recordOf: recordOfDocument,
|
|
15023
|
+
heldEntry(indexName, partitionId, seqNo, primaryTerm) {
|
|
15024
|
+
const history = indexes.get(indexName)?.get(partitionId)?.history;
|
|
15025
|
+
if (history === void 0) return "uncovered";
|
|
15026
|
+
if (seqNo > history.highestSeqNo) return "absent";
|
|
15027
|
+
const term = history.termOf(seqNo);
|
|
15028
|
+
if (term === null) return "uncovered";
|
|
15029
|
+
return term === primaryTerm ? "held" : "absent";
|
|
15030
|
+
},
|
|
15031
|
+
resetPartition(indexName, partitionId) {
|
|
15032
|
+
const held = indexes.get(indexName)?.get(partitionId);
|
|
15033
|
+
if (held === void 0) return;
|
|
15034
|
+
held.table.clear();
|
|
15035
|
+
held.history.clear();
|
|
15036
|
+
held.pagedSegments = [];
|
|
15037
|
+
},
|
|
15038
|
+
loadPagedSegment(indexName, partitionId, segment) {
|
|
15039
|
+
const held = partition(indexName, partitionId);
|
|
15040
|
+
for (const documentId of segment.paged?.snapshot.tombstones ?? []) held.table.remove(documentId);
|
|
15041
|
+
held.pagedSegments = [...held.pagedSegments, segment];
|
|
15042
|
+
},
|
|
15043
|
+
adoptPagedSegments(indexName, partitionId, segments, changed) {
|
|
15044
|
+
const held = partition(indexName, partitionId);
|
|
15045
|
+
const kept = [];
|
|
15046
|
+
for (const documentId of changed) {
|
|
15047
|
+
const record = held.table.recordOf(documentId);
|
|
15048
|
+
if (record !== null) kept.push([documentId, record]);
|
|
15049
|
+
}
|
|
15050
|
+
held.table.clear();
|
|
15051
|
+
for (const [documentId, record] of kept) held.table.set(documentId, record.seqNo, record.primaryTerm);
|
|
15052
|
+
held.pagedSegments = segments;
|
|
15053
|
+
},
|
|
15054
|
+
loadSegment(indexName, partitionId, segment, contents) {
|
|
15055
|
+
const { table } = partition(indexName, partitionId);
|
|
15056
|
+
for (const documentId of contents.tombstones) table.remove(documentId);
|
|
15057
|
+
const records = contents.writeRecords;
|
|
15058
|
+
for (let ordinal = 0; ordinal < segment.arrays.documentCount; ordinal++) {
|
|
15059
|
+
const documentId = segment.arrays.idTable.idAt(ordinal);
|
|
15060
|
+
const seqNo = records?.seqNos[ordinal] ?? 0;
|
|
15061
|
+
if (seqNo === 0 || records === void 0) table.remove(documentId);
|
|
15062
|
+
else table.set(documentId, seqNo, records.primaryTerms[ordinal]);
|
|
15063
|
+
}
|
|
15064
|
+
},
|
|
15065
|
+
captureSegments(indexName, partitionId, segments) {
|
|
15066
|
+
const captured = /* @__PURE__ */ new Map();
|
|
15067
|
+
if (indexes.get(indexName)?.get(partitionId) === void 0) return captured;
|
|
15068
|
+
for (const segment of segments) {
|
|
15069
|
+
if (segment.paged === null) captured.set(segment, captureSegment(indexName, partitionId, segment));
|
|
15070
|
+
}
|
|
15071
|
+
return captured;
|
|
15072
|
+
},
|
|
15073
|
+
termHistoryOf(indexName, partitionId, lastSeqNo) {
|
|
15074
|
+
const history = indexes.get(indexName)?.get(partitionId)?.history;
|
|
15075
|
+
if (history === void 0) return [];
|
|
15076
|
+
const oldest = oldestRetainedSeqNo(indexName, partitionId);
|
|
15077
|
+
if (oldest !== void 0) history.prune(Math.min(oldest, lastSeqNo));
|
|
15078
|
+
return history.starts(lastSeqNo);
|
|
15079
|
+
},
|
|
15080
|
+
loadTermHistory(indexName, partitionId, checkpoint) {
|
|
15081
|
+
partition(indexName, partitionId).history.replace(checkpoint?.termHistory ?? [], checkpoint?.lastSeqNo ?? 0);
|
|
15082
|
+
},
|
|
15083
|
+
dropIndex(indexName) {
|
|
15084
|
+
indexes.delete(indexName);
|
|
15085
|
+
}
|
|
15086
|
+
};
|
|
15087
|
+
}
|
|
15088
|
+
|
|
15089
|
+
// src/distribution/cluster-node/partition-publication.ts
|
|
15090
|
+
async function publishPartitionCopy(core, indexName, partitionId, records, load) {
|
|
15091
|
+
const current = core.requireManager(indexName);
|
|
15092
|
+
const analysis = current.analysis;
|
|
15093
|
+
const earlier = current.getPartition(partitionId);
|
|
15094
|
+
const detached = createDetachedIndex(indexName, current.config, current.language, current.synonyms, core.executor);
|
|
15095
|
+
const staged = detached.manager;
|
|
15096
|
+
const stagedRecords = createClusterWriteRecords();
|
|
15097
|
+
try {
|
|
15098
|
+
await load(staged, stagedRecords);
|
|
15099
|
+
await current.publication.write(() => {
|
|
15100
|
+
if (core.requireManager(indexName) !== current || current.analysis !== analysis || current.partitionAt(partitionId) !== earlier) {
|
|
15101
|
+
throw new NarsilError(
|
|
15102
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
15103
|
+
`Index "${indexName}" changed while its partition copy loaded`
|
|
15104
|
+
);
|
|
15105
|
+
}
|
|
15106
|
+
const replacement = staged.getPartition(partitionId);
|
|
15107
|
+
const vectors = current.getVectorIndexes();
|
|
15108
|
+
const ids = vectors.size === 0 ? [] : /* @__PURE__ */ new Set([...earlier.docIds(), ...replacement.docIds()]);
|
|
15109
|
+
const previous = /* @__PURE__ */ new Map();
|
|
15110
|
+
for (const docId of ids) {
|
|
15111
|
+
previous.set(docId, new Map([...vectors].map(([field, vector]) => [field, vector.getVector(docId)])));
|
|
15112
|
+
}
|
|
15113
|
+
const changed = [];
|
|
15114
|
+
try {
|
|
15115
|
+
for (const docId of ids) {
|
|
15116
|
+
const next = new Map(
|
|
15117
|
+
[...vectors.keys()].map((field) => [field, staged.getVectorIndexes().get(field)?.getVector(docId) ?? null])
|
|
15118
|
+
);
|
|
15119
|
+
updateDocumentVectors(docId, next, vectors, partitionId);
|
|
15120
|
+
changed.push(docId);
|
|
15121
|
+
}
|
|
15122
|
+
} catch (error) {
|
|
15123
|
+
for (const docId of changed) {
|
|
15124
|
+
const old = previous.get(docId);
|
|
15125
|
+
if (old !== void 0) updateDocumentVectors(docId, old, vectors, partitionId);
|
|
15126
|
+
}
|
|
15127
|
+
throw error;
|
|
15128
|
+
}
|
|
15129
|
+
const partitions = current.getAllPartitions();
|
|
15130
|
+
partitions[partitionId] = replacement;
|
|
15131
|
+
current.setPartitions(partitions);
|
|
15132
|
+
records.adoptPartition(indexName, partitionId, stagedRecords.partition(indexName, partitionId));
|
|
15133
|
+
staged.removePartition(partitionId);
|
|
15134
|
+
});
|
|
15135
|
+
for (const vector of current.getVectorIndexes().values()) vector.scheduleBuild();
|
|
15136
|
+
} finally {
|
|
15137
|
+
core.executor.discardDetachedIndex(detached);
|
|
15138
|
+
}
|
|
15139
|
+
}
|
|
15140
|
+
|
|
15141
|
+
// src/distribution/cluster-node/copy-reset.ts
|
|
15142
|
+
function clearPartition(manager, partitionId) {
|
|
15143
|
+
const held = [...manager.getPartition(partitionId).docIds()];
|
|
15144
|
+
for (const vectorIndex of manager.getVectorIndexes().values()) {
|
|
15145
|
+
for (const docId of held) if (vectorIndex.has(docId)) vectorIndex.remove(docId);
|
|
15146
|
+
}
|
|
15147
|
+
manager.deserializePartition(partitionId, emptySerializablePartition(manager, partitionId), deserializePartition);
|
|
15148
|
+
}
|
|
15149
|
+
async function withPartitionHeld(core, indexName, work) {
|
|
15150
|
+
const release = await core.indexState.acquire(indexName, false);
|
|
15151
|
+
try {
|
|
15152
|
+
const wasScaledOut = core.orchestrator.desyncIndex(indexName);
|
|
15153
|
+
try {
|
|
15154
|
+
await work();
|
|
15155
|
+
} finally {
|
|
15156
|
+
await core.orchestrator.resyncIndex(indexName, wasScaledOut);
|
|
15157
|
+
}
|
|
15158
|
+
} finally {
|
|
15159
|
+
release();
|
|
15160
|
+
}
|
|
15161
|
+
}
|
|
15162
|
+
async function emptyPartitionCopy(core, indexName, partitionId, writeRecords) {
|
|
15163
|
+
if (!core.indexRegistry.has(indexName)) return;
|
|
15164
|
+
await withPartitionHeld(core, indexName, async () => {
|
|
15165
|
+
const manager = core.requireManager(indexName);
|
|
15166
|
+
await manager.publication.write(() => {
|
|
15167
|
+
clearPartition(manager, partitionId);
|
|
15168
|
+
writeRecords.resetPartition(indexName, partitionId);
|
|
15169
|
+
});
|
|
15170
|
+
});
|
|
15171
|
+
const durable = core.durability?.manager;
|
|
15172
|
+
if (durable !== void 0) await restartPartitionLog(durable, indexName, partitionId, { seqNo: 0, primaryTerm: 0 });
|
|
15173
|
+
}
|
|
15174
|
+
async function loadPartitionVectors(store, manifest, manager, partitionId) {
|
|
15175
|
+
const partition = manager.getPartition(partitionId);
|
|
15176
|
+
for (const field of manifest.vectors) {
|
|
15177
|
+
const vectorIndex = manager.getVectorIndexes().get(field.fieldPath);
|
|
15178
|
+
if (vectorIndex === void 0) continue;
|
|
15179
|
+
for (const file of field.files) {
|
|
15180
|
+
const { payload } = await readVectorFile(store, file, field.fieldPath);
|
|
15181
|
+
const vectors = bytesToVectors(payload.vectors);
|
|
15182
|
+
for (let i = 0; i < payload.docIds.length; i++) {
|
|
15183
|
+
if (isDead(file.dead, i) || !partition.has(payload.docIds[i])) continue;
|
|
15184
|
+
vectorIndex.insert(
|
|
15185
|
+
payload.docIds[i],
|
|
15186
|
+
vectors.slice(i * payload.dimension, (i + 1) * payload.dimension),
|
|
15187
|
+
partitionId
|
|
15188
|
+
);
|
|
15189
|
+
}
|
|
15190
|
+
}
|
|
15191
|
+
}
|
|
15192
|
+
}
|
|
15193
|
+
async function reloadPartitionCopy(core, indexName, partitionId, writeRecords) {
|
|
15194
|
+
const durable = core.durability?.manager;
|
|
15195
|
+
const registry = durable?.partitionRegistry;
|
|
15196
|
+
if (durable === void 0 || registry === void 0) {
|
|
15197
|
+
throw new NarsilError(
|
|
15198
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
15199
|
+
`Partition ${partitionId} of index "${indexName}" has no files on disk to reload its copy from`,
|
|
15200
|
+
{ indexName, partitionId }
|
|
15201
|
+
);
|
|
15202
|
+
}
|
|
15203
|
+
await withPartitionHeld(core, indexName, async () => {
|
|
15204
|
+
await publishPartitionCopy(core, indexName, partitionId, writeRecords, async (manager, writeRecords2) => {
|
|
15205
|
+
const store = directoryCheckpointStore(registry.directory);
|
|
15206
|
+
const decoded = await readSegmentManifest(store, indexName);
|
|
15207
|
+
if (decoded !== null && decoded.kind !== "current") {
|
|
15208
|
+
throw new NarsilError(
|
|
15209
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
15210
|
+
`Index "${indexName}" holds the checkpoint layout of an earlier release, so partition ${partitionId} cannot reload alone`,
|
|
15211
|
+
{ indexName, partitionId }
|
|
15212
|
+
);
|
|
15213
|
+
}
|
|
15214
|
+
const manifest = decoded?.manifest ?? null;
|
|
15215
|
+
const listed = manifest?.partitions.find((entry) => entry.partitionId === partitionId);
|
|
15216
|
+
if (manifest !== null && listed !== void 0) {
|
|
15217
|
+
await loadPartitionSegments(store, listed, manager, {
|
|
15218
|
+
schema: manifest.schema,
|
|
15219
|
+
decodesEveryDocument: true,
|
|
15220
|
+
writeRecords: writeRecords2
|
|
15221
|
+
});
|
|
15222
|
+
await loadPartitionVectors(store, manifest, manager, partitionId);
|
|
15223
|
+
}
|
|
15224
|
+
const checkpoint = manifest?.checkpoint ?? [];
|
|
15225
|
+
writeRecords2.loadTermHistory(
|
|
15226
|
+
indexName,
|
|
15227
|
+
partitionId,
|
|
15228
|
+
checkpoint.find((entry) => entry.partitionId === partitionId)
|
|
15229
|
+
);
|
|
15230
|
+
await reopenPartitionLogAfterReplay(durable, indexName, partitionId, async () => {
|
|
15231
|
+
const replayed = await replaySyncedWal(
|
|
15232
|
+
registry.directory,
|
|
15233
|
+
indexName,
|
|
15234
|
+
partitionId,
|
|
15235
|
+
checkpointLastSeqNo(checkpoint, partitionId),
|
|
15236
|
+
{
|
|
15237
|
+
manager,
|
|
15238
|
+
vectorFieldPaths: core.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
15239
|
+
vectorIndexes: manager.getVectorIndexes(),
|
|
15240
|
+
writeRecords: writeRecords2
|
|
15241
|
+
}
|
|
15242
|
+
);
|
|
15243
|
+
return {
|
|
15244
|
+
seqNo: replayed.highestSeqNo,
|
|
15245
|
+
primaryTerm: Math.max(checkpointPrimaryTerm(checkpoint, partitionId), replayed.highestPrimaryTerm)
|
|
15246
|
+
};
|
|
15247
|
+
});
|
|
15248
|
+
});
|
|
15249
|
+
});
|
|
15250
|
+
}
|
|
15251
|
+
|
|
15252
|
+
// src/distribution/cluster-node/held-partitions.ts
|
|
15253
|
+
function createHeldPartitionRecord(core) {
|
|
15254
|
+
const writesByIndex = /* @__PURE__ */ new Map();
|
|
15255
|
+
const indexesBehindOnDisk = /* @__PURE__ */ new Set();
|
|
15256
|
+
function queueWrite(indexName, next) {
|
|
15257
|
+
const previous = writesByIndex.get(indexName) ?? Promise.resolve();
|
|
15258
|
+
const write = previous.then(() => writeHeldPartitions(core, indexName, next, indexesBehindOnDisk));
|
|
15259
|
+
writesByIndex.set(
|
|
15260
|
+
indexName,
|
|
15261
|
+
write.catch(() => void 0)
|
|
15262
|
+
);
|
|
15263
|
+
return write;
|
|
15264
|
+
}
|
|
15265
|
+
return {
|
|
15266
|
+
record(indexName, partitionId) {
|
|
15267
|
+
return queueWrite(indexName, (held) => held.includes(partitionId) ? null : [...held, partitionId]);
|
|
15268
|
+
},
|
|
15269
|
+
forget(indexName, partitionId) {
|
|
15270
|
+
return queueWrite(indexName, (held) => held.includes(partitionId) ? held.filter((id) => id !== partitionId) : null);
|
|
15271
|
+
},
|
|
15272
|
+
held(indexName) {
|
|
15273
|
+
return core.indexRegistry.get(indexName)?.heldPartitions ?? void 0;
|
|
15274
|
+
}
|
|
15275
|
+
};
|
|
15011
15276
|
}
|
|
15012
|
-
function
|
|
15013
|
-
const
|
|
15014
|
-
|
|
15015
|
-
|
|
15277
|
+
async function writeHeldPartitions(core, indexName, next, indexesBehindOnDisk) {
|
|
15278
|
+
const entry = core.indexRegistry.get(indexName);
|
|
15279
|
+
if (entry === void 0) {
|
|
15280
|
+
return;
|
|
15016
15281
|
}
|
|
15282
|
+
const updated = next(entry.heldPartitions ?? []);
|
|
15283
|
+
if (updated === null && !indexesBehindOnDisk.has(indexName)) {
|
|
15284
|
+
return;
|
|
15285
|
+
}
|
|
15286
|
+
if (updated !== null) {
|
|
15287
|
+
core.indexRegistry.set(indexName, { ...entry, heldPartitions: [...updated].sort((a, b) => a - b) });
|
|
15288
|
+
}
|
|
15289
|
+
if (core.durability === null) {
|
|
15290
|
+
return;
|
|
15291
|
+
}
|
|
15292
|
+
indexesBehindOnDisk.add(indexName);
|
|
15293
|
+
await core.durability.manager.persistMetadata(indexName);
|
|
15294
|
+
indexesBehindOnDisk.delete(indexName);
|
|
15017
15295
|
}
|
|
15018
15296
|
|
|
15019
15297
|
// src/distribution/cluster-node/local-replication.ts
|
|
@@ -15041,23 +15319,25 @@ async function seedReopenedPrimaryLogs(input) {
|
|
|
15041
15319
|
function requestIdOf(kind, entry) {
|
|
15042
15320
|
return `replicate-entry-${kind}-${entry.indexName}-${entry.partitionId}-${entry.seqNo}`;
|
|
15043
15321
|
}
|
|
15044
|
-
function
|
|
15045
|
-
const
|
|
15046
|
-
const held = manager.has(docId);
|
|
15047
|
-
const removedSpellings = held && entry.document !== null ? manager.removedSpellingsOf(docId) : void 0;
|
|
15048
|
-
try {
|
|
15049
|
-
applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes);
|
|
15050
|
-
} catch (err) {
|
|
15051
|
-
if (held && !manager.has(docId)) {
|
|
15052
|
-
core.orchestrator.recordMainWrite({ type: "remove", indexName, docId, requestId: requestIdOf("remove", entry) });
|
|
15053
|
-
}
|
|
15054
|
-
throw err;
|
|
15055
|
-
}
|
|
15056
|
-
const document = entry.document === null ? void 0 : manager.getRef(docId);
|
|
15322
|
+
function publishIndexEntry(core, manager, entry) {
|
|
15323
|
+
const document = entry.document === null ? void 0 : manager.getRef(entry.documentId);
|
|
15057
15324
|
if (document === void 0) return;
|
|
15058
|
-
core.orchestrator.
|
|
15059
|
-
|
|
15060
|
-
|
|
15325
|
+
core.orchestrator.publishToWorkers({
|
|
15326
|
+
type: "update",
|
|
15327
|
+
indexName: entry.indexName,
|
|
15328
|
+
docId: entry.documentId,
|
|
15329
|
+
document,
|
|
15330
|
+
requestId: requestIdOf("update", entry)
|
|
15331
|
+
});
|
|
15332
|
+
}
|
|
15333
|
+
function publishRemoval(core, entry, removedSpellings) {
|
|
15334
|
+
core.orchestrator.publishToWorkers({
|
|
15335
|
+
type: "remove",
|
|
15336
|
+
indexName: entry.indexName,
|
|
15337
|
+
docId: entry.documentId,
|
|
15338
|
+
requestId: requestIdOf("remove", entry),
|
|
15339
|
+
removedSpellings
|
|
15340
|
+
});
|
|
15061
15341
|
}
|
|
15062
15342
|
async function applyLocalReplicationEntry(core, entry, writeRecords) {
|
|
15063
15343
|
const { applied, error } = await applyLocalReplicationEntries(core, [entry], writeRecords);
|
|
@@ -15073,37 +15353,32 @@ async function applyLocalReplicationEntries(core, entries, writeRecords) {
|
|
|
15073
15353
|
const manager = core.requireManager(first.indexName);
|
|
15074
15354
|
const vecIndexes = manager.getVectorIndexes();
|
|
15075
15355
|
let stopped = null;
|
|
15076
|
-
const applyAt = (position) => {
|
|
15356
|
+
const applyAt = (position) => manager.publication.write(() => {
|
|
15077
15357
|
if (stopped !== null) throw stopped;
|
|
15078
15358
|
const entry = entries[position];
|
|
15359
|
+
const held = manager.has(entry.documentId);
|
|
15360
|
+
const removedSpellings = held ? manager.removedSpellingsOf(entry.documentId) : void 0;
|
|
15079
15361
|
try {
|
|
15080
15362
|
if (entry.operation === "INDEX") {
|
|
15081
|
-
|
|
15363
|
+
applyIndexEntry(entry, manager, indexEntry.vectorFieldPaths, vecIndexes, false);
|
|
15082
15364
|
} else {
|
|
15083
|
-
const held = manager.has(entry.documentId);
|
|
15084
|
-
const removedSpellings = manager.removedSpellingsOf(entry.documentId);
|
|
15085
15365
|
applyDeleteEntry(entry, manager, vecIndexes, removedSpellings);
|
|
15086
|
-
if (held)
|
|
15087
|
-
core.orchestrator.recordMainWrite({
|
|
15088
|
-
type: "remove",
|
|
15089
|
-
indexName: entry.indexName,
|
|
15090
|
-
docId: entry.documentId,
|
|
15091
|
-
requestId: requestIdOf("remove", entry),
|
|
15092
|
-
removedSpellings
|
|
15093
|
-
});
|
|
15094
15366
|
}
|
|
15095
15367
|
writeRecords.applied(entry);
|
|
15368
|
+
if (entry.operation === "INDEX") publishIndexEntry(core, manager, entry);
|
|
15369
|
+
else if (held) publishRemoval(core, entry, removedSpellings);
|
|
15096
15370
|
} catch (err) {
|
|
15371
|
+
if (held && !manager.has(entry.documentId)) publishRemoval(core, entry, removedSpellings);
|
|
15097
15372
|
stopped = err;
|
|
15098
15373
|
throw err;
|
|
15099
15374
|
}
|
|
15100
|
-
};
|
|
15375
|
+
});
|
|
15101
15376
|
let applied = 0;
|
|
15102
15377
|
let error = null;
|
|
15103
15378
|
const durable = core.durability?.manager;
|
|
15104
15379
|
if (durable === void 0) {
|
|
15105
15380
|
try {
|
|
15106
|
-
for (; applied < entries.length; applied++) applyAt(applied);
|
|
15381
|
+
for (; applied < entries.length; applied++) await applyAt(applied);
|
|
15107
15382
|
} catch (err) {
|
|
15108
15383
|
error = err;
|
|
15109
15384
|
}
|
|
@@ -15127,8 +15402,10 @@ async function applyLocalReplicationEntries(core, entries, writeRecords) {
|
|
|
15127
15402
|
applied++;
|
|
15128
15403
|
}
|
|
15129
15404
|
}
|
|
15130
|
-
|
|
15131
|
-
|
|
15405
|
+
if (applied > 0) {
|
|
15406
|
+
for (const vector of vecIndexes.values()) vector.scheduleBuild();
|
|
15407
|
+
}
|
|
15408
|
+
await core.orchestrator.settlePublications(first.indexName);
|
|
15132
15409
|
return { applied, error };
|
|
15133
15410
|
} finally {
|
|
15134
15411
|
release();
|
|
@@ -15248,7 +15525,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
15248
15525
|
);
|
|
15249
15526
|
}
|
|
15250
15527
|
try {
|
|
15251
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15528
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-G576XGVE.mjs');
|
|
15252
15529
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
15253
15530
|
return {
|
|
15254
15531
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -15304,6 +15581,7 @@ async function receiveArchive(indexName, partitionId, source, schema, directory)
|
|
|
15304
15581
|
let arrayPositionGap = null;
|
|
15305
15582
|
let received = null;
|
|
15306
15583
|
await readSnapshotArchive(createByteReader(source), {
|
|
15584
|
+
partitionId,
|
|
15307
15585
|
config: async (bytes) => {
|
|
15308
15586
|
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
15309
15587
|
envelopeFormatVersion: 5,
|
|
@@ -15370,7 +15648,7 @@ async function replacePartition(manager, partitionId, archive, writeRecords) {
|
|
|
15370
15648
|
for (const vectorIndex of vectorIndexes.values()) {
|
|
15371
15649
|
for (const docId of held) if (vectorIndex.has(docId)) vectorIndex.remove(docId);
|
|
15372
15650
|
}
|
|
15373
|
-
manager.deserializePartition(partitionId, emptySerializablePartition(manager, partitionId));
|
|
15651
|
+
manager.deserializePartition(partitionId, emptySerializablePartition(manager, partitionId), deserializePartition);
|
|
15374
15652
|
writeRecords.resetPartition(manager.indexName, partitionId);
|
|
15375
15653
|
const entry = archive.manifest.partitions[0];
|
|
15376
15654
|
if (entry !== void 0) {
|
|
@@ -15403,7 +15681,6 @@ async function replacePartition(manager, partitionId, archive, writeRecords) {
|
|
|
15403
15681
|
vectorIndex.insert(decoded.docIds[i], vector, partitionId);
|
|
15404
15682
|
}
|
|
15405
15683
|
}
|
|
15406
|
-
vectorIndex.scheduleBuild();
|
|
15407
15684
|
}
|
|
15408
15685
|
}
|
|
15409
15686
|
function recordReceivedAnalysis(core, indexName, receivedRevision, receivedGap) {
|
|
@@ -15452,12 +15729,18 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
15452
15729
|
}
|
|
15453
15730
|
const wasScaledOut = core.orchestrator.desyncIndex(indexName);
|
|
15454
15731
|
try {
|
|
15455
|
-
await
|
|
15732
|
+
await publishPartitionCopy(
|
|
15733
|
+
core,
|
|
15734
|
+
indexName,
|
|
15735
|
+
partitionId,
|
|
15736
|
+
writeRecords,
|
|
15737
|
+
(manager, stagedRecords) => replacePartition(manager, partitionId, archive, stagedRecords)
|
|
15738
|
+
);
|
|
15456
15739
|
recordReceivedAnalysis(core, indexName, archive.analysisRevision, archive.arrayPositionGap);
|
|
15457
15740
|
if (directory !== null) {
|
|
15458
15741
|
const durable = core.durability?.manager;
|
|
15459
15742
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
15460
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
15743
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-G576XGVE.mjs');
|
|
15461
15744
|
await removeReceivedFilesNothingReads(
|
|
15462
15745
|
directory,
|
|
15463
15746
|
core.requireManager(indexName),
|
|
@@ -15489,297 +15772,66 @@ function snapshotRankedDocuments(manager, result, into) {
|
|
|
15489
15772
|
if (into.has(docId)) return;
|
|
15490
15773
|
const snapshot = snapshotOf(manager, docId);
|
|
15491
15774
|
if (snapshot !== void 0) into.set(docId, snapshot);
|
|
15492
|
-
};
|
|
15493
|
-
for (const hit of result.hits) take(hit.id);
|
|
15494
|
-
for (const group of result.groups ?? []) {
|
|
15495
|
-
for (const hit of group.hits) take(hit.id);
|
|
15496
|
-
}
|
|
15497
|
-
}
|
|
15498
|
-
|
|
15499
|
-
// src/distribution/cluster-node/write-admission.ts
|
|
15500
|
-
function createAdmittingRecorder(core, admission, durable) {
|
|
15501
|
-
function admit(indexName, docId, document) {
|
|
15502
|
-
const partitionId = fnv1a(docId) % Math.max(1, core.requireManager(indexName).partitionCount);
|
|
15503
|
-
admission(indexName, partitionId, docId, document);
|
|
15504
|
-
return partitionId;
|
|
15505
|
-
}
|
|
15506
|
-
async function applyAdmitted(indexName, partitionId, apply) {
|
|
15507
|
-
await apply();
|
|
15508
|
-
return { indexName, partitionId, seqNo: 0 };
|
|
15509
|
-
}
|
|
15510
|
-
return {
|
|
15511
|
-
async recordInsertOrUpdate(indexName, docId, document, apply) {
|
|
15512
|
-
const partitionId = admit(indexName, docId, document);
|
|
15513
|
-
if (durable === null) return applyAdmitted(indexName, partitionId, apply);
|
|
15514
|
-
return durable.recordInsertOrUpdate(indexName, docId, document, apply);
|
|
15515
|
-
},
|
|
15516
|
-
async recordRemove(indexName, docId, apply) {
|
|
15517
|
-
const partitionId = admit(indexName, docId, null);
|
|
15518
|
-
if (durable === null) return applyAdmitted(indexName, partitionId, apply);
|
|
15519
|
-
return durable.recordRemove(indexName, docId, apply);
|
|
15520
|
-
},
|
|
15521
|
-
async recordMutationBatch(indexName, mutations) {
|
|
15522
|
-
const outcomes = new Array(mutations.length);
|
|
15523
|
-
const admitted = [];
|
|
15524
|
-
const positions = [];
|
|
15525
|
-
const partitionIds = [];
|
|
15526
|
-
for (let i = 0; i < mutations.length; i++) {
|
|
15527
|
-
try {
|
|
15528
|
-
partitionIds.push(admit(indexName, mutations[i].docId, mutations[i].document));
|
|
15529
|
-
admitted.push(mutations[i]);
|
|
15530
|
-
positions.push(i);
|
|
15531
|
-
} catch (error) {
|
|
15532
|
-
outcomes[i] = { ok: false, error };
|
|
15533
|
-
}
|
|
15534
|
-
}
|
|
15535
|
-
if (admitted.length === 0) return outcomes;
|
|
15536
|
-
if (durable !== null) {
|
|
15537
|
-
const recorded = await durable.recordMutationBatch(indexName, admitted);
|
|
15538
|
-
for (let k = 0; k < recorded.length; k++) outcomes[positions[k]] = recorded[k];
|
|
15539
|
-
return outcomes;
|
|
15540
|
-
}
|
|
15541
|
-
for (let k = 0; k < admitted.length; k++) {
|
|
15542
|
-
try {
|
|
15543
|
-
outcomes[positions[k]] = {
|
|
15544
|
-
ok: true,
|
|
15545
|
-
write: await applyAdmitted(indexName, partitionIds[k], admitted[k].apply)
|
|
15546
|
-
};
|
|
15547
|
-
} catch (error) {
|
|
15548
|
-
outcomes[positions[k]] = { ok: false, error };
|
|
15549
|
-
}
|
|
15550
|
-
}
|
|
15551
|
-
return outcomes;
|
|
15552
|
-
}
|
|
15553
|
-
};
|
|
15554
|
-
}
|
|
15555
|
-
|
|
15556
|
-
// src/distribution/cluster-node/write-records/constants.ts
|
|
15557
|
-
var INITIAL_WRITE_RECORD_SLOTS = 64;
|
|
15558
|
-
|
|
15559
|
-
// src/distribution/cluster-node/write-records/table.ts
|
|
15560
|
-
function createWriteRecordTable() {
|
|
15561
|
-
const slots = /* @__PURE__ */ new Map();
|
|
15562
|
-
let seqNos = new Float64Array(0);
|
|
15563
|
-
let primaryTerms = new Float64Array(0);
|
|
15564
|
-
let freeSlots = [];
|
|
15565
|
-
let nextSlot = 0;
|
|
15566
|
-
function grow() {
|
|
15567
|
-
const capacity = Math.max(INITIAL_WRITE_RECORD_SLOTS, seqNos.length * 2);
|
|
15568
|
-
const grownSeqNos = new Float64Array(capacity);
|
|
15569
|
-
const grownTerms = new Float64Array(capacity);
|
|
15570
|
-
grownSeqNos.set(seqNos);
|
|
15571
|
-
grownTerms.set(primaryTerms);
|
|
15572
|
-
seqNos = grownSeqNos;
|
|
15573
|
-
primaryTerms = grownTerms;
|
|
15574
|
-
}
|
|
15575
|
-
function claimSlot() {
|
|
15576
|
-
const reused = freeSlots.pop();
|
|
15577
|
-
if (reused !== void 0) return reused;
|
|
15578
|
-
if (nextSlot === seqNos.length) grow();
|
|
15579
|
-
return nextSlot++;
|
|
15580
|
-
}
|
|
15581
|
-
return {
|
|
15582
|
-
get count() {
|
|
15583
|
-
return slots.size;
|
|
15584
|
-
},
|
|
15585
|
-
set(documentId, seqNo, primaryTerm) {
|
|
15586
|
-
let slot = slots.get(documentId);
|
|
15587
|
-
if (slot === void 0) {
|
|
15588
|
-
slot = claimSlot();
|
|
15589
|
-
slots.set(documentId, slot);
|
|
15590
|
-
}
|
|
15591
|
-
seqNos[slot] = seqNo;
|
|
15592
|
-
primaryTerms[slot] = primaryTerm;
|
|
15593
|
-
},
|
|
15594
|
-
remove(documentId) {
|
|
15595
|
-
const slot = slots.get(documentId);
|
|
15596
|
-
if (slot === void 0) return;
|
|
15597
|
-
slots.delete(documentId);
|
|
15598
|
-
seqNos[slot] = 0;
|
|
15599
|
-
primaryTerms[slot] = 0;
|
|
15600
|
-
freeSlots.push(slot);
|
|
15601
|
-
},
|
|
15602
|
-
recordOf(documentId) {
|
|
15603
|
-
const slot = slots.get(documentId);
|
|
15604
|
-
if (slot === void 0) return null;
|
|
15605
|
-
return { seqNo: seqNos[slot], primaryTerm: primaryTerms[slot] };
|
|
15606
|
-
},
|
|
15607
|
-
seqNoOf(documentId) {
|
|
15608
|
-
const slot = slots.get(documentId);
|
|
15609
|
-
return slot === void 0 ? 0 : seqNos[slot];
|
|
15610
|
-
},
|
|
15611
|
-
primaryTermOf(documentId) {
|
|
15612
|
-
const slot = slots.get(documentId);
|
|
15613
|
-
return slot === void 0 ? 0 : primaryTerms[slot];
|
|
15614
|
-
},
|
|
15615
|
-
clear() {
|
|
15616
|
-
slots.clear();
|
|
15617
|
-
seqNos = new Float64Array(0);
|
|
15618
|
-
primaryTerms = new Float64Array(0);
|
|
15619
|
-
freeSlots = [];
|
|
15620
|
-
nextSlot = 0;
|
|
15621
|
-
}
|
|
15622
|
-
};
|
|
15623
|
-
}
|
|
15624
|
-
|
|
15625
|
-
// src/distribution/cluster-node/write-records/term-history.ts
|
|
15626
|
-
function createTermHistory() {
|
|
15627
|
-
let starts = [];
|
|
15628
|
-
let highestSeqNo = 0;
|
|
15629
|
-
return {
|
|
15630
|
-
get highestSeqNo() {
|
|
15631
|
-
return highestSeqNo;
|
|
15632
|
-
},
|
|
15633
|
-
noteEntry(seqNo, primaryTerm) {
|
|
15634
|
-
const last = starts.at(-1);
|
|
15635
|
-
if (last === void 0 || primaryTerm > last.primaryTerm) starts.push({ primaryTerm, firstSeqNo: seqNo });
|
|
15636
|
-
if (seqNo > highestSeqNo) highestSeqNo = seqNo;
|
|
15637
|
-
},
|
|
15638
|
-
termOf(seqNo) {
|
|
15639
|
-
if (seqNo > highestSeqNo) return null;
|
|
15640
|
-
let term = null;
|
|
15641
|
-
for (const start of starts) {
|
|
15642
|
-
if (start.firstSeqNo > seqNo) break;
|
|
15643
|
-
term = start.primaryTerm;
|
|
15644
|
-
}
|
|
15645
|
-
return term;
|
|
15646
|
-
},
|
|
15647
|
-
starts(upToSeqNo) {
|
|
15648
|
-
return starts.filter((start) => start.firstSeqNo <= upToSeqNo).map((start) => ({ ...start }));
|
|
15649
|
-
},
|
|
15650
|
-
replace(next, nextHighest) {
|
|
15651
|
-
starts = next.map((start) => ({ ...start }));
|
|
15652
|
-
highestSeqNo = nextHighest;
|
|
15653
|
-
},
|
|
15654
|
-
prune(floorSeqNo) {
|
|
15655
|
-
while (starts.length > 1 && starts[1].firstSeqNo <= floorSeqNo) starts.shift();
|
|
15656
|
-
},
|
|
15657
|
-
clear() {
|
|
15658
|
-
starts = [];
|
|
15659
|
-
highestSeqNo = 0;
|
|
15660
|
-
}
|
|
15661
|
-
};
|
|
15662
|
-
}
|
|
15663
|
-
|
|
15664
|
-
// src/distribution/cluster-node/write-records/keeper.ts
|
|
15665
|
-
function createClusterWriteRecords() {
|
|
15666
|
-
const indexes = /* @__PURE__ */ new Map();
|
|
15667
|
-
let oldestRetainedSeqNo = () => void 0;
|
|
15668
|
-
function partition(indexName, partitionId) {
|
|
15669
|
-
let partitions = indexes.get(indexName);
|
|
15670
|
-
if (partitions === void 0) {
|
|
15671
|
-
partitions = /* @__PURE__ */ new Map();
|
|
15672
|
-
indexes.set(indexName, partitions);
|
|
15673
|
-
}
|
|
15674
|
-
let held = partitions.get(partitionId);
|
|
15675
|
-
if (held === void 0) {
|
|
15676
|
-
held = { table: createWriteRecordTable(), history: createTermHistory(), pagedSegments: [] };
|
|
15677
|
-
partitions.set(partitionId, held);
|
|
15678
|
-
}
|
|
15679
|
-
return held;
|
|
15680
|
-
}
|
|
15681
|
-
function applied(entry) {
|
|
15682
|
-
const held = partition(entry.indexName, entry.partitionId);
|
|
15683
|
-
if (entry.operation === "INDEX") held.table.set(entry.documentId, entry.seqNo, entry.primaryTerm);
|
|
15684
|
-
else held.table.remove(entry.documentId);
|
|
15685
|
-
held.history.noteEntry(entry.seqNo, entry.primaryTerm);
|
|
15775
|
+
};
|
|
15776
|
+
for (const hit of result.hits) take(hit.id);
|
|
15777
|
+
for (const group of result.groups ?? []) {
|
|
15778
|
+
for (const hit of group.hits) take(hit.id);
|
|
15686
15779
|
}
|
|
15687
|
-
|
|
15688
|
-
|
|
15689
|
-
|
|
15690
|
-
|
|
15691
|
-
|
|
15692
|
-
|
|
15693
|
-
|
|
15694
|
-
|
|
15695
|
-
const stored = segment.paged.writeRecordAt(segment.arrays.idTable.ordinalOf(documentId));
|
|
15696
|
-
return stored.seqNo === 0 ? null : stored;
|
|
15697
|
-
}
|
|
15698
|
-
return null;
|
|
15780
|
+
}
|
|
15781
|
+
|
|
15782
|
+
// src/distribution/cluster-node/write-admission.ts
|
|
15783
|
+
function createAdmittingRecorder(core, admission, durable) {
|
|
15784
|
+
function admit(indexName, docId, document) {
|
|
15785
|
+
const partitionId = fnv1a(docId) % Math.max(1, core.requireManager(indexName).partitionCount);
|
|
15786
|
+
admission(indexName, partitionId, docId, document);
|
|
15787
|
+
return partitionId;
|
|
15699
15788
|
}
|
|
15700
|
-
function
|
|
15701
|
-
|
|
15702
|
-
|
|
15703
|
-
const primaryTerms = new Float64Array(count2);
|
|
15704
|
-
for (let ordinal = 0; ordinal < count2; ordinal++) {
|
|
15705
|
-
if (segment.isTombstoned(ordinal)) continue;
|
|
15706
|
-
const record = recordOfDocument(indexName, partitionId, segment.arrays.idTable.idAt(ordinal));
|
|
15707
|
-
if (record === null) continue;
|
|
15708
|
-
seqNos[ordinal] = record.seqNo;
|
|
15709
|
-
primaryTerms[ordinal] = record.primaryTerm;
|
|
15710
|
-
}
|
|
15711
|
-
return { seqNos, primaryTerms };
|
|
15789
|
+
async function applyAdmitted(indexName, partitionId, apply) {
|
|
15790
|
+
await apply();
|
|
15791
|
+
return { indexName, partitionId, seqNo: 0 };
|
|
15712
15792
|
}
|
|
15713
15793
|
return {
|
|
15714
|
-
|
|
15715
|
-
|
|
15716
|
-
|
|
15717
|
-
|
|
15718
|
-
replayed: applied,
|
|
15719
|
-
partition,
|
|
15720
|
-
recordOf: recordOfDocument,
|
|
15721
|
-
heldEntry(indexName, partitionId, seqNo, primaryTerm) {
|
|
15722
|
-
const history = indexes.get(indexName)?.get(partitionId)?.history;
|
|
15723
|
-
if (history === void 0) return "uncovered";
|
|
15724
|
-
if (seqNo > history.highestSeqNo) return "absent";
|
|
15725
|
-
const term = history.termOf(seqNo);
|
|
15726
|
-
if (term === null) return "uncovered";
|
|
15727
|
-
return term === primaryTerm ? "held" : "absent";
|
|
15728
|
-
},
|
|
15729
|
-
resetPartition(indexName, partitionId) {
|
|
15730
|
-
const held = indexes.get(indexName)?.get(partitionId);
|
|
15731
|
-
if (held === void 0) return;
|
|
15732
|
-
held.table.clear();
|
|
15733
|
-
held.history.clear();
|
|
15734
|
-
held.pagedSegments = [];
|
|
15794
|
+
async recordInsertOrUpdate(indexName, docId, document, apply) {
|
|
15795
|
+
const partitionId = admit(indexName, docId, document);
|
|
15796
|
+
if (durable === null) return applyAdmitted(indexName, partitionId, apply);
|
|
15797
|
+
return durable.recordInsertOrUpdate(indexName, docId, document, apply);
|
|
15735
15798
|
},
|
|
15736
|
-
|
|
15737
|
-
const
|
|
15738
|
-
|
|
15739
|
-
|
|
15799
|
+
async recordRemove(indexName, docId, apply) {
|
|
15800
|
+
const partitionId = admit(indexName, docId, null);
|
|
15801
|
+
if (durable === null) return applyAdmitted(indexName, partitionId, apply);
|
|
15802
|
+
return durable.recordRemove(indexName, docId, apply);
|
|
15740
15803
|
},
|
|
15741
|
-
|
|
15742
|
-
const
|
|
15743
|
-
const
|
|
15744
|
-
|
|
15745
|
-
|
|
15746
|
-
|
|
15804
|
+
async recordMutationBatch(indexName, mutations) {
|
|
15805
|
+
const outcomes = new Array(mutations.length);
|
|
15806
|
+
const admitted = [];
|
|
15807
|
+
const positions = [];
|
|
15808
|
+
const partitionIds = [];
|
|
15809
|
+
for (let i = 0; i < mutations.length; i++) {
|
|
15810
|
+
try {
|
|
15811
|
+
partitionIds.push(admit(indexName, mutations[i].docId, mutations[i].document));
|
|
15812
|
+
admitted.push(mutations[i]);
|
|
15813
|
+
positions.push(i);
|
|
15814
|
+
} catch (error) {
|
|
15815
|
+
outcomes[i] = { ok: false, error };
|
|
15816
|
+
}
|
|
15747
15817
|
}
|
|
15748
|
-
|
|
15749
|
-
|
|
15750
|
-
|
|
15751
|
-
|
|
15752
|
-
|
|
15753
|
-
const { table } = partition(indexName, partitionId);
|
|
15754
|
-
for (const documentId of contents.tombstones) table.remove(documentId);
|
|
15755
|
-
const records = contents.writeRecords;
|
|
15756
|
-
for (let ordinal = 0; ordinal < segment.arrays.documentCount; ordinal++) {
|
|
15757
|
-
const documentId = segment.arrays.idTable.idAt(ordinal);
|
|
15758
|
-
const seqNo = records?.seqNos[ordinal] ?? 0;
|
|
15759
|
-
if (seqNo === 0 || records === void 0) table.remove(documentId);
|
|
15760
|
-
else table.set(documentId, seqNo, records.primaryTerms[ordinal]);
|
|
15818
|
+
if (admitted.length === 0) return outcomes;
|
|
15819
|
+
if (durable !== null) {
|
|
15820
|
+
const recorded = await durable.recordMutationBatch(indexName, admitted);
|
|
15821
|
+
for (let k = 0; k < recorded.length; k++) outcomes[positions[k]] = recorded[k];
|
|
15822
|
+
return outcomes;
|
|
15761
15823
|
}
|
|
15762
|
-
|
|
15763
|
-
|
|
15764
|
-
|
|
15765
|
-
|
|
15766
|
-
|
|
15767
|
-
|
|
15824
|
+
for (let k = 0; k < admitted.length; k++) {
|
|
15825
|
+
try {
|
|
15826
|
+
outcomes[positions[k]] = {
|
|
15827
|
+
ok: true,
|
|
15828
|
+
write: await applyAdmitted(indexName, partitionIds[k], admitted[k].apply)
|
|
15829
|
+
};
|
|
15830
|
+
} catch (error) {
|
|
15831
|
+
outcomes[positions[k]] = { ok: false, error };
|
|
15832
|
+
}
|
|
15768
15833
|
}
|
|
15769
|
-
return
|
|
15770
|
-
},
|
|
15771
|
-
termHistoryOf(indexName, partitionId, lastSeqNo) {
|
|
15772
|
-
const history = indexes.get(indexName)?.get(partitionId)?.history;
|
|
15773
|
-
if (history === void 0) return [];
|
|
15774
|
-
const oldest = oldestRetainedSeqNo(indexName, partitionId);
|
|
15775
|
-
if (oldest !== void 0) history.prune(Math.min(oldest, lastSeqNo));
|
|
15776
|
-
return history.starts(lastSeqNo);
|
|
15777
|
-
},
|
|
15778
|
-
loadTermHistory(indexName, partitionId, checkpoint) {
|
|
15779
|
-
partition(indexName, partitionId).history.replace(checkpoint?.termHistory ?? [], checkpoint?.lastSeqNo ?? 0);
|
|
15780
|
-
},
|
|
15781
|
-
dropIndex(indexName) {
|
|
15782
|
-
indexes.delete(indexName);
|
|
15834
|
+
return outcomes;
|
|
15783
15835
|
}
|
|
15784
15836
|
};
|
|
15785
15837
|
}
|
|
@@ -15787,6 +15839,7 @@ function createClusterWriteRecords() {
|
|
|
15787
15839
|
// src/distribution/cluster-node/local-engine.ts
|
|
15788
15840
|
async function createClusterLocalEngine(config, hooks) {
|
|
15789
15841
|
requireKnownConfig(config);
|
|
15842
|
+
await Promise.all([importOptionalReads(), loadMessagePack()]);
|
|
15790
15843
|
const runnableConfig = await resolveRunnableConfig(config);
|
|
15791
15844
|
const writeRecords = createClusterWriteRecords();
|
|
15792
15845
|
const core = await createEngineCore(
|
|
@@ -15820,6 +15873,7 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
15820
15873
|
dropIndex: async (name) => {
|
|
15821
15874
|
await dropIndex(name);
|
|
15822
15875
|
writeRecords.dropIndex(name);
|
|
15876
|
+
hooks?.onIndexDrop?.(name);
|
|
15823
15877
|
},
|
|
15824
15878
|
restore: async (name, data) => {
|
|
15825
15879
|
try {
|
|
@@ -16216,7 +16270,15 @@ async function storeNewNodeId(directory, filePath) {
|
|
|
16216
16270
|
return nodeId;
|
|
16217
16271
|
}
|
|
16218
16272
|
async function resolveNodeId(explicitNodeId, directory) {
|
|
16219
|
-
if (explicitNodeId !== void 0)
|
|
16273
|
+
if (explicitNodeId !== void 0) {
|
|
16274
|
+
if (typeof explicitNodeId !== "string" || explicitNodeId.length === 0 || holdsUnpairedSurrogate(explicitNodeId)) {
|
|
16275
|
+
throw new NarsilError(
|
|
16276
|
+
ErrorCodes.CONFIG_INVALID,
|
|
16277
|
+
"ClusterNodeConfig.nodeId must be a non-empty string without an unpaired surrogate, the half of a character such as an emoji whose other half is missing, because the cluster hashes, sorts, and stores node ids as UTF-8"
|
|
16278
|
+
);
|
|
16279
|
+
}
|
|
16280
|
+
return explicitNodeId;
|
|
16281
|
+
}
|
|
16220
16282
|
if (directory === void 0) return generateId();
|
|
16221
16283
|
const pathModule = await getPath();
|
|
16222
16284
|
const root = pathModule.resolve(directory);
|
|
@@ -17514,7 +17576,7 @@ function convertLocalGroupToWire(group) {
|
|
|
17514
17576
|
}
|
|
17515
17577
|
return {
|
|
17516
17578
|
fields: [...group.fields],
|
|
17517
|
-
maxPerGroup: group.reduce !== void 0 ? MAX_LIMIT : hitsKeptPerGroup(group.maxPerGroup),
|
|
17579
|
+
maxPerGroup: group.reduce !== void 0 ? MAX_LIMIT : requireLoadedOptionalReads().hitsKeptPerGroup(group.maxPerGroup),
|
|
17518
17580
|
limit: group.limit ?? null
|
|
17519
17581
|
};
|
|
17520
17582
|
}
|
|
@@ -17577,7 +17639,7 @@ function convertWireFacetsToLocal(result, wireFacets, requested) {
|
|
|
17577
17639
|
for (const [field, buckets] of Object.entries(wireFacets)) {
|
|
17578
17640
|
const mergedBound = result.facetErrorBounds?.[field] ?? 0;
|
|
17579
17641
|
const kept = requested === void 0 ? buckets.length : Math.min(buckets.length, facetLimitOf(requested[field]));
|
|
17580
|
-
const values =
|
|
17642
|
+
const values = /* @__PURE__ */ Object.create(null);
|
|
17581
17643
|
for (let index = 0; index < kept; index++) values[buckets[index].value] = buckets[index].count;
|
|
17582
17644
|
let largestCut = 0;
|
|
17583
17645
|
for (let index = kept; index < buckets.length; index++) largestCut = Math.max(largestCut, buckets[index].count);
|
|
@@ -17867,199 +17929,220 @@ async function sendWithTimeout(transport, targets, message, timeout, validate, o
|
|
|
17867
17929
|
}
|
|
17868
17930
|
}
|
|
17869
17931
|
|
|
17870
|
-
// src/
|
|
17871
|
-
function
|
|
17872
|
-
|
|
17873
|
-
|
|
17874
|
-
|
|
17875
|
-
|
|
17876
|
-
|
|
17877
|
-
|
|
17878
|
-
|
|
17879
|
-
|
|
17880
|
-
|
|
17881
|
-
|
|
17882
|
-
|
|
17883
|
-
|
|
17884
|
-
|
|
17885
|
-
|
|
17886
|
-
|
|
17887
|
-
|
|
17888
|
-
if (
|
|
17889
|
-
if (
|
|
17890
|
-
|
|
17891
|
-
return
|
|
17892
|
-
}
|
|
17893
|
-
return heapMergeScoredEntries(nonEmpty, limit, compare);
|
|
17894
|
-
}
|
|
17895
|
-
function sequentialMergeScoredEntries(arrays, limit, compare) {
|
|
17896
|
-
let merged = arrays[0];
|
|
17897
|
-
for (let i = 1; i < arrays.length; i++) {
|
|
17898
|
-
merged = mergeTwoSortedScoredEntries(merged, arrays[i], compare);
|
|
17899
|
-
}
|
|
17900
|
-
return merged.slice(0, limit);
|
|
17901
|
-
}
|
|
17902
|
-
function mergeTwoSortedScoredEntries(a, b, compare) {
|
|
17903
|
-
const result = new Array(a.length + b.length);
|
|
17904
|
-
let ai = 0;
|
|
17905
|
-
let bi = 0;
|
|
17906
|
-
let ri = 0;
|
|
17907
|
-
while (ai < a.length && bi < b.length) {
|
|
17908
|
-
if (compare(a[ai], b[bi]) <= 0) {
|
|
17909
|
-
result[ri++] = a[ai++];
|
|
17910
|
-
} else {
|
|
17911
|
-
result[ri++] = b[bi++];
|
|
17912
|
-
}
|
|
17913
|
-
}
|
|
17914
|
-
while (ai < a.length) {
|
|
17915
|
-
result[ri++] = a[ai++];
|
|
17916
|
-
}
|
|
17917
|
-
while (bi < b.length) {
|
|
17918
|
-
result[ri++] = b[bi++];
|
|
17919
|
-
}
|
|
17920
|
-
return result;
|
|
17921
|
-
}
|
|
17922
|
-
function heapMergeScoredEntries(arrays, limit, compare) {
|
|
17923
|
-
const heap = [];
|
|
17924
|
-
const nodeFirst = (a, b) => compare(a.entry, b.entry) < 0;
|
|
17925
|
-
for (let i = 0; i < arrays.length; i++) {
|
|
17926
|
-
if (arrays[i].length > 0) {
|
|
17927
|
-
pushScoredHeap(heap, { entry: arrays[i][0], sourceIdx: i, resultIdx: 0 }, nodeFirst);
|
|
17928
|
-
}
|
|
17929
|
-
}
|
|
17930
|
-
const result = [];
|
|
17931
|
-
while (heap.length > 0 && result.length < limit) {
|
|
17932
|
-
const top = popScoredHeap(heap, nodeFirst);
|
|
17933
|
-
result.push(top.entry);
|
|
17934
|
-
const nextIdx = top.resultIdx + 1;
|
|
17935
|
-
if (nextIdx < arrays[top.sourceIdx].length) {
|
|
17936
|
-
pushScoredHeap(
|
|
17937
|
-
heap,
|
|
17938
|
-
{ entry: arrays[top.sourceIdx][nextIdx], sourceIdx: top.sourceIdx, resultIdx: nextIdx },
|
|
17939
|
-
nodeFirst
|
|
17940
|
-
);
|
|
17941
|
-
}
|
|
17942
|
-
}
|
|
17943
|
-
return result;
|
|
17944
|
-
}
|
|
17945
|
-
function pushScoredHeap(heap, node, nodeFirst) {
|
|
17946
|
-
heap.push(node);
|
|
17947
|
-
let idx = heap.length - 1;
|
|
17948
|
-
while (idx > 0) {
|
|
17949
|
-
const parentIdx = idx - 1 >> 1;
|
|
17950
|
-
if (nodeFirst(heap[idx], heap[parentIdx])) {
|
|
17951
|
-
const tmp = heap[idx];
|
|
17952
|
-
heap[idx] = heap[parentIdx];
|
|
17953
|
-
heap[parentIdx] = tmp;
|
|
17954
|
-
idx = parentIdx;
|
|
17955
|
-
} else {
|
|
17956
|
-
break;
|
|
17957
|
-
}
|
|
17932
|
+
// src/core/merges/groups.ts
|
|
17933
|
+
function unsigned(value) {
|
|
17934
|
+
return Number.isInteger(value) && value >= 0 && value <= 4294967295;
|
|
17935
|
+
}
|
|
17936
|
+
function mergeGroups(keys, offsets, scores, scorePresent, ranks, idRanks, sorted, maxPerGroup, limit, outKeys, outOffsets, outSources, outIndices) {
|
|
17937
|
+
const count2 = ranks.length;
|
|
17938
|
+
if (!unsigned(keys.length) || !unsigned(count2) || offsets.length !== keys.length + 1 || offsets[0] !== 0 || offsets[keys.length] !== count2 || idRanks.length !== count2 || typeof sorted !== "boolean" || !unsigned(maxPerGroup) || limit !== null && !unsigned(limit) || (scores === null ? scorePresent !== null : scores.length !== count2) || scorePresent !== null && scorePresent.length < Math.max(1, Math.ceil(count2 / 32)))
|
|
17939
|
+
return -1;
|
|
17940
|
+
const inputs = [keys, offsets, scores, scorePresent, ranks, idRanks];
|
|
17941
|
+
const outputs = [outKeys, outOffsets, outSources, outIndices];
|
|
17942
|
+
for (let target = 0; target < outputs.length; target++) {
|
|
17943
|
+
if (inputs.some((input) => input !== null && regionsOverlap(outputs[target], input))) return -1;
|
|
17944
|
+
for (let prior = 0; prior < target; prior++) if (regionsOverlap(outputs[target], outputs[prior])) return -1;
|
|
17945
|
+
}
|
|
17946
|
+
for (let at = 1; at < offsets.length; at++) if (offsets[at] < offsets[at - 1]) return -1;
|
|
17947
|
+
if (!sorted) {
|
|
17948
|
+
for (let at = 0; at < count2; at++) if (ranks[at] !== idRanks[at]) return -1;
|
|
17949
|
+
}
|
|
17950
|
+
if (outOffsets.length === 0) return -3;
|
|
17951
|
+
if (keys.length === 0 || limit === 0) {
|
|
17952
|
+
outOffsets[0] = 0;
|
|
17953
|
+
return 0;
|
|
17958
17954
|
}
|
|
17959
|
-
|
|
17960
|
-
|
|
17961
|
-
|
|
17962
|
-
|
|
17963
|
-
|
|
17964
|
-
|
|
17965
|
-
|
|
17966
|
-
|
|
17967
|
-
|
|
17968
|
-
|
|
17969
|
-
|
|
17970
|
-
|
|
17971
|
-
|
|
17972
|
-
|
|
17973
|
-
|
|
17974
|
-
|
|
17975
|
-
|
|
17976
|
-
|
|
17977
|
-
|
|
17978
|
-
|
|
17979
|
-
|
|
17980
|
-
|
|
17981
|
-
|
|
17982
|
-
|
|
17983
|
-
|
|
17984
|
-
|
|
17955
|
+
const scoreOf = (entry, absent) => scores === null || scorePresent !== null && (scorePresent[entry >>> 5] & 1 << (entry & 31)) === 0 ? absent : scores[entry];
|
|
17956
|
+
const before = (left, leftSource, right, rightSource) => {
|
|
17957
|
+
const score = sorted ? 0 : compareScores(scoreOf(left, Number.NEGATIVE_INFINITY), scoreOf(right, Number.NEGATIVE_INFINITY));
|
|
17958
|
+
return (score || ranks[left] - ranks[right] || leftSource - rightSource) < 0;
|
|
17959
|
+
};
|
|
17960
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
17961
|
+
for (let source = 0; source < keys.length; source++) {
|
|
17962
|
+
let group = byKey.get(keys[source]);
|
|
17963
|
+
if (group === void 0) {
|
|
17964
|
+
group = { key: keys[source], sources: [], retained: 0, head: -1, score: 0, idRank: 0 };
|
|
17965
|
+
byKey.set(group.key, group);
|
|
17966
|
+
}
|
|
17967
|
+
group.sources.push(source);
|
|
17968
|
+
const start = offsets[source];
|
|
17969
|
+
const entries = offsets[source + 1] - start;
|
|
17970
|
+
group.retained = Math.min(maxPerGroup, group.retained + entries);
|
|
17971
|
+
if (maxPerGroup > 0 && entries > 0 && (group.head === -1 || before(start, source, offsets[group.head], group.head))) {
|
|
17972
|
+
group.head = source;
|
|
17973
|
+
group.score = scoreOf(start, 0);
|
|
17974
|
+
group.idRank = idRanks[start];
|
|
17975
|
+
}
|
|
17976
|
+
}
|
|
17977
|
+
const groups = [...byKey.values()];
|
|
17978
|
+
groups.sort(
|
|
17979
|
+
(left, right) => compareScores(left.score, right.score) || Number(left.retained > 0) - Number(right.retained > 0) || left.idRank - right.idRank || left.sources[0] - right.sources[0]
|
|
17980
|
+
);
|
|
17981
|
+
const kept = Math.min(limit ?? groups.length, groups.length);
|
|
17982
|
+
if (outKeys.length < kept || outOffsets.length < kept + 1) return -3;
|
|
17983
|
+
let needed = 0;
|
|
17984
|
+
for (let at = 0; at < kept; at++) needed += groups[at].retained;
|
|
17985
|
+
if (outSources.length < needed || outIndices.length < needed) return -3;
|
|
17986
|
+
const positions = new Uint32Array(keys.length);
|
|
17987
|
+
const heap = new Uint32Array(keys.length);
|
|
17988
|
+
const sift = (parent, size) => {
|
|
17989
|
+
const source = heap[parent];
|
|
17990
|
+
while (parent < Math.floor(size / 2)) {
|
|
17991
|
+
let child = parent * 2 + 1;
|
|
17992
|
+
if (child + 1 < size && before(positions[heap[child + 1]], heap[child + 1], positions[heap[child]], heap[child]))
|
|
17993
|
+
child++;
|
|
17994
|
+
if (!before(positions[heap[child]], heap[child], positions[source], source)) break;
|
|
17995
|
+
heap[parent] = heap[child];
|
|
17996
|
+
parent = child;
|
|
17997
|
+
}
|
|
17998
|
+
heap[parent] = source;
|
|
17999
|
+
};
|
|
18000
|
+
let written = 0;
|
|
18001
|
+
outOffsets[0] = 0;
|
|
18002
|
+
for (let at = 0; at < kept; at++) {
|
|
18003
|
+
const group = groups[at];
|
|
18004
|
+
let size = 0;
|
|
18005
|
+
for (const source of group.sources) {
|
|
18006
|
+
if (offsets[source] === offsets[source + 1]) continue;
|
|
18007
|
+
heap[size++] = source;
|
|
18008
|
+
positions[source] = offsets[source];
|
|
18009
|
+
}
|
|
18010
|
+
for (let parent = Math.floor(size / 2); parent > 0; parent--) sift(parent - 1, size);
|
|
18011
|
+
for (let entry = 0; entry < group.retained; entry++) {
|
|
18012
|
+
const source = heap[0];
|
|
18013
|
+
const position = positions[source]++;
|
|
18014
|
+
outSources[written] = source;
|
|
18015
|
+
outIndices[written++] = position - offsets[source];
|
|
18016
|
+
if (position + 1 === offsets[source + 1]) heap[0] = heap[--size];
|
|
18017
|
+
if (size > 0) sift(0, size);
|
|
18018
|
+
}
|
|
18019
|
+
outKeys[at] = group.key;
|
|
18020
|
+
outOffsets[at + 1] = written;
|
|
17985
18021
|
}
|
|
17986
|
-
return
|
|
18022
|
+
return kept;
|
|
17987
18023
|
}
|
|
17988
|
-
|
|
17989
|
-
|
|
17990
|
-
|
|
17991
|
-
|
|
17992
|
-
|
|
17993
|
-
|
|
17994
|
-
|
|
17995
|
-
|
|
17996
|
-
|
|
17997
|
-
|
|
17998
|
-
|
|
17999
|
-
|
|
18000
|
-
|
|
18001
|
-
|
|
18002
|
-
|
|
18003
|
-
|
|
18004
|
-
|
|
18005
|
-
|
|
18006
|
-
|
|
18007
|
-
|
|
18008
|
-
|
|
18009
|
-
|
|
18010
|
-
|
|
18011
|
-
|
|
18012
|
-
|
|
18013
|
-
|
|
18014
|
-
|
|
18015
|
-
|
|
18016
|
-
|
|
18017
|
-
|
|
18018
|
-
|
|
18019
|
-
|
|
18020
|
-
|
|
18021
|
-
});
|
|
18022
|
-
|
|
18023
|
-
|
|
18024
|
-
|
|
18025
|
-
|
|
18026
|
-
|
|
18027
|
-
}
|
|
18028
|
-
undercounts[field] = undercount;
|
|
18029
|
-
errorBounds[field] = undercount + largestLeftOut;
|
|
18024
|
+
|
|
18025
|
+
// src/core/merges/groups.native.ts
|
|
18026
|
+
var mergeGroups2 = (...args) => {
|
|
18027
|
+
if (!nativeSearchCoreIsEnabled()) return mergeGroups(...args);
|
|
18028
|
+
const core = loadNativeCore();
|
|
18029
|
+
if (core !== null) return core.mergeGroups(...args);
|
|
18030
|
+
noteFallback("mergeGroups");
|
|
18031
|
+
return mergeGroups(...args);
|
|
18032
|
+
};
|
|
18033
|
+
|
|
18034
|
+
// src/distribution/query/group-identities.ts
|
|
18035
|
+
function groupIdentities(sources, fields) {
|
|
18036
|
+
const count2 = sources.length;
|
|
18037
|
+
const stored = fields.map((field) => buildStoredValues(sources.map((source) => source.values[field])));
|
|
18038
|
+
const ordinals = Uint32Array.from(sources, (_, position) => position);
|
|
18039
|
+
const scores = new Float64Array(count2);
|
|
18040
|
+
const heads = new Uint32Array(count2);
|
|
18041
|
+
const offsets = new Uint32Array(count2 + 1);
|
|
18042
|
+
const positions = new Uint32Array(count2);
|
|
18043
|
+
const members = new Uint32Array(count2);
|
|
18044
|
+
const found = requireLoadedOptionalReads().groupHits(
|
|
18045
|
+
stored,
|
|
18046
|
+
ordinals,
|
|
18047
|
+
scores,
|
|
18048
|
+
ordinals,
|
|
18049
|
+
count2,
|
|
18050
|
+
null,
|
|
18051
|
+
heads,
|
|
18052
|
+
offsets,
|
|
18053
|
+
positions,
|
|
18054
|
+
members
|
|
18055
|
+
);
|
|
18056
|
+
if (found < 0)
|
|
18057
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The group values cannot be compared", { status: found });
|
|
18058
|
+
const keys = new Uint32Array(count2);
|
|
18059
|
+
const values = new Array(found);
|
|
18060
|
+
for (let group = 0; group < found; group++) {
|
|
18061
|
+
values[group] = sources[heads[group]].values;
|
|
18062
|
+
for (let at = offsets[group]; at < offsets[group + 1]; at++) keys[positions[at]] = group;
|
|
18030
18063
|
}
|
|
18031
|
-
return {
|
|
18064
|
+
return { keys, values };
|
|
18032
18065
|
}
|
|
18033
18066
|
|
|
18034
18067
|
// src/distribution/query/group-merge.ts
|
|
18068
|
+
function entryRanks(entries, compare) {
|
|
18069
|
+
const order = Uint32Array.from(entries, (_, position) => position);
|
|
18070
|
+
order.sort((left, right) => compare(entries[left], entries[right]));
|
|
18071
|
+
const ranks = new Uint32Array(entries.length);
|
|
18072
|
+
let rank = 0;
|
|
18073
|
+
for (let at = 0; at < order.length; at++) {
|
|
18074
|
+
if (at > 0 && compare(entries[order[at - 1]], entries[order[at]]) !== 0) rank++;
|
|
18075
|
+
ranks[order[at]] = rank;
|
|
18076
|
+
}
|
|
18077
|
+
return ranks;
|
|
18078
|
+
}
|
|
18035
18079
|
function mergeDistributedGroups(perNode, fields, maxPerGroup, limit, sortDirections) {
|
|
18036
|
-
const
|
|
18080
|
+
const { compareSortValues } = requireLoadedOptionalReads();
|
|
18081
|
+
const sources = [];
|
|
18082
|
+
let count2 = 0;
|
|
18037
18083
|
for (const groups of perNode) {
|
|
18038
18084
|
for (const group of groups) {
|
|
18039
|
-
|
|
18040
|
-
|
|
18041
|
-
|
|
18042
|
-
|
|
18085
|
+
sources.push(group);
|
|
18086
|
+
count2 += group.scored.length;
|
|
18087
|
+
}
|
|
18088
|
+
}
|
|
18089
|
+
if (count2 > 4294967295 || sources.length > 4294967295) {
|
|
18090
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The grouped results exceed the merge layout");
|
|
18091
|
+
}
|
|
18092
|
+
const { keys, values } = groupIdentities(sources, fields);
|
|
18093
|
+
const offsets = new Uint32Array(sources.length + 1);
|
|
18094
|
+
const entries = new Array(count2);
|
|
18095
|
+
const scores = new Float64Array(count2);
|
|
18096
|
+
let presence = null;
|
|
18097
|
+
let present = 0;
|
|
18098
|
+
let position = 0;
|
|
18099
|
+
for (let source = 0; source < sources.length; source++) {
|
|
18100
|
+
offsets[source] = position;
|
|
18101
|
+
for (const entry of sources[source].scored) {
|
|
18102
|
+
entries[position] = entry;
|
|
18103
|
+
if (entry.score === null) {
|
|
18104
|
+
presence ??= new Uint32Array(Math.max(1, Math.ceil(count2 / 32))).fill(4294967295);
|
|
18105
|
+
presence[position >>> 5] &= ~(1 << (position & 31));
|
|
18043
18106
|
} else {
|
|
18044
|
-
|
|
18107
|
+
scores[position] = entry.score;
|
|
18108
|
+
present++;
|
|
18045
18109
|
}
|
|
18110
|
+
position++;
|
|
18046
18111
|
}
|
|
18047
18112
|
}
|
|
18048
|
-
|
|
18049
|
-
|
|
18050
|
-
|
|
18051
|
-
|
|
18052
|
-
|
|
18053
|
-
|
|
18054
|
-
|
|
18055
|
-
|
|
18056
|
-
|
|
18057
|
-
|
|
18058
|
-
|
|
18059
|
-
|
|
18060
|
-
|
|
18061
|
-
|
|
18062
|
-
|
|
18113
|
+
offsets[sources.length] = position;
|
|
18114
|
+
const idRanks = entryRanks(entries, (left, right) => compareCodePoints(left.docId, right.docId));
|
|
18115
|
+
const ranks = sortDirections === null ? idRanks : entryRanks(
|
|
18116
|
+
entries,
|
|
18117
|
+
(left, right) => compareSortValues(left.sortValues ?? [], right.sortValues ?? [], sortDirections) || compareCodePoints(left.docId, right.docId)
|
|
18118
|
+
);
|
|
18119
|
+
const outKeys = new Uint32Array(values.length);
|
|
18120
|
+
const outOffsets = new Uint32Array(values.length + 1);
|
|
18121
|
+
const outSources = new Uint32Array(count2);
|
|
18122
|
+
const outIndices = new Uint32Array(count2);
|
|
18123
|
+
const found = mergeGroups2(
|
|
18124
|
+
keys,
|
|
18125
|
+
offsets,
|
|
18126
|
+
present === 0 ? null : scores,
|
|
18127
|
+
present === 0 ? null : presence,
|
|
18128
|
+
ranks,
|
|
18129
|
+
idRanks,
|
|
18130
|
+
sortDirections !== null,
|
|
18131
|
+
maxPerGroup,
|
|
18132
|
+
limit,
|
|
18133
|
+
outKeys,
|
|
18134
|
+
outOffsets,
|
|
18135
|
+
outSources,
|
|
18136
|
+
outIndices
|
|
18137
|
+
);
|
|
18138
|
+
if (found < 0) throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The groups cannot be merged", { status: found });
|
|
18139
|
+
const merged = new Array(found);
|
|
18140
|
+
for (let group = 0; group < found; group++) {
|
|
18141
|
+
const scored = [];
|
|
18142
|
+
for (let at = outOffsets[group]; at < outOffsets[group + 1]; at++) {
|
|
18143
|
+
scored.push(sources[outSources[at]].scored[outIndices[at]]);
|
|
18144
|
+
}
|
|
18145
|
+
merged[group] = { values: values[outKeys[group]], scored };
|
|
18063
18146
|
}
|
|
18064
18147
|
return merged;
|
|
18065
18148
|
}
|
|
@@ -18077,70 +18160,54 @@ function mergeGroupsFor(params, allGroups, sortFields) {
|
|
|
18077
18160
|
}
|
|
18078
18161
|
|
|
18079
18162
|
// src/distribution/query/fusion.ts
|
|
18080
|
-
function compareFusedEntries(a, b) {
|
|
18081
|
-
const aScore = a.score ?? 0;
|
|
18082
|
-
const bScore = b.score ?? 0;
|
|
18083
|
-
if (aScore !== bScore) {
|
|
18084
|
-
return bScore - aScore;
|
|
18085
|
-
}
|
|
18086
|
-
return compareCodePoints(a.docId, b.docId);
|
|
18087
|
-
}
|
|
18088
18163
|
function distributedRRF(lists, options) {
|
|
18089
|
-
|
|
18090
|
-
const scores = /* @__PURE__ */ new Map();
|
|
18091
|
-
for (const list of lists) {
|
|
18092
|
-
for (let rank = 0; rank < list.length; rank++) {
|
|
18093
|
-
const entry = list[rank];
|
|
18094
|
-
const rrfContribution = 1 / (k + rank + 1);
|
|
18095
|
-
scores.set(entry.docId, (scores.get(entry.docId) ?? 0) + rrfContribution);
|
|
18096
|
-
}
|
|
18097
|
-
}
|
|
18098
|
-
const result = [];
|
|
18099
|
-
for (const [docId, score] of scores) {
|
|
18100
|
-
result.push({ docId, score, sortValues: null });
|
|
18101
|
-
}
|
|
18102
|
-
result.sort(compareFusedEntries);
|
|
18103
|
-
return result;
|
|
18164
|
+
return fuseRankings(lists[0], lists[1], 0, options.k, DEFAULT_LINEAR_ALPHA, withFusedScore);
|
|
18104
18165
|
}
|
|
18105
18166
|
function distributedLinearCombination(textResults, vectorResults, options) {
|
|
18106
|
-
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
|
|
18119
|
-
|
|
18120
|
-
|
|
18121
|
-
|
|
18122
|
-
|
|
18123
|
-
|
|
18124
|
-
|
|
18125
|
-
|
|
18126
|
-
|
|
18127
|
-
|
|
18128
|
-
|
|
18167
|
+
return fuseRankings(textResults, vectorResults, 1, DEFAULT_RANK_CONSTANT, options.alpha, withFusedScore);
|
|
18168
|
+
}
|
|
18169
|
+
function withFusedScore(entry, score) {
|
|
18170
|
+
return {
|
|
18171
|
+
docId: entry.docId,
|
|
18172
|
+
score,
|
|
18173
|
+
sortValues: null
|
|
18174
|
+
};
|
|
18175
|
+
}
|
|
18176
|
+
|
|
18177
|
+
// src/distribution/query/merge.ts
|
|
18178
|
+
function ownField(record, field) {
|
|
18179
|
+
return record !== null && record !== void 0 && Object.hasOwn(record, field) ? record[field] : void 0;
|
|
18180
|
+
}
|
|
18181
|
+
function mergeAndTruncateScoredEntries(arrays, limit) {
|
|
18182
|
+
return mergeEntries(arrays, limit, (left, right) => compareCodePoints(left.docId, right.docId), true);
|
|
18183
|
+
}
|
|
18184
|
+
function mergeAndTruncateSortedEntries(arrays, limit, directions) {
|
|
18185
|
+
const { compareSortValues } = requireLoadedOptionalReads();
|
|
18186
|
+
return mergeEntries(
|
|
18187
|
+
arrays,
|
|
18188
|
+
limit,
|
|
18189
|
+
(left, right) => compareSortValues(left.sortValues ?? [], right.sortValues ?? [], directions) || compareCodePoints(left.docId, right.docId),
|
|
18190
|
+
false
|
|
18191
|
+
);
|
|
18129
18192
|
}
|
|
18130
|
-
function
|
|
18131
|
-
|
|
18132
|
-
|
|
18133
|
-
|
|
18134
|
-
for (let i = 1; i < entries.length; i++) {
|
|
18135
|
-
const score = entries[i].score ?? 0;
|
|
18136
|
-
if (score < min) min = score;
|
|
18137
|
-
if (score > max) max = score;
|
|
18193
|
+
function mergeDistributedFacets(allFacets, allBounds, maxBuckets = DEFAULT_MAX_FACET_BUCKETS, ascending = /* @__PURE__ */ new Set()) {
|
|
18194
|
+
const fields = /* @__PURE__ */ new Set();
|
|
18195
|
+
for (const facetMap of allFacets) {
|
|
18196
|
+
for (const field of Object.keys(facetMap)) fields.add(field);
|
|
18138
18197
|
}
|
|
18139
|
-
const
|
|
18140
|
-
|
|
18141
|
-
|
|
18198
|
+
const facets = /* @__PURE__ */ Object.create(null);
|
|
18199
|
+
const errorBounds = /* @__PURE__ */ Object.create(null);
|
|
18200
|
+
const undercounts = /* @__PURE__ */ Object.create(null);
|
|
18201
|
+
for (const field of fields) {
|
|
18202
|
+
const count2 = Math.max(allFacets.length, allBounds.length);
|
|
18203
|
+
const lists = Array.from({ length: count2 }, (_, source) => ownField(allFacets[source], field) ?? []);
|
|
18204
|
+
const bounds = Array.from({ length: count2 }, (_, source) => ownField(allBounds[source], field) ?? 0);
|
|
18205
|
+
const merged = requireLoadedOptionalReads().mergeFacetBuckets(lists, bounds, ascending.has(field), maxBuckets);
|
|
18206
|
+
facets[field] = merged.buckets;
|
|
18207
|
+
undercounts[field] = merged.undercount;
|
|
18208
|
+
errorBounds[field] = merged.errorBound;
|
|
18142
18209
|
}
|
|
18143
|
-
return
|
|
18210
|
+
return { facets, errorBounds, undercounts };
|
|
18144
18211
|
}
|
|
18145
18212
|
|
|
18146
18213
|
// src/distribution/query/pinning.ts
|
|
@@ -18154,26 +18221,27 @@ function lastOrganicEntry(page, pinned) {
|
|
|
18154
18221
|
}
|
|
18155
18222
|
return void 0;
|
|
18156
18223
|
}
|
|
18157
|
-
function placePinnedEntries(merged, pinned, depth, allMatchesPresent) {
|
|
18158
|
-
const
|
|
18159
|
-
const
|
|
18160
|
-
const
|
|
18161
|
-
|
|
18162
|
-
|
|
18163
|
-
|
|
18164
|
-
|
|
18165
|
-
|
|
18166
|
-
result.push(entry);
|
|
18167
|
-
}
|
|
18168
|
-
}
|
|
18169
|
-
const ordered = deduped.sort((a, b) => a.position - b.position);
|
|
18224
|
+
function placePinnedEntries(merged, pinned, depth, allMatchesPresent, stored) {
|
|
18225
|
+
const { dedupePinnedEntries, pinnedOrder } = requireLoadedOptionalReads();
|
|
18226
|
+
const deduped = dedupePinnedEntries(stored === void 0 ? pinned : pinned.filter((entry) => stored.has(entry.docId)));
|
|
18227
|
+
const { order, ranked } = pinnedOrder(
|
|
18228
|
+
merged.map((entry) => entry.docId),
|
|
18229
|
+
deduped,
|
|
18230
|
+
allMatchesPresent ? null : depth
|
|
18231
|
+
);
|
|
18232
|
+
const result = new Array(order.length);
|
|
18170
18233
|
const placedFromOutside = [];
|
|
18171
|
-
for (
|
|
18172
|
-
|
|
18173
|
-
|
|
18234
|
+
for (let at = 0; at < order.length; at++) {
|
|
18235
|
+
const source = order[at];
|
|
18236
|
+
if (source >= 0) {
|
|
18237
|
+
result[at] = merged[source];
|
|
18238
|
+
continue;
|
|
18239
|
+
}
|
|
18240
|
+
const pin = -1 - source;
|
|
18241
|
+
const entry = deduped[pin];
|
|
18242
|
+
const organic = ranked[pin] < 0 ? void 0 : merged[ranked[pin]];
|
|
18174
18243
|
if (organic === void 0) placedFromOutside.push(entry.docId);
|
|
18175
|
-
|
|
18176
|
-
result.splice(Math.max(0, Math.min(entry.position, result.length)), 0, placed);
|
|
18244
|
+
result[at] = { docId: entry.docId, score: 0, sortValues: organic?.sortValues ?? null };
|
|
18177
18245
|
}
|
|
18178
18246
|
return { entries: result, placedFromOutside };
|
|
18179
18247
|
}
|
|
@@ -18221,9 +18289,9 @@ function fixedLegsOf(params, depth) {
|
|
|
18221
18289
|
}
|
|
18222
18290
|
};
|
|
18223
18291
|
}
|
|
18224
|
-
function pageFixedDepth(placed, fused, pins, allMatchesPresent, start, page, depth) {
|
|
18292
|
+
function pageFixedDepth(pageRows, { placed, fused, pins, allMatchesPresent, start, page, depth }) {
|
|
18225
18293
|
const placedIds = pins === null ? void 0 : new Set(pins.filter((entry) => allMatchesPresent || entry.position < depth).map((entry) => entry.docId));
|
|
18226
|
-
const paged =
|
|
18294
|
+
const paged = pageRows(placed.map(positionedOf), fused.map(positionedOf), {
|
|
18227
18295
|
start,
|
|
18228
18296
|
limit: page.limit,
|
|
18229
18297
|
pinnedIds: placedIds,
|
|
@@ -18234,9 +18302,12 @@ function pageFixedDepth(placed, fused, pins, allMatchesPresent, start, page, dep
|
|
|
18234
18302
|
}
|
|
18235
18303
|
async function executeHybridQuery(indexName, params, page, facetShardSize, facetSize, totalPartitions, routing, deps, config) {
|
|
18236
18304
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
18237
|
-
const
|
|
18305
|
+
const fixedDepthCode = (params.hybrid?.depth ?? null) === null ? null : await loadRerankRuntime();
|
|
18306
|
+
const fixedDepth = fixedDepthCode?.fixedHybridDepthOf(params.hybrid);
|
|
18238
18307
|
const cursorDepth = fixedDepth === void 0 || params.searchAfter === null ? 0 : decodePageCursor(params.searchAfter).depth ?? 0;
|
|
18239
|
-
if (fixedDepth !== void 0)
|
|
18308
|
+
if (fixedDepthCode !== null && fixedDepth !== void 0) {
|
|
18309
|
+
fixedDepthCode.requireWithinHybridDepth(cursorDepth, page.offset, fixedDepth);
|
|
18310
|
+
}
|
|
18240
18311
|
const depth = fixedDepth ?? page.limit + page.offset;
|
|
18241
18312
|
const { text: textParams, vector: vectorParams } = fixedDepth === void 0 ? pageLegsOf(params, depth) : fixedLegsOf(params, depth);
|
|
18242
18313
|
let textGlobalStats = null;
|
|
@@ -18311,8 +18382,16 @@ async function executeHybridQuery(indexName, params, page, facetShardSize, facet
|
|
|
18311
18382
|
if (fixedDepth !== void 0 && fused.length > depth) fused = fused.slice(0, depth);
|
|
18312
18383
|
const allMatchesPresent = coverage.queriedPartitions === coverage.totalPartitions && totalHits <= mergedText.length && (fixedDepth === void 0 ? mergedVector.length < depth : params.vector?.similarity === null && vectorHits <= mergedVector.length);
|
|
18313
18384
|
const pins = fixedDepth === void 0 || params.searchAfter === null ? params.pinned : null;
|
|
18314
|
-
const placement = pins !== null ? placePinnedEntries(fused, pins, depth, allMatchesPresent) : { entries: fused, placedFromOutside: [] };
|
|
18315
|
-
const paged = fixedDepth === void 0 ? { page: placement.entries.slice(page.offset, depth), cursor: null } : pageFixedDepth(
|
|
18385
|
+
const placement = pins !== null ? placePinnedEntries(fused, pins, depth, allMatchesPresent, deps.pinnedDocuments) : { entries: fused, placedFromOutside: [] };
|
|
18386
|
+
const paged = fixedDepthCode === null || fixedDepth === void 0 ? { page: placement.entries.slice(page.offset, depth), cursor: null } : pageFixedDepth(fixedDepthCode.pageByPosition, {
|
|
18387
|
+
placed: placement.entries,
|
|
18388
|
+
fused,
|
|
18389
|
+
pins,
|
|
18390
|
+
allMatchesPresent,
|
|
18391
|
+
start: cursorDepth + page.offset,
|
|
18392
|
+
page,
|
|
18393
|
+
depth
|
|
18394
|
+
});
|
|
18316
18395
|
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize, ascendingFacetFields(params.facets)) : null;
|
|
18317
18396
|
return {
|
|
18318
18397
|
scored: paged.page,
|
|
@@ -18430,7 +18509,13 @@ async function executeSingleFanOut(indexName, params, binding, limit, offset, fa
|
|
|
18430
18509
|
globalStats = await collectDistributedStats(indexName, params, routing, deps, config);
|
|
18431
18510
|
}
|
|
18432
18511
|
const depth = limit + offset;
|
|
18433
|
-
const
|
|
18512
|
+
const sortedGroups = params.group !== null && params.sort !== null && params.sort.length > 0;
|
|
18513
|
+
const searchParams = {
|
|
18514
|
+
...params,
|
|
18515
|
+
limit: depth,
|
|
18516
|
+
offset: 0,
|
|
18517
|
+
includeScores: sortedGroups ? true : params.includeScores
|
|
18518
|
+
};
|
|
18434
18519
|
const keepAliveMs = config.searchContexts ? searchContextKeepAlive(routing, config.partitionTimeout, deps.transport) : null;
|
|
18435
18520
|
const outcomes = await fanOutSearch(
|
|
18436
18521
|
indexName,
|
|
@@ -18484,7 +18569,7 @@ async function executeSingleFanOut(indexName, params, binding, limit, offset, fa
|
|
|
18484
18569
|
sortFields.map((field) => field.direction)
|
|
18485
18570
|
) : mergeAndTruncateScoredEntries(allScored, depth);
|
|
18486
18571
|
const allMatchesPresent = coverage.queriedPartitions === coverage.totalPartitions && totalHits <= merged.length;
|
|
18487
|
-
const placement = params.pinned !== null && params.searchAfter === null ? placePinnedEntries(merged, params.pinned, depth, allMatchesPresent) : { entries: merged, placedFromOutside: [] };
|
|
18572
|
+
const placement = params.pinned !== null && params.searchAfter === null ? placePinnedEntries(merged, params.pinned, depth, allMatchesPresent, deps.pinnedDocuments) : { entries: merged, placedFromOutside: [] };
|
|
18488
18573
|
const mergedScored = placement.entries.slice(offset, depth);
|
|
18489
18574
|
const mergedFacets = allFacets.length > 0 ? mergeDistributedFacets(allFacets, allFacetBounds, facetSize, ascendingFacetFields(params.facets)) : null;
|
|
18490
18575
|
const mergedGroups = mergeGroupsFor(params, allGroups, sortFields);
|
|
@@ -18575,7 +18660,7 @@ async function assembleDistributedGroups(params, distributed, projection, readDo
|
|
|
18575
18660
|
}
|
|
18576
18661
|
const reduce = group.reduce;
|
|
18577
18662
|
const needsDocuments = reduce !== void 0 || projection.kind !== "none";
|
|
18578
|
-
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
18663
|
+
const keptPerGroup = requireLoadedOptionalReads().hitsKeptPerGroup(group.maxPerGroup);
|
|
18579
18664
|
const docIdsRead = (entry) => (reduce === void 0 ? entry.scored.slice(0, keptPerGroup) : entry.scored).map((scored) => scored.docId);
|
|
18580
18665
|
const results = [];
|
|
18581
18666
|
let batch = [];
|
|
@@ -18603,7 +18688,7 @@ function assembleGroup(entry, documents, keptPerGroup, projection, reduce) {
|
|
|
18603
18688
|
}));
|
|
18604
18689
|
const result = { values: entry.values, hits };
|
|
18605
18690
|
if (reduce === void 0) return result;
|
|
18606
|
-
const folded = foldGroupReducer(
|
|
18691
|
+
const folded = requireLoadedOptionalReads().foldGroupReducer(
|
|
18607
18692
|
reduce,
|
|
18608
18693
|
entry.scored.map((scored) => ({ document: documents.get(scored.docId), score: scored.score ?? 0 }))
|
|
18609
18694
|
);
|
|
@@ -18893,6 +18978,8 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
18893
18978
|
}
|
|
18894
18979
|
};
|
|
18895
18980
|
}
|
|
18981
|
+
|
|
18982
|
+
// src/distribution/cluster-node/reads/scatter.ts
|
|
18896
18983
|
async function gatherFromEveryPartition(allocation, indexName, read) {
|
|
18897
18984
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
18898
18985
|
if (routing.unavailablePartitions.length > 0) {
|
|
@@ -19044,6 +19131,13 @@ async function listIndexesCluster(deps) {
|
|
|
19044
19131
|
return listed;
|
|
19045
19132
|
}
|
|
19046
19133
|
|
|
19134
|
+
// src/search/sort-keys.ts
|
|
19135
|
+
function readSortValues(document, fields, modes) {
|
|
19136
|
+
if (!document) return fields.map(() => null);
|
|
19137
|
+
const { reduceSortValue } = requireLoadedOptionalReads();
|
|
19138
|
+
return fields.map((field, index) => reduceSortValue(readSortField(document, field), modes[index] ?? "min"));
|
|
19139
|
+
}
|
|
19140
|
+
|
|
19047
19141
|
// src/distribution/cluster-node/reads/list.ts
|
|
19048
19142
|
function clampListLimit(limit) {
|
|
19049
19143
|
const clamped = Math.max(1, clampLimit(limit));
|
|
@@ -19138,6 +19232,7 @@ async function listCluster(deps, indexName, params) {
|
|
|
19138
19232
|
});
|
|
19139
19233
|
}
|
|
19140
19234
|
}
|
|
19235
|
+
const { compareComparableKeys } = requireLoadedOptionalReads();
|
|
19141
19236
|
merged.sort((a, b) => {
|
|
19142
19237
|
if (sortFieldNames !== null) {
|
|
19143
19238
|
const byKey = compareComparableKeys(a.key ?? [], b.key ?? [], directions);
|
|
@@ -19327,7 +19422,7 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
19327
19422
|
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
19328
19423
|
);
|
|
19329
19424
|
}
|
|
19330
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
19425
|
+
const { rerankClusterQuery } = await import('./rerank-Y6MWBMTT.mjs');
|
|
19331
19426
|
return withRankedReader(
|
|
19332
19427
|
deps,
|
|
19333
19428
|
indexName,
|
|
@@ -19350,13 +19445,15 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
19350
19445
|
const readsDocuments = readsLater || projection.kind !== "none" || groupsRead;
|
|
19351
19446
|
const { wireParams, carriesBm25 } = heldBm25RoutingFor(deps, settingName, params) ?? await bm25RoutingFor(deps, settingName, params);
|
|
19352
19447
|
wireParams.analysisGeneration = deps.analysisView.held(indexName) ?? await deps.analysisView.generationFor(indexName);
|
|
19448
|
+
const pinPresence = await readPinPresence(deps, indexName, params, allocation);
|
|
19353
19449
|
const queryDeps = {
|
|
19354
19450
|
transport: deps.config.transport,
|
|
19355
19451
|
sourceNodeId: deps.nodeId,
|
|
19356
19452
|
getAllocation: (idx) => deps.config.coordinator.getAllocation(idx),
|
|
19357
19453
|
resolveNodeTargets: deps.resolveNodeTargets,
|
|
19358
19454
|
listNodes: () => deps.config.coordinator.listNodes(),
|
|
19359
|
-
carriesBm25
|
|
19455
|
+
carriesBm25,
|
|
19456
|
+
pinnedDocuments: pinPresence === null ? void 0 : /* @__PURE__ */ new Set([...pinPresence.stored, ...pinPresence.unverifiable])
|
|
19360
19457
|
};
|
|
19361
19458
|
const distributed = await distributedQuery(
|
|
19362
19459
|
indexName,
|
|
@@ -19373,7 +19470,6 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
19373
19470
|
);
|
|
19374
19471
|
for (const [docId, document] of found) documents.set(docId, applyProjection(document, projection));
|
|
19375
19472
|
}
|
|
19376
|
-
const pinPresence = await readPinPresence(deps, indexName, params, allocation);
|
|
19377
19473
|
const scored = dropUnstoredPinnedEntries(distributed, pinPresence);
|
|
19378
19474
|
const pins = countStoredPinsFromOutside(distributed, pinPresence);
|
|
19379
19475
|
let countExact = distributedCountIsExact(params, distributed.coverage);
|
|
@@ -19398,6 +19494,10 @@ async function searchCluster(deps, indexName, settingName, params, allocation, r
|
|
|
19398
19494
|
if (groups !== void 0) {
|
|
19399
19495
|
result.groups = groups;
|
|
19400
19496
|
}
|
|
19497
|
+
if (wireParams.sort !== null && wireParams.sort.length > 0 && params.includeScores !== true) {
|
|
19498
|
+
for (const hit of result.hits) hit.score = void 0;
|
|
19499
|
+
for (const group of groups ?? []) for (const hit of group.hits) hit.score = void 0;
|
|
19500
|
+
}
|
|
19401
19501
|
return result;
|
|
19402
19502
|
}
|
|
19403
19503
|
|
|
@@ -19433,6 +19533,8 @@ async function clearCluster(readDeps, writeDeps, indexName) {
|
|
|
19433
19533
|
}
|
|
19434
19534
|
}
|
|
19435
19535
|
}
|
|
19536
|
+
|
|
19537
|
+
// src/distribution/cluster-node/drop-acknowledgement.ts
|
|
19436
19538
|
function pause2() {
|
|
19437
19539
|
return new Promise((resolve) => setTimeout(resolve, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS));
|
|
19438
19540
|
}
|
|
@@ -19498,6 +19600,8 @@ async function awaitTornDownCopy(engine, coordinator, indexName) {
|
|
|
19498
19600
|
await pause2();
|
|
19499
19601
|
}
|
|
19500
19602
|
}
|
|
19603
|
+
|
|
19604
|
+
// src/distribution/cluster-node/message-handler/partition-stores.ts
|
|
19501
19605
|
function heldPartitionIdsOf(engine, indexName) {
|
|
19502
19606
|
const held = new Set(engine.heldPartitionsOf(indexName) ?? []);
|
|
19503
19607
|
for (const partition of engine.getPartitionStats(indexName)) {
|
|
@@ -21012,6 +21116,8 @@ async function handleReshardPurge(message, respond, deps) {
|
|
|
21012
21116
|
const result = { removed };
|
|
21013
21117
|
await respond(reply2(deps, message, result));
|
|
21014
21118
|
}
|
|
21119
|
+
|
|
21120
|
+
// src/distribution/cluster-node/message-handler/index-settings.ts
|
|
21015
21121
|
async function handleIndexSettings(message, respond, deps) {
|
|
21016
21122
|
const payload = validateIndexSettingsPayload(decode(message.payload));
|
|
21017
21123
|
const applied = payload !== null && await applyStoredSynonyms(deps.engine, deps.coordinator, payload.indexName, payload.indexUuid);
|
|
@@ -21202,6 +21308,8 @@ function convertLocalFacetsToWire2(facets) {
|
|
|
21202
21308
|
}
|
|
21203
21309
|
return result;
|
|
21204
21310
|
}
|
|
21311
|
+
|
|
21312
|
+
// src/distribution/cluster-node/message-handler/read-handlers.ts
|
|
21205
21313
|
async function handleCount(message, respond, deps) {
|
|
21206
21314
|
const payload = validateCountPayload(decode(message.payload));
|
|
21207
21315
|
const requested = new Set(payload.partitionIds);
|
|
@@ -22956,7 +23064,7 @@ function createClusterQuerySuggestions(deps) {
|
|
|
22956
23064
|
isController: deps.isController,
|
|
22957
23065
|
emit: (name, payload) => deps.engine.emitEvent(name, payload)
|
|
22958
23066
|
});
|
|
22959
|
-
const started =
|
|
23067
|
+
const started = await createQuerySuggestionsRuntime(host, key);
|
|
22960
23068
|
runtime = started;
|
|
22961
23069
|
await started.start();
|
|
22962
23070
|
leaseTimer = setInterval(() => {
|
|
@@ -23292,6 +23400,7 @@ async function createClusterNode(callerConfig) {
|
|
|
23292
23400
|
const replicationLogs = /* @__PURE__ */ new Map();
|
|
23293
23401
|
const reshard = createReshardState();
|
|
23294
23402
|
let localEngine = null;
|
|
23403
|
+
const copies = { replicationLogs, reshard, bootstrapSync: bootstrapSyncState, catchUp: () => writeDeps.catchUp };
|
|
23295
23404
|
const engine = await createClusterLocalEngine(config.engine, {
|
|
23296
23405
|
async onIndexOpen(indexName) {
|
|
23297
23406
|
if (localEngine === null) return;
|
|
@@ -23304,10 +23413,8 @@ async function createClusterNode(callerConfig) {
|
|
|
23304
23413
|
replicationConfig: config.replication
|
|
23305
23414
|
});
|
|
23306
23415
|
},
|
|
23307
|
-
onIndexClose: (indexName) =>
|
|
23308
|
-
|
|
23309
|
-
forgetReshardIndex(reshard, indexName);
|
|
23310
|
-
}
|
|
23416
|
+
onIndexClose: (indexName) => forgetClosedCopy(copies, indexName),
|
|
23417
|
+
onIndexDrop: (indexName) => forgetDroppedCopy(copies, indexName)
|
|
23311
23418
|
});
|
|
23312
23419
|
localEngine = engine;
|
|
23313
23420
|
const snapshotSpool = await openSnapshotSpoolDirectory(config.engine?.durability?.directory).catch(async (err) => {
|