@delali/narsil 0.3.1-next.152 → 0.3.1-next.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-4GCRVRBW.mjs} +27 -18
- package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KYGSXW4K.mjs} +1 -1
- package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-IXWPXXJJ.mjs} +17 -7
- package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-FCSHFQSS.mjs} +1 -1
- package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-2FX6ROWR.mjs} +15 -6
- package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-2SHAKW2Z.mjs} +119 -1922
- package/dist/browser/{chunk-55WYRLDZ.mjs → chunk-3U3QR5YP.mjs} +2 -2
- package/dist/browser/chunk-4PD752F6.mjs +85 -0
- package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
- package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-7MEYN7V6.mjs} +1 -151
- package/dist/browser/chunk-7TTWXUH6.mjs +1828 -0
- package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-AJ4HHOVT.mjs} +4 -4
- package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-AYY6PIKL.mjs} +2 -2
- package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-C4KS2QNF.mjs} +9 -8
- package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
- package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
- package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-EUNBNLKM.mjs} +6 -5
- package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-G7LUMBSG.mjs} +137 -253
- package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-H6QFBMVE.mjs} +3 -2
- package/dist/browser/chunk-J433A7P3.mjs +241 -0
- package/dist/browser/chunk-ML6HY7T7.mjs +96 -0
- package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
- package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-MYITLCHT.mjs} +7 -1
- package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-QXTPSX2H.mjs} +9 -150
- package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
- package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SIR3LMTV.mjs} +2 -2
- package/dist/browser/{chunk-LURXVCHG.mjs → chunk-SJB6VIZG.mjs} +1 -1
- package/dist/browser/chunk-TTD27VXX.mjs +298 -0
- package/dist/browser/chunk-U2E3HMYW.mjs +151 -0
- package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-VCOYGVP7.mjs} +5 -31
- package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
- package/dist/browser/chunk-W5X2UFWQ.mjs +102 -0
- package/dist/browser/{chunk-HT7J73QL.mjs → chunk-XRV72TJD.mjs} +2 -2
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
- package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-ZPTA75ZC.mjs} +2 -79
- package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-VTUGDYNY.mjs} +20 -14
- package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-GL4A7KDV.mjs} +1886 -1364
- package/dist/browser/import-YI5GNB4H.mjs +17 -0
- package/dist/browser/methods-LMUTS7OI.mjs +30 -0
- package/dist/browser/{module-XQLEOYNM.mjs → module-EC3FYW7N.mjs} +9 -8
- package/dist/browser/{module-KAMRAJLE.mjs → module-M4JNBV6V.mjs} +16 -10
- package/dist/browser/operations-WVYLF72Q.mjs +5 -0
- package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-NPYUVX75.mjs} +1 -1
- package/dist/browser/{pool-DTYSPBMQ.mjs → pool-WCGDZHJP.mjs} +3 -2
- package/dist/{restore-queue-AOF5OTQV.mjs → browser/restore-queue-TYIGDL5L.mjs} +2 -2
- package/dist/browser/retention-6QMRS5VK.mjs +207 -0
- package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-3EW2AGB6.mjs} +1 -1
- package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-DN3CRVUQ.mjs} +38 -307
- package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
- package/dist/browser/{write-I5P5MW4O.mjs → write-WAWAFGKQ.mjs} +16 -11
- package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-B3Brs1Rc.d.ts} +62 -7
- package/dist/{chunk-DGRM4EZZ.mjs → chunk-37IVJLP6.mjs} +1 -1
- package/dist/{chunk-KYVRZAYP.mjs → chunk-3A5KI24C.mjs} +2 -69
- package/dist/{chunk-POUYICOU.mjs → chunk-3OFU3NJT.mjs} +1 -1
- package/dist/{chunk-3TVDTACC.mjs → chunk-3Y3J4V2R.mjs} +1 -1
- package/dist/{chunk-FNBNZCGP.mjs → chunk-466PYR4P.mjs} +2 -2
- package/dist/{chunk-SEAJS664.mjs → chunk-4HWZV5MK.mjs} +5 -5
- package/dist/{chunk-UGVUW27S.mjs → chunk-5FUOVJKG.mjs} +13 -9
- package/dist/{chunk-ZHAXP7SD.mjs → chunk-5KGYQT6W.mjs} +6 -235
- package/dist/{chunk-NPXVQS33.mjs → chunk-62G63NMH.mjs} +4 -3
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-7GXJUJ24.mjs +49 -0
- package/dist/{chunk-3E4RRF5J.mjs → chunk-7HCZWC7B.mjs} +58 -8
- package/dist/{chunk-6RKSY3BX.mjs → chunk-7NTMUDC5.mjs} +5 -5
- package/dist/chunk-7QUGDVFC.mjs +101 -0
- package/dist/{chunk-G5KHSM6E.mjs → chunk-AO2FAYIZ.mjs} +1 -1
- package/dist/chunk-B7S6YEBP.mjs +102 -0
- package/dist/{chunk-AWNHDBKV.mjs → chunk-BPSKLCS7.mjs} +15 -7
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/{chunk-T4CJDBBE.mjs → chunk-C6ASEFZI.mjs} +1 -1
- package/dist/{chunk-NITCPDFL.mjs → chunk-C73JLU4E.mjs} +2 -2
- package/dist/{chunk-NYSONUB7.mjs → chunk-CP5RRPJR.mjs} +1 -1
- package/dist/{chunk-TTCKT2RT.mjs → chunk-DMQPFLGI.mjs} +134 -11
- package/dist/{chunk-3SBJVQLL.mjs → chunk-DVWYODEB.mjs} +2 -2
- package/dist/chunk-ECX4SGX6.mjs +167 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/{chunk-NVTFQFUL.mjs → chunk-HVKXNBFL.mjs} +1 -1
- package/dist/chunk-HVYICDY3.mjs +214 -0
- package/dist/{chunk-3HV7M5OJ.mjs → chunk-HWEDFLB7.mjs} +9 -133
- package/dist/{chunk-52QQWEBS.mjs → chunk-IBNLW3AV.mjs} +21 -6
- package/dist/chunk-IJU7LL7D.mjs +242 -0
- package/dist/chunk-IPN7S4KX.mjs +130 -0
- package/dist/{chunk-5QMHMELB.mjs → chunk-IQ4R2I6N.mjs} +4 -1
- package/dist/chunk-JCLTESFN.mjs +96 -0
- package/dist/{chunk-J4BDAH2W.mjs → chunk-JFNKIBDT.mjs} +2 -2
- package/dist/{chunk-LDGGR7YT.mjs → chunk-K4B7OH65.mjs} +2 -2
- package/dist/{chunk-7TE2AJIH.mjs → chunk-K7KLL263.mjs} +13 -10
- package/dist/{chunk-CSHHU7PA.mjs → chunk-KCUIFVVW.mjs} +11 -30
- package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-KUZKUONM.mjs} +11 -2
- package/dist/chunk-KYYABCSW.mjs +72 -0
- package/dist/{chunk-B2FTZNNF.mjs → chunk-LKUE5GXS.mjs} +34 -14
- package/dist/{chunk-2PNYKG2D.mjs → chunk-LSM2YTKM.mjs} +100 -335
- package/dist/{chunk-DMNPYF54.mjs → chunk-LTUGDMV4.mjs} +2 -2
- package/dist/{chunk-FDG4KXX5.mjs → chunk-MLHNSAPK.mjs} +5 -3
- package/dist/{chunk-VP6XRMWR.mjs → chunk-MMBEANB2.mjs} +6 -32
- package/dist/{chunk-LJ4ENXOD.mjs → chunk-MMEMEUOO.mjs} +2 -2
- package/dist/{chunk-2HJWQPR6.mjs → chunk-NTZOXVUZ.mjs} +6 -122
- package/dist/{chunk-5WB2A3XU.mjs → chunk-NZMAXR2S.mjs} +397 -24
- package/dist/{chunk-IZT3G24Q.mjs → chunk-O43Z4MS6.mjs} +2 -2
- package/dist/{chunk-U46IZNCW.mjs → chunk-OKX2BBNW.mjs} +3 -2
- package/dist/{chunk-RBEHLIKX.mjs → chunk-PFKM7VNZ.mjs} +14 -8
- package/dist/chunk-PGVPE7GJ.mjs +96 -0
- package/dist/chunk-Q3TCTQCC.mjs +85 -0
- package/dist/chunk-Q7MS6P2J.mjs +74 -0
- package/dist/chunk-QDZJM7NI.mjs +23 -0
- package/dist/{chunk-ZS6CMXAE.mjs → chunk-QKZHLADX.mjs} +89 -250
- package/dist/{chunk-IJSFWGVW.mjs → chunk-REL7LFLR.mjs} +3 -3
- package/dist/chunk-RVKFI3A2.mjs +237 -0
- package/dist/chunk-S2N75OTN.mjs +83 -0
- package/dist/chunk-TRISKD4Q.mjs +50 -0
- package/dist/{chunk-6F6USYII.mjs → chunk-TV2L3BN7.mjs} +13 -11
- package/dist/{chunk-6BWLDLS3.mjs → chunk-URPCYQVJ.mjs} +1 -1
- package/dist/chunk-USR7FQRZ.mjs +34 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/{chunk-SOUZMPME.mjs → chunk-VMXSVVXG.mjs} +7 -194
- package/dist/{chunk-4QCSHLNM.mjs → chunk-VREK3MYG.mjs} +3 -3
- package/dist/{chunk-YGQOPB2Z.mjs → chunk-WCBA2LVZ.mjs} +1 -1
- package/dist/{chunk-KQJVNPRM.mjs → chunk-X6NSJYQ3.mjs} +19 -16
- package/dist/chunk-Y4WPNHMC.mjs +50 -0
- package/dist/{chunk-IRWP524D.mjs → chunk-YBJDOA2T.mjs} +1 -1
- package/dist/{chunk-IJPK5PGM.mjs → chunk-YPUEV6ZC.mjs} +1 -1
- package/dist/{chunk-J7M73VY7.mjs → chunk-YROWNOMH.mjs} +1 -1
- package/dist/chunk-Z3TNWCI5.mjs +196 -0
- package/dist/{chunk-65WCV5JO.mjs → chunk-ZIFJXCKA.mjs} +1 -1
- package/dist/chunk-ZYAY26KG.mjs +29 -0
- package/dist/{client-Cn3gNxUU.d.ts → client-Df76GNKm.d.ts} +76 -6
- package/dist/client.d.ts +8 -7
- package/dist/client.mjs +56 -5
- package/dist/{cold-module-PYSSBIL7.mjs → cold-module-YLWMTTZG.mjs} +38 -28
- package/dist/{config-CfhH1IWQ.d.ts → config-05G_hy-1.d.ts} +3 -3
- package/dist/config-B8DP2VwR.d.ts +463 -0
- package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-FI5GJHWX.mjs} +14 -9
- package/dist/distribution/coordinator/etcd.d.ts +11 -2
- package/dist/distribution/coordinator/etcd.mjs +268 -107
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +41 -9
- package/dist/distribution.mjs +1296 -117
- package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-RHLXDIVF.mjs} +1751 -1226
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CJRUHXgf.d.ts → engine-DYu-e0oR.d.ts} +50 -40
- package/dist/{import-GJOMB2HC.mjs → import-SRBNC6TE.mjs} +27 -19
- package/dist/{index-MsF7e5hL.d.ts → index-nRqDM6uS.d.ts} +1 -1
- package/dist/index.browser.mjs +8 -4
- package/dist/index.d.ts +7 -6
- package/dist/index.mjs +24 -16
- package/dist/index.react-native.mjs +17 -11
- package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-EMYTGSJP.mjs} +28 -15
- package/dist/methods-ABM5ELBR.mjs +30 -0
- package/dist/{module-AVQEIB7O.mjs → module-3OMJZUFT.mjs} +12 -12
- package/dist/{module-6RLIKSZQ.mjs → module-MG7JMV2U.mjs} +33 -23
- package/dist/operations-PGOAY2KS.mjs +5 -0
- package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-UOLHOEZJ.mjs} +37 -26
- package/dist/partial-words.mjs +6 -4
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +9 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +34 -23
- package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-O3MDRQXP.mjs} +1 -1
- package/dist/{pool-UPV5PVPY.mjs → pool-CQXYPFLG.mjs} +2 -2
- package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-EAYO3Y6Q.mjs} +27 -15
- package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-TNLUC6IZ.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-2CGSQM4O.mjs} +12 -10
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
- package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-3O57C2ZO.mjs} +2 -2
- package/dist/react-native/chunks/chunk-3XKMRNRE.mjs +298 -0
- package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-63YSZOXB.mjs} +97 -333
- package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-75VWIY35.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7GWPJDOA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-AUT7PMA4.mjs} +6 -190
- package/dist/react-native/chunks/chunk-C2H2QQFW.mjs +288 -0
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
- package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-CUNGPR5R.mjs} +11 -2
- package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-DCKMICOB.mjs} +2 -79
- package/dist/react-native/chunks/chunk-DH3TVWWK.mjs +102 -0
- package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-EH5CFV6Z.mjs} +2 -2
- package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
- package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-GFYFQLJD.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-I5PGWQJF.mjs} +142 -257
- package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-IOY2SM2P.mjs} +1 -1
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
- package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-JPAJWO6D.mjs} +1 -1
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
- package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-JX2RIAXH.mjs} +4 -4
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
- package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-NCJS3WIY.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-NKIZX26F.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-NP7DD4G5.mjs} +5 -4
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
- package/dist/react-native/chunks/chunk-O32RB3BL.mjs +191 -0
- package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-OQQVRWZ2.mjs} +2 -69
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-RVAEBN74.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-RYMDHGL4.mjs} +16 -8
- package/dist/react-native/chunks/chunk-SJWDBRNP.mjs +85 -0
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
- package/dist/react-native/chunks/chunk-UWDNZDWK.mjs +242 -0
- package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-VQSKZ3EZ.mjs} +9 -152
- package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-VVFYOADG.mjs} +7 -6
- package/dist/react-native/chunks/{chunk-VGUWIZFQ.mjs → chunk-YBEW6YTR.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-YHKZSLOC.mjs} +7 -33
- package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-ZBSWRSBG.mjs} +4 -30
- package/dist/react-native/chunks/chunk-ZYDKJXQJ.mjs +96 -0
- package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-TJL6AFAD.mjs} +29 -21
- package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-RRDYJLLM.mjs} +1882 -1358
- package/dist/react-native/chunks/gzip.react-native-DNB5YNFA.mjs +3 -0
- package/dist/react-native/chunks/import-JYBMEJPQ.mjs +29 -0
- package/dist/react-native/chunks/methods-X6F36SJE.mjs +30 -0
- package/dist/react-native/chunks/module-QT5HZAY4.mjs +33 -0
- package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-S4DKSWMF.mjs} +10 -9
- package/dist/react-native/chunks/operations-RYKTSZQA.mjs +5 -0
- package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-JZ37TB7N.mjs} +1 -1
- package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-RTGXJXPU.mjs} +3 -2
- package/dist/{browser/restore-queue-UQ6AAUVX.mjs → react-native/chunks/restore-queue-ZBVIKJWI.mjs} +2 -2
- package/dist/react-native/chunks/retention-PPCTP2VY.mjs +219 -0
- package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-KUATOLYB.mjs} +1 -1
- package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-ZBIEOZTZ.mjs} +47 -314
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
- package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-2W4FTX7J.mjs} +25 -18
- package/dist/react-native/partial-words.mjs +6 -4
- package/dist/react-native/pattern.mjs +4 -3
- package/dist/react-native/sync.mjs +2236 -0
- package/dist/react-native/synonyms.mjs +7 -4
- package/dist/react.d.ts +10 -9
- package/dist/react.mjs +7 -1
- package/dist/{rerank-3W54N6P2.mjs → rerank-IZOB6NII.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{react-native/chunks/restore-queue-H5VCN453.mjs → restore-queue-COI2GSSK.mjs} +2 -2
- package/dist/retention-55WA7GPU.mjs +58 -0
- package/dist/{runtime-L2DZYY6N.mjs → runtime-UZOC3NPY.mjs} +5 -5
- package/dist/{schema-D33CyXCy.d.ts → schema-DDauvDOv.d.ts} +1 -1
- package/dist/server.d.ts +22 -19
- package/dist/server.mjs +868 -50
- package/dist/{snapshot-WMX6JCLP.mjs → snapshot-NFX2EYN2.mjs} +62 -36
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +271 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +237 -0
- package/dist/sync-state-ZWHLAFLE.mjs +15 -0
- package/dist/{config-aIsy7mcL.d.ts → sync-uXR7MmM7.d.ts} +53 -444
- package/dist/sync.browser.mjs +2231 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2206 -0
- package/dist/synonyms.mjs +5 -3
- package/dist/{tasks-C4ElXr4P.d.ts → tasks-BShx8pdK.d.ts} +4 -4
- package/dist/thread-P2G2FWM2.mjs +44 -0
- package/dist/types-SnO7_tAI.d.ts +228 -0
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-SKJ6HW7O.mjs → write-FGB62BHQ.mjs} +31 -23
- package/openapi.json +119 -3
- package/package.json +28 -8
- package/dist/browser/import-QOY6ZH6X.mjs +0 -12
- package/dist/chunk-7LDMWKP6.mjs +0 -209
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
- package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
- package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
- package/dist/thread-I7DVHHW7.mjs +0 -31
package/dist/distribution.mjs
CHANGED
|
@@ -1,72 +1,90 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { closePartitionWriter, replaySyncedWal } from './chunk-4QCSHLNM.mjs';
|
|
8
|
-
import { indexMetadataOf, loadPartitionSegments, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentManifest } from './chunk-ZS6CMXAE.mjs';
|
|
9
|
-
import { readVectorFile } from './chunk-6RKSY3BX.mjs';
|
|
10
|
-
import { bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, backupKeys, validateBackupName, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-52QQWEBS.mjs';
|
|
11
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, MAX_SOURCE_ID_LENGTH, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-LDGGR7YT.mjs';
|
|
12
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-LDGGR7YT.mjs';
|
|
13
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigFromSettings, metadataWithBm25, indexConfigKey, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, TEARDOWN_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-3E4RRF5J.mjs';
|
|
14
|
-
import { restoredConfigFields, restoredEmbedding } from './chunk-IRWP524D.mjs';
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolvePrimaryAssignment, resolveNodeTargets, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-JFNKIBDT.mjs';
|
|
2
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-C6ASEFZI.mjs';
|
|
3
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-VREK3MYG.mjs';
|
|
4
|
+
import { archiveStream, buildArchiveFromMemory, memoryCheckpointStore, writeArchiveFile } from './chunk-RVKFI3A2.mjs';
|
|
5
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest } from './chunk-QKZHLADX.mjs';
|
|
6
|
+
import { readVectorFile } from './chunk-7NTMUDC5.mjs';
|
|
15
7
|
import './chunk-ZXNNR73R.mjs';
|
|
16
|
-
import './chunk-DMNPYF54.mjs';
|
|
17
8
|
import { addBackgroundAdmission, workWaitingForMemory } from './chunk-PELHVTO7.mjs';
|
|
18
|
-
import './chunk-
|
|
9
|
+
import './chunk-REL7LFLR.mjs';
|
|
19
10
|
import './chunk-MATMVH65.mjs';
|
|
20
11
|
import './chunk-VD7J552I.mjs';
|
|
21
|
-
import {
|
|
12
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, resolveAnalysisTarget, namedSettingsOf, holdsSettings } from './chunk-PFKM7VNZ.mjs';
|
|
13
|
+
import { requireRestoreMemory, archiveFootprint, storedBytesOfSegments } from './chunk-MMBEANB2.mjs';
|
|
14
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-IBNLW3AV.mjs';
|
|
15
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, MAX_SOURCE_ID_LENGTH, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-K4B7OH65.mjs';
|
|
16
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-K4B7OH65.mjs';
|
|
17
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigKey, metadataWithSync, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, isRecord2, decodeIndexSettings, HIDDEN_GENERATION_PREFIX, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, TEARDOWN_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-7HCZWC7B.mjs';
|
|
18
|
+
import { estimatedArchiveBytes, archiveLayout } from './chunk-Q7MS6P2J.mjs';
|
|
19
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-YBJDOA2T.mjs';
|
|
20
|
+
import { requireSyncSetting } from './chunk-B7S6YEBP.mjs';
|
|
21
|
+
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
22
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS } from './chunk-ZYAY26KG.mjs';
|
|
23
|
+
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
24
|
+
import { readSnapshotArchive, sourceIndexNameOf } from './chunk-5KGYQT6W.mjs';
|
|
25
|
+
import { openManifestPayload, listedKeysOf, decodeManifestPayload } from './chunk-IJU7LL7D.mjs';
|
|
26
|
+
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
27
|
+
import './chunk-LTUGDMV4.mjs';
|
|
28
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-AO2FAYIZ.mjs';
|
|
22
29
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
23
30
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
24
|
-
import './chunk-
|
|
25
|
-
import './chunk-37CD3OKS.mjs';
|
|
31
|
+
import './chunk-BPSKLCS7.mjs';
|
|
26
32
|
import './chunk-DJHHF2L5.mjs';
|
|
27
|
-
import
|
|
28
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-
|
|
33
|
+
import './chunk-37CD3OKS.mjs';
|
|
34
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-X6NSJYQ3.mjs';
|
|
35
|
+
import { encodeSegmentManifest } from './chunk-TRISKD4Q.mjs';
|
|
29
36
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
30
|
-
import './chunk-
|
|
31
|
-
import
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
37
|
+
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
38
|
+
import './chunk-KCUIFVVW.mjs';
|
|
39
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-62G63NMH.mjs';
|
|
40
|
+
import { directoryCheckpointStore, readListedFile } from './chunk-S2N75OTN.mjs';
|
|
41
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-Q3TCTQCC.mjs';
|
|
42
|
+
import './chunk-QDZJM7NI.mjs';
|
|
43
|
+
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-IQ4R2I6N.mjs';
|
|
44
|
+
import { bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, applyStoredIndexBm25, createEngineIndex, encodeIndexConfiguration, memoryFractionsOf, createMemoryMonitor, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, storedArrayPositionGap, emptySerializablePartition, arrayPositionsAreStale, yieldToEventLoop, providedDocId, engineCoreOf } from './chunk-LSM2YTKM.mjs';
|
|
45
|
+
import { runtimeHost, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, readRecentMemoryFigures, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, now, listBindingOf, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, hitsKeptPerGroup, queryBindingOf, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-TV2L3BN7.mjs';
|
|
39
46
|
import './chunk-HJSN7UYG.mjs';
|
|
40
|
-
import './chunk-
|
|
47
|
+
import './chunk-URPCYQVJ.mjs';
|
|
41
48
|
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-XIHUI4KB.mjs';
|
|
42
|
-
import './chunk-Q2UL2TPC.mjs';
|
|
43
49
|
import './chunk-3MXV3VHR.mjs';
|
|
44
50
|
import { BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-EEBCDP36.mjs';
|
|
45
|
-
import {
|
|
51
|
+
import { synonymModule, generateId } from './chunk-5FUOVJKG.mjs';
|
|
52
|
+
import { oversampledShardSize, bm25SetsValue, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, holdsQuotedText, sortSignatureEntry, mergePartitionStats } from './chunk-HWEDFLB7.mjs';
|
|
53
|
+
import { decodeIndexConfiguration } from './chunk-USR7FQRZ.mjs';
|
|
54
|
+
import { openChecksummedEnvelope } from './chunk-HVYICDY3.mjs';
|
|
55
|
+
import './chunk-IPN7S4KX.mjs';
|
|
56
|
+
import { fnv1a, fnv1aUint32s } from './chunk-KAKQ74LS.mjs';
|
|
57
|
+
import { crc32Init, crc32Final, crc32Update } from './chunk-7GXJUJ24.mjs';
|
|
58
|
+
import './chunk-Q2UL2TPC.mjs';
|
|
59
|
+
import { configOfNamedSettings } from './chunk-NTZOXVUZ.mjs';
|
|
46
60
|
import './chunk-X5CV2E2B.mjs';
|
|
61
|
+
import './chunk-Y4WPNHMC.mjs';
|
|
62
|
+
import { getLanguage, resolveIndexAnalysis, hasLanguage } from './chunk-JCLTESFN.mjs';
|
|
47
63
|
import './chunk-TGP5M5R7.mjs';
|
|
48
64
|
import './chunk-U2BKYEC7.mjs';
|
|
49
|
-
import './chunk-DGRM4EZZ.mjs';
|
|
50
65
|
import './chunk-QHPYS427.mjs';
|
|
51
66
|
import { collectAndReturnPages } from './chunk-QYRNLLLI.mjs';
|
|
67
|
+
import './chunk-37IVJLP6.mjs';
|
|
52
68
|
import './chunk-ZXNQEW43.mjs';
|
|
53
69
|
import './chunk-KOKM4PQN.mjs';
|
|
54
|
-
import { MIN_OVERSAMPLE } from './chunk-
|
|
55
|
-
import './chunk-LQG5G2LY.mjs';
|
|
70
|
+
import { MIN_OVERSAMPLE } from './chunk-HVKXNBFL.mjs';
|
|
56
71
|
import './chunk-BTPGQC6Z.mjs';
|
|
72
|
+
import './chunk-LQG5G2LY.mjs';
|
|
57
73
|
import './chunk-RFBLQLLZ.mjs';
|
|
58
|
-
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE } from './chunk-
|
|
74
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE } from './chunk-OKX2BBNW.mjs';
|
|
59
75
|
import './chunk-JUKYYXSP.mjs';
|
|
60
|
-
import {
|
|
76
|
+
import { getNestedValue, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-VMXSVVXG.mjs';
|
|
77
|
+
import { encodeDocument, validateSchema, requireCreatableSchema, encodeDocumentReply, schemaFieldsOf } from './chunk-Z3TNWCI5.mjs';
|
|
61
78
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
62
|
-
import { visitTokens } from './chunk-
|
|
79
|
+
import { visitTokens } from './chunk-ZIFJXCKA.mjs';
|
|
63
80
|
import './chunk-4UMZ7KQL.mjs';
|
|
64
81
|
import './chunk-ZXX5GZEX.mjs';
|
|
65
|
-
import './chunk-
|
|
82
|
+
import './chunk-GES7U2PM.mjs';
|
|
83
|
+
import './chunk-3A5KI24C.mjs';
|
|
66
84
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
67
|
-
import { joinChunks, readWhole } from './chunk-
|
|
85
|
+
import { joinChunks, readWhole } from './chunk-3Y3J4V2R.mjs';
|
|
68
86
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
69
|
-
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-
|
|
87
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-KUZKUONM.mjs';
|
|
70
88
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
71
89
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
72
90
|
import { encode, decode } from '@msgpack/msgpack';
|
|
@@ -243,10 +261,11 @@ async function clearRebalanceProgress(coordinator, indexName, partitionId) {
|
|
|
243
261
|
}
|
|
244
262
|
|
|
245
263
|
// src/distribution/cluster/version.ts
|
|
246
|
-
var SPEC_VERSION = "1.
|
|
264
|
+
var SPEC_VERSION = "1.3";
|
|
247
265
|
var PHRASE_SEARCH_SPEC_VERSION = "1.1";
|
|
248
266
|
var BM25_TUNING_SPEC_VERSION = "1.1";
|
|
249
267
|
var BACKUP_SPEC_VERSION = "1.2";
|
|
268
|
+
var DEVICE_SYNC_SPEC_VERSION = "1.3";
|
|
250
269
|
var VERSION_PATTERN = /^(\d+)\.(\d+)$/;
|
|
251
270
|
function versionPartsOf(version) {
|
|
252
271
|
const match = VERSION_PATTERN.exec(version);
|
|
@@ -679,6 +698,9 @@ async function readEveryListed(store) {
|
|
|
679
698
|
}
|
|
680
699
|
return reads;
|
|
681
700
|
}
|
|
701
|
+
async function readEveryReadableManifest(store) {
|
|
702
|
+
return (await readEveryListed(store)).flatMap((read) => "error" in read ? [] : [read.manifest]);
|
|
703
|
+
}
|
|
682
704
|
async function readEveryManifest(store) {
|
|
683
705
|
const manifests = [];
|
|
684
706
|
for (const read of await readEveryListed(store)) {
|
|
@@ -6638,11 +6660,13 @@ async function requireIndexMetadata(context, indexName, generation) {
|
|
|
6638
6660
|
}
|
|
6639
6661
|
return metadata;
|
|
6640
6662
|
}
|
|
6641
|
-
async function runBackupCreation(context) {
|
|
6663
|
+
async function runBackupCreation(context, takenBy) {
|
|
6642
6664
|
const { coordinator } = context.config;
|
|
6643
6665
|
const { store, job } = context;
|
|
6644
6666
|
const indexName = jobIndexNameOf(context);
|
|
6645
6667
|
const generation = await servingNameOf(coordinator, indexName);
|
|
6668
|
+
const named = (await readIndexMetadataSnapshot(coordinator, indexName))?.settings;
|
|
6669
|
+
const syncEpoch = named?.sync?.enabled === true ? named.syncEpoch : void 0;
|
|
6646
6670
|
const metadata = await requireIndexMetadata(context, indexName, generation);
|
|
6647
6671
|
const clusterUuid = await clusterUuidOf(coordinator);
|
|
6648
6672
|
await reportJobProgress(context, 0, metadata.partitionCount);
|
|
@@ -6655,6 +6679,8 @@ async function runBackupCreation(context) {
|
|
|
6655
6679
|
startedAt: job.startedAt,
|
|
6656
6680
|
completedAt: Date.now(),
|
|
6657
6681
|
partitionCount: metadata.partitionCount,
|
|
6682
|
+
...syncEpoch !== void 0 ? { syncEpoch } : {},
|
|
6683
|
+
...takenBy !== void 0 ? { takenBy } : {},
|
|
6658
6684
|
replicationFactor: metadata.replicationFactor,
|
|
6659
6685
|
constraints: metadata.constraints,
|
|
6660
6686
|
config: sameConfigOf(results, indexName),
|
|
@@ -6920,6 +6946,15 @@ async function runBackupRestore(context, request) {
|
|
|
6920
6946
|
await publishRestoredName(context, generation, nameIndexUuid);
|
|
6921
6947
|
}
|
|
6922
6948
|
|
|
6949
|
+
// src/distribution/backups/sync-retention.ts
|
|
6950
|
+
async function surplusSyncBackups(store, indexName) {
|
|
6951
|
+
const taken = (await readEveryReadableManifest(store)).filter(
|
|
6952
|
+
(manifest) => manifest.indexName === indexName && manifest.takenBy === SYNC_BACKUP_TAKER
|
|
6953
|
+
);
|
|
6954
|
+
taken.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
|
|
6955
|
+
return taken.slice(SYNC_BACKUPS_KEPT).map((manifest) => manifest.name);
|
|
6956
|
+
}
|
|
6957
|
+
|
|
6923
6958
|
// src/distribution/backups/controller-messages.ts
|
|
6924
6959
|
var KIND_BY_TYPE = /* @__PURE__ */ new Map([
|
|
6925
6960
|
[ClusterMessageTypes.BACKUP, "backup"],
|
|
@@ -6973,12 +7008,12 @@ function driveBackupJob(deps, store, jobId, job, run) {
|
|
|
6973
7008
|
jobId,
|
|
6974
7009
|
job
|
|
6975
7010
|
};
|
|
6976
|
-
|
|
7011
|
+
return run(context).then(
|
|
6977
7012
|
() => finishBackupJob(deps.config.coordinator, jobId, null),
|
|
6978
7013
|
(error) => deps.isActive() ? finishBackupJob(deps.config.coordinator, jobId, error) : deps.onError(error)
|
|
6979
7014
|
).catch(deps.onError).finally(() => deps.driving.delete(jobId));
|
|
6980
7015
|
}
|
|
6981
|
-
async function startJob(deps, store, job, run) {
|
|
7016
|
+
async function startJob(deps, store, job, run, afterJob) {
|
|
6982
7017
|
const jobId = generateId();
|
|
6983
7018
|
deps.driving.add(jobId);
|
|
6984
7019
|
try {
|
|
@@ -6987,8 +7022,28 @@ async function startJob(deps, store, job, run) {
|
|
|
6987
7022
|
deps.driving.delete(jobId);
|
|
6988
7023
|
throw error;
|
|
6989
7024
|
}
|
|
6990
|
-
driveBackupJob(deps, store, jobId, job, run);
|
|
6991
|
-
|
|
7025
|
+
const driven = driveBackupJob(deps, store, jobId, job, run);
|
|
7026
|
+
const ended = afterJob === void 0 ? driven : driven.then(afterJob).catch(deps.onError);
|
|
7027
|
+
return { jobId, ended };
|
|
7028
|
+
}
|
|
7029
|
+
function takenByOf(request) {
|
|
7030
|
+
if (request.takenBy === void 0) return void 0;
|
|
7031
|
+
if (request.takenBy !== SYNC_BACKUP_TAKER) throw malformed3("backup", `"takenBy" must be "${SYNC_BACKUP_TAKER}"`);
|
|
7032
|
+
return SYNC_BACKUP_TAKER;
|
|
7033
|
+
}
|
|
7034
|
+
function quietRefusal(error) {
|
|
7035
|
+
return error instanceof NarsilError && (error.code === ErrorCodes.BACKUP_IN_PROGRESS || error.code === ErrorCodes.BACKUP_NOT_FOUND);
|
|
7036
|
+
}
|
|
7037
|
+
async function deleteSurplusSyncBackups(deps, store, indexName) {
|
|
7038
|
+
for (const backupName of await surplusSyncBackups(store, indexName)) {
|
|
7039
|
+
if (!deps.isActive()) return;
|
|
7040
|
+
try {
|
|
7041
|
+
await admitDeletion(deps.config.coordinator, store, backupName);
|
|
7042
|
+
await (await startJob(deps, store, runningJob("delete", backupName), runBackupDeletion)).ended;
|
|
7043
|
+
} catch (error) {
|
|
7044
|
+
if (!quietRefusal(error)) throw error;
|
|
7045
|
+
}
|
|
7046
|
+
}
|
|
6992
7047
|
}
|
|
6993
7048
|
function runningJob(kind, backupName, indexName) {
|
|
6994
7049
|
return {
|
|
@@ -7013,18 +7068,33 @@ async function acceptRequest(deps, kind, message) {
|
|
|
7013
7068
|
const { coordinator } = deps.config;
|
|
7014
7069
|
if (kind === "backup") {
|
|
7015
7070
|
const indexName = indexNameOf(kind, request);
|
|
7071
|
+
const takenBy = takenByOf(request);
|
|
7016
7072
|
await admitBackup(coordinator, store, backupName, indexName);
|
|
7017
|
-
|
|
7073
|
+
const afterJob = takenBy === void 0 ? void 0 : () => deleteSurplusSyncBackups(deps, store, indexName);
|
|
7074
|
+
return jobIdOf(
|
|
7075
|
+
startJob(
|
|
7076
|
+
deps,
|
|
7077
|
+
store,
|
|
7078
|
+
runningJob(kind, backupName, indexName),
|
|
7079
|
+
(context) => runBackupCreation(context, takenBy),
|
|
7080
|
+
afterJob
|
|
7081
|
+
)
|
|
7082
|
+
);
|
|
7018
7083
|
}
|
|
7019
7084
|
if (kind === "restore") {
|
|
7020
7085
|
const indexName = indexNameOf(kind, request);
|
|
7021
7086
|
const options = restoreOptionsOf(request);
|
|
7022
7087
|
const manifest = await admitRestore(coordinator, store, backupName, indexName);
|
|
7023
7088
|
const restore = { manifest, ...options };
|
|
7024
|
-
return
|
|
7089
|
+
return jobIdOf(
|
|
7090
|
+
startJob(deps, store, runningJob(kind, backupName, indexName), (context) => runBackupRestore(context, restore))
|
|
7091
|
+
);
|
|
7025
7092
|
}
|
|
7026
7093
|
await admitDeletion(coordinator, store, backupName);
|
|
7027
|
-
return startJob(deps, store, runningJob(kind, backupName), runBackupDeletion);
|
|
7094
|
+
return jobIdOf(startJob(deps, store, runningJob(kind, backupName), runBackupDeletion));
|
|
7095
|
+
}
|
|
7096
|
+
async function jobIdOf(started) {
|
|
7097
|
+
return { jobId: (await started).jobId };
|
|
7028
7098
|
}
|
|
7029
7099
|
function answer(deps, kind, message, respond) {
|
|
7030
7100
|
acceptRequest(deps, kind, message).then(
|
|
@@ -7527,11 +7597,18 @@ async function runJob(deps, request, options) {
|
|
|
7527
7597
|
onProgress: options?.onProgress
|
|
7528
7598
|
});
|
|
7529
7599
|
}
|
|
7530
|
-
|
|
7600
|
+
function requestBackup(deps, indexName, backupName, options) {
|
|
7601
|
+
return backUp(deps, indexName, backupName, {}, options);
|
|
7602
|
+
}
|
|
7603
|
+
async function requestSyncBackup(deps, indexName, backupName) {
|
|
7604
|
+
await backUp(deps, indexName, backupName, { takenBy: SYNC_BACKUP_TAKER });
|
|
7605
|
+
}
|
|
7606
|
+
async function backUp(deps, indexName, backupName, taker, options) {
|
|
7531
7607
|
const store = requireBackupStore(deps.store);
|
|
7532
7608
|
const payload = {
|
|
7533
7609
|
backupName: validateBackupName(backupName),
|
|
7534
|
-
indexName: requireIndexName2(indexName)
|
|
7610
|
+
indexName: requireIndexName2(indexName),
|
|
7611
|
+
...taker
|
|
7535
7612
|
};
|
|
7536
7613
|
await runJob(
|
|
7537
7614
|
deps,
|
|
@@ -7576,7 +7653,7 @@ function abandonedBackup(jobId, job) {
|
|
|
7576
7653
|
}
|
|
7577
7654
|
function takeOverJob(deps, store, jobId, job) {
|
|
7578
7655
|
if (job.kind === "backup") return finishBackupJob(deps.config.coordinator, jobId, abandonedBackup(jobId, job));
|
|
7579
|
-
driveBackupJob(deps, store, jobId, job, job.kind === "delete" ? runBackupDeletion : resumeRestore);
|
|
7656
|
+
void driveBackupJob(deps, store, jobId, job, job.kind === "delete" ? runBackupDeletion : resumeRestore);
|
|
7580
7657
|
return Promise.resolve();
|
|
7581
7658
|
}
|
|
7582
7659
|
function createBackupService(deps) {
|
|
@@ -11741,7 +11818,8 @@ function metadataPublishingAnalysis(snapshot, pending) {
|
|
|
11741
11818
|
analysisGeneration: pending.generation,
|
|
11742
11819
|
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
11743
11820
|
...pending.stopWords !== void 0 ? { stopWords: pending.stopWords } : {},
|
|
11744
|
-
...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {}
|
|
11821
|
+
...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {},
|
|
11822
|
+
...isRecord2(rest.sync) && rest.sync.enabled === true ? { syncActive: false, syncEpoch: newSyncEpoch() } : {}
|
|
11745
11823
|
});
|
|
11746
11824
|
}
|
|
11747
11825
|
function metadataWithAnalysisFailure(snapshot, failure) {
|
|
@@ -12280,6 +12358,19 @@ async function clearEveryProgress(coordinator, indexName, partitionCount) {
|
|
|
12280
12358
|
}
|
|
12281
12359
|
}
|
|
12282
12360
|
|
|
12361
|
+
// src/distribution/cluster/sync-epoch-renewal.ts
|
|
12362
|
+
async function renewClusterSyncEpoch(coordinator, indexName) {
|
|
12363
|
+
for (let attempt = 1; attempt <= SYNC_EPOCH_CAS_ATTEMPTS; attempt++) {
|
|
12364
|
+
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
12365
|
+
if (current === null) return false;
|
|
12366
|
+
const sync = current.settings?.sync;
|
|
12367
|
+
if (sync?.enabled !== true) return true;
|
|
12368
|
+
const next = metadataWithSync(current, { sync, syncActive: false, syncEpoch: newSyncEpoch() });
|
|
12369
|
+
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return true;
|
|
12370
|
+
}
|
|
12371
|
+
return false;
|
|
12372
|
+
}
|
|
12373
|
+
|
|
12283
12374
|
// src/distribution/cluster/controller/event-loop/rebalance/primaries.ts
|
|
12284
12375
|
async function targetsOf(ctx, nodeId) {
|
|
12285
12376
|
const nodes = await ctx.coordinator.listNodes();
|
|
@@ -12395,7 +12486,7 @@ function blockStillHolds(source, blocked) {
|
|
|
12395
12486
|
}
|
|
12396
12487
|
return true;
|
|
12397
12488
|
}
|
|
12398
|
-
async function swapIndexName(ctx, indexName, record, blocked) {
|
|
12489
|
+
async function swapIndexName(ctx, indexName, record, blocked, epoch) {
|
|
12399
12490
|
const nameTable = await ctx.coordinator.getAllocation(indexName);
|
|
12400
12491
|
if (nameTable === null) {
|
|
12401
12492
|
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' was dropped during cutover`, { indexName });
|
|
@@ -12414,6 +12505,10 @@ async function swapIndexName(ctx, indexName, record, blocked) {
|
|
|
12414
12505
|
if (nameTable.indexUuid !== void 0) pointer.indexUuid = nameTable.indexUuid;
|
|
12415
12506
|
const rewritten = await rewriteForCurrentIndex(ctx.coordinator, pointer);
|
|
12416
12507
|
if (rewritten === null) return false;
|
|
12508
|
+
if (!epoch.renewed) {
|
|
12509
|
+
if (!await renewClusterSyncEpoch(ctx.coordinator, indexName)) return false;
|
|
12510
|
+
epoch.renewed = true;
|
|
12511
|
+
}
|
|
12417
12512
|
return ctx.coordinator.putAllocation(indexName, rewritten, nameTable.version, ctx.nodeId);
|
|
12418
12513
|
}
|
|
12419
12514
|
async function attemptCutover(ctx, indexName, record, started) {
|
|
@@ -12426,12 +12521,13 @@ async function attemptCutover(ctx, indexName, record, started) {
|
|
|
12426
12521
|
if (cut === null) return "gone";
|
|
12427
12522
|
if (cut.phase !== "cutover") return "retry";
|
|
12428
12523
|
const blocked = /* @__PURE__ */ new Map();
|
|
12524
|
+
const epoch = { renewed: false };
|
|
12429
12525
|
const deadline = Date.now() + RESHARD_CUTOVER_TIMEOUT_MS;
|
|
12430
12526
|
while (Date.now() < deadline && ctx.isLive()) {
|
|
12431
12527
|
const latest = await readRebalanceRecord(ctx.coordinator, indexName);
|
|
12432
12528
|
if (latest === null || latest.jobId !== cut.jobId) return "gone";
|
|
12433
12529
|
if (latest.phase === "aborting") return "retry";
|
|
12434
|
-
if (await everyPartitionCaughtUp(ctx, indexName, cut, blocked, started) && await swapIndexName(ctx, indexName, cut, blocked)) {
|
|
12530
|
+
if (await everyPartitionCaughtUp(ctx, indexName, cut, blocked, started) && await swapIndexName(ctx, indexName, cut, blocked, epoch)) {
|
|
12435
12531
|
return "swapped";
|
|
12436
12532
|
}
|
|
12437
12533
|
await pause(RESHARD_CONTROLLER_POLL_INTERVAL_MS);
|
|
@@ -13190,6 +13286,1039 @@ async function decommissionNode(deps, signal) {
|
|
|
13190
13286
|
await deps.node.shutdown();
|
|
13191
13287
|
}
|
|
13192
13288
|
|
|
13289
|
+
// src/distribution/cluster-node/device-sync/constants.ts
|
|
13290
|
+
var DEVICE_SYNC_RECONCILE_INTERVAL_MS = 1e4;
|
|
13291
|
+
var DEVICE_SYNC_HEAD_POLL_MS = 1e3;
|
|
13292
|
+
var DEVICE_SYNC_BOUNDS_REUSE_MS = 250;
|
|
13293
|
+
var DEVICE_SYNC_ARCHIVE_IDLE_MS = 6e5;
|
|
13294
|
+
var DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS = 5e3;
|
|
13295
|
+
var DEVICE_SYNC_BACKUP_NAME_PREFIX = "sync-";
|
|
13296
|
+
var CACHED_BACKUP_MANIFEST_LIMIT = 4096;
|
|
13297
|
+
|
|
13298
|
+
// src/distribution/cluster-node/reshard/leadership.ts
|
|
13299
|
+
async function leadingAssignment2(deps, indexName, partitionId) {
|
|
13300
|
+
let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
|
|
13301
|
+
if (assignment?.primary !== deps.nodeId && deps.refreshAllocation !== void 0) {
|
|
13302
|
+
assignment = (await deps.refreshAllocation(indexName))?.assignments.get(partitionId);
|
|
13303
|
+
}
|
|
13304
|
+
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
13305
|
+
throw new NarsilError(
|
|
13306
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
13307
|
+
`Node '${deps.nodeId}' does not lead partition ${partitionId} of index '${indexName}'`,
|
|
13308
|
+
{ indexName, partitionId, primaryNodeId: assignment?.primary ?? null, localNodeId: deps.nodeId }
|
|
13309
|
+
);
|
|
13310
|
+
}
|
|
13311
|
+
return assignment;
|
|
13312
|
+
}
|
|
13313
|
+
function forwardableEnd(log, primaryTerm) {
|
|
13314
|
+
let end = log.commitPoint;
|
|
13315
|
+
for (; ; ) {
|
|
13316
|
+
const next = log.getEntry(end + 1);
|
|
13317
|
+
if (next === void 0 || next.primaryTerm >= primaryTerm) return end;
|
|
13318
|
+
end = next.seqNo;
|
|
13319
|
+
}
|
|
13320
|
+
}
|
|
13321
|
+
var UINT32_MAX = 4294967295;
|
|
13322
|
+
var FIXMAP_OF_FIVE = 133;
|
|
13323
|
+
var FIXARRAY_LIMIT = 16;
|
|
13324
|
+
var ARRAY16_LIMIT = 65536;
|
|
13325
|
+
function isSeqNo(value) {
|
|
13326
|
+
return isValidInteger(value) && value >= 0;
|
|
13327
|
+
}
|
|
13328
|
+
function validateSyncReadRequest(decoded) {
|
|
13329
|
+
if (!isRecord(decoded)) return null;
|
|
13330
|
+
const { indexName, indexUuid, partitionId, afterSeqNo, maxBytes } = decoded;
|
|
13331
|
+
if (typeof indexName !== "string" || indexName.length === 0) return null;
|
|
13332
|
+
if (typeof indexUuid !== "string" || indexUuid.length === 0) return null;
|
|
13333
|
+
if (!isValidInteger(partitionId) || partitionId < 0 || partitionId > UINT32_MAX) return null;
|
|
13334
|
+
if (!isSeqNo(afterSeqNo)) return null;
|
|
13335
|
+
if (!isValidInteger(maxBytes) || maxBytes < 1 || maxBytes > UINT32_MAX) return null;
|
|
13336
|
+
return { indexName, indexUuid, partitionId, afterSeqNo, maxBytes };
|
|
13337
|
+
}
|
|
13338
|
+
function createSyncReadMessage(request, sourceId, requestId) {
|
|
13339
|
+
return { type: SyncMessageTypes.READ, sourceId, requestId, payload: encode(request) };
|
|
13340
|
+
}
|
|
13341
|
+
function arrayHeader(count2) {
|
|
13342
|
+
if (count2 < FIXARRAY_LIMIT) return Uint8Array.of(144 | count2);
|
|
13343
|
+
if (count2 < ARRAY16_LIMIT) return Uint8Array.of(220, count2 >>> 8, count2 & 255);
|
|
13344
|
+
return Uint8Array.of(221, count2 >>> 24, count2 >>> 16 & 255, count2 >>> 8 & 255, count2 & 255);
|
|
13345
|
+
}
|
|
13346
|
+
function concat(parts) {
|
|
13347
|
+
let length = 0;
|
|
13348
|
+
for (const part of parts) length += part.byteLength;
|
|
13349
|
+
const out = new Uint8Array(length);
|
|
13350
|
+
let at = 0;
|
|
13351
|
+
for (const part of parts) {
|
|
13352
|
+
out.set(part, at);
|
|
13353
|
+
at += part.byteLength;
|
|
13354
|
+
}
|
|
13355
|
+
return out;
|
|
13356
|
+
}
|
|
13357
|
+
function encodeSyncReadReply(reply3) {
|
|
13358
|
+
return concat([
|
|
13359
|
+
Uint8Array.of(FIXMAP_OF_FIVE),
|
|
13360
|
+
encode("entries"),
|
|
13361
|
+
arrayHeader(reply3.frames.length),
|
|
13362
|
+
...reply3.frames.map((frame) => frame.payload),
|
|
13363
|
+
encode("head"),
|
|
13364
|
+
encode(reply3.head),
|
|
13365
|
+
encode("highestSeqNo"),
|
|
13366
|
+
encode(reply3.highestSeqNo),
|
|
13367
|
+
encode("oldestSeqNo"),
|
|
13368
|
+
encode(reply3.oldestSeqNo),
|
|
13369
|
+
encode("bytesBehind"),
|
|
13370
|
+
encode(reply3.bytesBehind)
|
|
13371
|
+
]);
|
|
13372
|
+
}
|
|
13373
|
+
function decodeSyncReadReply(decoded, partitionId, afterSeqNo) {
|
|
13374
|
+
if (!isRecord(decoded) || !Array.isArray(decoded.entries)) return null;
|
|
13375
|
+
const { head, highestSeqNo, oldestSeqNo, bytesBehind } = decoded;
|
|
13376
|
+
if (!isSeqNo(head) || !isSeqNo(highestSeqNo) || !isSeqNo(oldestSeqNo) || !isSeqNo(bytesBehind)) return null;
|
|
13377
|
+
const frames = [];
|
|
13378
|
+
let previous = afterSeqNo;
|
|
13379
|
+
for (const entry of decoded.entries) {
|
|
13380
|
+
if (!isRecord(entry) || !isSeqNo(entry.seqNo) || entry.seqNo <= previous || entry.seqNo > head) return null;
|
|
13381
|
+
if (entry.partitionId !== partitionId) return null;
|
|
13382
|
+
const payload = encode(entry);
|
|
13383
|
+
frames.push({ seqNo: entry.seqNo, payload, frameBytes: payload.byteLength });
|
|
13384
|
+
previous = entry.seqNo;
|
|
13385
|
+
}
|
|
13386
|
+
return { frames, head, highestSeqNo, oldestSeqNo, bytesBehind };
|
|
13387
|
+
}
|
|
13388
|
+
|
|
13389
|
+
// src/distribution/cluster-node/device-sync/read-handler.ts
|
|
13390
|
+
async function answerSyncRead(deps, request) {
|
|
13391
|
+
const { indexName, indexUuid, partitionId, afterSeqNo, maxBytes } = request;
|
|
13392
|
+
await leadingAssignment2(deps.writeDeps, indexName, partitionId);
|
|
13393
|
+
if (deps.engine.indexUuidOf(indexName) !== indexUuid) {
|
|
13394
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Node '${deps.nodeId}' holds no copy of index '${indexName}'`, {
|
|
13395
|
+
indexName,
|
|
13396
|
+
indexUuid
|
|
13397
|
+
});
|
|
13398
|
+
}
|
|
13399
|
+
const storage = deps.engine.syncStorage();
|
|
13400
|
+
if (storage === null) {
|
|
13401
|
+
throw new NarsilError(
|
|
13402
|
+
ErrorCodes.SYNC_DURABILITY_REQUIRED,
|
|
13403
|
+
`Node '${deps.nodeId}' has no write-ahead log durability, so it can't read the log for a device feed`,
|
|
13404
|
+
{ indexName }
|
|
13405
|
+
);
|
|
13406
|
+
}
|
|
13407
|
+
const replicationLog = deps.writeDeps.getReplicationLog(indexName, partitionId);
|
|
13408
|
+
const log = await storage.openPartitionLog(indexName, partitionId);
|
|
13409
|
+
const head = Math.min(replicationLog.commitPoint, log.head);
|
|
13410
|
+
const highestSeqNo = Math.max(log.head, replicationLog.localLogEnd);
|
|
13411
|
+
const read = afterSeqNo < head ? await storage.readPartitionLog(indexName, partitionId, afterSeqNo, head, maxBytes) : { frames: [], gone: false };
|
|
13412
|
+
const bytesBehind = afterSeqNo < head ? await storage.logBytesAfter(log, afterSeqNo) : 0;
|
|
13413
|
+
const oldestSeqNo = read.gone ? Math.max(log.oldestSeqNo, afterSeqNo + 2) : log.oldestSeqNo;
|
|
13414
|
+
return { frames: read.frames, head, highestSeqNo, oldestSeqNo, bytesBehind };
|
|
13415
|
+
}
|
|
13416
|
+
async function handleSyncRead(message, respond, deps) {
|
|
13417
|
+
const request = validateSyncReadRequest(decode(message.payload));
|
|
13418
|
+
if (request === null) {
|
|
13419
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A sync.read request is malformed", {
|
|
13420
|
+
sourceNodeId: message.sourceId
|
|
13421
|
+
});
|
|
13422
|
+
}
|
|
13423
|
+
const reply3 = await answerSyncRead(deps, request);
|
|
13424
|
+
await respond({
|
|
13425
|
+
type: SyncMessageTypes.READ,
|
|
13426
|
+
sourceId: deps.nodeId,
|
|
13427
|
+
requestId: message.requestId,
|
|
13428
|
+
payload: encodeSyncReadReply(reply3)
|
|
13429
|
+
});
|
|
13430
|
+
}
|
|
13431
|
+
|
|
13432
|
+
// src/distribution/cluster-node/device-sync/feed-source.ts
|
|
13433
|
+
var NO_ENTRIES_AFTER = Number.MAX_SAFE_INTEGER;
|
|
13434
|
+
var PRIMARY_MOVED_CODES = /* @__PURE__ */ new Set([ErrorCodes.PARTITION_NOT_PRIMARY]);
|
|
13435
|
+
function settledEither() {
|
|
13436
|
+
}
|
|
13437
|
+
function replyError(nodeId, decoded, indexName, partitionId) {
|
|
13438
|
+
const reply3 = isRecord(decoded) ? decoded : {};
|
|
13439
|
+
return new NarsilError(
|
|
13440
|
+
typeof reply3.code === "string" ? reply3.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
13441
|
+
typeof reply3.message === "string" ? reply3.message : `Node '${nodeId}' did not answer sync.read for partition ${partitionId} of index '${indexName}'`,
|
|
13442
|
+
{ indexName, partitionId, targetNodeId: nodeId }
|
|
13443
|
+
);
|
|
13444
|
+
}
|
|
13445
|
+
function createClusterFeedSource(deps) {
|
|
13446
|
+
const { view, coordinator, config } = deps;
|
|
13447
|
+
const reusedBounds = /* @__PURE__ */ new Map();
|
|
13448
|
+
const queuedBounds = /* @__PURE__ */ new Map();
|
|
13449
|
+
const watched = /* @__PURE__ */ new Map();
|
|
13450
|
+
const lastHeads = /* @__PURE__ */ new Map();
|
|
13451
|
+
let pollTimer = null;
|
|
13452
|
+
let polling = false;
|
|
13453
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
13454
|
+
function emit(change) {
|
|
13455
|
+
for (const listener of listeners) listener(change);
|
|
13456
|
+
}
|
|
13457
|
+
function serving(indexName) {
|
|
13458
|
+
const held = view.held(indexName);
|
|
13459
|
+
if (held === void 0) {
|
|
13460
|
+
throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
13461
|
+
}
|
|
13462
|
+
return held.serving;
|
|
13463
|
+
}
|
|
13464
|
+
async function askPrimary(primary, indexName, request) {
|
|
13465
|
+
if (primary === deps.nodeId) return answerSyncRead(deps, request);
|
|
13466
|
+
const message = createSyncReadMessage(request, deps.nodeId, generateId());
|
|
13467
|
+
let lastError;
|
|
13468
|
+
for (const target of await resolveNodeTargets2(config, primary)) {
|
|
13469
|
+
let response;
|
|
13470
|
+
try {
|
|
13471
|
+
response = await config.transport.send(target, message);
|
|
13472
|
+
} catch (error) {
|
|
13473
|
+
lastError = error;
|
|
13474
|
+
continue;
|
|
13475
|
+
}
|
|
13476
|
+
const decoded = decode(response.payload);
|
|
13477
|
+
if (response.type.endsWith(".error")) throw replyError(primary, decoded, indexName, request.partitionId);
|
|
13478
|
+
const reply3 = decodeSyncReadReply(decoded, request.partitionId, request.afterSeqNo);
|
|
13479
|
+
if (reply3 === null) throw replyError(primary, null, indexName, request.partitionId);
|
|
13480
|
+
return reply3;
|
|
13481
|
+
}
|
|
13482
|
+
throw new NarsilError(
|
|
13483
|
+
ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
13484
|
+
`Node '${primary}' did not answer sync.read for partition ${request.partitionId} of index '${indexName}'`,
|
|
13485
|
+
{
|
|
13486
|
+
indexName,
|
|
13487
|
+
partitionId: request.partitionId,
|
|
13488
|
+
...lastError instanceof TransportError ? { transportCode: lastError.code } : {}
|
|
13489
|
+
}
|
|
13490
|
+
);
|
|
13491
|
+
}
|
|
13492
|
+
async function readFromPrimary(indexName, partitionId, afterSeqNo, maxBytes) {
|
|
13493
|
+
const layout = serving(indexName);
|
|
13494
|
+
const request = { indexName: layout.servingName, indexUuid: layout.indexUuid, partitionId, afterSeqNo, maxBytes };
|
|
13495
|
+
for (let attempt = 0; ; attempt++) {
|
|
13496
|
+
const table = await coordinator.getAllocation(layout.servingName);
|
|
13497
|
+
const primary = table?.assignments.get(partitionId)?.primary ?? null;
|
|
13498
|
+
if (primary === null) {
|
|
13499
|
+
throw new NarsilError(
|
|
13500
|
+
ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
13501
|
+
`Partition ${partitionId} of index '${indexName}' has no primary to read the feed from`,
|
|
13502
|
+
{ indexName, partitionId }
|
|
13503
|
+
);
|
|
13504
|
+
}
|
|
13505
|
+
try {
|
|
13506
|
+
return await askPrimary(primary, indexName, request);
|
|
13507
|
+
} catch (error) {
|
|
13508
|
+
if (attempt > 0 || !(error instanceof NarsilError) || !PRIMARY_MOVED_CODES.has(error.code)) throw error;
|
|
13509
|
+
await deps.refreshAllocation(layout.servingName);
|
|
13510
|
+
}
|
|
13511
|
+
}
|
|
13512
|
+
}
|
|
13513
|
+
function startBoundsRead(indexName, partitionId, key) {
|
|
13514
|
+
const reply3 = readFromPrimary(indexName, partitionId, NO_ENTRIES_AFTER, 1);
|
|
13515
|
+
const started = { at: performance.now(), reply: reply3, settled: false };
|
|
13516
|
+
reusedBounds.set(key, started);
|
|
13517
|
+
reply3.then(
|
|
13518
|
+
() => {
|
|
13519
|
+
started.settled = true;
|
|
13520
|
+
},
|
|
13521
|
+
() => {
|
|
13522
|
+
started.settled = true;
|
|
13523
|
+
if (reusedBounds.get(key) === started) reusedBounds.delete(key);
|
|
13524
|
+
}
|
|
13525
|
+
);
|
|
13526
|
+
return reply3;
|
|
13527
|
+
}
|
|
13528
|
+
function bounds(indexName, partitionId, freshness = "recent") {
|
|
13529
|
+
const key = `${indexName}\0${partitionId}`;
|
|
13530
|
+
const latest = reusedBounds.get(key);
|
|
13531
|
+
if (freshness === "recent") {
|
|
13532
|
+
if (latest !== void 0 && performance.now() - latest.at < DEVICE_SYNC_BOUNDS_REUSE_MS) return latest.reply;
|
|
13533
|
+
return startBoundsRead(indexName, partitionId, key);
|
|
13534
|
+
}
|
|
13535
|
+
if (latest === void 0 || latest.settled) return startBoundsRead(indexName, partitionId, key);
|
|
13536
|
+
const queued = queuedBounds.get(key);
|
|
13537
|
+
if (queued !== void 0) return queued;
|
|
13538
|
+
const next = latest.reply.then(settledEither, settledEither).then(() => {
|
|
13539
|
+
queuedBounds.delete(key);
|
|
13540
|
+
return startBoundsRead(indexName, partitionId, key);
|
|
13541
|
+
});
|
|
13542
|
+
queuedBounds.set(key, next);
|
|
13543
|
+
return next;
|
|
13544
|
+
}
|
|
13545
|
+
async function pollHeads() {
|
|
13546
|
+
if (polling) return;
|
|
13547
|
+
polling = true;
|
|
13548
|
+
try {
|
|
13549
|
+
for (const indexName of watched.keys()) {
|
|
13550
|
+
const held = view.held(indexName);
|
|
13551
|
+
if (held === void 0) continue;
|
|
13552
|
+
let rose = false;
|
|
13553
|
+
for (let partitionId = 0; partitionId < held.serving.partitionCount; partitionId++) {
|
|
13554
|
+
const key = `${indexName}\0${partitionId}`;
|
|
13555
|
+
const head = (await bounds(indexName, partitionId).catch(() => null))?.head;
|
|
13556
|
+
if (head === void 0) continue;
|
|
13557
|
+
if (head > (lastHeads.get(key) ?? -1)) rose = true;
|
|
13558
|
+
lastHeads.set(key, head);
|
|
13559
|
+
}
|
|
13560
|
+
if (rose) emit({ kind: "advanced", indexName });
|
|
13561
|
+
}
|
|
13562
|
+
} finally {
|
|
13563
|
+
polling = false;
|
|
13564
|
+
}
|
|
13565
|
+
}
|
|
13566
|
+
function namesServedBy(localName) {
|
|
13567
|
+
const names = [];
|
|
13568
|
+
for (const indexName of watched.keys()) {
|
|
13569
|
+
if (view.held(indexName)?.serving.servingName === localName) names.push(indexName);
|
|
13570
|
+
}
|
|
13571
|
+
return names;
|
|
13572
|
+
}
|
|
13573
|
+
const onLocalWrite = (payload) => {
|
|
13574
|
+
for (const indexName of namesServedBy(payload.indexName)) {
|
|
13575
|
+
reusedBounds.forEach((_, key) => {
|
|
13576
|
+
if (key.startsWith(`${indexName}\0`)) reusedBounds.delete(key);
|
|
13577
|
+
});
|
|
13578
|
+
emit({ kind: "advanced", indexName });
|
|
13579
|
+
}
|
|
13580
|
+
};
|
|
13581
|
+
return {
|
|
13582
|
+
indexState(indexName) {
|
|
13583
|
+
const held = view.held(indexName);
|
|
13584
|
+
const sync = held?.settings;
|
|
13585
|
+
if (held === void 0 || sync?.sync?.enabled !== true || sync.syncEpoch === void 0) return null;
|
|
13586
|
+
return { epoch: sync.syncEpoch, partitionCount: held.serving.partitionCount, settings: held.feedSettings };
|
|
13587
|
+
},
|
|
13588
|
+
async bounds(indexName, partitionId, freshness) {
|
|
13589
|
+
const reply3 = await bounds(indexName, partitionId, freshness);
|
|
13590
|
+
return { oldestSeqNo: reply3.oldestSeqNo, head: reply3.head, highestSeqNo: reply3.highestSeqNo };
|
|
13591
|
+
},
|
|
13592
|
+
async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
13593
|
+
const reply3 = await readFromPrimary(indexName, partitionId, afterSeqNo, maxBytes);
|
|
13594
|
+
const gone = afterSeqNo + 1 < reply3.oldestSeqNo && afterSeqNo < reply3.head;
|
|
13595
|
+
return { frames: gone ? [] : reply3.frames.filter((frame) => frame.seqNo <= upToSeqNo), gone };
|
|
13596
|
+
},
|
|
13597
|
+
async bytesBehind(indexName, seqNos) {
|
|
13598
|
+
let bytes = 0;
|
|
13599
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
13600
|
+
bytes += (await readFromPrimary(indexName, partitionId, seqNos[partitionId], 1)).bytesBehind;
|
|
13601
|
+
}
|
|
13602
|
+
return bytes;
|
|
13603
|
+
},
|
|
13604
|
+
snapshotBytes: (indexName) => deps.snapshotBytes(indexName),
|
|
13605
|
+
pendingBeyondHead: () => false,
|
|
13606
|
+
listen(onChange) {
|
|
13607
|
+
listeners.add(onChange);
|
|
13608
|
+
const stopView = view.listen(onChange);
|
|
13609
|
+
if (listeners.size === 1) deps.engine.on("write", onLocalWrite);
|
|
13610
|
+
return () => {
|
|
13611
|
+
stopView();
|
|
13612
|
+
listeners.delete(onChange);
|
|
13613
|
+
if (listeners.size === 0) deps.engine.off("write", onLocalWrite);
|
|
13614
|
+
};
|
|
13615
|
+
},
|
|
13616
|
+
watchIndex(indexName) {
|
|
13617
|
+
watched.set(indexName, (watched.get(indexName) ?? 0) + 1);
|
|
13618
|
+
if (pollTimer === null) {
|
|
13619
|
+
pollTimer = setInterval(() => void pollHeads(), DEVICE_SYNC_HEAD_POLL_MS);
|
|
13620
|
+
pollTimer.unref?.();
|
|
13621
|
+
}
|
|
13622
|
+
let watching = true;
|
|
13623
|
+
return () => {
|
|
13624
|
+
if (!watching) return;
|
|
13625
|
+
watching = false;
|
|
13626
|
+
const count2 = (watched.get(indexName) ?? 1) - 1;
|
|
13627
|
+
if (count2 > 0) {
|
|
13628
|
+
watched.set(indexName, count2);
|
|
13629
|
+
return;
|
|
13630
|
+
}
|
|
13631
|
+
watched.delete(indexName);
|
|
13632
|
+
for (const key of [...lastHeads.keys()]) if (key.startsWith(`${indexName}\0`)) lastHeads.delete(key);
|
|
13633
|
+
if (watched.size === 0 && pollTimer !== null) {
|
|
13634
|
+
clearInterval(pollTimer);
|
|
13635
|
+
pollTimer = null;
|
|
13636
|
+
}
|
|
13637
|
+
};
|
|
13638
|
+
}
|
|
13639
|
+
};
|
|
13640
|
+
}
|
|
13641
|
+
|
|
13642
|
+
// src/distribution/cluster-node/device-sync/state.ts
|
|
13643
|
+
function syncStateOfSettings(settings) {
|
|
13644
|
+
const setting = settings?.sync;
|
|
13645
|
+
if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
|
|
13646
|
+
return {
|
|
13647
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
13648
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
|
|
13649
|
+
active: settings?.syncActive === true,
|
|
13650
|
+
epoch: settings.syncEpoch
|
|
13651
|
+
};
|
|
13652
|
+
}
|
|
13653
|
+
function syncStatusOfSettings(settings) {
|
|
13654
|
+
const state = syncStateOfSettings(settings);
|
|
13655
|
+
if (state === null) return { enabled: false, active: false, epoch: null };
|
|
13656
|
+
return { enabled: true, ...state };
|
|
13657
|
+
}
|
|
13658
|
+
function analysisRevisionOf(language) {
|
|
13659
|
+
if (!hasLanguage(language)) return void 0;
|
|
13660
|
+
return getLanguage(language).revision;
|
|
13661
|
+
}
|
|
13662
|
+
function feedSettingsOf(settings, synonyms) {
|
|
13663
|
+
const language = settings?.language ?? getLanguage("english").name;
|
|
13664
|
+
const revision = analysisRevisionOf(language);
|
|
13665
|
+
return {
|
|
13666
|
+
language,
|
|
13667
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
13668
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
13669
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {},
|
|
13670
|
+
...revision !== void 0 ? { analysisRevision: revision } : {},
|
|
13671
|
+
...settings?.bm25 !== void 0 ? { bm25: settings.bm25 } : {},
|
|
13672
|
+
...synonyms.length > 0 ? { synonyms: [...synonyms] } : {}
|
|
13673
|
+
};
|
|
13674
|
+
}
|
|
13675
|
+
function sameFeedSettings(held, next) {
|
|
13676
|
+
const a = held.settings;
|
|
13677
|
+
const b = next.settings;
|
|
13678
|
+
return held.synonymsGeneration === next.synonymsGeneration && a?.language === b?.language && a?.tokenizer === b?.tokenizer && a?.stopWords === b?.stopWords && JSON.stringify(a?.stopWordList ?? null) === JSON.stringify(b?.stopWordList ?? null) && (a?.bm25Revision ?? 0) === (b?.bm25Revision ?? 0);
|
|
13679
|
+
}
|
|
13680
|
+
function changeBetween(held, next) {
|
|
13681
|
+
const before = held === void 0 ? null : syncStateOfSettings(held.settings);
|
|
13682
|
+
if (held === void 0 || before === null) return null;
|
|
13683
|
+
if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
|
|
13684
|
+
const after = syncStateOfSettings(next.settings);
|
|
13685
|
+
if (after === null) return "off";
|
|
13686
|
+
if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
|
|
13687
|
+
return sameFeedSettings(held, next) ? null : "settings";
|
|
13688
|
+
}
|
|
13689
|
+
|
|
13690
|
+
// src/distribution/cluster-node/device-sync/settings.ts
|
|
13691
|
+
function indexNotFound(indexName) {
|
|
13692
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
13693
|
+
indexName
|
|
13694
|
+
});
|
|
13695
|
+
}
|
|
13696
|
+
function syncNotReady(subject, olderNodes, indexName) {
|
|
13697
|
+
return new NarsilError(
|
|
13698
|
+
ErrorCodes.SYNC_NOT_READY,
|
|
13699
|
+
`${subject} ${olderNodes.length} node(s) below specification version ${DEVICE_SYNC_SPEC_VERSION}, which can't read the log for a device feed. Try again once every node registers that version`,
|
|
13700
|
+
{ indexName, nodeIds: [...olderNodes].sort(), minimumVersion: DEVICE_SYNC_SPEC_VERSION }
|
|
13701
|
+
);
|
|
13702
|
+
}
|
|
13703
|
+
function registeredNodesBelowDeviceSync(nodes) {
|
|
13704
|
+
return nodes.filter((node) => !reachesVersion(node.version, DEVICE_SYNC_SPEC_VERSION)).map((node) => node.nodeId);
|
|
13705
|
+
}
|
|
13706
|
+
function storedSetting(setting) {
|
|
13707
|
+
return {
|
|
13708
|
+
enabled: true,
|
|
13709
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
13710
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {}
|
|
13711
|
+
};
|
|
13712
|
+
}
|
|
13713
|
+
function nextSyncSettings(current, setting) {
|
|
13714
|
+
const wasOn = current.sync?.enabled === true && current.syncEpoch !== void 0;
|
|
13715
|
+
if (!setting.enabled) return wasOn || current.sync !== void 0 ? {} : null;
|
|
13716
|
+
if (!wasOn) return { sync: storedSetting(setting), syncActive: false, syncEpoch: newSyncEpoch() };
|
|
13717
|
+
return {
|
|
13718
|
+
sync: storedSetting(setting),
|
|
13719
|
+
...current.syncActive !== void 0 ? { syncActive: current.syncActive } : {},
|
|
13720
|
+
...current.syncEpoch !== void 0 ? { syncEpoch: current.syncEpoch } : {}
|
|
13721
|
+
};
|
|
13722
|
+
}
|
|
13723
|
+
async function updateClusterSync(deps, indexName, value) {
|
|
13724
|
+
refuseHiddenGeneration(indexName);
|
|
13725
|
+
validateIndexName(indexName);
|
|
13726
|
+
const setting = requireSyncSetting(value);
|
|
13727
|
+
if (setting.enabled) {
|
|
13728
|
+
if (!deps.hasDurability()) {
|
|
13729
|
+
throw new NarsilError(
|
|
13730
|
+
ErrorCodes.SYNC_DURABILITY_REQUIRED,
|
|
13731
|
+
`Index "${indexName}" needs write-ahead log durability on every node for device sync, because the feed reads the log`,
|
|
13732
|
+
{ indexName }
|
|
13733
|
+
);
|
|
13734
|
+
}
|
|
13735
|
+
if (!deps.hasBackupStore) {
|
|
13736
|
+
throw new NarsilError(
|
|
13737
|
+
ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
|
|
13738
|
+
`Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
|
|
13739
|
+
{ indexName }
|
|
13740
|
+
);
|
|
13741
|
+
}
|
|
13742
|
+
const olderNodes = registeredNodesBelowDeviceSync(await deps.coordinator.listNodes());
|
|
13743
|
+
if (olderNodes.length > 0) throw syncNotReady("The cluster has", olderNodes, indexName);
|
|
13744
|
+
}
|
|
13745
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
13746
|
+
const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
13747
|
+
if (current === null) throw indexNotFound(indexName);
|
|
13748
|
+
const next = nextSyncSettings(current.settings ?? {}, setting);
|
|
13749
|
+
if (next === null) return;
|
|
13750
|
+
if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, metadataWithSync(current, next))) {
|
|
13751
|
+
await deps.view.refresh(indexName);
|
|
13752
|
+
return;
|
|
13753
|
+
}
|
|
13754
|
+
}
|
|
13755
|
+
throw new NarsilError(
|
|
13756
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
13757
|
+
`Index '${indexName}' kept changing while its sync setting was replaced ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the new setting was not stored. Retry the request`,
|
|
13758
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
13759
|
+
);
|
|
13760
|
+
}
|
|
13761
|
+
async function readClusterSync(deps, indexName) {
|
|
13762
|
+
refuseHiddenGeneration(indexName);
|
|
13763
|
+
validateIndexName(indexName);
|
|
13764
|
+
const snapshot = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
13765
|
+
if (snapshot === null) throw indexNotFound(indexName);
|
|
13766
|
+
return syncStatusOfSettings(snapshot.settings);
|
|
13767
|
+
}
|
|
13768
|
+
async function markClusterSyncActive(deps, indexName) {
|
|
13769
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
13770
|
+
const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
13771
|
+
if (current === null) throw indexNotFound(indexName);
|
|
13772
|
+
const settings = current.settings ?? {};
|
|
13773
|
+
if (settings.sync?.enabled !== true || settings.syncEpoch === void 0) {
|
|
13774
|
+
throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index '${indexName}' has device sync off`, { indexName });
|
|
13775
|
+
}
|
|
13776
|
+
if (settings.syncActive === true) return;
|
|
13777
|
+
const next = metadataWithSync(current, { sync: settings.sync, syncActive: true, syncEpoch: settings.syncEpoch });
|
|
13778
|
+
if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
13779
|
+
await deps.view.refresh(indexName);
|
|
13780
|
+
return;
|
|
13781
|
+
}
|
|
13782
|
+
}
|
|
13783
|
+
throw new NarsilError(
|
|
13784
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
13785
|
+
`Index '${indexName}' kept changing while a device asked for its snapshot, so ask again`,
|
|
13786
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
13787
|
+
);
|
|
13788
|
+
}
|
|
13789
|
+
function damaged2(backupName, reason) {
|
|
13790
|
+
return new NarsilError(ErrorCodes.BACKUP_DAMAGED, `Backup '${backupName}' ${reason}, so a device can't load it`, {
|
|
13791
|
+
backupName
|
|
13792
|
+
});
|
|
13793
|
+
}
|
|
13794
|
+
function mergedVectors(fields, sourceIndexName) {
|
|
13795
|
+
const vectors = [];
|
|
13796
|
+
for (const [fieldPath2, parts] of fields) {
|
|
13797
|
+
const merged = { fieldPath: fieldPath2, nextFileId: 0, files: [], graphGeneration: 0, graphParts: [] };
|
|
13798
|
+
for (const part of parts) {
|
|
13799
|
+
for (const file of part.files) {
|
|
13800
|
+
const id = merged.nextFileId;
|
|
13801
|
+
merged.nextFileId += 1;
|
|
13802
|
+
merged.files.push({ ...file, id, key: vectorFileKey(sourceIndexName, fieldPath2, id) });
|
|
13803
|
+
}
|
|
13804
|
+
}
|
|
13805
|
+
vectors.push(merged);
|
|
13806
|
+
}
|
|
13807
|
+
return vectors;
|
|
13808
|
+
}
|
|
13809
|
+
async function mergedManifest(store, backup) {
|
|
13810
|
+
let first = null;
|
|
13811
|
+
const merged = {
|
|
13812
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
13813
|
+
schema: {},
|
|
13814
|
+
language: "",
|
|
13815
|
+
checkpoint: [],
|
|
13816
|
+
partitions: [],
|
|
13817
|
+
vectors: []
|
|
13818
|
+
};
|
|
13819
|
+
const fields = /* @__PURE__ */ new Map();
|
|
13820
|
+
const segmentFiles = [];
|
|
13821
|
+
const vectorFiles = /* @__PURE__ */ new Map();
|
|
13822
|
+
for (const partition of backup.partitions) {
|
|
13823
|
+
const read = await readPartitionManifest(store, backup.name, partition);
|
|
13824
|
+
first ??= read;
|
|
13825
|
+
const { manifest: manifest2 } = read;
|
|
13826
|
+
const name = first.sourceIndexName;
|
|
13827
|
+
const listedByKey = new Map(partition.files.map((file) => [`${read.sourceIndexName}/${file.key}`, file]));
|
|
13828
|
+
const listedOf = (ref) => {
|
|
13829
|
+
const listed = listedByKey.get(ref.key);
|
|
13830
|
+
if (listed === void 0) throw damaged2(backup.name, `lists no file for "${ref.key}"`);
|
|
13831
|
+
return listed;
|
|
13832
|
+
};
|
|
13833
|
+
for (const entry of manifest2.partitions) {
|
|
13834
|
+
const segments = entry.segments.map((segment) => {
|
|
13835
|
+
const key = segmentKey(name, entry.partitionId, segment.id);
|
|
13836
|
+
segmentFiles.push({ ref: { ...segment, key }, listed: listedOf(segment) });
|
|
13837
|
+
return { ...segment, key };
|
|
13838
|
+
});
|
|
13839
|
+
merged.partitions.push({ ...entry, segments });
|
|
13840
|
+
}
|
|
13841
|
+
merged.checkpoint.push(...manifest2.checkpoint);
|
|
13842
|
+
for (const field of manifest2.vectors) {
|
|
13843
|
+
const parts = fields.get(field.fieldPath) ?? [];
|
|
13844
|
+
parts.push(field);
|
|
13845
|
+
fields.set(field.fieldPath, parts);
|
|
13846
|
+
const listed = vectorFiles.get(field.fieldPath) ?? [];
|
|
13847
|
+
for (const file of field.files) listed.push({ ref: file, listed: listedOf(file) });
|
|
13848
|
+
vectorFiles.set(field.fieldPath, listed);
|
|
13849
|
+
}
|
|
13850
|
+
}
|
|
13851
|
+
if (first === null) throw damaged2(backup.name, "holds no partition");
|
|
13852
|
+
const vectors = mergedVectors(fields, first.sourceIndexName);
|
|
13853
|
+
const files = [...segmentFiles];
|
|
13854
|
+
for (const field of vectors) {
|
|
13855
|
+
const originals = vectorFiles.get(field.fieldPath) ?? [];
|
|
13856
|
+
field.files.forEach((file, at) => {
|
|
13857
|
+
const original = originals[at];
|
|
13858
|
+
if (original === void 0) throw damaged2(backup.name, `lists too few files for "${field.fieldPath}"`);
|
|
13859
|
+
files.push({ ref: file, listed: original.listed });
|
|
13860
|
+
});
|
|
13861
|
+
}
|
|
13862
|
+
const manifest = {
|
|
13863
|
+
...merged,
|
|
13864
|
+
schema: first.manifest.schema,
|
|
13865
|
+
language: first.manifest.language,
|
|
13866
|
+
...first.manifest.analysisGeneration !== void 0 ? { analysisGeneration: first.manifest.analysisGeneration } : {},
|
|
13867
|
+
vectors
|
|
13868
|
+
};
|
|
13869
|
+
return { manifest, sourceIndexName: first.sourceIndexName, files };
|
|
13870
|
+
}
|
|
13871
|
+
async function spoolFile(store, backupName, listed, path) {
|
|
13872
|
+
const handle = await open(path, "wx");
|
|
13873
|
+
try {
|
|
13874
|
+
await loadCheckedFile(store, listed, backupName, async (chunk) => {
|
|
13875
|
+
await handle.write(chunk);
|
|
13876
|
+
});
|
|
13877
|
+
await handle.sync();
|
|
13878
|
+
} finally {
|
|
13879
|
+
await handle.close();
|
|
13880
|
+
}
|
|
13881
|
+
}
|
|
13882
|
+
async function readSpooled2(path, offset, length) {
|
|
13883
|
+
const handle = await open(path, "r");
|
|
13884
|
+
try {
|
|
13885
|
+
const bytes = new Uint8Array(length);
|
|
13886
|
+
let filled = 0;
|
|
13887
|
+
while (filled < length) {
|
|
13888
|
+
const { bytesRead } = await handle.read(bytes, filled, length - filled, offset + filled);
|
|
13889
|
+
if (bytesRead === 0) {
|
|
13890
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "A spooled snapshot file ended early", { path });
|
|
13891
|
+
}
|
|
13892
|
+
filled += bytesRead;
|
|
13893
|
+
}
|
|
13894
|
+
return bytes;
|
|
13895
|
+
} finally {
|
|
13896
|
+
await handle.close();
|
|
13897
|
+
}
|
|
13898
|
+
}
|
|
13899
|
+
async function spoolBackupArchive(store, backup, spoolDirectory) {
|
|
13900
|
+
const manifestBytes = await store.adapter.load(store.keys.manifest(backup.name));
|
|
13901
|
+
if (manifestBytes === null) throw damaged2(backup.name, "lost its manifest");
|
|
13902
|
+
const entityTag = sha256Hex(manifestBytes);
|
|
13903
|
+
const config = await loadCheckedBytes(store, backup.config, backup.name);
|
|
13904
|
+
const { manifest, sourceIndexName, files } = await mergedManifest(store, backup);
|
|
13905
|
+
const encoded2 = await encodeSegmentManifest(manifest);
|
|
13906
|
+
const packed = joinChunks(encoded2.chunks, encoded2.byteLength);
|
|
13907
|
+
const directory = join(spoolDirectory, `device-sync-${randomUUID()}`);
|
|
13908
|
+
await mkdir(directory, { recursive: true });
|
|
13909
|
+
const dispose = () => rm(directory, { recursive: true, force: true });
|
|
13910
|
+
try {
|
|
13911
|
+
const pathByDigest = /* @__PURE__ */ new Map();
|
|
13912
|
+
const unique = files.filter((file) => {
|
|
13913
|
+
if (pathByDigest.has(file.listed.sha256)) return false;
|
|
13914
|
+
pathByDigest.set(file.listed.sha256, join(directory, file.listed.sha256));
|
|
13915
|
+
return true;
|
|
13916
|
+
});
|
|
13917
|
+
let next = 0;
|
|
13918
|
+
const worker = async () => {
|
|
13919
|
+
while (next < unique.length) {
|
|
13920
|
+
const file = unique[next++];
|
|
13921
|
+
const path = pathByDigest.get(file.listed.sha256);
|
|
13922
|
+
if (path !== void 0) await spoolFile(store, backup.name, file.listed, path);
|
|
13923
|
+
}
|
|
13924
|
+
};
|
|
13925
|
+
await Promise.all(Array.from({ length: Math.min(BACKUP_FILES_AT_ONCE, unique.length) }, worker));
|
|
13926
|
+
const prefix = `${sourceIndexName}/`;
|
|
13927
|
+
const ordered = manifestFileRefs(manifest);
|
|
13928
|
+
const byKey = new Map(files.map((file) => [file.ref.key, file]));
|
|
13929
|
+
const layout = archiveLayout(
|
|
13930
|
+
config,
|
|
13931
|
+
packed,
|
|
13932
|
+
ordered.map((ref) => {
|
|
13933
|
+
const file = byKey.get(ref.key);
|
|
13934
|
+
const path = file === void 0 ? void 0 : pathByDigest.get(file.listed.sha256);
|
|
13935
|
+
if (path === void 0) throw damaged2(backup.name, `lost the spooled copy of "${ref.key}"`);
|
|
13936
|
+
return {
|
|
13937
|
+
name: ref.key.slice(prefix.length),
|
|
13938
|
+
bytes: ref.bytes,
|
|
13939
|
+
read: (offset, length) => readSpooled2(path, offset, length)
|
|
13940
|
+
};
|
|
13941
|
+
})
|
|
13942
|
+
);
|
|
13943
|
+
return { backupName: backup.name, entityTag, layout, dispose };
|
|
13944
|
+
} catch (error) {
|
|
13945
|
+
await dispose();
|
|
13946
|
+
throw error;
|
|
13947
|
+
}
|
|
13948
|
+
}
|
|
13949
|
+
|
|
13950
|
+
// src/distribution/cluster-node/device-sync/snapshots.ts
|
|
13951
|
+
function syncBackupPrefix(indexName) {
|
|
13952
|
+
return `${DEVICE_SYNC_BACKUP_NAME_PREFIX}${sha256Hex(new TextEncoder().encode(indexName)).slice(0, 32)}-`;
|
|
13953
|
+
}
|
|
13954
|
+
function syncOff(indexName) {
|
|
13955
|
+
return new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
13956
|
+
}
|
|
13957
|
+
function preparing(indexName) {
|
|
13958
|
+
return new NarsilError(
|
|
13959
|
+
ErrorCodes.SYNC_SNAPSHOT_PREPARING,
|
|
13960
|
+
`The cluster is taking the backup that the snapshot of index "${indexName}" needs. Ask again later`,
|
|
13961
|
+
{ indexName, retryAfterMs: DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS }
|
|
13962
|
+
);
|
|
13963
|
+
}
|
|
13964
|
+
function createClusterSnapshots(deps) {
|
|
13965
|
+
const manifests = /* @__PURE__ */ new Map();
|
|
13966
|
+
const spools = /* @__PURE__ */ new Map();
|
|
13967
|
+
const chosen = /* @__PURE__ */ new Map();
|
|
13968
|
+
const backingUp = /* @__PURE__ */ new Map();
|
|
13969
|
+
let closed = false;
|
|
13970
|
+
function requireStore2(indexName) {
|
|
13971
|
+
if (deps.store === null) {
|
|
13972
|
+
throw new NarsilError(
|
|
13973
|
+
ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
|
|
13974
|
+
`Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
|
|
13975
|
+
{ indexName }
|
|
13976
|
+
);
|
|
13977
|
+
}
|
|
13978
|
+
return deps.store;
|
|
13979
|
+
}
|
|
13980
|
+
async function manifestOf(store, name) {
|
|
13981
|
+
if (manifests.has(name)) return manifests.get(name) ?? null;
|
|
13982
|
+
let read;
|
|
13983
|
+
try {
|
|
13984
|
+
read = await readBackupManifest(store, name);
|
|
13985
|
+
} catch (error) {
|
|
13986
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
13987
|
+
if (error.code !== ErrorCodes.BACKUP_DAMAGED && error.code !== ErrorCodes.BACKUP_NOT_FOUND) throw error;
|
|
13988
|
+
read = null;
|
|
13989
|
+
}
|
|
13990
|
+
if (manifests.size >= CACHED_BACKUP_MANIFEST_LIMIT) {
|
|
13991
|
+
const oldest = manifests.keys().next().value;
|
|
13992
|
+
if (oldest !== void 0) manifests.delete(oldest);
|
|
13993
|
+
}
|
|
13994
|
+
manifests.set(name, read);
|
|
13995
|
+
return read;
|
|
13996
|
+
}
|
|
13997
|
+
async function backupsOf(indexName, held, epoch) {
|
|
13998
|
+
const store = requireStore2(indexName);
|
|
13999
|
+
const prefix = syncBackupPrefix(indexName);
|
|
14000
|
+
const names = (await listBackupNames(store)).filter(
|
|
14001
|
+
(name) => name.startsWith(prefix) || !name.startsWith(DEVICE_SYNC_BACKUP_NAME_PREFIX)
|
|
14002
|
+
);
|
|
14003
|
+
const matching = [];
|
|
14004
|
+
for (const name of names) {
|
|
14005
|
+
const manifest = await manifestOf(store, name);
|
|
14006
|
+
if (manifest !== null && manifest.indexName === indexName && manifest.syncEpoch === epoch && manifest.partitionCount === held.serving.partitionCount) {
|
|
14007
|
+
matching.push(manifest);
|
|
14008
|
+
}
|
|
14009
|
+
}
|
|
14010
|
+
return matching.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
|
|
14011
|
+
}
|
|
14012
|
+
async function qualifyingBackup(indexName, held, epoch) {
|
|
14013
|
+
const candidates = await backupsOf(indexName, held, epoch);
|
|
14014
|
+
if (candidates.length === 0) return null;
|
|
14015
|
+
const oldest = [];
|
|
14016
|
+
for (let partitionId = 0; partitionId < held.serving.partitionCount; partitionId++) {
|
|
14017
|
+
oldest.push((await deps.bounds(indexName, partitionId)).oldestSeqNo);
|
|
14018
|
+
}
|
|
14019
|
+
return candidates.find(
|
|
14020
|
+
(backup) => backup.partitions.every((partition) => partition.lastSeqNo >= (oldest[partition.partitionId] ?? 0) - 1)
|
|
14021
|
+
) ?? null;
|
|
14022
|
+
}
|
|
14023
|
+
async function startBackup(indexName) {
|
|
14024
|
+
if (backingUp.has(indexName) || closed) return;
|
|
14025
|
+
for (const job of (await readBackupJobs(deps.coordinator)).values()) {
|
|
14026
|
+
if (job.kind === "backup" && job.status === "running" && job.indexName === indexName) return;
|
|
14027
|
+
}
|
|
14028
|
+
if (backingUp.has(indexName)) return;
|
|
14029
|
+
const backupName = `${syncBackupPrefix(indexName)}${generateId()}`;
|
|
14030
|
+
const run = requestSyncBackup(deps.requests, indexName, backupName).catch(deps.onError).finally(() => backingUp.delete(indexName));
|
|
14031
|
+
backingUp.set(indexName, run);
|
|
14032
|
+
}
|
|
14033
|
+
function evictIdle(now3) {
|
|
14034
|
+
for (const [name, entry] of spools) {
|
|
14035
|
+
if (entry.users > 0) continue;
|
|
14036
|
+
const superseded = chosen.get(entry.indexName) !== name;
|
|
14037
|
+
if (!superseded && now3 - entry.lastUsedAt < DEVICE_SYNC_ARCHIVE_IDLE_MS) continue;
|
|
14038
|
+
spools.delete(name);
|
|
14039
|
+
void entry.ready.then((archive) => archive.dispose()).catch(() => void 0);
|
|
14040
|
+
}
|
|
14041
|
+
}
|
|
14042
|
+
function spoolOf(store, indexName, backup) {
|
|
14043
|
+
let entry = spools.get(backup.name);
|
|
14044
|
+
if (entry === void 0) {
|
|
14045
|
+
const ready = spoolBackupArchive(store, backup, deps.spoolDirectory);
|
|
14046
|
+
entry = { indexName, ready, users: 0, lastUsedAt: Date.now() };
|
|
14047
|
+
spools.set(backup.name, entry);
|
|
14048
|
+
const created = entry;
|
|
14049
|
+
ready.catch(() => {
|
|
14050
|
+
if (spools.get(backup.name) === created) spools.delete(backup.name);
|
|
14051
|
+
});
|
|
14052
|
+
}
|
|
14053
|
+
return entry;
|
|
14054
|
+
}
|
|
14055
|
+
return {
|
|
14056
|
+
async snapshot(indexName) {
|
|
14057
|
+
let held = await deps.settings.view.current(indexName);
|
|
14058
|
+
const state = held === null ? null : syncStateOfSettings(held.settings);
|
|
14059
|
+
if (held === null || state === null) throw syncOff(indexName);
|
|
14060
|
+
if (!state.active) {
|
|
14061
|
+
await markClusterSyncActive(deps.settings, indexName);
|
|
14062
|
+
held = deps.settings.view.held(indexName) ?? held;
|
|
14063
|
+
}
|
|
14064
|
+
const store = requireStore2(indexName);
|
|
14065
|
+
const backup = await qualifyingBackup(indexName, held, state.epoch);
|
|
14066
|
+
if (backup === null) {
|
|
14067
|
+
await startBackup(indexName);
|
|
14068
|
+
throw preparing(indexName);
|
|
14069
|
+
}
|
|
14070
|
+
chosen.set(indexName, backup.name);
|
|
14071
|
+
const now3 = Date.now();
|
|
14072
|
+
evictIdle(now3);
|
|
14073
|
+
const entry = spoolOf(store, indexName, backup);
|
|
14074
|
+
entry.users += 1;
|
|
14075
|
+
let archive;
|
|
14076
|
+
try {
|
|
14077
|
+
archive = await entry.ready;
|
|
14078
|
+
} catch (error) {
|
|
14079
|
+
entry.users -= 1;
|
|
14080
|
+
throw error;
|
|
14081
|
+
}
|
|
14082
|
+
let released = false;
|
|
14083
|
+
return {
|
|
14084
|
+
entityTag: archive.entityTag,
|
|
14085
|
+
layout: archive.layout,
|
|
14086
|
+
release() {
|
|
14087
|
+
if (released) return;
|
|
14088
|
+
released = true;
|
|
14089
|
+
entry.users -= 1;
|
|
14090
|
+
entry.lastUsedAt = Date.now();
|
|
14091
|
+
}
|
|
14092
|
+
};
|
|
14093
|
+
},
|
|
14094
|
+
async snapshotBytes(indexName) {
|
|
14095
|
+
const held = deps.settings.view.held(indexName);
|
|
14096
|
+
const state = held === void 0 ? null : syncStateOfSettings(held.settings);
|
|
14097
|
+
if (held === void 0 || state === null || deps.store === null) return 0;
|
|
14098
|
+
const spooled = spools.get(chosen.get(indexName) ?? "");
|
|
14099
|
+
if (spooled !== void 0) {
|
|
14100
|
+
const archive = await spooled.ready.catch(() => null);
|
|
14101
|
+
if (archive !== null) return archive.layout.byteLength;
|
|
14102
|
+
}
|
|
14103
|
+
const [newest] = await backupsOf(indexName, held, state.epoch);
|
|
14104
|
+
if (newest === void 0) return 0;
|
|
14105
|
+
return estimatedArchiveBytes(storedFilesOf(newest));
|
|
14106
|
+
},
|
|
14107
|
+
async close() {
|
|
14108
|
+
closed = true;
|
|
14109
|
+
await Promise.all([...backingUp.values()]);
|
|
14110
|
+
const entries = [...spools.values()];
|
|
14111
|
+
spools.clear();
|
|
14112
|
+
await Promise.all(entries.map((entry) => entry.ready.then((archive) => archive.dispose()).catch(() => void 0)));
|
|
14113
|
+
}
|
|
14114
|
+
};
|
|
14115
|
+
}
|
|
14116
|
+
|
|
14117
|
+
// src/distribution/cluster-node/device-sync/view.ts
|
|
14118
|
+
async function readHeld(coordinator, indexName) {
|
|
14119
|
+
const named = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
14120
|
+
if (named === null) return null;
|
|
14121
|
+
const servingName = (await coordinator.getAllocation(indexName))?.generation ?? indexName;
|
|
14122
|
+
const state = await readIndexState(coordinator, servingName);
|
|
14123
|
+
if (state === null) return null;
|
|
14124
|
+
const servingSettings = state.settings ?? named.settings;
|
|
14125
|
+
const feedShape = servingName === indexName || servingSettings === void 0 ? servingSettings : { ...servingSettings, bm25: named.settings?.bm25 };
|
|
14126
|
+
return {
|
|
14127
|
+
indexUuid: named.record.indexUuid,
|
|
14128
|
+
settings: named.settings,
|
|
14129
|
+
serving: { servingName, indexUuid: state.record.indexUuid, partitionCount: state.record.partitionCount },
|
|
14130
|
+
feedSettings: feedSettingsOf(feedShape, state.synonyms),
|
|
14131
|
+
synonymsGeneration: servingSettings?.synonyms?.generation ?? null
|
|
14132
|
+
};
|
|
14133
|
+
}
|
|
14134
|
+
function createDeviceSyncView(deps) {
|
|
14135
|
+
const { coordinator, engine, onError } = deps;
|
|
14136
|
+
const held = /* @__PURE__ */ new Map();
|
|
14137
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
14138
|
+
const refreshing = /* @__PURE__ */ new Map();
|
|
14139
|
+
const again = /* @__PURE__ */ new Set();
|
|
14140
|
+
const unwatchers = [];
|
|
14141
|
+
let reconcileTimer = null;
|
|
14142
|
+
let reconciling = false;
|
|
14143
|
+
let stopped = true;
|
|
14144
|
+
function emit(change) {
|
|
14145
|
+
for (const listener of listeners) {
|
|
14146
|
+
try {
|
|
14147
|
+
listener(change);
|
|
14148
|
+
} catch (error) {
|
|
14149
|
+
onError(error);
|
|
14150
|
+
}
|
|
14151
|
+
}
|
|
14152
|
+
}
|
|
14153
|
+
async function adoptOnLocalCopies(indexName, next) {
|
|
14154
|
+
const state = next === null ? null : syncStateOfSettings(next.settings);
|
|
14155
|
+
const local = new Set(engine.listIndexes().map((index) => index.name));
|
|
14156
|
+
const servingName = next?.serving.servingName ?? indexName;
|
|
14157
|
+
if (local.has(servingName)) await engine.adoptSyncState(servingName, state);
|
|
14158
|
+
if (servingName !== indexName && local.has(indexName)) await engine.adoptSyncState(indexName, null);
|
|
14159
|
+
}
|
|
14160
|
+
async function reread(indexName) {
|
|
14161
|
+
const previous = held.get(indexName);
|
|
14162
|
+
const next = await readHeld(coordinator, indexName);
|
|
14163
|
+
if (next === null) held.delete(indexName);
|
|
14164
|
+
else held.set(indexName, next);
|
|
14165
|
+
await adoptOnLocalCopies(indexName, next);
|
|
14166
|
+
const change = changeBetween(previous, next);
|
|
14167
|
+
if (change !== null) emit({ kind: change, indexName });
|
|
14168
|
+
return next;
|
|
14169
|
+
}
|
|
14170
|
+
function refresh(indexName) {
|
|
14171
|
+
const running = refreshing.get(indexName);
|
|
14172
|
+
if (running !== void 0) {
|
|
14173
|
+
again.add(indexName);
|
|
14174
|
+
return running;
|
|
14175
|
+
}
|
|
14176
|
+
const run = (async () => {
|
|
14177
|
+
let latest;
|
|
14178
|
+
do {
|
|
14179
|
+
again.delete(indexName);
|
|
14180
|
+
latest = await reread(indexName);
|
|
14181
|
+
} while (again.has(indexName) && !stopped);
|
|
14182
|
+
return latest;
|
|
14183
|
+
})().finally(() => refreshing.delete(indexName));
|
|
14184
|
+
refreshing.set(indexName, run);
|
|
14185
|
+
return run;
|
|
14186
|
+
}
|
|
14187
|
+
function namesServedBy(changedName) {
|
|
14188
|
+
if (!isHiddenGeneration(changedName)) return [changedName];
|
|
14189
|
+
const names = [];
|
|
14190
|
+
for (const [indexName, index] of held) {
|
|
14191
|
+
if (index.serving.servingName === changedName) names.push(indexName);
|
|
14192
|
+
}
|
|
14193
|
+
return names;
|
|
14194
|
+
}
|
|
14195
|
+
function onMetadataChange(changedName) {
|
|
14196
|
+
for (const indexName of namesServedBy(changedName)) {
|
|
14197
|
+
void refresh(indexName).catch(onError);
|
|
14198
|
+
}
|
|
14199
|
+
}
|
|
14200
|
+
async function namesOfServedGenerations() {
|
|
14201
|
+
const generations = engine.listIndexes().map((index) => index.name).filter((name) => isHiddenGeneration(name) && namesServedBy(name).length === 0);
|
|
14202
|
+
if (generations.length === 0) return;
|
|
14203
|
+
const wanted = new Set(generations);
|
|
14204
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
14205
|
+
if (stopped || wanted.size === 0) return;
|
|
14206
|
+
if (isHiddenGeneration(indexName) || held.has(indexName)) continue;
|
|
14207
|
+
const generation = (await coordinator.getAllocation(indexName))?.generation;
|
|
14208
|
+
if (generation === void 0 || !wanted.has(generation)) continue;
|
|
14209
|
+
wanted.delete(generation);
|
|
14210
|
+
await refresh(indexName).catch(onError);
|
|
14211
|
+
}
|
|
14212
|
+
}
|
|
14213
|
+
async function reconcile() {
|
|
14214
|
+
if (reconciling || stopped) return;
|
|
14215
|
+
reconciling = true;
|
|
14216
|
+
try {
|
|
14217
|
+
for (const index of engine.listIndexes()) {
|
|
14218
|
+
if (stopped) return;
|
|
14219
|
+
if (isHiddenGeneration(index.name)) continue;
|
|
14220
|
+
const known = held.get(index.name);
|
|
14221
|
+
if (known === void 0) await refresh(index.name).catch(onError);
|
|
14222
|
+
else await adoptOnLocalCopies(index.name, known).catch(onError);
|
|
14223
|
+
}
|
|
14224
|
+
for (const [indexName, known] of held) {
|
|
14225
|
+
if (stopped) return;
|
|
14226
|
+
if (known.serving.servingName !== indexName) await adoptOnLocalCopies(indexName, known).catch(onError);
|
|
14227
|
+
}
|
|
14228
|
+
} finally {
|
|
14229
|
+
reconciling = false;
|
|
14230
|
+
}
|
|
14231
|
+
}
|
|
14232
|
+
return {
|
|
14233
|
+
held: (indexName) => held.get(indexName),
|
|
14234
|
+
async current(indexName) {
|
|
14235
|
+
return held.get(indexName) ?? refresh(indexName);
|
|
14236
|
+
},
|
|
14237
|
+
refresh,
|
|
14238
|
+
listen(onChange) {
|
|
14239
|
+
listeners.add(onChange);
|
|
14240
|
+
return () => listeners.delete(onChange);
|
|
14241
|
+
},
|
|
14242
|
+
async start() {
|
|
14243
|
+
if (!stopped) return;
|
|
14244
|
+
stopped = false;
|
|
14245
|
+
unwatchers.push(await coordinator.watchIndexMetadata((event) => onMetadataChange(event.indexName)));
|
|
14246
|
+
unwatchers.push(await coordinator.watchAllocation((event) => onMetadataChange(event.indexName)));
|
|
14247
|
+
await namesOfServedGenerations().catch(onError);
|
|
14248
|
+
await reconcile().catch(onError);
|
|
14249
|
+
reconcileTimer = setInterval(() => void reconcile().catch(onError), DEVICE_SYNC_RECONCILE_INTERVAL_MS);
|
|
14250
|
+
reconcileTimer.unref?.();
|
|
14251
|
+
},
|
|
14252
|
+
stop() {
|
|
14253
|
+
stopped = true;
|
|
14254
|
+
for (const unwatch of unwatchers.splice(0)) unwatch();
|
|
14255
|
+
if (reconcileTimer !== null) clearInterval(reconcileTimer);
|
|
14256
|
+
reconcileTimer = null;
|
|
14257
|
+
held.clear();
|
|
14258
|
+
listeners.clear();
|
|
14259
|
+
}
|
|
14260
|
+
};
|
|
14261
|
+
}
|
|
14262
|
+
|
|
14263
|
+
// src/distribution/cluster-node/device-sync/index.ts
|
|
14264
|
+
function createClusterDeviceSync(deps) {
|
|
14265
|
+
const { config, coordinator, engine, onError } = deps;
|
|
14266
|
+
const view = createDeviceSyncView({ coordinator, engine, onError });
|
|
14267
|
+
const store = config.backups === void 0 ? null : openBackupStore(config.backups);
|
|
14268
|
+
const settings = {
|
|
14269
|
+
coordinator,
|
|
14270
|
+
view,
|
|
14271
|
+
hasBackupStore: store !== null,
|
|
14272
|
+
hasDurability: () => engine.syncStorage() !== null
|
|
14273
|
+
};
|
|
14274
|
+
const requests = { config, nodeId: deps.nodeId, guard: deps.guard, store };
|
|
14275
|
+
let snapshots = null;
|
|
14276
|
+
let serving = null;
|
|
14277
|
+
return {
|
|
14278
|
+
view,
|
|
14279
|
+
updateSync: (indexName, setting) => updateClusterSync(settings, indexName, setting),
|
|
14280
|
+
getSync: (indexName) => readClusterSync(settings, indexName),
|
|
14281
|
+
serving() {
|
|
14282
|
+
if (serving !== null) return serving;
|
|
14283
|
+
const feed = createClusterFeedSource({
|
|
14284
|
+
nodeId: deps.nodeId,
|
|
14285
|
+
writeDeps: deps.writeDeps,
|
|
14286
|
+
config,
|
|
14287
|
+
coordinator,
|
|
14288
|
+
engine,
|
|
14289
|
+
view,
|
|
14290
|
+
refreshAllocation: deps.refreshAllocation,
|
|
14291
|
+
snapshotBytes: (indexName) => snapshots?.snapshotBytes(indexName) ?? Promise.resolve(0)
|
|
14292
|
+
});
|
|
14293
|
+
const built = createClusterSnapshots({
|
|
14294
|
+
coordinator,
|
|
14295
|
+
settings,
|
|
14296
|
+
store,
|
|
14297
|
+
requests,
|
|
14298
|
+
spoolDirectory: join(deps.spoolDirectory, "device-sync"),
|
|
14299
|
+
bounds: (indexName, partitionId) => feed.bounds(indexName, partitionId),
|
|
14300
|
+
onError
|
|
14301
|
+
});
|
|
14302
|
+
snapshots = built;
|
|
14303
|
+
serving = {
|
|
14304
|
+
snapshot: (indexName) => built.snapshot(indexName),
|
|
14305
|
+
feed,
|
|
14306
|
+
async admitFeed(indexName) {
|
|
14307
|
+
const olderNodes = registeredNodesBelowDeviceSync(await coordinator.listNodes());
|
|
14308
|
+
if (olderNodes.length > 0) throw syncNotReady("The cluster has", olderNodes, indexName);
|
|
14309
|
+
await view.current(indexName);
|
|
14310
|
+
}
|
|
14311
|
+
};
|
|
14312
|
+
return serving;
|
|
14313
|
+
},
|
|
14314
|
+
start: () => view.start(),
|
|
14315
|
+
async stop() {
|
|
14316
|
+
view.stop();
|
|
14317
|
+
await snapshots?.close();
|
|
14318
|
+
}
|
|
14319
|
+
};
|
|
14320
|
+
}
|
|
14321
|
+
|
|
13193
14322
|
// src/distribution/cluster-node/error-forwarding.ts
|
|
13194
14323
|
function createErrorForwarder(onError) {
|
|
13195
14324
|
return (error) => {
|
|
@@ -13234,6 +14363,32 @@ function vectorGrowthBytesOf(core, indexName, documents) {
|
|
|
13234
14363
|
return bytes;
|
|
13235
14364
|
}
|
|
13236
14365
|
|
|
14366
|
+
// src/engine/sync/adopt.ts
|
|
14367
|
+
function sameState(held, next) {
|
|
14368
|
+
if (held === null || next === null) return held === next;
|
|
14369
|
+
return held.epoch === next.epoch && held.active === next.active && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
|
|
14370
|
+
}
|
|
14371
|
+
async function adoptIndexSyncState(core, indexName, next) {
|
|
14372
|
+
return oneSyncChangeAtATime(core.indexRegistry, indexName, async () => {
|
|
14373
|
+
const entry = core.indexRegistry.get(indexName);
|
|
14374
|
+
if (entry === void 0) return false;
|
|
14375
|
+
const held = entry.sync;
|
|
14376
|
+
if (sameState(held, next)) return false;
|
|
14377
|
+
const storage = core.durability?.manager.syncStorage?.();
|
|
14378
|
+
const anotherEpoch = held !== null && next !== null && held.epoch !== next.epoch;
|
|
14379
|
+
if (anotherEpoch) await storage?.release(indexName);
|
|
14380
|
+
entry.sync = next;
|
|
14381
|
+
try {
|
|
14382
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
14383
|
+
} catch (error) {
|
|
14384
|
+
entry.sync = held;
|
|
14385
|
+
throw error;
|
|
14386
|
+
}
|
|
14387
|
+
if (next === null) await storage?.release(indexName);
|
|
14388
|
+
return true;
|
|
14389
|
+
});
|
|
14390
|
+
}
|
|
14391
|
+
|
|
13237
14392
|
// src/persistence/durability/partition-log.ts
|
|
13238
14393
|
function persistedPosition(manager, indexName, partitionId) {
|
|
13239
14394
|
const partition = manager.partitionRegistry?.indexes.get(indexName)?.partitions.get(partitionId);
|
|
@@ -13267,6 +14422,7 @@ async function restartPartitionLog(manager, indexName, partitionId, position) {
|
|
|
13267
14422
|
const registry = manager.partitionRegistry;
|
|
13268
14423
|
if (registry === void 0 || manager.checkpointFromMemory === void 0) return;
|
|
13269
14424
|
await replacePartitionLog(registry, indexName, partitionId, position);
|
|
14425
|
+
await manager.syncStorage?.().removeHistory(indexName, partitionId);
|
|
13270
14426
|
await manager.checkpointFromMemory(indexName);
|
|
13271
14427
|
}
|
|
13272
14428
|
|
|
@@ -13304,7 +14460,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
13304
14460
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
13305
14461
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
13306
14462
|
const config = await encodeIndexConfiguration(entry);
|
|
13307
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
14463
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-UOLHOEZJ.mjs');
|
|
13308
14464
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
13309
14465
|
handedOver = true;
|
|
13310
14466
|
try {
|
|
@@ -13983,7 +15139,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
13983
15139
|
);
|
|
13984
15140
|
}
|
|
13985
15141
|
try {
|
|
13986
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15142
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-UOLHOEZJ.mjs');
|
|
13987
15143
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
13988
15144
|
return {
|
|
13989
15145
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -14192,7 +15348,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
14192
15348
|
if (directory !== null) {
|
|
14193
15349
|
const durable = core.durability?.manager;
|
|
14194
15350
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
14195
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
15351
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-UOLHOEZJ.mjs');
|
|
14196
15352
|
await removeReceivedFilesNothingReads(
|
|
14197
15353
|
directory,
|
|
14198
15354
|
core.requireManager(indexName),
|
|
@@ -14568,6 +15724,9 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
14568
15724
|
stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
|
|
14569
15725
|
applyStoredSynonyms: (indexName, load) => synonymModule().applyLoaded(core, indexName, load),
|
|
14570
15726
|
applyStoredBm25: (indexName, load) => applyStoredIndexBm25(core, indexName, load),
|
|
15727
|
+
syncStateOf: (indexName) => core.indexRegistry.get(indexName)?.sync,
|
|
15728
|
+
adoptSyncState: (indexName, state) => adoptIndexSyncState(core, indexName, state),
|
|
15729
|
+
syncStorage: () => core.durability?.manager.syncStorage?.() ?? null,
|
|
14571
15730
|
analysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.analysisGeneration,
|
|
14572
15731
|
pendingAnalysisGenerationOf: (indexName) => core.indexRegistry.get(indexName)?.pendingAnalysis?.generation ?? null,
|
|
14573
15732
|
buildHeldAnalysis: (indexName, pending) => buildHeldAnalysisChange(core, indexName, pending),
|
|
@@ -14670,7 +15829,7 @@ function createIndexSettingsResultMessage(result, sourceNodeId, requestId) {
|
|
|
14670
15829
|
}
|
|
14671
15830
|
|
|
14672
15831
|
// src/distribution/cluster-node/synonym-sets.ts
|
|
14673
|
-
function
|
|
15832
|
+
function indexNotFound2(indexName) {
|
|
14674
15833
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
14675
15834
|
indexName
|
|
14676
15835
|
});
|
|
@@ -14678,7 +15837,7 @@ function indexNotFound(indexName) {
|
|
|
14678
15837
|
async function readClusterSynonyms(coordinator, indexName) {
|
|
14679
15838
|
validateIndexName(indexName);
|
|
14680
15839
|
const state = await readIndexState(coordinator, await servingNameOf(coordinator, indexName));
|
|
14681
|
-
if (state === null) throw
|
|
15840
|
+
if (state === null) throw indexNotFound2(indexName);
|
|
14682
15841
|
return state.synonyms;
|
|
14683
15842
|
}
|
|
14684
15843
|
async function requireSettledPartitionCount(coordinator, indexName, servingName) {
|
|
@@ -14712,7 +15871,7 @@ async function swapStoredSet(coordinator, indexName, first, ref) {
|
|
|
14712
15871
|
const { indexUuid } = first.record;
|
|
14713
15872
|
let current = first;
|
|
14714
15873
|
for (let attempt = 1; ; attempt++) {
|
|
14715
|
-
if (current === null || current.record.indexUuid !== indexUuid) throw
|
|
15874
|
+
if (current === null || current.record.indexUuid !== indexUuid) throw indexNotFound2(indexName);
|
|
14716
15875
|
const replaced = current.settings?.synonyms ?? null;
|
|
14717
15876
|
const next = metadataWithSynonymSet(current, ref);
|
|
14718
15877
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return replaced;
|
|
@@ -14764,7 +15923,7 @@ async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
|
14764
15923
|
const servingName = await servingNameOf(coordinator, indexName);
|
|
14765
15924
|
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
14766
15925
|
const first = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
14767
|
-
if (first === null) throw
|
|
15926
|
+
if (first === null) throw indexNotFound2(indexName);
|
|
14768
15927
|
const encoded2 = preparedSet(first, servingName, synonyms);
|
|
14769
15928
|
const { indexUuid } = first.record;
|
|
14770
15929
|
const ref = encoded2 === null ? null : await storeSynonymSet(coordinator, indexUuid, encoded2);
|
|
@@ -15133,30 +16292,6 @@ async function purgeGeneration(deps, job) {
|
|
|
15133
16292
|
);
|
|
15134
16293
|
}
|
|
15135
16294
|
|
|
15136
|
-
// src/distribution/cluster-node/reshard/leadership.ts
|
|
15137
|
-
async function leadingAssignment2(deps, indexName, partitionId) {
|
|
15138
|
-
let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
|
|
15139
|
-
if (assignment?.primary !== deps.nodeId && deps.refreshAllocation !== void 0) {
|
|
15140
|
-
assignment = (await deps.refreshAllocation(indexName))?.assignments.get(partitionId);
|
|
15141
|
-
}
|
|
15142
|
-
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
15143
|
-
throw new NarsilError(
|
|
15144
|
-
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
15145
|
-
`Node '${deps.nodeId}' does not lead partition ${partitionId} of index '${indexName}'`,
|
|
15146
|
-
{ indexName, partitionId, primaryNodeId: assignment?.primary ?? null, localNodeId: deps.nodeId }
|
|
15147
|
-
);
|
|
15148
|
-
}
|
|
15149
|
-
return assignment;
|
|
15150
|
-
}
|
|
15151
|
-
function forwardableEnd(log, primaryTerm) {
|
|
15152
|
-
let end = log.commitPoint;
|
|
15153
|
-
for (; ; ) {
|
|
15154
|
-
const next = log.getEntry(end + 1);
|
|
15155
|
-
if (next === void 0 || next.primaryTerm >= primaryTerm) return end;
|
|
15156
|
-
end = next.seqNo;
|
|
15157
|
-
}
|
|
15158
|
-
}
|
|
15159
|
-
|
|
15160
16295
|
// src/distribution/cluster-node/reshard/copy.ts
|
|
15161
16296
|
async function stillLeading(deps, job) {
|
|
15162
16297
|
const table = await deps.writeDeps.coordinator.getAllocation(job.sourceGeneration);
|
|
@@ -17073,6 +18208,9 @@ function createDataNodeHandler(deps) {
|
|
|
17073
18208
|
case ClusterMessageTypes.RESTORE_PARTITION:
|
|
17074
18209
|
await handleRestorePartition(message, respond, deps);
|
|
17075
18210
|
return;
|
|
18211
|
+
case SyncMessageTypes.READ:
|
|
18212
|
+
await handleSyncRead(message, respond, deps);
|
|
18213
|
+
return;
|
|
17076
18214
|
default:
|
|
17077
18215
|
return;
|
|
17078
18216
|
}
|
|
@@ -17088,6 +18226,16 @@ function createDataNodeHandler(deps) {
|
|
|
17088
18226
|
};
|
|
17089
18227
|
}
|
|
17090
18228
|
|
|
18229
|
+
// src/distribution/cluster-node/node-bindings.ts
|
|
18230
|
+
function bindNodeSources(node, sources) {
|
|
18231
|
+
bindWriteBlockLookup(node, async (indexName) => {
|
|
18232
|
+
const table = await routableAllocation(sources.coordinator, indexName).catch(() => null);
|
|
18233
|
+
return table === null ? null : findWriteBlock(table, sources.memoryView);
|
|
18234
|
+
});
|
|
18235
|
+
bindRerankWindowSource(node, () => rerankWindowCapOf(sources.engine));
|
|
18236
|
+
bindSyncServing(node, sources.syncServing);
|
|
18237
|
+
}
|
|
18238
|
+
|
|
17091
18239
|
// src/distribution/cluster-node/node-identity.ts
|
|
17092
18240
|
var NODE_ID_FILE_NAME = ".narsil-node-id";
|
|
17093
18241
|
var STORED_NODE_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
@@ -18134,7 +19282,7 @@ function createNodeLifecycle(deps) {
|
|
|
18134
19282
|
}
|
|
18135
19283
|
|
|
18136
19284
|
// src/distribution/cluster-node/analysis-change/request.ts
|
|
18137
|
-
function
|
|
19285
|
+
function indexNotFound3(indexName) {
|
|
18138
19286
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
18139
19287
|
indexName
|
|
18140
19288
|
});
|
|
@@ -18155,10 +19303,10 @@ async function requireTargetAcceptable(coordinator, indexName, target) {
|
|
|
18155
19303
|
}
|
|
18156
19304
|
async function writePending(coordinator, indexName, target) {
|
|
18157
19305
|
const schema = await coordinator.getSchema(indexName);
|
|
18158
|
-
if (schema === null) throw
|
|
19306
|
+
if (schema === null) throw indexNotFound3(indexName);
|
|
18159
19307
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
18160
19308
|
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
18161
|
-
if (snapshot === null) throw
|
|
19309
|
+
if (snapshot === null) throw indexNotFound3(indexName);
|
|
18162
19310
|
const running = snapshot.settings?.pendingAnalysis;
|
|
18163
19311
|
if (running !== void 0 && asksForPending(running, target)) return running;
|
|
18164
19312
|
const current = indexConfigFromSettings(schema, snapshot.settings, indexName);
|
|
@@ -18204,7 +19352,7 @@ async function changeClusterAnalysis(request) {
|
|
|
18204
19352
|
validateIndexName(servingName);
|
|
18205
19353
|
const schema = await coordinator.getSchema(servingName);
|
|
18206
19354
|
const metadata = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
18207
|
-
if (schema === null || metadata === null) throw
|
|
19355
|
+
if (schema === null || metadata === null) throw indexNotFound3(publicName);
|
|
18208
19356
|
const target = resolveAnalysisTarget(
|
|
18209
19357
|
publicName,
|
|
18210
19358
|
indexConfigFromSettings(schema, metadata.settings, servingName),
|
|
@@ -18232,7 +19380,7 @@ async function changeClusterAnalysis(request) {
|
|
|
18232
19380
|
for (; ; ) {
|
|
18233
19381
|
guard();
|
|
18234
19382
|
const snapshot = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
18235
|
-
if (snapshot === null) throw
|
|
19383
|
+
if (snapshot === null) throw indexNotFound3(publicName);
|
|
18236
19384
|
if (outcomeOf(snapshot, pending, publicName) === "published") break;
|
|
18237
19385
|
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
18238
19386
|
}
|
|
@@ -18241,7 +19389,7 @@ async function changeClusterAnalysis(request) {
|
|
|
18241
19389
|
}
|
|
18242
19390
|
|
|
18243
19391
|
// src/distribution/cluster-node/bm25-change.ts
|
|
18244
|
-
function
|
|
19392
|
+
function indexNotFound4(indexName) {
|
|
18245
19393
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
18246
19394
|
indexName
|
|
18247
19395
|
});
|
|
@@ -18249,14 +19397,14 @@ function indexNotFound3(indexName) {
|
|
|
18249
19397
|
async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
18250
19398
|
validateIndexName(indexName);
|
|
18251
19399
|
const schema = await coordinator.getSchema(indexName);
|
|
18252
|
-
if (schema === null) throw
|
|
19400
|
+
if (schema === null) throw indexNotFound4(indexName);
|
|
18253
19401
|
validateBM25Params(value, schema, ErrorCodes.CONFIG_INVALID);
|
|
18254
19402
|
const bm25 = storedBM25Params(value);
|
|
18255
19403
|
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
18256
19404
|
if (olderNodes.length > 0) throw bm25NotReady("The cluster has", olderNodes, indexName);
|
|
18257
19405
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
18258
19406
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
18259
|
-
if (current === null) throw
|
|
19407
|
+
if (current === null) throw indexNotFound4(indexName);
|
|
18260
19408
|
const revision = (current.settings?.bm25Revision ?? 0) + 1;
|
|
18261
19409
|
const next = metadataWithBm25(current, bm25, revision);
|
|
18262
19410
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
@@ -19367,12 +20515,12 @@ function lostContextOf(error, context, indexName) {
|
|
|
19367
20515
|
}
|
|
19368
20516
|
function createRankedDocumentReader(deps, indexName, allocation) {
|
|
19369
20517
|
const sources = /* @__PURE__ */ new Map();
|
|
19370
|
-
const
|
|
20518
|
+
const open3 = /* @__PURE__ */ new Map();
|
|
19371
20519
|
const partitionCount = allocation.assignments.size;
|
|
19372
20520
|
function releaseQuietly(contexts) {
|
|
19373
20521
|
const remote = [];
|
|
19374
20522
|
for (const context of contexts) {
|
|
19375
|
-
|
|
20523
|
+
open3.delete(context.contextId);
|
|
19376
20524
|
if (context.nodeId === deps.nodeId && deps.searchContexts !== void 0) {
|
|
19377
20525
|
deps.searchContexts.release(context.contextId);
|
|
19378
20526
|
} else {
|
|
@@ -19390,7 +20538,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19390
20538
|
const found = registry?.read(context.contextId, docIds) ?? null;
|
|
19391
20539
|
if (last) {
|
|
19392
20540
|
registry?.release(context.contextId);
|
|
19393
|
-
|
|
20541
|
+
open3.delete(context.contextId);
|
|
19394
20542
|
}
|
|
19395
20543
|
if (found === null) throw contextLost(context, indexName);
|
|
19396
20544
|
const documents = /* @__PURE__ */ new Map();
|
|
@@ -19422,7 +20570,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19422
20570
|
} catch (error) {
|
|
19423
20571
|
throw lostContextOf(error, context, indexName) ?? error;
|
|
19424
20572
|
}
|
|
19425
|
-
if (releaseContext)
|
|
20573
|
+
if (releaseContext) open3.delete(context.contextId);
|
|
19426
20574
|
refuseDecodeWithoutRoom(response.payload);
|
|
19427
20575
|
const decoded = decode(response.payload);
|
|
19428
20576
|
if (response.type.endsWith(".error") || decoded.error === true) {
|
|
@@ -19438,7 +20586,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19438
20586
|
return {
|
|
19439
20587
|
adopt(contexts) {
|
|
19440
20588
|
if (contexts === void 0) return;
|
|
19441
|
-
for (const context of contexts.opened)
|
|
20589
|
+
for (const context of contexts.opened) open3.set(context.contextId, context);
|
|
19442
20590
|
for (const [docId, context] of contexts.sources) if (!sources.has(docId)) sources.set(docId, context);
|
|
19443
20591
|
},
|
|
19444
20592
|
async read(docIds, options) {
|
|
@@ -19446,7 +20594,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19446
20594
|
const unsourced = [];
|
|
19447
20595
|
for (const docId of new Set(docIds)) {
|
|
19448
20596
|
const context = sources.get(docId);
|
|
19449
|
-
if (context === void 0 || !
|
|
20597
|
+
if (context === void 0 || !open3.has(context.contextId)) {
|
|
19450
20598
|
unsourced.push(docId);
|
|
19451
20599
|
continue;
|
|
19452
20600
|
}
|
|
@@ -19455,7 +20603,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19455
20603
|
else group.docIds.push(docId);
|
|
19456
20604
|
}
|
|
19457
20605
|
if (options.last) {
|
|
19458
|
-
releaseQuietly([...
|
|
20606
|
+
releaseQuietly([...open3.values()].filter((context) => !byContext.has(context.contextId)));
|
|
19459
20607
|
}
|
|
19460
20608
|
const reads = [];
|
|
19461
20609
|
for (const { context, docIds: contextIds } of byContext.values()) {
|
|
@@ -19484,7 +20632,7 @@ function createRankedDocumentReader(deps, indexName, allocation) {
|
|
|
19484
20632
|
return documents;
|
|
19485
20633
|
},
|
|
19486
20634
|
release() {
|
|
19487
|
-
releaseQuietly([...
|
|
20635
|
+
releaseQuietly([...open3.values()]);
|
|
19488
20636
|
}
|
|
19489
20637
|
};
|
|
19490
20638
|
}
|
|
@@ -19922,7 +21070,7 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
19922
21070
|
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
19923
21071
|
);
|
|
19924
21072
|
}
|
|
19925
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
21073
|
+
const { rerankClusterQuery } = await import('./rerank-IZOB6NII.mjs');
|
|
19926
21074
|
return withRankedReader(
|
|
19927
21075
|
deps,
|
|
19928
21076
|
indexName,
|
|
@@ -20928,6 +22076,21 @@ async function requestRebalance(config, nodeId, indexName, targetPartitionCount)
|
|
|
20928
22076
|
return { jobId: reply3.jobId, targetGeneration: reply3.targetGeneration };
|
|
20929
22077
|
}
|
|
20930
22078
|
|
|
22079
|
+
// src/distribution/cluster-node/settling-writes.ts
|
|
22080
|
+
function createSettlingWrites(onError) {
|
|
22081
|
+
const writes = /* @__PURE__ */ new Set();
|
|
22082
|
+
return {
|
|
22083
|
+
track(write) {
|
|
22084
|
+
const settled = write.catch(onError);
|
|
22085
|
+
writes.add(settled);
|
|
22086
|
+
void settled.finally(() => writes.delete(settled));
|
|
22087
|
+
},
|
|
22088
|
+
async settled() {
|
|
22089
|
+
await Promise.all([...writes]);
|
|
22090
|
+
}
|
|
22091
|
+
};
|
|
22092
|
+
}
|
|
22093
|
+
|
|
20931
22094
|
// src/distribution/cluster-node/transport-listener.ts
|
|
20932
22095
|
var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
20933
22096
|
ReplicationMessageTypes.INSYNC_ADD,
|
|
@@ -21162,12 +22325,7 @@ async function createClusterNode(callerConfig) {
|
|
|
21162
22325
|
const hasControllerRole = roles.includes("controller");
|
|
21163
22326
|
const controllerTransport = hasDataRole ? createMultiplexedControllerTransport(config.transport, nodeId) : null;
|
|
21164
22327
|
const orphanedIndexes = /* @__PURE__ */ new Set();
|
|
21165
|
-
const heldPartitionWrites =
|
|
21166
|
-
function trackHeldPartitionWrite(write) {
|
|
21167
|
-
const settled = write.catch(forwardOnError);
|
|
21168
|
-
heldPartitionWrites.add(settled);
|
|
21169
|
-
void settled.finally(() => heldPartitionWrites.delete(settled));
|
|
21170
|
-
}
|
|
22328
|
+
const heldPartitionWrites = createSettlingWrites(forwardOnError);
|
|
21171
22329
|
let controller = null;
|
|
21172
22330
|
let unregisterHandler = null;
|
|
21173
22331
|
let unregisterNotReadyHandler = null;
|
|
@@ -21213,7 +22371,7 @@ async function createClusterNode(callerConfig) {
|
|
|
21213
22371
|
replicationLogs,
|
|
21214
22372
|
copyFailures,
|
|
21215
22373
|
bootstrapPartition: bootstrapPartition2,
|
|
21216
|
-
trackHeldPartitionWrite,
|
|
22374
|
+
trackHeldPartitionWrite: heldPartitionWrites.track,
|
|
21217
22375
|
cleanupDeps
|
|
21218
22376
|
});
|
|
21219
22377
|
function guardShutdown() {
|
|
@@ -21247,6 +22405,14 @@ async function createClusterNode(callerConfig) {
|
|
|
21247
22405
|
const analysisView = createAnalysisView({ coordinator: config.coordinator, engine, onError: forwardOnError });
|
|
21248
22406
|
const readDeps = { config: routingConfig2, nodeId, engine, resolveNodeTargets: resolveNodeTargets3, searchContexts, bm25View, analysisView };
|
|
21249
22407
|
const servingName = (indexName) => servingNameOf(allocationView.routing, indexName);
|
|
22408
|
+
const deviceSync = createClusterDeviceSync({
|
|
22409
|
+
...cleanupDeps,
|
|
22410
|
+
config,
|
|
22411
|
+
writeDeps,
|
|
22412
|
+
refreshAllocation: allocationView.refresh,
|
|
22413
|
+
spoolDirectory: snapshotSpool.path,
|
|
22414
|
+
guard: guardShutdown
|
|
22415
|
+
});
|
|
21250
22416
|
const cluster = createClusterNamespace({
|
|
21251
22417
|
nodeId,
|
|
21252
22418
|
roles,
|
|
@@ -21295,12 +22461,15 @@ async function createClusterNode(callerConfig) {
|
|
|
21295
22461
|
},
|
|
21296
22462
|
isOrphaned: (indexName) => orphanedIndexes.has(indexName)
|
|
21297
22463
|
}),
|
|
22464
|
+
updateSync: (indexName, setting) => trackOp(indexName, () => deviceSync.updateSync(indexName, setting)),
|
|
22465
|
+
getSync: (indexName) => trackOp(indexName, () => deviceSync.getSync(indexName)),
|
|
21298
22466
|
cluster,
|
|
21299
22467
|
async start() {
|
|
21300
22468
|
guardShutdown();
|
|
21301
22469
|
await allocationView.start();
|
|
21302
22470
|
await bm25View.start();
|
|
21303
22471
|
await analysisView.start();
|
|
22472
|
+
await deviceSync.start();
|
|
21304
22473
|
const dispositions = await reconcileLocalIndexes({
|
|
21305
22474
|
engine,
|
|
21306
22475
|
coordinator: config.coordinator,
|
|
@@ -21367,6 +22536,7 @@ async function createClusterNode(callerConfig) {
|
|
|
21367
22536
|
allocationView.stop();
|
|
21368
22537
|
bm25View.stop();
|
|
21369
22538
|
analysisView.stop();
|
|
22539
|
+
await deviceSync.stop();
|
|
21370
22540
|
unregisterHandler?.();
|
|
21371
22541
|
unregisterHandler = null;
|
|
21372
22542
|
unregisterNotReadyHandler?.();
|
|
@@ -21379,16 +22549,17 @@ async function createClusterNode(callerConfig) {
|
|
|
21379
22549
|
if (controller !== null) {
|
|
21380
22550
|
await controller.shutdown();
|
|
21381
22551
|
}
|
|
21382
|
-
await
|
|
22552
|
+
await heldPartitionWrites.settled();
|
|
21383
22553
|
await engine.shutdown();
|
|
21384
22554
|
await snapshotSpool.dispose();
|
|
21385
22555
|
}
|
|
21386
22556
|
};
|
|
21387
|
-
|
|
21388
|
-
|
|
21389
|
-
|
|
22557
|
+
bindNodeSources(node, {
|
|
22558
|
+
coordinator: writeDeps.coordinator,
|
|
22559
|
+
engine,
|
|
22560
|
+
memoryView: memoryReporting.view,
|
|
22561
|
+
syncServing: deviceSync.serving
|
|
21390
22562
|
});
|
|
21391
|
-
bindRerankWindowSource(node, () => rerankWindowCapOf(engine));
|
|
21392
22563
|
return node;
|
|
21393
22564
|
}
|
|
21394
22565
|
|
|
@@ -21447,6 +22618,12 @@ function clusterNodeEngine(node, options) {
|
|
|
21447
22618
|
async updateBM25(indexName, bm25) {
|
|
21448
22619
|
return node.updateBM25(indexName, bm25);
|
|
21449
22620
|
},
|
|
22621
|
+
updateSync(indexName, setting) {
|
|
22622
|
+
return node.updateSync(indexName, setting);
|
|
22623
|
+
},
|
|
22624
|
+
getSync(indexName) {
|
|
22625
|
+
return node.getSync(indexName);
|
|
22626
|
+
},
|
|
21450
22627
|
async getSynonyms(indexName) {
|
|
21451
22628
|
return node.getSynonyms(indexName);
|
|
21452
22629
|
},
|
|
@@ -21523,6 +22700,8 @@ function clusterNodeEngine(node, options) {
|
|
|
21523
22700
|
getPartitionStats: (indexName) => node.getPartitionStats(indexName)
|
|
21524
22701
|
});
|
|
21525
22702
|
bindRerankWindowSource(engine, () => rerankWindowCapOf(node));
|
|
22703
|
+
const syncServing = syncServingSourceOf(node);
|
|
22704
|
+
if (syncServing !== void 0) bindSyncServing(engine, syncServing);
|
|
21526
22705
|
return engine;
|
|
21527
22706
|
}
|
|
21528
22707
|
|