@delali/narsil 0.3.1-next.153 → 0.3.1-next.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
- package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
- package/dist/archive-history-E4R73RWD.mjs +29 -0
- package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
- package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
- package/dist/browser/chunk-23XRXIBW.mjs +85 -0
- package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
- package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
- package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
- package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
- package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
- package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
- package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
- package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
- package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
- package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
- package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
- package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
- package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
- package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
- package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
- package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
- package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
- package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
- package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
- package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
- package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
- package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
- package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
- package/dist/browser/chunk-KKAMCABR.mjs +432 -0
- package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
- package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
- package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
- package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
- package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
- package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
- package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
- package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
- package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
- package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
- package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
- package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
- package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
- package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
- package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
- package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
- package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
- package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
- package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
- package/dist/browser/import-OM6QAW45.mjs +18 -0
- package/dist/browser/methods-KTQCSIN4.mjs +30 -0
- package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
- package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
- package/dist/browser/operations-OJ3BXS55.mjs +5 -0
- package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
- package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
- package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
- package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
- package/dist/browser/retention-ASWUHAYI.mjs +209 -0
- package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
- package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
- package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
- package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
- package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
- package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
- package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
- package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
- package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
- package/dist/chunk-3HB3PEC6.mjs +240 -0
- package/dist/chunk-3HHQLEEQ.mjs +1092 -0
- package/dist/chunk-3OH67G7R.mjs +214 -0
- package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
- package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
- package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
- package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
- package/dist/chunk-5FVU47TB.mjs +20 -0
- package/dist/chunk-5YNRKI2C.mjs +81 -0
- package/dist/chunk-6L3XJXGU.mjs +470 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
- package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
- package/dist/chunk-7GXJUJ24.mjs +49 -0
- package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
- package/dist/chunk-BDRIVYOK.mjs +72 -0
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6ZDGYS.mjs +50 -0
- package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
- package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
- package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-GH4WQ22V.mjs +34 -0
- package/dist/chunk-GQPTDQCL.mjs +54 -0
- package/dist/chunk-H4KHPLHJ.mjs +382 -0
- package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
- package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
- package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
- package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
- package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
- package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
- package/dist/chunk-IPN7S4KX.mjs +130 -0
- package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
- package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
- package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
- package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
- package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
- package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
- package/dist/chunk-KJ3RTZDJ.mjs +101 -0
- package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
- package/dist/chunk-KP7TYHWT.mjs +92 -0
- package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
- package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
- package/dist/chunk-KYYABCSW.mjs +72 -0
- package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
- package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
- package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
- package/dist/chunk-MT2NRTA6.mjs +216 -0
- package/dist/chunk-NF4DT3IH.mjs +85 -0
- package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
- package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
- package/dist/chunk-PHQNACDT.mjs +63 -0
- package/dist/chunk-Q622CTMC.mjs +74 -0
- package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
- package/dist/chunk-QDZJM7NI.mjs +23 -0
- package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
- package/dist/chunk-QJVOLKU6.mjs +111 -0
- package/dist/chunk-QUJHMPTP.mjs +30 -0
- package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
- package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
- package/dist/chunk-RZWWB2XR.mjs +96 -0
- package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
- package/dist/chunk-S5LN746F.mjs +10 -0
- package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
- package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
- package/dist/chunk-SY3EWKQY.mjs +242 -0
- package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
- package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
- package/dist/chunk-UHICS3JI.mjs +83 -0
- package/dist/chunk-UZRZKNNZ.mjs +101 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
- package/dist/chunk-VHBNF4MA.mjs +543 -0
- package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
- package/dist/chunk-WQRN64Q6.mjs +196 -0
- package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
- package/dist/chunk-Y4WPNHMC.mjs +50 -0
- package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
- package/dist/chunk-YEKL4JBK.mjs +96 -0
- package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
- package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
- package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
- package/dist/chunk-ZTW5WTGG.mjs +167 -0
- package/dist/chunk-ZYAY26KG.mjs +29 -0
- package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
- package/dist/client.d.ts +9 -7
- package/dist/client.mjs +120 -7
- package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
- package/dist/config-CoSig5Ox.d.ts +482 -0
- package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
- package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +41 -9
- package/dist/distribution.mjs +9304 -7271
- package/dist/durability-config-GE4U7MCZ.mjs +89 -0
- package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
- package/dist/engine-hooks-DHCMOADY.mjs +47 -0
- package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
- package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
- package/dist/index.browser.mjs +9 -4
- package/dist/index.d.ts +7 -6
- package/dist/index.mjs +29 -21
- package/dist/index.react-native.mjs +19 -12
- package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
- package/dist/methods-7IOINERF.mjs +30 -0
- package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
- package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
- package/dist/operations-G5EP6JPX.mjs +5 -0
- package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
- package/dist/partial-words.mjs +7 -5
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
- package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
- package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +81 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +246 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +62 -0
- package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
- package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
- package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
- package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
- package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
- package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
- package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
- package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
- package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
- package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
- package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
- package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
- package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
- package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
- package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
- package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
- package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
- package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
- package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
- package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
- package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
- package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
- package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
- package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
- package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
- package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
- package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
- package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
- package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
- package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
- package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
- package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
- package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
- package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
- package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
- package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
- package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
- package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
- package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
- package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
- package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
- package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
- package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
- package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
- package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
- package/dist/react-native/partial-words.mjs +8 -5
- package/dist/react-native/pattern.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +258 -0
- package/dist/react-native/sync.mjs +2299 -0
- package/dist/react-native/synonyms.mjs +7 -4
- package/dist/react.d.ts +11 -10
- package/dist/react.mjs +8 -1
- package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
- package/dist/restore-queue-JVGENGGE.mjs +55 -0
- package/dist/retention-HROPV2QU.mjs +59 -0
- package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
- package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
- package/dist/server.d.ts +119 -20
- package/dist/server.mjs +1123 -58
- package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
- package/dist/start-UTWHGH42.mjs +59 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +271 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +237 -0
- package/dist/sync-state-ZWHLAFLE.mjs +15 -0
- package/dist/sync.browser.mjs +2294 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2268 -0
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
- package/dist/thread-6MOXY2KD.mjs +45 -0
- package/dist/types-BZCGjfLn.d.ts +227 -0
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +7 -7
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
- package/openapi.json +912 -5
- package/package.json +45 -8
- package/dist/browser/import-QOY6ZH6X.mjs +0 -12
- package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
- package/dist/chunk-7LDMWKP6.mjs +0 -209
- package/dist/engine-CJRUHXgf.d.ts +0 -664
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
- package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
- package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
- package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
- package/dist/restore-queue-AOF5OTQV.mjs +0 -41
- package/dist/thread-I7DVHHW7.mjs +0 -31
package/dist/server.mjs
CHANGED
|
@@ -1,56 +1,582 @@
|
|
|
1
|
-
import { answerRequest } from './chunk-
|
|
2
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture,
|
|
3
|
-
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
4
|
-
import { rerankOnHost } from './chunk-
|
|
5
|
-
import {
|
|
6
|
-
import './chunk-
|
|
7
|
-
import {
|
|
8
|
-
import './chunk-
|
|
1
|
+
import { answerRequest } from './chunk-SMVH6JBE.mjs';
|
|
2
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, toHttpError, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, isThreadFeedNote, ENTRIES_MESSAGE_LIMIT_BYTES, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-DCNKVEDB.mjs';
|
|
3
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-DCNKVEDB.mjs';
|
|
4
|
+
import { rerankOnHost } from './chunk-KHKIRS5D.mjs';
|
|
5
|
+
import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-ZTW5WTGG.mjs';
|
|
6
|
+
import './chunk-YEKL4JBK.mjs';
|
|
7
|
+
import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, SYNC_CLOSE_GOING_AWAY } from './chunk-KYYABCSW.mjs';
|
|
8
|
+
import { syncServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-QH3YIQP4.mjs';
|
|
9
|
+
import './chunk-2MANFZJ3.mjs';
|
|
10
|
+
import { isRecord, validateIndexName } from './chunk-RSVULW4Q.mjs';
|
|
11
|
+
import './chunk-INJYO3BU.mjs';
|
|
12
|
+
import { estimatedArchiveBytes } from './chunk-Q622CTMC.mjs';
|
|
13
|
+
import './chunk-QJVOLKU6.mjs';
|
|
14
|
+
import './chunk-ZYAY26KG.mjs';
|
|
15
|
+
import './chunk-U4BS74YN.mjs';
|
|
16
|
+
import './chunk-SY3EWKQY.mjs';
|
|
17
|
+
import './chunk-K4ZIGL2O.mjs';
|
|
18
|
+
import './chunk-BUMREPLU.mjs';
|
|
9
19
|
import './chunk-DJHHF2L5.mjs';
|
|
20
|
+
import './chunk-37CD3OKS.mjs';
|
|
10
21
|
import './chunk-JDDXV6UF.mjs';
|
|
11
|
-
import {
|
|
12
|
-
import
|
|
13
|
-
import './chunk-
|
|
14
|
-
import './chunk-
|
|
15
|
-
import './chunk-
|
|
22
|
+
import { manifestFileRefs } from './chunk-NF4DT3IH.mjs';
|
|
23
|
+
import './chunk-QDZJM7NI.mjs';
|
|
24
|
+
import { querySuggestions } from './chunk-5FVU47TB.mjs';
|
|
25
|
+
import './chunk-QUJHMPTP.mjs';
|
|
26
|
+
import './chunk-PHQNACDT.mjs';
|
|
27
|
+
import './chunk-S5LN746F.mjs';
|
|
28
|
+
import './chunk-MT2NRTA6.mjs';
|
|
29
|
+
import './chunk-3HHQLEEQ.mjs';
|
|
30
|
+
import './chunk-VHBNF4MA.mjs';
|
|
31
|
+
import './chunk-KP7TYHWT.mjs';
|
|
32
|
+
import './chunk-LNGE7SLY.mjs';
|
|
33
|
+
import './chunk-UZRZKNNZ.mjs';
|
|
34
|
+
import './chunk-H4KHPLHJ.mjs';
|
|
35
|
+
import './chunk-V6EUBAIT.mjs';
|
|
36
|
+
import './chunk-BEZH7VEH.mjs';
|
|
37
|
+
import './chunk-BDRIVYOK.mjs';
|
|
38
|
+
import './chunk-6L3XJXGU.mjs';
|
|
39
|
+
import './chunk-GQPTDQCL.mjs';
|
|
40
|
+
import './chunk-5YNRKI2C.mjs';
|
|
41
|
+
import { rerankWindowCapOf, engineCoreOf, rulesOfStoredSynonyms, providedDocId, resolveVectorText } from './chunk-KMUNHGBQ.mjs';
|
|
42
|
+
import { readMemoryFigures } from './chunk-L7ZD4L3K.mjs';
|
|
16
43
|
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
17
|
-
import { detectRuntime } from './chunk-
|
|
44
|
+
import { detectRuntime } from './chunk-LHKOSI2J.mjs';
|
|
18
45
|
import './chunk-XIHUI4KB.mjs';
|
|
19
|
-
import './chunk-Q2UL2TPC.mjs';
|
|
20
46
|
import './chunk-3MXV3VHR.mjs';
|
|
21
47
|
import './chunk-EEBCDP36.mjs';
|
|
22
|
-
import './chunk-
|
|
48
|
+
import './chunk-Z2BE2MAM.mjs';
|
|
49
|
+
import './chunk-ZAUJFLEN.mjs';
|
|
50
|
+
import './chunk-GH4WQ22V.mjs';
|
|
51
|
+
import './chunk-3OH67G7R.mjs';
|
|
52
|
+
import './chunk-IPN7S4KX.mjs';
|
|
53
|
+
import './chunk-KAKQ74LS.mjs';
|
|
54
|
+
import './chunk-7GXJUJ24.mjs';
|
|
55
|
+
import './chunk-Q2UL2TPC.mjs';
|
|
56
|
+
import './chunk-LC3FDWTX.mjs';
|
|
57
|
+
import './chunk-JUKYYXSP.mjs';
|
|
58
|
+
import './chunk-4KRC2APY.mjs';
|
|
59
|
+
import './chunk-FQBFZAWG.mjs';
|
|
60
|
+
import './chunk-WQRN64Q6.mjs';
|
|
61
|
+
import './chunk-GES7U2PM.mjs';
|
|
62
|
+
import './chunk-I54T6CQP.mjs';
|
|
63
|
+
import './chunk-YEDIDCR7.mjs';
|
|
23
64
|
import './chunk-X5CV2E2B.mjs';
|
|
65
|
+
import { listenForSyncChanges } from './chunk-Y4WPNHMC.mjs';
|
|
66
|
+
import './chunk-RZWWB2XR.mjs';
|
|
24
67
|
import './chunk-TGP5M5R7.mjs';
|
|
68
|
+
import './chunk-4BYWRYPM.mjs';
|
|
69
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
25
70
|
import './chunk-U2BKYEC7.mjs';
|
|
26
|
-
import './chunk-DGRM4EZZ.mjs';
|
|
27
71
|
import './chunk-QHPYS427.mjs';
|
|
28
72
|
import { startIdleHeapCleanup } from './chunk-QYRNLLLI.mjs';
|
|
73
|
+
import './chunk-HNVVEQP7.mjs';
|
|
29
74
|
import './chunk-ZXNQEW43.mjs';
|
|
75
|
+
import './chunk-L5ORED7V.mjs';
|
|
30
76
|
import './chunk-KOKM4PQN.mjs';
|
|
31
|
-
import './chunk-
|
|
32
|
-
import './chunk-LQG5G2LY.mjs';
|
|
77
|
+
import './chunk-Q7G3BV3Y.mjs';
|
|
33
78
|
import './chunk-BTPGQC6Z.mjs';
|
|
34
|
-
import './chunk-
|
|
35
|
-
import './chunk-U46IZNCW.mjs';
|
|
36
|
-
import './chunk-JUKYYXSP.mjs';
|
|
37
|
-
import './chunk-SOUZMPME.mjs';
|
|
38
|
-
import './chunk-FQBFZAWG.mjs';
|
|
39
|
-
import './chunk-65WCV5JO.mjs';
|
|
40
|
-
import './chunk-4UMZ7KQL.mjs';
|
|
79
|
+
import './chunk-LQG5G2LY.mjs';
|
|
41
80
|
import './chunk-ZXX5GZEX.mjs';
|
|
42
|
-
import './chunk-
|
|
43
|
-
import './chunk-
|
|
44
|
-
import './chunk-3TVDTACC.mjs';
|
|
81
|
+
import './chunk-RFBLQLLZ.mjs';
|
|
82
|
+
import './chunk-3C7MTCCI.mjs';
|
|
45
83
|
import './chunk-U476IXKQ.mjs';
|
|
46
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-
|
|
47
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
84
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-MP7ABPTD.mjs';
|
|
85
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-MP7ABPTD.mjs';
|
|
48
86
|
import './chunk-R6A773WK.mjs';
|
|
49
|
-
import './chunk-YTJCGKM4.mjs';
|
|
87
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
50
88
|
import { randomUUID } from 'crypto';
|
|
89
|
+
import { encode, decode } from '@msgpack/msgpack';
|
|
51
90
|
import { createReadStream } from 'fs';
|
|
52
91
|
import { MessageChannel } from 'worker_threads';
|
|
53
92
|
|
|
93
|
+
// src/sync/feed/limits.ts
|
|
94
|
+
function createSemaphore(limit) {
|
|
95
|
+
let active = 0;
|
|
96
|
+
const waiting = [];
|
|
97
|
+
const release = () => {
|
|
98
|
+
active -= 1;
|
|
99
|
+
waiting.shift()?.();
|
|
100
|
+
};
|
|
101
|
+
return {
|
|
102
|
+
async run(task) {
|
|
103
|
+
if (active >= limit) await new Promise((resolve) => waiting.push(resolve));
|
|
104
|
+
active += 1;
|
|
105
|
+
try {
|
|
106
|
+
return await task();
|
|
107
|
+
} finally {
|
|
108
|
+
release();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function createByteBudget(limit) {
|
|
114
|
+
let used = 0;
|
|
115
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
116
|
+
return {
|
|
117
|
+
take(bytes) {
|
|
118
|
+
if (used > 0 && used + bytes > limit) return false;
|
|
119
|
+
used += bytes;
|
|
120
|
+
return true;
|
|
121
|
+
},
|
|
122
|
+
give(bytes) {
|
|
123
|
+
used = Math.max(0, used - bytes);
|
|
124
|
+
for (const listener of [...listeners]) listener();
|
|
125
|
+
},
|
|
126
|
+
onFreed(listener) {
|
|
127
|
+
listeners.add(listener);
|
|
128
|
+
return () => {
|
|
129
|
+
listeners.delete(listener);
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
var FIXARRAY_LIMIT = 16;
|
|
135
|
+
var ARRAY16_LIMIT = 65536;
|
|
136
|
+
var FIXARRAY_TAG = 144;
|
|
137
|
+
var ARRAY16_TAG = 220;
|
|
138
|
+
var ARRAY32_TAG = 221;
|
|
139
|
+
var FIXMAP_OF_THREE = 131;
|
|
140
|
+
var entriesKeyPrefix = (() => {
|
|
141
|
+
const parts = [Uint8Array.of(FIXMAP_OF_THREE), encode("type"), encode("entries"), encode("partitionId")];
|
|
142
|
+
return concat(parts);
|
|
143
|
+
})();
|
|
144
|
+
var entriesKey = encode("entries");
|
|
145
|
+
function concat(parts) {
|
|
146
|
+
let length = 0;
|
|
147
|
+
for (const part of parts) length += part.byteLength;
|
|
148
|
+
const out = new Uint8Array(length);
|
|
149
|
+
let at = 0;
|
|
150
|
+
for (const part of parts) {
|
|
151
|
+
out.set(part, at);
|
|
152
|
+
at += part.byteLength;
|
|
153
|
+
}
|
|
154
|
+
return out;
|
|
155
|
+
}
|
|
156
|
+
function arrayHeader(count) {
|
|
157
|
+
if (count < FIXARRAY_LIMIT) return Uint8Array.of(FIXARRAY_TAG | count);
|
|
158
|
+
if (count < ARRAY16_LIMIT) return Uint8Array.of(ARRAY16_TAG, count >>> 8, count & 255);
|
|
159
|
+
return Uint8Array.of(ARRAY32_TAG, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
|
|
160
|
+
}
|
|
161
|
+
function encodeEntriesMessage(partitionId, payloads) {
|
|
162
|
+
return concat([entriesKeyPrefix, encode(partitionId), entriesKey, arrayHeader(payloads.length), ...payloads]);
|
|
163
|
+
}
|
|
164
|
+
function encodeServerMessage(message) {
|
|
165
|
+
return encode(message, { ignoreUndefined: true });
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// src/sync/feed/session.ts
|
|
169
|
+
var FeedSession = class {
|
|
170
|
+
constructor(deps) {
|
|
171
|
+
this.deps = deps;
|
|
172
|
+
this.stopBudgetWatch = deps.buffered.onFreed(() => {
|
|
173
|
+
if (this.state === "subscribed" && this.charged === 0) this.advance();
|
|
174
|
+
});
|
|
175
|
+
this.armExpiry();
|
|
176
|
+
}
|
|
177
|
+
state = "awaiting";
|
|
178
|
+
epoch = "";
|
|
179
|
+
sent = [];
|
|
180
|
+
heads = [];
|
|
181
|
+
unacked = [];
|
|
182
|
+
unackedBytes = 0;
|
|
183
|
+
charged = 0;
|
|
184
|
+
pumping = false;
|
|
185
|
+
pumpAgain = false;
|
|
186
|
+
retry = null;
|
|
187
|
+
expiry = null;
|
|
188
|
+
incoming = Promise.resolve();
|
|
189
|
+
stopBudgetWatch;
|
|
190
|
+
armExpiry() {
|
|
191
|
+
const { expiresAt } = this.deps;
|
|
192
|
+
if (expiresAt === null || this.state === "closed") return;
|
|
193
|
+
const remaining = expiresAt - Date.now();
|
|
194
|
+
if (remaining <= 0) {
|
|
195
|
+
queueMicrotask(() => this.close(SYNC_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
this.expiry = setTimeout(
|
|
199
|
+
() => {
|
|
200
|
+
this.expiry = null;
|
|
201
|
+
this.armExpiry();
|
|
202
|
+
},
|
|
203
|
+
Math.min(remaining, TIMER_DELAY_LIMIT_MS)
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
get indexName() {
|
|
207
|
+
return this.deps.indexName;
|
|
208
|
+
}
|
|
209
|
+
receive(bytes, binary = true) {
|
|
210
|
+
if (this.state === "closed") return;
|
|
211
|
+
if (!binary) {
|
|
212
|
+
this.close(SYNC_CLOSE_POLICY, "feed messages travel in binary frames");
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (bytes.byteLength > MAX_DEVICE_MESSAGE_BYTES) {
|
|
216
|
+
this.close(SYNC_CLOSE_TOO_LARGE, "message too large");
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
let message;
|
|
220
|
+
try {
|
|
221
|
+
message = deviceMessageOf(decode(bytes));
|
|
222
|
+
} catch {
|
|
223
|
+
message = null;
|
|
224
|
+
}
|
|
225
|
+
if (message === null) {
|
|
226
|
+
this.close(SYNC_CLOSE_POLICY, "invalid message");
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
const received = message;
|
|
230
|
+
this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
|
|
231
|
+
}
|
|
232
|
+
advance() {
|
|
233
|
+
if (this.state !== "subscribed") return;
|
|
234
|
+
void this.pump();
|
|
235
|
+
}
|
|
236
|
+
sendSettings() {
|
|
237
|
+
if (this.state !== "subscribed") return;
|
|
238
|
+
const state = this.deps.source.indexState(this.deps.indexName);
|
|
239
|
+
if (state === null) return;
|
|
240
|
+
this.send(encodeServerMessage({ type: "settings", settings: state.settings }));
|
|
241
|
+
}
|
|
242
|
+
drained() {
|
|
243
|
+
this.settleBuffered();
|
|
244
|
+
this.advance();
|
|
245
|
+
}
|
|
246
|
+
resync(reason) {
|
|
247
|
+
if (this.state === "closed") return;
|
|
248
|
+
this.send(encodeServerMessage({ type: "resync", reason }));
|
|
249
|
+
this.close(SYNC_CLOSE_RESYNC, `resync ${reason}`);
|
|
250
|
+
}
|
|
251
|
+
close(code, reason) {
|
|
252
|
+
if (this.state === "closed") return;
|
|
253
|
+
this.state = "closed";
|
|
254
|
+
this.cleanUp();
|
|
255
|
+
this.deps.socket.close(code, reason);
|
|
256
|
+
this.deps.onClosed(this);
|
|
257
|
+
}
|
|
258
|
+
closedByPeer() {
|
|
259
|
+
if (this.state === "closed") return;
|
|
260
|
+
this.state = "closed";
|
|
261
|
+
this.cleanUp();
|
|
262
|
+
this.deps.onClosed(this);
|
|
263
|
+
}
|
|
264
|
+
cleanUp() {
|
|
265
|
+
if (this.retry !== null) clearTimeout(this.retry);
|
|
266
|
+
this.retry = null;
|
|
267
|
+
if (this.expiry !== null) clearTimeout(this.expiry);
|
|
268
|
+
this.expiry = null;
|
|
269
|
+
this.stopBudgetWatch();
|
|
270
|
+
this.deps.buffered.give(this.charged);
|
|
271
|
+
this.charged = 0;
|
|
272
|
+
}
|
|
273
|
+
fail(error) {
|
|
274
|
+
this.deps.onError(error);
|
|
275
|
+
this.close(SYNC_CLOSE_SERVER_ERROR, "feed read failed");
|
|
276
|
+
}
|
|
277
|
+
send(bytes) {
|
|
278
|
+
if (this.state === "closed") return;
|
|
279
|
+
this.deps.socket.send(bytes);
|
|
280
|
+
this.settleBuffered();
|
|
281
|
+
}
|
|
282
|
+
settleBuffered() {
|
|
283
|
+
const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
|
|
284
|
+
if (now < this.charged) this.deps.buffered.give(this.charged - now);
|
|
285
|
+
this.charged = Math.min(this.charged, now);
|
|
286
|
+
}
|
|
287
|
+
async handle(message) {
|
|
288
|
+
if (this.state === "closed") return;
|
|
289
|
+
if (message.type === "subscribe") {
|
|
290
|
+
if (this.state !== "awaiting") return this.close(SYNC_CLOSE_POLICY, "subscribe after the first message");
|
|
291
|
+
return this.subscribe(message.epoch, message.positions);
|
|
292
|
+
}
|
|
293
|
+
if (this.state !== "subscribed") return this.close(SYNC_CLOSE_POLICY, "subscribe must come first");
|
|
294
|
+
if (message.type === "ack") return this.acknowledge(message.positions);
|
|
295
|
+
this.answerCheckpoint(message.id).catch((error) => this.fail(error));
|
|
296
|
+
}
|
|
297
|
+
async currentBounds() {
|
|
298
|
+
const all = [];
|
|
299
|
+
for (let partitionId = 0; partitionId < this.sent.length; partitionId++) {
|
|
300
|
+
all.push(await this.deps.source.bounds(this.deps.indexName, partitionId, "current"));
|
|
301
|
+
}
|
|
302
|
+
return all;
|
|
303
|
+
}
|
|
304
|
+
async answerCheckpoint(id) {
|
|
305
|
+
const covering = (await this.currentBounds()).map((bounds) => bounds.highestSeqNo);
|
|
306
|
+
while (this.state === "subscribed") {
|
|
307
|
+
const all = await this.currentBounds();
|
|
308
|
+
let covered = true;
|
|
309
|
+
for (let partitionId = 0; partitionId < all.length; partitionId++) {
|
|
310
|
+
covering[partitionId] = Math.min(covering[partitionId], all[partitionId].highestSeqNo);
|
|
311
|
+
const reachable = Math.max(all[partitionId].head, this.sent[partitionId]);
|
|
312
|
+
if (reachable < covering[partitionId]) covered = false;
|
|
313
|
+
}
|
|
314
|
+
if (covered) {
|
|
315
|
+
const heads = all.map((bounds) => bounds.head);
|
|
316
|
+
this.send(encodeServerMessage({ type: "checkpoint", id, heads: positionList(heads) }));
|
|
317
|
+
this.advance();
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
await new Promise((resolve) => setTimeout(resolve, FEED_HEAD_RETRY_MS));
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
async subscribe(epoch, positions) {
|
|
324
|
+
const { source, indexName } = this.deps;
|
|
325
|
+
const state = source.indexState(indexName);
|
|
326
|
+
if (state === null) return this.close(SYNC_CLOSE_RESYNC, "sync is off");
|
|
327
|
+
if (epoch !== state.epoch) return this.resync("epoch");
|
|
328
|
+
const seqNos = positionsByPartition(positions, state.partitionCount);
|
|
329
|
+
if (seqNos === null) return this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
|
|
330
|
+
const heads = [];
|
|
331
|
+
for (let partitionId = 0; partitionId < state.partitionCount; partitionId++) {
|
|
332
|
+
const bounds = await this.deps.reads.run(() => source.bounds(indexName, partitionId, "current"));
|
|
333
|
+
if (seqNos[partitionId] < bounds.oldestSeqNo - 1) return this.resync("history");
|
|
334
|
+
if (seqNos[partitionId] > bounds.highestSeqNo) return this.resync("ahead");
|
|
335
|
+
heads.push(bounds.head);
|
|
336
|
+
}
|
|
337
|
+
const bytesBehind = await this.deps.reads.run(() => source.bytesBehind(indexName, seqNos));
|
|
338
|
+
const snapshotBytes = await this.deps.reads.run(() => source.snapshotBytes(indexName));
|
|
339
|
+
if (this.state === "closed") return;
|
|
340
|
+
if (source.indexState(indexName)?.epoch !== epoch) return this.resync("epoch");
|
|
341
|
+
this.epoch = epoch;
|
|
342
|
+
this.sent = seqNos;
|
|
343
|
+
this.heads = heads;
|
|
344
|
+
for (let i = 0; i < state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
|
|
345
|
+
this.state = "subscribed";
|
|
346
|
+
this.send(
|
|
347
|
+
encodeServerMessage({
|
|
348
|
+
type: "subscribed",
|
|
349
|
+
epoch,
|
|
350
|
+
partitionCount: state.partitionCount,
|
|
351
|
+
heads: positionList(heads),
|
|
352
|
+
bytesBehind,
|
|
353
|
+
snapshotBytes,
|
|
354
|
+
windowBytes: this.deps.windowBytes,
|
|
355
|
+
settings: state.settings
|
|
356
|
+
})
|
|
357
|
+
);
|
|
358
|
+
void this.pump();
|
|
359
|
+
}
|
|
360
|
+
acknowledge(positions) {
|
|
361
|
+
const seqNos = positionsByPartition(positions, this.sent.length);
|
|
362
|
+
if (seqNos === null) {
|
|
363
|
+
this.close(SYNC_CLOSE_POLICY, "positions must name each partition once");
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
367
|
+
if (seqNos[partitionId] > this.sent[partitionId]) {
|
|
368
|
+
this.close(SYNC_CLOSE_POLICY, "acknowledgement above a sent entry");
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
373
|
+
const queue = this.unacked[partitionId];
|
|
374
|
+
while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
|
|
375
|
+
this.unackedBytes -= queue.items[queue.head].bytes;
|
|
376
|
+
queue.head += 1;
|
|
377
|
+
}
|
|
378
|
+
if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
|
|
379
|
+
queue.items = queue.items.slice(queue.head);
|
|
380
|
+
queue.head = 0;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
void this.pump();
|
|
384
|
+
}
|
|
385
|
+
windowAllows(nextBytes) {
|
|
386
|
+
return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
|
|
387
|
+
}
|
|
388
|
+
async pump() {
|
|
389
|
+
if (this.pumping) {
|
|
390
|
+
this.pumpAgain = true;
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
this.pumping = true;
|
|
394
|
+
try {
|
|
395
|
+
do {
|
|
396
|
+
this.pumpAgain = false;
|
|
397
|
+
await this.refreshHeads();
|
|
398
|
+
await this.sendBehind();
|
|
399
|
+
} while (this.pumpAgain && this.state === "subscribed");
|
|
400
|
+
this.scheduleRetryWhileUndurable();
|
|
401
|
+
} catch (error) {
|
|
402
|
+
this.fail(error);
|
|
403
|
+
} finally {
|
|
404
|
+
this.pumping = false;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
async refreshHeads() {
|
|
408
|
+
const { source, indexName } = this.deps;
|
|
409
|
+
if (source.indexState(indexName)?.epoch !== this.epoch) return this.resync("epoch");
|
|
410
|
+
for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
|
|
411
|
+
this.heads[partitionId] = Math.max(this.heads[partitionId], (await source.bounds(indexName, partitionId)).head);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
async sendBehind() {
|
|
415
|
+
const { source, indexName } = this.deps;
|
|
416
|
+
let progressed = true;
|
|
417
|
+
while (progressed && this.state === "subscribed") {
|
|
418
|
+
progressed = false;
|
|
419
|
+
for (let partitionId = 0; partitionId < this.heads.length; partitionId++) {
|
|
420
|
+
if (this.state !== "subscribed" || this.sent[partitionId] >= this.heads[partitionId]) continue;
|
|
421
|
+
if (!this.windowAllows(1) || this.charged > 0) return;
|
|
422
|
+
const room = this.unackedBytes === 0 ? this.deps.windowBytes : this.deps.windowBytes - this.unackedBytes;
|
|
423
|
+
const maxBytes = Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES));
|
|
424
|
+
const after = this.sent[partitionId];
|
|
425
|
+
const read = await this.deps.reads.run(
|
|
426
|
+
() => source.read(indexName, partitionId, after, this.heads[partitionId], maxBytes)
|
|
427
|
+
);
|
|
428
|
+
if (this.state !== "subscribed") return;
|
|
429
|
+
if (read.gone) return this.resync("history");
|
|
430
|
+
const frames = read.frames.filter((frame) => frame.seqNo > this.sent[partitionId]);
|
|
431
|
+
if (frames.length === 0) continue;
|
|
432
|
+
let bytes = 0;
|
|
433
|
+
for (const frame of frames) bytes += frame.payload.byteLength;
|
|
434
|
+
if (!this.windowAllows(bytes)) return;
|
|
435
|
+
const message = encodeEntriesMessage(
|
|
436
|
+
partitionId,
|
|
437
|
+
frames.map((frame) => frame.payload)
|
|
438
|
+
);
|
|
439
|
+
if (!this.deps.buffered.take(message.byteLength)) return;
|
|
440
|
+
this.charged += message.byteLength;
|
|
441
|
+
const queue = this.unacked[partitionId].items;
|
|
442
|
+
for (const frame of frames) queue.push({ seqNo: frame.seqNo, bytes: frame.payload.byteLength });
|
|
443
|
+
this.unackedBytes += bytes;
|
|
444
|
+
this.sent[partitionId] = frames[frames.length - 1].seqNo;
|
|
445
|
+
this.send(message);
|
|
446
|
+
progressed = true;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
scheduleRetryWhileUndurable() {
|
|
451
|
+
if (this.state !== "subscribed" || this.retry !== null) return;
|
|
452
|
+
const { source, indexName } = this.deps;
|
|
453
|
+
const waiting = this.heads.some((head, partitionId) => source.pendingBeyondHead(indexName, partitionId, head));
|
|
454
|
+
if (!waiting) return;
|
|
455
|
+
this.retry = setTimeout(() => {
|
|
456
|
+
this.retry = null;
|
|
457
|
+
this.advance();
|
|
458
|
+
}, FEED_HEAD_RETRY_MS);
|
|
459
|
+
}
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
// src/sync/feed/hub.ts
|
|
463
|
+
function createFeedHub(source, options = {}) {
|
|
464
|
+
const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
|
|
465
|
+
const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
|
|
466
|
+
const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
|
|
467
|
+
const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
|
|
468
|
+
const sessionsByIndex = /* @__PURE__ */ new Map();
|
|
469
|
+
const unwatchByIndex = /* @__PURE__ */ new Map();
|
|
470
|
+
let connections = 0;
|
|
471
|
+
let stopListening = null;
|
|
472
|
+
let closed = false;
|
|
473
|
+
function each(indexName, action) {
|
|
474
|
+
const sessions = sessionsByIndex.get(indexName);
|
|
475
|
+
if (sessions === void 0) return;
|
|
476
|
+
for (const session of [...sessions]) action(session);
|
|
477
|
+
}
|
|
478
|
+
function onChange(change) {
|
|
479
|
+
switch (change.kind) {
|
|
480
|
+
case "advanced":
|
|
481
|
+
each(change.indexName, (session) => session.advance());
|
|
482
|
+
return;
|
|
483
|
+
case "settings":
|
|
484
|
+
each(change.indexName, (session) => session.sendSettings());
|
|
485
|
+
return;
|
|
486
|
+
case "epoch":
|
|
487
|
+
each(change.indexName, (session) => session.resync("epoch"));
|
|
488
|
+
return;
|
|
489
|
+
case "off":
|
|
490
|
+
case "dropped":
|
|
491
|
+
each(change.indexName, (session) => session.close(SYNC_CLOSE_RESYNC, "sync is off"));
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
function forget(session) {
|
|
496
|
+
const sessions = sessionsByIndex.get(session.indexName);
|
|
497
|
+
if (sessions === void 0 || !sessions.delete(session)) return;
|
|
498
|
+
if (sessions.size === 0) {
|
|
499
|
+
sessionsByIndex.delete(session.indexName);
|
|
500
|
+
unwatchByIndex.get(session.indexName)?.();
|
|
501
|
+
unwatchByIndex.delete(session.indexName);
|
|
502
|
+
}
|
|
503
|
+
release();
|
|
504
|
+
}
|
|
505
|
+
function release() {
|
|
506
|
+
connections -= 1;
|
|
507
|
+
if (connections === 0 && stopListening !== null) {
|
|
508
|
+
stopListening();
|
|
509
|
+
stopListening = null;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
function open(indexName, expiresAt, socket) {
|
|
513
|
+
if (stopListening === null) stopListening = source.listen(onChange);
|
|
514
|
+
const session = new FeedSession({
|
|
515
|
+
source,
|
|
516
|
+
indexName,
|
|
517
|
+
socket,
|
|
518
|
+
windowBytes,
|
|
519
|
+
expiresAt,
|
|
520
|
+
reads,
|
|
521
|
+
buffered,
|
|
522
|
+
onClosed: forget,
|
|
523
|
+
onError: (error) => options.onError?.(error)
|
|
524
|
+
});
|
|
525
|
+
let sessions = sessionsByIndex.get(indexName);
|
|
526
|
+
if (sessions === void 0) {
|
|
527
|
+
sessions = /* @__PURE__ */ new Set();
|
|
528
|
+
sessionsByIndex.set(indexName, sessions);
|
|
529
|
+
const unwatch = source.watchIndex?.(indexName);
|
|
530
|
+
if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
|
|
531
|
+
}
|
|
532
|
+
sessions.add(session);
|
|
533
|
+
return session;
|
|
534
|
+
}
|
|
535
|
+
return {
|
|
536
|
+
admit(indexName, expiresAt) {
|
|
537
|
+
if (closed) {
|
|
538
|
+
throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
|
|
539
|
+
}
|
|
540
|
+
if (source.indexState(indexName) === null) {
|
|
541
|
+
throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
542
|
+
}
|
|
543
|
+
if (connections >= maxConnections) {
|
|
544
|
+
throw new NarsilError(
|
|
545
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
546
|
+
`The server holds ${connections} feed connections, the most that it accepts`,
|
|
547
|
+
{ maximum: maxConnections }
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
connections += 1;
|
|
551
|
+
let settled = false;
|
|
552
|
+
return {
|
|
553
|
+
open(socket) {
|
|
554
|
+
if (settled) throw new Error("A feed admission opens one connection");
|
|
555
|
+
settled = true;
|
|
556
|
+
return open(indexName, expiresAt, socket);
|
|
557
|
+
},
|
|
558
|
+
cancel() {
|
|
559
|
+
if (settled) return;
|
|
560
|
+
settled = true;
|
|
561
|
+
release();
|
|
562
|
+
}
|
|
563
|
+
};
|
|
564
|
+
},
|
|
565
|
+
connectionCount: () => connections,
|
|
566
|
+
close() {
|
|
567
|
+
closed = true;
|
|
568
|
+
for (const indexName of [...sessionsByIndex.keys()]) {
|
|
569
|
+
each(indexName, (session) => session.close(SYNC_CLOSE_GOING_AWAY, "server closing"));
|
|
570
|
+
}
|
|
571
|
+
stopListening?.();
|
|
572
|
+
stopListening = null;
|
|
573
|
+
}
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
function snapshotHistoryOf(raw) {
|
|
577
|
+
if (raw === null || raw === "false") return false;
|
|
578
|
+
return raw === "true" ? true : null;
|
|
579
|
+
}
|
|
54
580
|
function createAdminHandlers(deps) {
|
|
55
581
|
const { engine, tasks, limits } = deps;
|
|
56
582
|
async function checkpoint(ctx) {
|
|
@@ -62,8 +588,15 @@ function createAdminHandlers(deps) {
|
|
|
62
588
|
}
|
|
63
589
|
}
|
|
64
590
|
async function snapshot(ctx) {
|
|
591
|
+
const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
|
|
592
|
+
if (queryHistory === null) {
|
|
593
|
+
badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
|
|
594
|
+
value: ctx.query.get("queryHistory")
|
|
595
|
+
});
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
65
598
|
try {
|
|
66
|
-
const archive = await engine.snapshot(ctx.params[0]);
|
|
599
|
+
const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
|
|
67
600
|
if (ctx.abort.aborted) {
|
|
68
601
|
await archive.cancel();
|
|
69
602
|
return;
|
|
@@ -439,12 +972,17 @@ function createBackupHandlers(deps) {
|
|
|
439
972
|
badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
|
|
440
973
|
return;
|
|
441
974
|
}
|
|
975
|
+
if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
|
|
976
|
+
badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
442
979
|
const backupName = validateBackupName(body.backup);
|
|
980
|
+
const queryHistory = body.queryHistory === true;
|
|
443
981
|
const record = await deps.tasks.start(
|
|
444
982
|
"backup",
|
|
445
983
|
indexName,
|
|
446
984
|
async (context) => {
|
|
447
|
-
await cluster.createBackup(indexName, backupName, stepsOf(context));
|
|
985
|
+
await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
|
|
448
986
|
},
|
|
449
987
|
void 0,
|
|
450
988
|
backupName
|
|
@@ -1134,6 +1672,196 @@ function createMetricsHandler(deps) {
|
|
|
1134
1672
|
};
|
|
1135
1673
|
}
|
|
1136
1674
|
|
|
1675
|
+
// src/server/handlers/query-suggestions.ts
|
|
1676
|
+
var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
|
|
1677
|
+
function readObject(ctx) {
|
|
1678
|
+
const body = parseJson(ctx);
|
|
1679
|
+
if (body === void 0) return void 0;
|
|
1680
|
+
if (!isRecord(body)) {
|
|
1681
|
+
badRequest(ctx.res, "The body must be a JSON object");
|
|
1682
|
+
return void 0;
|
|
1683
|
+
}
|
|
1684
|
+
return body;
|
|
1685
|
+
}
|
|
1686
|
+
function readQuery(ctx, body) {
|
|
1687
|
+
if (typeof body.query === "string") return body.query;
|
|
1688
|
+
badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
|
|
1689
|
+
return void 0;
|
|
1690
|
+
}
|
|
1691
|
+
function isStringList(value) {
|
|
1692
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
1693
|
+
}
|
|
1694
|
+
function isIdentity(value) {
|
|
1695
|
+
return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
|
|
1696
|
+
}
|
|
1697
|
+
function previewLimitOf(raw) {
|
|
1698
|
+
if (raw === null) return void 0;
|
|
1699
|
+
return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
|
|
1700
|
+
}
|
|
1701
|
+
async function identityOf(options, ctx) {
|
|
1702
|
+
if (ctx.hookContext === null) return void 0;
|
|
1703
|
+
try {
|
|
1704
|
+
const identity = await options.identify(ctx.hookContext);
|
|
1705
|
+
if (identity === null) return null;
|
|
1706
|
+
return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
|
|
1707
|
+
} catch {
|
|
1708
|
+
return void 0;
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
function noIdentify(indexName) {
|
|
1712
|
+
return new NarsilError(
|
|
1713
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
1714
|
+
"This server has no identify function for popular query suggestions, so it counts no search",
|
|
1715
|
+
{ indexName }
|
|
1716
|
+
);
|
|
1717
|
+
}
|
|
1718
|
+
function createQuerySuggestionHandlers(deps, options) {
|
|
1719
|
+
const { engine, tasks } = deps;
|
|
1720
|
+
const suggestions = () => querySuggestions(engine);
|
|
1721
|
+
async function update(ctx) {
|
|
1722
|
+
const body = parseJson(ctx);
|
|
1723
|
+
if (body === void 0) return;
|
|
1724
|
+
try {
|
|
1725
|
+
await suggestions().updateSetting(ctx.params[0], body);
|
|
1726
|
+
respondJson(ctx, { ok: true });
|
|
1727
|
+
} catch (err) {
|
|
1728
|
+
respondError(ctx, err);
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
async function status(ctx) {
|
|
1732
|
+
try {
|
|
1733
|
+
respondJson(ctx, await suggestions().status(ctx.params[0]));
|
|
1734
|
+
} catch (err) {
|
|
1735
|
+
respondError(ctx, err);
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
async function record(ctx) {
|
|
1739
|
+
const body = readObject(ctx);
|
|
1740
|
+
if (body === void 0) return;
|
|
1741
|
+
const extra = Object.keys(body).filter((key) => key !== "query");
|
|
1742
|
+
if (extra.length > 0) {
|
|
1743
|
+
badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
|
|
1744
|
+
fields: extra
|
|
1745
|
+
});
|
|
1746
|
+
return;
|
|
1747
|
+
}
|
|
1748
|
+
const query = readQuery(ctx, body);
|
|
1749
|
+
if (query === void 0) return;
|
|
1750
|
+
const indexName = ctx.params[0];
|
|
1751
|
+
try {
|
|
1752
|
+
if (options === void 0) throw noIdentify(indexName);
|
|
1753
|
+
const identity = await identityOf(options, ctx);
|
|
1754
|
+
if (identity === void 0) {
|
|
1755
|
+
if (!ctx.abort.aborted) {
|
|
1756
|
+
sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
|
|
1757
|
+
}
|
|
1758
|
+
return;
|
|
1759
|
+
}
|
|
1760
|
+
if (identity === null) {
|
|
1761
|
+
respondJson(ctx, { recorded: false, outcome: "unidentified" });
|
|
1762
|
+
return;
|
|
1763
|
+
}
|
|
1764
|
+
respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
|
|
1765
|
+
} catch (err) {
|
|
1766
|
+
respondError(ctx, err);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
async function build(ctx) {
|
|
1770
|
+
const indexName = ctx.params[0];
|
|
1771
|
+
try {
|
|
1772
|
+
const operations = suggestions();
|
|
1773
|
+
const current = await operations.status(indexName);
|
|
1774
|
+
if (current.setting?.enabled !== true) {
|
|
1775
|
+
throw new NarsilError(
|
|
1776
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
1777
|
+
`Index "${indexName}" has popular query suggestions off`,
|
|
1778
|
+
{ indexName }
|
|
1779
|
+
);
|
|
1780
|
+
}
|
|
1781
|
+
const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
|
|
1782
|
+
const finished = await operations.build(indexName);
|
|
1783
|
+
context.reportBuild(finished);
|
|
1784
|
+
if (finished.status === "failed") {
|
|
1785
|
+
throw new NarsilError(
|
|
1786
|
+
finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
|
|
1787
|
+
"The build of popular query suggestions failed",
|
|
1788
|
+
{ indexName, buildId: finished.buildId }
|
|
1789
|
+
);
|
|
1790
|
+
}
|
|
1791
|
+
});
|
|
1792
|
+
respondJson(ctx, task, 202);
|
|
1793
|
+
} catch (err) {
|
|
1794
|
+
respondError(ctx, err);
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
async function preview(ctx) {
|
|
1798
|
+
const limit = previewLimitOf(ctx.query.get("limit"));
|
|
1799
|
+
if (limit === null) {
|
|
1800
|
+
badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
|
|
1801
|
+
return;
|
|
1802
|
+
}
|
|
1803
|
+
try {
|
|
1804
|
+
respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
|
|
1805
|
+
} catch (err) {
|
|
1806
|
+
respondError(ctx, err);
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
async function readBans(ctx) {
|
|
1810
|
+
try {
|
|
1811
|
+
respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
|
|
1812
|
+
} catch (err) {
|
|
1813
|
+
respondError(ctx, err);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
async function replaceBans(ctx) {
|
|
1817
|
+
const body = readObject(ctx);
|
|
1818
|
+
if (body === void 0) return;
|
|
1819
|
+
if (!isStringList(body.bans)) {
|
|
1820
|
+
badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
|
|
1821
|
+
return;
|
|
1822
|
+
}
|
|
1823
|
+
try {
|
|
1824
|
+
respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
|
|
1825
|
+
} catch (err) {
|
|
1826
|
+
respondError(ctx, err);
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
async function remove(ctx) {
|
|
1830
|
+
const body = readObject(ctx);
|
|
1831
|
+
if (body === void 0) return;
|
|
1832
|
+
const query = readQuery(ctx, body);
|
|
1833
|
+
if (query === void 0) return;
|
|
1834
|
+
try {
|
|
1835
|
+
const held = await suggestions().removeQuery(ctx.params[0], query);
|
|
1836
|
+
respondJson(ctx, { suggestionIndexes: held });
|
|
1837
|
+
} catch (err) {
|
|
1838
|
+
respondError(ctx, err);
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
async function manual(ctx) {
|
|
1842
|
+
const body = readObject(ctx);
|
|
1843
|
+
if (body === void 0) return;
|
|
1844
|
+
const query = readQuery(ctx, body);
|
|
1845
|
+
if (query === void 0) return;
|
|
1846
|
+
const { group, count } = body;
|
|
1847
|
+
if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
|
|
1848
|
+
badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
|
|
1849
|
+
return;
|
|
1850
|
+
}
|
|
1851
|
+
try {
|
|
1852
|
+
const name = await suggestions().addSuggestion(ctx.params[0], {
|
|
1853
|
+
group,
|
|
1854
|
+
query,
|
|
1855
|
+
...count === void 0 ? {} : { count }
|
|
1856
|
+
});
|
|
1857
|
+
respondJson(ctx, { indexName: name });
|
|
1858
|
+
} catch (err) {
|
|
1859
|
+
respondError(ctx, err);
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1137
1865
|
// src/server/handlers/search.ts
|
|
1138
1866
|
function createSearchHandlers(deps) {
|
|
1139
1867
|
const { engine, limits, allowRegex } = deps;
|
|
@@ -1256,6 +1984,16 @@ function assertServerLimits(limits, engineRerankWindow) {
|
|
|
1256
1984
|
assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
|
|
1257
1985
|
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
1258
1986
|
}
|
|
1987
|
+
function assertQuerySuggestionsOptions(options) {
|
|
1988
|
+
if (options === void 0) return;
|
|
1989
|
+
const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
|
|
1990
|
+
if (typeof identify === "function") return;
|
|
1991
|
+
throw new NarsilError(
|
|
1992
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1993
|
+
"querySuggestions must carry an identify function that returns the person and the group of a request",
|
|
1994
|
+
{ field: "querySuggestions.identify" }
|
|
1995
|
+
);
|
|
1996
|
+
}
|
|
1259
1997
|
function assertAllowRegex(allowRegex) {
|
|
1260
1998
|
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
1261
1999
|
throw new NarsilError(
|
|
@@ -1264,34 +2002,28 @@ function assertAllowRegex(allowRegex) {
|
|
|
1264
2002
|
{ field: "allowRegex", value: String(allowRegex) }
|
|
1265
2003
|
);
|
|
1266
2004
|
}
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
"authorize",
|
|
1270
|
-
"embed",
|
|
1271
|
-
"rerank",
|
|
1272
|
-
"await-writes",
|
|
1273
|
-
"stream-credit",
|
|
1274
|
-
"stream-cancel"
|
|
1275
|
-
]);
|
|
1276
|
-
function isRelayRequest(message) {
|
|
1277
|
-
if (typeof message !== "object" || message === null) return false;
|
|
1278
|
-
const { type, requestId } = message;
|
|
1279
|
-
if (type === "touch" || type === "leaving") return true;
|
|
1280
|
-
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
1281
|
-
}
|
|
2005
|
+
|
|
2006
|
+
// src/server/request-threads/relay-answers.ts
|
|
1282
2007
|
async function answerAuthorize(authorize, request) {
|
|
1283
2008
|
if (authorize === void 0) {
|
|
1284
|
-
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false };
|
|
2009
|
+
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
|
|
1285
2010
|
}
|
|
1286
2011
|
const authorization = await authorize(request.context);
|
|
1287
2012
|
if (authorization.allowed) {
|
|
1288
|
-
return {
|
|
2013
|
+
return {
|
|
2014
|
+
type: "authorized",
|
|
2015
|
+
requestId: request.requestId,
|
|
2016
|
+
denial: null,
|
|
2017
|
+
hookError: false,
|
|
2018
|
+
expiresAt: authorization.expiresAt
|
|
2019
|
+
};
|
|
1289
2020
|
}
|
|
1290
2021
|
return {
|
|
1291
2022
|
type: "authorized",
|
|
1292
2023
|
requestId: request.requestId,
|
|
1293
2024
|
denial: authorization.denial,
|
|
1294
|
-
hookError: authorization.denial === null
|
|
2025
|
+
hookError: authorization.denial === null,
|
|
2026
|
+
expiresAt: null
|
|
1295
2027
|
};
|
|
1296
2028
|
}
|
|
1297
2029
|
async function answerEmbed(core, request) {
|
|
@@ -1328,6 +2060,33 @@ async function answerAwaitWrites(core, request) {
|
|
|
1328
2060
|
await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
|
|
1329
2061
|
return { type: "writes-applied", requestId: request.requestId };
|
|
1330
2062
|
}
|
|
2063
|
+
async function answerFeedAdmit(feeds, request) {
|
|
2064
|
+
if (feeds !== null) return feeds.admit(request);
|
|
2065
|
+
return {
|
|
2066
|
+
type: "feed-admitted",
|
|
2067
|
+
requestId: request.requestId,
|
|
2068
|
+
feedId: null,
|
|
2069
|
+
error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
|
|
2070
|
+
};
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
// src/server/request-threads/host.ts
|
|
2074
|
+
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
2075
|
+
"request",
|
|
2076
|
+
"authorize",
|
|
2077
|
+
"embed",
|
|
2078
|
+
"rerank",
|
|
2079
|
+
"await-writes",
|
|
2080
|
+
"stream-credit",
|
|
2081
|
+
"stream-cancel",
|
|
2082
|
+
"feed-admit"
|
|
2083
|
+
]);
|
|
2084
|
+
function isRelayRequest(message) {
|
|
2085
|
+
if (typeof message !== "object" || message === null) return false;
|
|
2086
|
+
const { type, requestId } = message;
|
|
2087
|
+
if (type === "touch" || type === "leaving") return true;
|
|
2088
|
+
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
2089
|
+
}
|
|
1331
2090
|
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
1332
2091
|
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
1333
2092
|
}
|
|
@@ -1347,6 +2106,7 @@ async function startRequestThreads(options) {
|
|
|
1347
2106
|
closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
|
|
1348
2107
|
}
|
|
1349
2108
|
function releaseDescriptor(thread) {
|
|
2109
|
+
thread.feeds?.closeAll();
|
|
1350
2110
|
closeAtExit.delete(thread.closeAtExit);
|
|
1351
2111
|
if (thread.descriptor !== null) {
|
|
1352
2112
|
try {
|
|
@@ -1359,7 +2119,12 @@ async function startRequestThreads(options) {
|
|
|
1359
2119
|
function relayFrom(thread) {
|
|
1360
2120
|
const { port } = thread;
|
|
1361
2121
|
const streams = /* @__PURE__ */ new Map();
|
|
2122
|
+
port.on("close", () => thread.feeds?.closeAll());
|
|
1362
2123
|
port.on("message", (message) => {
|
|
2124
|
+
if (isThreadFeedNote(message)) {
|
|
2125
|
+
thread.feeds?.handle(message);
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
1363
2128
|
if (!isRelayRequest(message)) return;
|
|
1364
2129
|
if (message.type === "leaving") {
|
|
1365
2130
|
releaseDescriptor(thread);
|
|
@@ -1377,7 +2142,7 @@ async function startRequestThreads(options) {
|
|
|
1377
2142
|
streams.get(message.requestId)?.cancel();
|
|
1378
2143
|
return;
|
|
1379
2144
|
}
|
|
1380
|
-
const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "embed" ? answerEmbed(core, message) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message);
|
|
2145
|
+
const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message);
|
|
1381
2146
|
void reply.then((answer) => {
|
|
1382
2147
|
if (answer !== null) port.postMessage(answer);
|
|
1383
2148
|
});
|
|
@@ -1407,6 +2172,7 @@ async function startRequestThreads(options) {
|
|
|
1407
2172
|
dropLongQueryThread();
|
|
1408
2173
|
const channel = new MessageChannel();
|
|
1409
2174
|
const relaying = {
|
|
2175
|
+
feeds: null,
|
|
1410
2176
|
descriptor: null,
|
|
1411
2177
|
port: channel.port2,
|
|
1412
2178
|
executor,
|
|
@@ -1446,6 +2212,7 @@ async function startRequestThreads(options) {
|
|
|
1446
2212
|
if (stopped) return;
|
|
1447
2213
|
const channel = new MessageChannel();
|
|
1448
2214
|
const thread = {
|
|
2215
|
+
feeds: options.feeds?.forThread(channel.port2) ?? null,
|
|
1449
2216
|
descriptor: null,
|
|
1450
2217
|
port: channel.port2,
|
|
1451
2218
|
executor,
|
|
@@ -1480,6 +2247,7 @@ async function startRequestThreads(options) {
|
|
|
1480
2247
|
patternWorkCap: options.core.patternWorkCap,
|
|
1481
2248
|
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1482
2249
|
allowRegex: options.allowRegex,
|
|
2250
|
+
sync: options.feeds !== null,
|
|
1483
2251
|
longQueriesInFlight,
|
|
1484
2252
|
longQueryPort: longQueryChannel?.port1 ?? null
|
|
1485
2253
|
}
|
|
@@ -1553,6 +2321,260 @@ async function startRequestThreads(options) {
|
|
|
1553
2321
|
};
|
|
1554
2322
|
}
|
|
1555
2323
|
|
|
2324
|
+
// src/sync/feed/engine-source.ts
|
|
2325
|
+
function syncSettingsOf(entry) {
|
|
2326
|
+
const { config } = entry;
|
|
2327
|
+
const synonyms = entry.synonyms === null ? [] : rulesOfStoredSynonyms(entry.synonyms);
|
|
2328
|
+
return {
|
|
2329
|
+
language: entry.language.name,
|
|
2330
|
+
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
2331
|
+
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
2332
|
+
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
2333
|
+
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
2334
|
+
...config.bm25 !== void 0 ? { bm25: config.bm25 } : {},
|
|
2335
|
+
...synonyms.length > 0 ? { synonyms } : {}
|
|
2336
|
+
};
|
|
2337
|
+
}
|
|
2338
|
+
function storageOf(core) {
|
|
2339
|
+
return core.durability?.manager.syncStorage?.() ?? null;
|
|
2340
|
+
}
|
|
2341
|
+
function engineFeedSource(engine, core) {
|
|
2342
|
+
const requireStorage = () => {
|
|
2343
|
+
const storage = storageOf(core);
|
|
2344
|
+
if (storage === null) {
|
|
2345
|
+
throw new NarsilError(ErrorCodes.SYNC_DURABILITY_REQUIRED, "Device sync needs write-ahead log durability");
|
|
2346
|
+
}
|
|
2347
|
+
return storage;
|
|
2348
|
+
};
|
|
2349
|
+
return {
|
|
2350
|
+
indexState(indexName) {
|
|
2351
|
+
const entry = core.indexRegistry.get(indexName);
|
|
2352
|
+
if (entry === void 0 || entry.sync === null || storageOf(core) === null) return null;
|
|
2353
|
+
return { epoch: entry.sync.epoch, partitionCount: entry.partitionCount, settings: syncSettingsOf(entry) };
|
|
2354
|
+
},
|
|
2355
|
+
async bounds(indexName, partitionId) {
|
|
2356
|
+
const log = await requireStorage().openPartitionLog(indexName, partitionId);
|
|
2357
|
+
const applied = core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0;
|
|
2358
|
+
return { oldestSeqNo: log.oldestSeqNo, head: log.head, highestSeqNo: Math.max(log.head, applied) };
|
|
2359
|
+
},
|
|
2360
|
+
async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
2361
|
+
const read = await requireStorage().readPartitionLog(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes);
|
|
2362
|
+
return { frames: read.frames, gone: read.gone };
|
|
2363
|
+
},
|
|
2364
|
+
async bytesBehind(indexName, seqNos) {
|
|
2365
|
+
const storage = requireStorage();
|
|
2366
|
+
let bytes = 0;
|
|
2367
|
+
for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
|
|
2368
|
+
const log = await storage.openPartitionLog(indexName, partitionId);
|
|
2369
|
+
bytes += await storage.logBytesAfter(log, seqNos[partitionId]);
|
|
2370
|
+
}
|
|
2371
|
+
return bytes;
|
|
2372
|
+
},
|
|
2373
|
+
async snapshotBytes(indexName) {
|
|
2374
|
+
const storage = requireStorage();
|
|
2375
|
+
const published = await storage.published(indexName);
|
|
2376
|
+
if (published !== null) return publishedArchiveLayout(indexName, published, storage).byteLength;
|
|
2377
|
+
const manifest = await storage.currentManifest(indexName);
|
|
2378
|
+
if (manifest === null) return 0;
|
|
2379
|
+
return estimatedArchiveBytes(manifestFileRefs(manifest));
|
|
2380
|
+
},
|
|
2381
|
+
pendingBeyondHead(indexName, partitionId, head) {
|
|
2382
|
+
return (core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0) > head;
|
|
2383
|
+
},
|
|
2384
|
+
listen(onChange) {
|
|
2385
|
+
const onWrite = (payload) => onChange({ kind: "advanced", indexName: payload.indexName });
|
|
2386
|
+
engine.on("write", onWrite);
|
|
2387
|
+
const stopSyncChanges = listenForSyncChanges(core.indexRegistry, onChange);
|
|
2388
|
+
return () => {
|
|
2389
|
+
engine.off("write", onWrite);
|
|
2390
|
+
stopSyncChanges();
|
|
2391
|
+
};
|
|
2392
|
+
}
|
|
2393
|
+
};
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
// src/server/sync/engine-serving.ts
|
|
2397
|
+
function engineSyncServing(engine, core) {
|
|
2398
|
+
startSyncRetentionSweep(core);
|
|
2399
|
+
return {
|
|
2400
|
+
async snapshot(indexName) {
|
|
2401
|
+
const published = await publishedSyncSnapshot(core, indexName);
|
|
2402
|
+
try {
|
|
2403
|
+
return {
|
|
2404
|
+
entityTag: published.snapshot.entityTag,
|
|
2405
|
+
layout: publishedArchiveLayout(indexName, published.snapshot, published.storage),
|
|
2406
|
+
release: published.unpin
|
|
2407
|
+
};
|
|
2408
|
+
} catch (error) {
|
|
2409
|
+
published.unpin();
|
|
2410
|
+
throw error;
|
|
2411
|
+
}
|
|
2412
|
+
},
|
|
2413
|
+
feed: engineFeedSource(engine, core),
|
|
2414
|
+
admitFeed: async () => void 0
|
|
2415
|
+
};
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
// src/server/sync/feed-host.ts
|
|
2419
|
+
var RelayedFeedSocket = class {
|
|
2420
|
+
constructor(port, feedId) {
|
|
2421
|
+
this.port = port;
|
|
2422
|
+
this.feedId = feedId;
|
|
2423
|
+
}
|
|
2424
|
+
inTransit = 0;
|
|
2425
|
+
threadBuffered = 0;
|
|
2426
|
+
closed = false;
|
|
2427
|
+
send(bytes) {
|
|
2428
|
+
if (this.closed) return;
|
|
2429
|
+
const copy = bytes.slice();
|
|
2430
|
+
this.inTransit += copy.byteLength;
|
|
2431
|
+
this.post({ type: "feed-send", feedId: this.feedId, bytes: copy }, [copy.buffer]);
|
|
2432
|
+
}
|
|
2433
|
+
bufferedAmount() {
|
|
2434
|
+
return this.inTransit + this.threadBuffered;
|
|
2435
|
+
}
|
|
2436
|
+
close(code, reason) {
|
|
2437
|
+
if (this.closed) return;
|
|
2438
|
+
this.closed = true;
|
|
2439
|
+
this.post({ type: "feed-close", feedId: this.feedId, code, reason }, []);
|
|
2440
|
+
}
|
|
2441
|
+
sent(bytes, buffered) {
|
|
2442
|
+
this.inTransit = Math.max(0, this.inTransit - bytes);
|
|
2443
|
+
this.threadBuffered = buffered;
|
|
2444
|
+
}
|
|
2445
|
+
markClosed() {
|
|
2446
|
+
this.closed = true;
|
|
2447
|
+
}
|
|
2448
|
+
post(note, transfer) {
|
|
2449
|
+
try {
|
|
2450
|
+
this.port.postMessage(note, transfer);
|
|
2451
|
+
} catch {
|
|
2452
|
+
this.closed = true;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
};
|
|
2456
|
+
function createFeedHost(hub, serving, authorize) {
|
|
2457
|
+
async function admit(indexName, expiresAt) {
|
|
2458
|
+
await serving.admitFeed(indexName);
|
|
2459
|
+
return hub.admit(indexName, expiresAt);
|
|
2460
|
+
}
|
|
2461
|
+
const gateway = {
|
|
2462
|
+
authorize: (context) => authorize === void 0 ? Promise.resolve({ allowed: true, expiresAt: null }) : authorize(context),
|
|
2463
|
+
async admit(indexName, expiresAt) {
|
|
2464
|
+
const admission = await admit(indexName, expiresAt);
|
|
2465
|
+
return { open: (socket) => admission.open(socket), cancel: () => admission.cancel() };
|
|
2466
|
+
}
|
|
2467
|
+
};
|
|
2468
|
+
function forThread(port) {
|
|
2469
|
+
const feeds = /* @__PURE__ */ new Map();
|
|
2470
|
+
let nextFeedId = 0;
|
|
2471
|
+
let closed = false;
|
|
2472
|
+
function closeFeed(feedId) {
|
|
2473
|
+
const feed = feeds.get(feedId);
|
|
2474
|
+
if (feed === void 0) return;
|
|
2475
|
+
feeds.delete(feedId);
|
|
2476
|
+
feed.socket.markClosed();
|
|
2477
|
+
if (feed.session === null) feed.admission.cancel();
|
|
2478
|
+
else feed.session.closedByPeer();
|
|
2479
|
+
}
|
|
2480
|
+
return {
|
|
2481
|
+
async admit(request) {
|
|
2482
|
+
try {
|
|
2483
|
+
const admission = await admit(request.indexName, request.expiresAt);
|
|
2484
|
+
if (closed) {
|
|
2485
|
+
admission.cancel();
|
|
2486
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: null };
|
|
2487
|
+
}
|
|
2488
|
+
nextFeedId += 1;
|
|
2489
|
+
const feedId = nextFeedId;
|
|
2490
|
+
feeds.set(feedId, { admission, socket: new RelayedFeedSocket(port, feedId), session: null });
|
|
2491
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId, error: null };
|
|
2492
|
+
} catch (err) {
|
|
2493
|
+
return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: toHttpError(err).body };
|
|
2494
|
+
}
|
|
2495
|
+
},
|
|
2496
|
+
handle(note) {
|
|
2497
|
+
const feed = feeds.get(note.feedId);
|
|
2498
|
+
if (feed === void 0) return;
|
|
2499
|
+
switch (note.type) {
|
|
2500
|
+
case "feed-open":
|
|
2501
|
+
if (feed.session === null) feed.session = feed.admission.open(feed.socket);
|
|
2502
|
+
return;
|
|
2503
|
+
case "feed-message":
|
|
2504
|
+
feed.session?.receive(note.bytes, note.binary);
|
|
2505
|
+
return;
|
|
2506
|
+
case "feed-sent":
|
|
2507
|
+
feed.socket.sent(note.bytes, note.buffered);
|
|
2508
|
+
feed.session?.drained();
|
|
2509
|
+
return;
|
|
2510
|
+
case "feed-cancel":
|
|
2511
|
+
case "feed-closed":
|
|
2512
|
+
closeFeed(note.feedId);
|
|
2513
|
+
return;
|
|
2514
|
+
}
|
|
2515
|
+
},
|
|
2516
|
+
closeAll() {
|
|
2517
|
+
closed = true;
|
|
2518
|
+
for (const feedId of [...feeds.keys()]) closeFeed(feedId);
|
|
2519
|
+
}
|
|
2520
|
+
};
|
|
2521
|
+
}
|
|
2522
|
+
return { gateway, forThread, close: () => hub.close() };
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
// src/server/sync/options.ts
|
|
2526
|
+
var SYNC_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
|
|
2527
|
+
function assertCount(field, value) {
|
|
2528
|
+
if (value === void 0) return;
|
|
2529
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
|
|
2530
|
+
throw new NarsilError(
|
|
2531
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2532
|
+
`sync.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
|
|
2533
|
+
{ field, value: String(value) }
|
|
2534
|
+
);
|
|
2535
|
+
}
|
|
2536
|
+
function resolveSyncOptions(sync) {
|
|
2537
|
+
if (sync === void 0 || sync === false) return null;
|
|
2538
|
+
if (sync !== true && (typeof sync !== "object" || sync === null)) {
|
|
2539
|
+
throw new NarsilError(
|
|
2540
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2541
|
+
`sync must be true, false, or an object of settings; however, this configuration sets it to ${String(sync)}`,
|
|
2542
|
+
{ field: "sync", value: String(sync) }
|
|
2543
|
+
);
|
|
2544
|
+
}
|
|
2545
|
+
const settings = sync === true ? {} : sync;
|
|
2546
|
+
for (const field of SYNC_OPTION_FIELDS) assertCount(field, settings[field]);
|
|
2547
|
+
return {
|
|
2548
|
+
windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
|
|
2549
|
+
maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
|
|
2550
|
+
maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
|
|
2551
|
+
maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
|
|
2552
|
+
};
|
|
2553
|
+
}
|
|
2554
|
+
function assertSyncHasAuthentication(sync, options) {
|
|
2555
|
+
if (sync === null || options.onRequest !== void 0) return;
|
|
2556
|
+
throw new NarsilError(
|
|
2557
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2558
|
+
"Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
|
|
2559
|
+
{ field: "sync" }
|
|
2560
|
+
);
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
// src/server/sync/setting-handlers.ts
|
|
2564
|
+
function createSyncSettingHandlers(engine) {
|
|
2565
|
+
async function updateSync(ctx) {
|
|
2566
|
+
const body = parseJson(ctx);
|
|
2567
|
+
if (body === void 0) return;
|
|
2568
|
+
try {
|
|
2569
|
+
await engine.updateSync(ctx.params[0], body);
|
|
2570
|
+
respondJson(ctx, { ok: true });
|
|
2571
|
+
} catch (err) {
|
|
2572
|
+
respondError(ctx, err);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
return { updateSync };
|
|
2576
|
+
}
|
|
2577
|
+
|
|
1556
2578
|
// src/server/task-status.ts
|
|
1557
2579
|
var TERMINAL_TASK_STATUSES = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled"]);
|
|
1558
2580
|
function isTerminalTaskStatus(status) {
|
|
@@ -1754,6 +2776,9 @@ var TaskRegistry = class {
|
|
|
1754
2776
|
reportSteps: (steps) => this.reportProgress(task, { steps }),
|
|
1755
2777
|
reportResult: (result) => {
|
|
1756
2778
|
task.record = { ...task.record, result };
|
|
2779
|
+
},
|
|
2780
|
+
reportBuild: (build) => {
|
|
2781
|
+
task.record = { ...task.record, build };
|
|
1757
2782
|
}
|
|
1758
2783
|
};
|
|
1759
2784
|
let settled;
|
|
@@ -1817,7 +2842,7 @@ function assertSecureBinding(host, options) {
|
|
|
1817
2842
|
{ host }
|
|
1818
2843
|
);
|
|
1819
2844
|
}
|
|
1820
|
-
function mainThreadHandlers(deps) {
|
|
2845
|
+
function mainThreadHandlers(deps, syncServing) {
|
|
1821
2846
|
const idx = createIndexHandlers(deps);
|
|
1822
2847
|
const doc = createDocumentHandlers(deps);
|
|
1823
2848
|
const search = createSearchHandlers(deps);
|
|
@@ -1828,6 +2853,8 @@ function mainThreadHandlers(deps) {
|
|
|
1828
2853
|
const backups = createBackupHandlers(deps);
|
|
1829
2854
|
const version = createVersionHandler(deps);
|
|
1830
2855
|
const capabilities = createCapabilitiesHandler();
|
|
2856
|
+
const syncSetting = createSyncSettingHandlers(deps.engine);
|
|
2857
|
+
const suggestions = createQuerySuggestionHandlers(deps, deps.querySuggestions);
|
|
1831
2858
|
return {
|
|
1832
2859
|
livez: health.livez,
|
|
1833
2860
|
readyz: health.readyz,
|
|
@@ -1883,7 +2910,18 @@ function mainThreadHandlers(deps) {
|
|
|
1883
2910
|
updateBm25: admin.updateBm25,
|
|
1884
2911
|
restore: admin.restore,
|
|
1885
2912
|
exportDocuments: documentExport.exportDocuments,
|
|
1886
|
-
importExport: documentExport.importExport
|
|
2913
|
+
importExport: documentExport.importExport,
|
|
2914
|
+
updateSync: syncSetting.updateSync,
|
|
2915
|
+
syncSnapshot: createSyncSnapshotHandler(syncServing),
|
|
2916
|
+
updateQuerySuggestions: suggestions.update,
|
|
2917
|
+
querySuggestionsStatus: suggestions.status,
|
|
2918
|
+
recordQuery: suggestions.record,
|
|
2919
|
+
buildQuerySuggestions: suggestions.build,
|
|
2920
|
+
previewQuerySuggestions: suggestions.preview,
|
|
2921
|
+
readQueryBans: suggestions.readBans,
|
|
2922
|
+
replaceQueryBans: suggestions.replaceBans,
|
|
2923
|
+
removeQuery: suggestions.remove,
|
|
2924
|
+
addManualSuggestion: suggestions.manual
|
|
1887
2925
|
};
|
|
1888
2926
|
}
|
|
1889
2927
|
var NarsilHttpServer = class {
|
|
@@ -1896,12 +2934,17 @@ var NarsilHttpServer = class {
|
|
|
1896
2934
|
listenSocket = null;
|
|
1897
2935
|
ready = false;
|
|
1898
2936
|
threads = null;
|
|
2937
|
+
sync;
|
|
2938
|
+
syncServing = null;
|
|
2939
|
+
feeds = null;
|
|
1899
2940
|
stopIdleHeapCleanup = () => void 0;
|
|
1900
2941
|
constructor(engine, options = {}) {
|
|
1901
2942
|
assertServerEngine(engine);
|
|
1902
2943
|
const engineRerankWindow = rerankWindowCapOf(engine);
|
|
1903
2944
|
assertServerLimits(options.limits, engineRerankWindow);
|
|
1904
2945
|
assertAllowRegex(options.allowRegex);
|
|
2946
|
+
assertQuerySuggestionsOptions(options.querySuggestions);
|
|
2947
|
+
this.sync = resolveSyncOptions(options.sync);
|
|
1905
2948
|
this.host = options.host ?? "127.0.0.1";
|
|
1906
2949
|
this.port = options.port ?? 9876;
|
|
1907
2950
|
this.options = options;
|
|
@@ -1924,10 +2967,12 @@ var NarsilHttpServer = class {
|
|
|
1924
2967
|
isReady: () => this.ready,
|
|
1925
2968
|
build: resolveBuild(options.build),
|
|
1926
2969
|
cluster: options.cluster,
|
|
1927
|
-
requestThreadCount: () => this.requestThreadCount
|
|
2970
|
+
requestThreadCount: () => this.requestThreadCount,
|
|
2971
|
+
querySuggestions: options.querySuggestions
|
|
1928
2972
|
};
|
|
1929
2973
|
}
|
|
1930
2974
|
async listen() {
|
|
2975
|
+
assertSyncHasAuthentication(this.sync, this.options);
|
|
1931
2976
|
assertSecureBinding(this.host, this.options);
|
|
1932
2977
|
const uws = await loadUWebSockets();
|
|
1933
2978
|
this.uws = uws;
|
|
@@ -1955,6 +3000,9 @@ var NarsilHttpServer = class {
|
|
|
1955
3000
|
this.uws.us_listen_socket_close(this.listenSocket);
|
|
1956
3001
|
this.listenSocket = null;
|
|
1957
3002
|
}
|
|
3003
|
+
this.feeds?.close();
|
|
3004
|
+
this.feeds = null;
|
|
3005
|
+
this.syncServing = null;
|
|
1958
3006
|
if (this.threads !== null) {
|
|
1959
3007
|
const threads = this.threads;
|
|
1960
3008
|
this.threads = null;
|
|
@@ -1972,6 +3020,21 @@ var NarsilHttpServer = class {
|
|
|
1972
3020
|
get requestThreadCount() {
|
|
1973
3021
|
return this.threads?.threadCount ?? 0;
|
|
1974
3022
|
}
|
|
3023
|
+
startSync(app, authorize, core) {
|
|
3024
|
+
if (this.sync === null) return;
|
|
3025
|
+
const servingOfNode = syncServingSourceOf(this.engine);
|
|
3026
|
+
let serving;
|
|
3027
|
+
if (servingOfNode !== void 0) serving = servingOfNode();
|
|
3028
|
+
else if (core !== void 0) serving = engineSyncServing(this.engine, core);
|
|
3029
|
+
else return;
|
|
3030
|
+
const hub = createFeedHub(serving.feed, {
|
|
3031
|
+
...this.sync,
|
|
3032
|
+
onError: (error) => console.warn("A device sync feed failed to read its index:", error)
|
|
3033
|
+
});
|
|
3034
|
+
this.syncServing = serving;
|
|
3035
|
+
this.feeds = createFeedHost(hub, serving, authorize);
|
|
3036
|
+
registerFeedRoute(app, this.feeds.gateway);
|
|
3037
|
+
}
|
|
1975
3038
|
async registerRoutes(app) {
|
|
1976
3039
|
const cors = resolveCors(this.options.cors);
|
|
1977
3040
|
const authorize = this.options.onRequest ? authorizerFor(this.options.onRequest) : void 0;
|
|
@@ -1980,7 +3043,8 @@ var NarsilHttpServer = class {
|
|
|
1980
3043
|
const gate = createSharedGateBuffer(gateSlotOfWorker(threadCount));
|
|
1981
3044
|
const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
|
|
1982
3045
|
const { limits, build, allowRegex } = this.deps;
|
|
1983
|
-
const handlers = mainThreadHandlers(this.deps);
|
|
3046
|
+
const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
|
|
3047
|
+
this.startSync(app, authorize, core);
|
|
1984
3048
|
const run = createRouteRunner({
|
|
1985
3049
|
authorize,
|
|
1986
3050
|
gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
|
|
@@ -2000,7 +3064,8 @@ var NarsilHttpServer = class {
|
|
|
2000
3064
|
limits,
|
|
2001
3065
|
allowRegex,
|
|
2002
3066
|
build,
|
|
2003
|
-
cors
|
|
3067
|
+
cors,
|
|
3068
|
+
feeds: this.feeds
|
|
2004
3069
|
});
|
|
2005
3070
|
} catch (err) {
|
|
2006
3071
|
console.warn("The server could not start request threads, so the main thread answers every request:", err);
|