@delali/narsil 0.3.1-next.153 → 0.3.1-next.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
- package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
- package/dist/archive-history-E4R73RWD.mjs +29 -0
- package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
- package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
- package/dist/browser/chunk-23XRXIBW.mjs +85 -0
- package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
- package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
- package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
- package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
- package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
- package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
- package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
- package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
- package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
- package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
- package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
- package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
- package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
- package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
- package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
- package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
- package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
- package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
- package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
- package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
- package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
- package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
- package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
- package/dist/browser/chunk-KKAMCABR.mjs +432 -0
- package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
- package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
- package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
- package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
- package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
- package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
- package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
- package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
- package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
- package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
- package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
- package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
- package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
- package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
- package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
- package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
- package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
- package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
- package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
- package/dist/browser/import-OM6QAW45.mjs +18 -0
- package/dist/browser/methods-KTQCSIN4.mjs +30 -0
- package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
- package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
- package/dist/browser/operations-OJ3BXS55.mjs +5 -0
- package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
- package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
- package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
- package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
- package/dist/browser/retention-ASWUHAYI.mjs +209 -0
- package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
- package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
- package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
- package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
- package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
- package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
- package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
- package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
- package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
- package/dist/chunk-3HB3PEC6.mjs +240 -0
- package/dist/chunk-3HHQLEEQ.mjs +1092 -0
- package/dist/chunk-3OH67G7R.mjs +214 -0
- package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
- package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
- package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
- package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
- package/dist/chunk-5FVU47TB.mjs +20 -0
- package/dist/chunk-5YNRKI2C.mjs +81 -0
- package/dist/chunk-6L3XJXGU.mjs +470 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
- package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
- package/dist/chunk-7GXJUJ24.mjs +49 -0
- package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
- package/dist/chunk-BDRIVYOK.mjs +72 -0
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6ZDGYS.mjs +50 -0
- package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
- package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
- package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-GH4WQ22V.mjs +34 -0
- package/dist/chunk-GQPTDQCL.mjs +54 -0
- package/dist/chunk-H4KHPLHJ.mjs +382 -0
- package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
- package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
- package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
- package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
- package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
- package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
- package/dist/chunk-IPN7S4KX.mjs +130 -0
- package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
- package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
- package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
- package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
- package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
- package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
- package/dist/chunk-KJ3RTZDJ.mjs +101 -0
- package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
- package/dist/chunk-KP7TYHWT.mjs +92 -0
- package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
- package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
- package/dist/chunk-KYYABCSW.mjs +72 -0
- package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
- package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
- package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
- package/dist/chunk-MT2NRTA6.mjs +216 -0
- package/dist/chunk-NF4DT3IH.mjs +85 -0
- package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
- package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
- package/dist/chunk-PHQNACDT.mjs +63 -0
- package/dist/chunk-Q622CTMC.mjs +74 -0
- package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
- package/dist/chunk-QDZJM7NI.mjs +23 -0
- package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
- package/dist/chunk-QJVOLKU6.mjs +111 -0
- package/dist/chunk-QUJHMPTP.mjs +30 -0
- package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
- package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
- package/dist/chunk-RZWWB2XR.mjs +96 -0
- package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
- package/dist/chunk-S5LN746F.mjs +10 -0
- package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
- package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
- package/dist/chunk-SY3EWKQY.mjs +242 -0
- package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
- package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
- package/dist/chunk-UHICS3JI.mjs +83 -0
- package/dist/chunk-UZRZKNNZ.mjs +101 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
- package/dist/chunk-VHBNF4MA.mjs +543 -0
- package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
- package/dist/chunk-WQRN64Q6.mjs +196 -0
- package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
- package/dist/chunk-Y4WPNHMC.mjs +50 -0
- package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
- package/dist/chunk-YEKL4JBK.mjs +96 -0
- package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
- package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
- package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
- package/dist/chunk-ZTW5WTGG.mjs +167 -0
- package/dist/chunk-ZYAY26KG.mjs +29 -0
- package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
- package/dist/client.d.ts +9 -7
- package/dist/client.mjs +120 -7
- package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
- package/dist/config-CoSig5Ox.d.ts +482 -0
- package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
- package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +41 -9
- package/dist/distribution.mjs +9304 -7271
- package/dist/durability-config-GE4U7MCZ.mjs +89 -0
- package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
- package/dist/engine-hooks-DHCMOADY.mjs +47 -0
- package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
- package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
- package/dist/index.browser.mjs +9 -4
- package/dist/index.d.ts +7 -6
- package/dist/index.mjs +29 -21
- package/dist/index.react-native.mjs +19 -12
- package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
- package/dist/methods-7IOINERF.mjs +30 -0
- package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
- package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
- package/dist/operations-G5EP6JPX.mjs +5 -0
- package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
- package/dist/partial-words.mjs +7 -5
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
- package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
- package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +81 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +246 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +62 -0
- package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
- package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
- package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
- package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
- package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
- package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
- package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
- package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
- package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
- package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
- package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
- package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
- package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
- package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
- package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
- package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
- package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
- package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
- package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
- package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
- package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
- package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
- package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
- package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
- package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
- package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
- package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
- package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
- package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
- package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
- package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
- package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
- package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
- package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
- package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
- package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
- package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
- package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
- package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
- package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
- package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
- package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
- package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
- package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
- package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
- package/dist/react-native/partial-words.mjs +8 -5
- package/dist/react-native/pattern.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +258 -0
- package/dist/react-native/sync.mjs +2299 -0
- package/dist/react-native/synonyms.mjs +7 -4
- package/dist/react.d.ts +11 -10
- package/dist/react.mjs +8 -1
- package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
- package/dist/restore-queue-JVGENGGE.mjs +55 -0
- package/dist/retention-HROPV2QU.mjs +59 -0
- package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
- package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
- package/dist/server.d.ts +119 -20
- package/dist/server.mjs +1123 -58
- package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
- package/dist/start-UTWHGH42.mjs +59 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +271 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +237 -0
- package/dist/sync-state-ZWHLAFLE.mjs +15 -0
- package/dist/sync.browser.mjs +2294 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2268 -0
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
- package/dist/thread-6MOXY2KD.mjs +45 -0
- package/dist/types-BZCGjfLn.d.ts +227 -0
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +7 -7
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
- package/openapi.json +912 -5
- package/package.json +45 -8
- package/dist/browser/import-QOY6ZH6X.mjs +0 -12
- package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
- package/dist/chunk-7LDMWKP6.mjs +0 -209
- package/dist/engine-CJRUHXgf.d.ts +0 -664
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
- package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
- package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
- package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
- package/dist/restore-queue-AOF5OTQV.mjs +0 -41
- package/dist/thread-I7DVHHW7.mjs +0 -31
|
@@ -1,713 +1,316 @@
|
|
|
1
|
-
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { deserializePayloadV2, documentsOfPayloadV2 } from './chunk-AILVAOLX.mjs';
|
|
2
|
+
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-YEKL4JBK.mjs';
|
|
3
|
+
import { closePartitionWriter, listPersistedIndexes, removeLeftoverRewriteFiles, drainIndexStateForUnload, loadMetadata, snapshotCheckpointFor, replayWal, createMarkerWriter } from './chunk-DUOLGBTE.mjs';
|
|
4
|
+
import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, unpackEnvelopeBytes, writeLiveVectors, growPartitionsTo, metadataConfigOf, durableMetadataConfigOf } from './chunk-3R4UZWQG.mjs';
|
|
5
|
+
import { VECTOR_FILE_ENVELOPE_VERSION } from './chunk-KBDBPVLN.mjs';
|
|
5
6
|
import './chunk-ZXNNR73R.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
+
import './chunk-WZ7YV54N.mjs';
|
|
7
8
|
import './chunk-MATMVH65.mjs';
|
|
8
9
|
import './chunk-VD7J552I.mjs';
|
|
9
|
-
import {
|
|
10
|
+
import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-SY3EWKQY.mjs';
|
|
11
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-K4ZIGL2O.mjs';
|
|
12
|
+
import './chunk-BUMREPLU.mjs';
|
|
10
13
|
import './chunk-37CD3OKS.mjs';
|
|
11
|
-
import { createDurableDirectory } from './chunk-
|
|
12
|
-
import {
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
14
|
+
import { createDurableDirectory } from './chunk-ZHCUZZE5.mjs';
|
|
15
|
+
import { SEGMENT_HEADER_SIZE, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, commitCheckpointManifest, removeCheckpointGarbage, writeSegmentHeader, buildEntry, captureWholePartition, checkpointLastSeqNo } from './chunk-S2EQU2GE.mjs';
|
|
16
|
+
import './chunk-BY6ZDGYS.mjs';
|
|
17
|
+
import { readSegmentFile } from './chunk-J4IWBIVY.mjs';
|
|
18
|
+
import './chunk-KXCVQWOR.mjs';
|
|
19
|
+
import { adapterCheckpointStore, directoryCheckpointStore, readListedFile } from './chunk-UHICS3JI.mjs';
|
|
20
|
+
import './chunk-5FLRXF73.mjs';
|
|
15
21
|
import './chunk-JDDXV6UF.mjs';
|
|
16
|
-
import {
|
|
17
|
-
import './chunk-
|
|
18
|
-
import {
|
|
19
|
-
import './chunk-
|
|
20
|
-
import
|
|
21
|
-
import './chunk-
|
|
22
|
-
import {
|
|
23
|
-
import './chunk-
|
|
24
|
-
import {
|
|
22
|
+
import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-NF4DT3IH.mjs';
|
|
23
|
+
import { getFs } from './chunk-HLPRHOSO.mjs';
|
|
24
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-QDZJM7NI.mjs';
|
|
25
|
+
import { sha256 } from './chunk-LNGE7SLY.mjs';
|
|
26
|
+
import './chunk-UZRZKNNZ.mjs';
|
|
27
|
+
import './chunk-H4KHPLHJ.mjs';
|
|
28
|
+
import { spawnNodeWorker } from './chunk-BEZH7VEH.mjs';
|
|
29
|
+
import './chunk-GQPTDQCL.mjs';
|
|
30
|
+
import { QUERY_HISTORY_FOLDER } from './chunk-5YNRKI2C.mjs';
|
|
31
|
+
import { toError, packChecksummedEnvelope } from './chunk-KMUNHGBQ.mjs';
|
|
32
|
+
import './chunk-L7ZD4L3K.mjs';
|
|
25
33
|
import './chunk-HJSN7UYG.mjs';
|
|
26
|
-
import { detectRuntime, resolveWorkerEntry } from './chunk-
|
|
34
|
+
import { detectRuntime, resolveWorkerEntry } from './chunk-LHKOSI2J.mjs';
|
|
27
35
|
import './chunk-XIHUI4KB.mjs';
|
|
28
|
-
import './chunk-Q2UL2TPC.mjs';
|
|
29
36
|
import './chunk-3MXV3VHR.mjs';
|
|
30
37
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-EEBCDP36.mjs';
|
|
31
|
-
import './chunk-
|
|
38
|
+
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-Z2BE2MAM.mjs';
|
|
39
|
+
import './chunk-ZAUJFLEN.mjs';
|
|
40
|
+
import './chunk-GH4WQ22V.mjs';
|
|
41
|
+
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-3OH67G7R.mjs';
|
|
42
|
+
import './chunk-IPN7S4KX.mjs';
|
|
43
|
+
import { fnv1a } from './chunk-KAKQ74LS.mjs';
|
|
44
|
+
import { crc32 } from './chunk-7GXJUJ24.mjs';
|
|
45
|
+
import './chunk-Q2UL2TPC.mjs';
|
|
46
|
+
import './chunk-LC3FDWTX.mjs';
|
|
47
|
+
import './chunk-JUKYYXSP.mjs';
|
|
48
|
+
import './chunk-4KRC2APY.mjs';
|
|
49
|
+
import './chunk-FQBFZAWG.mjs';
|
|
50
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-WQRN64Q6.mjs';
|
|
51
|
+
import './chunk-GES7U2PM.mjs';
|
|
52
|
+
import './chunk-I54T6CQP.mjs';
|
|
53
|
+
import './chunk-YEDIDCR7.mjs';
|
|
32
54
|
import './chunk-X5CV2E2B.mjs';
|
|
55
|
+
import './chunk-Y4WPNHMC.mjs';
|
|
56
|
+
import './chunk-RZWWB2XR.mjs';
|
|
33
57
|
import './chunk-TGP5M5R7.mjs';
|
|
58
|
+
import './chunk-4BYWRYPM.mjs';
|
|
59
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
34
60
|
import './chunk-U2BKYEC7.mjs';
|
|
35
|
-
import './chunk-DGRM4EZZ.mjs';
|
|
36
61
|
import { createRequestId } from './chunk-QHPYS427.mjs';
|
|
37
62
|
import './chunk-QYRNLLLI.mjs';
|
|
63
|
+
import './chunk-HNVVEQP7.mjs';
|
|
38
64
|
import './chunk-ZXNQEW43.mjs';
|
|
65
|
+
import './chunk-L5ORED7V.mjs';
|
|
39
66
|
import './chunk-KOKM4PQN.mjs';
|
|
40
|
-
import './chunk-
|
|
41
|
-
import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
|
|
67
|
+
import './chunk-Q7G3BV3Y.mjs';
|
|
42
68
|
import './chunk-BTPGQC6Z.mjs';
|
|
43
|
-
import './chunk-
|
|
44
|
-
import './chunk-U46IZNCW.mjs';
|
|
45
|
-
import './chunk-JUKYYXSP.mjs';
|
|
46
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-SOUZMPME.mjs';
|
|
47
|
-
import './chunk-FQBFZAWG.mjs';
|
|
48
|
-
import './chunk-65WCV5JO.mjs';
|
|
49
|
-
import './chunk-4UMZ7KQL.mjs';
|
|
69
|
+
import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite } from './chunk-LQG5G2LY.mjs';
|
|
50
70
|
import { MIN_M } from './chunk-ZXX5GZEX.mjs';
|
|
51
|
-
import './chunk-
|
|
52
|
-
import './chunk-L5ORED7V.mjs';
|
|
71
|
+
import './chunk-RFBLQLLZ.mjs';
|
|
53
72
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
54
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
73
|
+
import { NarsilError, ErrorCodes } from './chunk-MP7ABPTD.mjs';
|
|
55
74
|
import './chunk-R6A773WK.mjs';
|
|
56
75
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
57
|
-
import { decode } from '@msgpack/msgpack';
|
|
76
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
58
77
|
|
|
59
|
-
// src/persistence/durability/
|
|
60
|
-
var
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
3049323471,
|
|
64
|
-
3921009573,
|
|
65
|
-
961987163,
|
|
66
|
-
1508970993,
|
|
67
|
-
2453635748,
|
|
68
|
-
2870763221,
|
|
69
|
-
3624381080,
|
|
70
|
-
310598401,
|
|
71
|
-
607225278,
|
|
72
|
-
1426881987,
|
|
73
|
-
1925078388,
|
|
74
|
-
2162078206,
|
|
75
|
-
2614888103,
|
|
76
|
-
3248222580,
|
|
77
|
-
3835390401,
|
|
78
|
-
4022224774,
|
|
79
|
-
264347078,
|
|
80
|
-
604807628,
|
|
81
|
-
770255983,
|
|
82
|
-
1249150122,
|
|
83
|
-
1555081692,
|
|
84
|
-
1996064986,
|
|
85
|
-
2554220882,
|
|
86
|
-
2821834349,
|
|
87
|
-
2952996808,
|
|
88
|
-
3210313671,
|
|
89
|
-
3336571891,
|
|
90
|
-
3584528711,
|
|
91
|
-
113926993,
|
|
92
|
-
338241895,
|
|
93
|
-
666307205,
|
|
94
|
-
773529912,
|
|
95
|
-
1294757372,
|
|
96
|
-
1396182291,
|
|
97
|
-
1695183700,
|
|
98
|
-
1986661051,
|
|
99
|
-
2177026350,
|
|
100
|
-
2456956037,
|
|
101
|
-
2730485921,
|
|
102
|
-
2820302411,
|
|
103
|
-
3259730800,
|
|
104
|
-
3345764771,
|
|
105
|
-
3516065817,
|
|
106
|
-
3600352804,
|
|
107
|
-
4094571909,
|
|
108
|
-
275423344,
|
|
109
|
-
430227734,
|
|
110
|
-
506948616,
|
|
111
|
-
659060556,
|
|
112
|
-
883997877,
|
|
113
|
-
958139571,
|
|
114
|
-
1322822218,
|
|
115
|
-
1537002063,
|
|
116
|
-
1747873779,
|
|
117
|
-
1955562222,
|
|
118
|
-
2024104815,
|
|
119
|
-
2227730452,
|
|
120
|
-
2361852424,
|
|
121
|
-
2428436474,
|
|
122
|
-
2756734187,
|
|
123
|
-
3204031479,
|
|
124
|
-
3329325298
|
|
125
|
-
]);
|
|
126
|
-
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
127
|
-
var BLOCK_BYTES = 64;
|
|
128
|
-
var LENGTH_BYTES = 8;
|
|
129
|
-
function rotateRight(value, bits) {
|
|
130
|
-
return value >>> bits | value << 32 - bits;
|
|
131
|
-
}
|
|
132
|
-
function padded(bytes) {
|
|
133
|
-
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
134
|
-
const message = new Uint8Array(total);
|
|
135
|
-
message.set(bytes);
|
|
136
|
-
message[bytes.byteLength] = 128;
|
|
137
|
-
const view = new DataView(message.buffer);
|
|
138
|
-
const bitLength = bytes.byteLength * 8;
|
|
139
|
-
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
140
|
-
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
141
|
-
return view;
|
|
142
|
-
}
|
|
143
|
-
function sha256(bytes) {
|
|
144
|
-
const view = padded(bytes);
|
|
145
|
-
const state = INITIAL_STATE.slice();
|
|
146
|
-
const words = new Uint32Array(64);
|
|
147
|
-
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
148
|
-
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
149
|
-
for (let i = 16; i < 64; i++) {
|
|
150
|
-
const w15 = words[i - 15];
|
|
151
|
-
const w2 = words[i - 2];
|
|
152
|
-
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
153
|
-
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
154
|
-
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
155
|
-
}
|
|
156
|
-
let [a, b, c, d, e, f, g, h] = state;
|
|
157
|
-
for (let i = 0; i < 64; i++) {
|
|
158
|
-
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
159
|
-
const choice = e & f ^ ~e & g;
|
|
160
|
-
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
161
|
-
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
162
|
-
const majority = a & b ^ a & c ^ b & c;
|
|
163
|
-
const t2 = s0 + majority >>> 0;
|
|
164
|
-
h = g;
|
|
165
|
-
g = f;
|
|
166
|
-
f = e;
|
|
167
|
-
e = d + t1 >>> 0;
|
|
168
|
-
d = c;
|
|
169
|
-
c = b;
|
|
170
|
-
b = a;
|
|
171
|
-
a = t1 + t2 >>> 0;
|
|
172
|
-
}
|
|
173
|
-
state[0] = state[0] + a >>> 0;
|
|
174
|
-
state[1] = state[1] + b >>> 0;
|
|
175
|
-
state[2] = state[2] + c >>> 0;
|
|
176
|
-
state[3] = state[3] + d >>> 0;
|
|
177
|
-
state[4] = state[4] + e >>> 0;
|
|
178
|
-
state[5] = state[5] + f >>> 0;
|
|
179
|
-
state[6] = state[6] + g >>> 0;
|
|
180
|
-
state[7] = state[7] + h >>> 0;
|
|
181
|
-
}
|
|
182
|
-
const digest = new Uint8Array(32);
|
|
183
|
-
const out = new DataView(digest.buffer);
|
|
184
|
-
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
185
|
-
return digest;
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
// src/persistence/durability/restore-journal.ts
|
|
189
|
-
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
190
|
-
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
191
|
-
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
192
|
-
var LOCK_FILE_KEY = ".narsil.lock";
|
|
193
|
-
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
194
|
-
function digestOf(indexName) {
|
|
195
|
-
let hex = "";
|
|
196
|
-
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
197
|
-
return hex;
|
|
78
|
+
// src/persistence/durability/sync/layout.ts
|
|
79
|
+
var SYNC_KEY_NUMBER_WIDTH = 16;
|
|
80
|
+
function syncDirectoryPrefix(indexName) {
|
|
81
|
+
return `${indexName}/sync/`;
|
|
198
82
|
}
|
|
199
|
-
function
|
|
200
|
-
return `${
|
|
83
|
+
function publishedSnapshotKey(indexName) {
|
|
84
|
+
return `${syncDirectoryPrefix(indexName)}published`;
|
|
201
85
|
}
|
|
202
|
-
function
|
|
203
|
-
return `${
|
|
86
|
+
function historyRoot(indexName) {
|
|
87
|
+
return `${indexName}/history/`;
|
|
204
88
|
}
|
|
205
|
-
function
|
|
206
|
-
return `${
|
|
89
|
+
function historyPrefix(indexName, partitionId) {
|
|
90
|
+
return `${historyRoot(indexName)}${partitionId}/`;
|
|
207
91
|
}
|
|
208
|
-
function
|
|
209
|
-
return `${
|
|
92
|
+
function walSegmentPrefix(indexName, partitionId) {
|
|
93
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
210
94
|
}
|
|
211
|
-
function
|
|
212
|
-
return
|
|
213
|
-
|
|
95
|
+
function formatSyncKeyNumber(value) {
|
|
96
|
+
return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
|
|
97
|
+
}
|
|
98
|
+
function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
|
|
99
|
+
return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
|
|
100
|
+
}
|
|
101
|
+
function walSegmentKey(indexName, partitionId, startSeqNo) {
|
|
102
|
+
return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
|
|
103
|
+
}
|
|
104
|
+
var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
|
|
105
|
+
function parseHistoryKey(indexName, key) {
|
|
106
|
+
const root = historyRoot(indexName);
|
|
107
|
+
if (!key.startsWith(root)) return null;
|
|
108
|
+
const match = HISTORY_TAIL.exec(key.slice(root.length));
|
|
109
|
+
if (match === null) return null;
|
|
110
|
+
const partitionId = Number.parseInt(match[1], 10);
|
|
111
|
+
const startSeqNo = Number.parseInt(match[2], 10);
|
|
112
|
+
const movedAtMs = Number.parseInt(match[3], 10);
|
|
113
|
+
if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
return { key, partitionId, startSeqNo, movedAtMs };
|
|
214
117
|
}
|
|
215
|
-
function
|
|
216
|
-
|
|
118
|
+
function parseWalSegmentStart(indexName, partitionId, key) {
|
|
119
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
120
|
+
if (!key.startsWith(prefix)) return null;
|
|
121
|
+
const tail = key.slice(prefix.length);
|
|
122
|
+
if (!/^\d{16}$/.test(tail)) return null;
|
|
123
|
+
const value = Number.parseInt(tail, 10);
|
|
124
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
217
125
|
}
|
|
218
|
-
|
|
219
|
-
|
|
126
|
+
|
|
127
|
+
// src/persistence/durability/sync/history.ts
|
|
128
|
+
function compareHistorySegments(a, b) {
|
|
129
|
+
return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
|
|
130
|
+
}
|
|
131
|
+
async function listHistorySegments(directory, indexName, partitionId) {
|
|
132
|
+
const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
|
|
133
|
+
const segments = [];
|
|
134
|
+
for (const key of await directory.list(prefix)) {
|
|
135
|
+
const segment = parseHistoryKey(indexName, key);
|
|
136
|
+
if (segment !== null) segments.push(segment);
|
|
137
|
+
}
|
|
138
|
+
return segments.sort(compareHistorySegments);
|
|
139
|
+
}
|
|
140
|
+
async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
|
|
141
|
+
await directory.move(
|
|
142
|
+
walSegmentKey(indexName, partitionId, startSeqNo),
|
|
143
|
+
historyKey(indexName, partitionId, startSeqNo, movedAtMs)
|
|
144
|
+
);
|
|
220
145
|
}
|
|
221
|
-
function
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
225
|
-
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
226
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
227
|
-
corruptJournal(key, "names an index that no index name can match");
|
|
228
|
-
}
|
|
229
|
-
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
230
|
-
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
231
|
-
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
146
|
+
async function removeHistory(directory, indexName, partitionId) {
|
|
147
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
148
|
+
await directory.remove(segment.key);
|
|
232
149
|
}
|
|
233
|
-
return { kind: "journal", indexName, stagingPrefix };
|
|
234
150
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
240
|
-
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
241
|
-
{ indexName, stagingPrefix }
|
|
242
|
-
);
|
|
243
|
-
}
|
|
244
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
245
|
-
if (decoded.kind !== "current") {
|
|
246
|
-
throw new NarsilError(
|
|
247
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
248
|
-
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
249
|
-
{ indexName }
|
|
250
|
-
);
|
|
251
|
-
}
|
|
252
|
-
const root = `${indexName}/`;
|
|
253
|
-
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
254
|
-
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
255
|
-
return keys;
|
|
151
|
+
|
|
152
|
+
// src/persistence/durability/checkpoint.ts
|
|
153
|
+
function walPrefix(indexName, partitionId) {
|
|
154
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
256
155
|
}
|
|
257
|
-
function
|
|
258
|
-
|
|
156
|
+
async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
|
|
157
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
158
|
+
const prefix = walPrefix(indexName, partitionId);
|
|
159
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
160
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
161
|
+
if (marker === null) {
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
165
|
+
const segments = await directory.list(prefix);
|
|
166
|
+
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
167
|
+
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
168
|
+
const start = startSeqNos[i];
|
|
169
|
+
if (start >= activeSegmentSeqNo) {
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
const nextStart = startSeqNos[i + 1];
|
|
173
|
+
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
174
|
+
if (segmentCoversBeyondCheckpoint) {
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
178
|
+
else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
259
181
|
}
|
|
260
|
-
async function
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
268
|
-
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
269
|
-
} else {
|
|
270
|
-
const stagedKeys = new Set(staged);
|
|
271
|
-
for (const key of listed) {
|
|
272
|
-
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
182
|
+
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
183
|
+
for (const [partitionId, partition] of [...partitions]) {
|
|
184
|
+
if (partitionId < partitionCount) continue;
|
|
185
|
+
try {
|
|
186
|
+
await partition.walWriter.close();
|
|
187
|
+
} catch (err) {
|
|
188
|
+
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
273
189
|
}
|
|
190
|
+
partitions.delete(partitionId);
|
|
274
191
|
}
|
|
275
|
-
|
|
276
|
-
|
|
192
|
+
const walRoot = `${indexName}/wal/`;
|
|
193
|
+
for (const key of await directory.list(walRoot)) {
|
|
194
|
+
const partitionId = walKeyPartitionId(key, walRoot);
|
|
195
|
+
if (partitionId === null || partitionId < partitionCount) continue;
|
|
196
|
+
await directory.remove(key);
|
|
277
197
|
}
|
|
278
|
-
await storage.remove(journalKey(indexName));
|
|
279
|
-
return retiredPrefix;
|
|
280
198
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
199
|
+
function walKeyPartitionId(key, walRoot) {
|
|
200
|
+
const tail = key.slice(walRoot.length);
|
|
201
|
+
const separator = tail.indexOf("/");
|
|
202
|
+
if (separator <= 0) {
|
|
203
|
+
return null;
|
|
285
204
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
290
|
-
}
|
|
291
|
-
function isTopLevelJournal(key) {
|
|
292
|
-
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
293
|
-
}
|
|
294
|
-
async function removeStagingPrefixes(storage) {
|
|
295
|
-
const prefixes = /* @__PURE__ */ new Set();
|
|
296
|
-
for (const key of await storage.list(".")) {
|
|
297
|
-
const slash = key.indexOf("/");
|
|
298
|
-
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
205
|
+
const segment = tail.slice(0, separator);
|
|
206
|
+
if (!/^\d+$/.test(segment)) {
|
|
207
|
+
return null;
|
|
299
208
|
}
|
|
300
|
-
|
|
209
|
+
const value = Number.parseInt(segment, 10);
|
|
210
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
301
211
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const bytes = await storage.read(key);
|
|
306
|
-
if (bytes === null) continue;
|
|
307
|
-
const reading = readJournal(key, bytes);
|
|
308
|
-
if (reading.kind === "leftover") {
|
|
309
|
-
await storage.remove(key);
|
|
310
|
-
continue;
|
|
311
|
-
}
|
|
312
|
-
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
212
|
+
function parseSegmentStartSeqNo(key, prefix) {
|
|
213
|
+
if (!key.startsWith(prefix)) {
|
|
214
|
+
return null;
|
|
313
215
|
}
|
|
314
|
-
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
return
|
|
216
|
+
const tail = key.slice(prefix.length);
|
|
217
|
+
if (!/^\d{16}$/.test(tail)) {
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
const value = Number.parseInt(tail, 10);
|
|
221
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
320
222
|
}
|
|
321
|
-
function
|
|
322
|
-
return
|
|
223
|
+
function formatSegmentStart(startSeqNo) {
|
|
224
|
+
return startSeqNo.toString().padStart(16, "0");
|
|
323
225
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
pathOf: (key) => directory.pathOf(key),
|
|
334
|
-
async moveTree(from, to) {
|
|
335
|
-
const source = await directory.pathOf(from);
|
|
336
|
-
const target = await directory.pathOf(to);
|
|
337
|
-
const fs = await getFs();
|
|
338
|
-
try {
|
|
339
|
-
await fs.rename(source, target);
|
|
340
|
-
} catch (error) {
|
|
341
|
-
if (missing(error)) return;
|
|
342
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
343
|
-
}
|
|
344
|
-
await directory.syncDirectoryOf(to);
|
|
345
|
-
},
|
|
346
|
-
async removeTree(prefix) {
|
|
347
|
-
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
348
|
-
const fs = await getFs();
|
|
226
|
+
|
|
227
|
+
// src/persistence/durability/checkpoint-capture.ts
|
|
228
|
+
function whileNoMutationApplies(partitions, capture2) {
|
|
229
|
+
return new Promise((resolve, reject) => {
|
|
230
|
+
let release = () => void 0;
|
|
231
|
+
const released = new Promise((settle) => {
|
|
232
|
+
release = settle;
|
|
233
|
+
});
|
|
234
|
+
function captureNow() {
|
|
349
235
|
try {
|
|
350
|
-
|
|
236
|
+
resolve(capture2());
|
|
351
237
|
} catch (error) {
|
|
352
|
-
|
|
238
|
+
reject(error);
|
|
239
|
+
} finally {
|
|
240
|
+
release();
|
|
353
241
|
}
|
|
354
|
-
await directory.syncDirectoryOf(key);
|
|
355
242
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// src/persistence/durability/restore-staging.ts
|
|
363
|
-
function toError2(error) {
|
|
364
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
365
|
-
}
|
|
366
|
-
function pathUnder(path, root) {
|
|
367
|
-
if (!path.startsWith(root)) return false;
|
|
368
|
-
const next = path.charAt(root.length);
|
|
369
|
-
return next === "" || next === "/" || next === "\\";
|
|
370
|
-
}
|
|
371
|
-
function stagedDirectory(directory, staged, unstaged) {
|
|
372
|
-
const { readRange, byteLength } = directory;
|
|
373
|
-
return {
|
|
374
|
-
root: directory.root,
|
|
375
|
-
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
376
|
-
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
377
|
-
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
378
|
-
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
379
|
-
read: (key) => directory.read(staged(key)),
|
|
380
|
-
remove: (key) => directory.remove(staged(key)),
|
|
381
|
-
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
382
|
-
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
383
|
-
pathOf: (key) => directory.pathOf(staged(key)),
|
|
384
|
-
...readRange === void 0 ? {} : {
|
|
385
|
-
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
386
|
-
},
|
|
387
|
-
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
388
|
-
};
|
|
389
|
-
}
|
|
390
|
-
function createRestoreStaging(storage, indexName, options = {}) {
|
|
391
|
-
const stagingPrefix = newStagingPrefix();
|
|
392
|
-
const indexRoot = `${indexName}/`;
|
|
393
|
-
let committed = false;
|
|
394
|
-
const staged = (key) => {
|
|
395
|
-
if (!key.startsWith(indexRoot)) {
|
|
396
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
397
|
-
key
|
|
398
|
-
});
|
|
243
|
+
let waiting = partitions.length;
|
|
244
|
+
if (waiting === 0) {
|
|
245
|
+
captureNow();
|
|
246
|
+
return;
|
|
399
247
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
remove: (key) => storage.remove(staged(key)),
|
|
407
|
-
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
408
|
-
};
|
|
409
|
-
const pathOf = storage.pathOf;
|
|
410
|
-
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
411
|
-
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
412
|
-
let retiredPrefix = null;
|
|
413
|
-
async function finishOnStorage() {
|
|
414
|
-
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
415
|
-
}
|
|
416
|
-
return {
|
|
417
|
-
store,
|
|
418
|
-
async relocation() {
|
|
419
|
-
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
420
|
-
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
421
|
-
const to = await pathOf.call(storage, indexName);
|
|
422
|
-
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
423
|
-
},
|
|
424
|
-
async commit() {
|
|
425
|
-
try {
|
|
426
|
-
await commitJournal(storage, indexName, stagingPrefix);
|
|
427
|
-
} catch (error) {
|
|
428
|
-
try {
|
|
429
|
-
await storage.remove(journalKey(indexName));
|
|
430
|
-
} catch (removal) {
|
|
431
|
-
committed = true;
|
|
432
|
-
options.onStorageLost?.(toError2(removal));
|
|
433
|
-
}
|
|
434
|
-
throw error;
|
|
435
|
-
}
|
|
436
|
-
committed = true;
|
|
437
|
-
},
|
|
438
|
-
async finish() {
|
|
439
|
-
try {
|
|
440
|
-
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
441
|
-
} catch (error) {
|
|
442
|
-
options.onStorageLost?.(toError2(error));
|
|
443
|
-
}
|
|
444
|
-
},
|
|
445
|
-
async removeRetired() {
|
|
446
|
-
const prefix = retiredPrefix;
|
|
447
|
-
retiredPrefix = null;
|
|
448
|
-
if (prefix !== null) await removePrefix(storage, prefix);
|
|
449
|
-
},
|
|
450
|
-
async discard() {
|
|
451
|
-
if (committed) return;
|
|
452
|
-
await removePrefix(storage, stagingPrefix);
|
|
248
|
+
for (const partition of partitions) {
|
|
249
|
+
partition.appendChain = partition.appendChain.then(() => {
|
|
250
|
+
waiting -= 1;
|
|
251
|
+
if (waiting === 0) captureNow();
|
|
252
|
+
return released;
|
|
253
|
+
});
|
|
453
254
|
}
|
|
454
|
-
};
|
|
255
|
+
});
|
|
455
256
|
}
|
|
456
|
-
function
|
|
457
|
-
const
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
async moveKey(from, to) {
|
|
465
|
-
const bytes = await store.read(from);
|
|
466
|
-
if (bytes === null) return;
|
|
467
|
-
await store.write(to, [bytes]);
|
|
468
|
-
await store.remove(from);
|
|
257
|
+
async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
|
|
258
|
+
for (const [partitionId, partition] of [...indexState.partitions]) {
|
|
259
|
+
await partition.appendChain;
|
|
260
|
+
if (partition.failed !== null) continue;
|
|
261
|
+
try {
|
|
262
|
+
await partition.walWriter.commit();
|
|
263
|
+
} catch (error) {
|
|
264
|
+
failPartition2(partitionId, partition, error);
|
|
469
265
|
}
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
// src/persistence/durability/archive-checkpoint.ts
|
|
474
|
-
function toError3(error) {
|
|
475
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
266
|
+
}
|
|
476
267
|
}
|
|
477
|
-
function
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
268
|
+
async function openNewLogSegments(indexState, failPartition2) {
|
|
269
|
+
const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
|
|
270
|
+
try {
|
|
271
|
+
if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
|
|
272
|
+
} catch (error) {
|
|
273
|
+
failPartition2(partitionId, partition, error);
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
|
|
277
|
+
failPartition2(partitionId, partition, error);
|
|
278
|
+
});
|
|
279
|
+
partition.appendChain = rolled;
|
|
280
|
+
await rolled;
|
|
488
281
|
});
|
|
282
|
+
await Promise.all(rolls);
|
|
489
283
|
}
|
|
490
|
-
function
|
|
491
|
-
|
|
492
|
-
return () => {
|
|
493
|
-
if (released) return;
|
|
494
|
-
released = true;
|
|
495
|
-
const count = (held.get(indexName) ?? 1) - 1;
|
|
496
|
-
if (count <= 0) held.delete(indexName);
|
|
497
|
-
else held.set(indexName, count);
|
|
498
|
-
};
|
|
284
|
+
function aTargetFailed(indexState, targets) {
|
|
285
|
+
return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
|
|
499
286
|
}
|
|
500
|
-
function
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
525
|
-
readRange: async (ref, offset, length) => {
|
|
526
|
-
const readRange = deps.directory.readRange;
|
|
527
|
-
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
528
|
-
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
529
|
-
return whole.slice(offset, offset + length);
|
|
530
|
-
},
|
|
531
|
-
release,
|
|
532
|
-
directory: deps.directory
|
|
533
|
-
};
|
|
534
|
-
} catch (error) {
|
|
535
|
-
release();
|
|
536
|
-
throw error;
|
|
537
|
-
}
|
|
538
|
-
},
|
|
539
|
-
stageRestore(indexName) {
|
|
540
|
-
return stageDirectoryRestore(deps, indexName);
|
|
541
|
-
},
|
|
542
|
-
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
543
|
-
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
544
|
-
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
287
|
+
function noWholePartitions() {
|
|
288
|
+
return { contents: [], captured: [] };
|
|
289
|
+
}
|
|
290
|
+
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
291
|
+
const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
|
|
292
|
+
return (targets) => {
|
|
293
|
+
const whole = { contents: [], captured: [] };
|
|
294
|
+
const changed = targets.filter(
|
|
295
|
+
(target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
296
|
+
);
|
|
297
|
+
const mostOfEachChanged = changed.every((target) => {
|
|
298
|
+
const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
|
|
299
|
+
const documents = manager.getPartition(target.partitionId).count();
|
|
300
|
+
return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
|
|
301
|
+
});
|
|
302
|
+
const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
|
|
303
|
+
if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
|
|
304
|
+
for (const { partitionId } of changed) {
|
|
305
|
+
const capture2 = captureRecords?.(partitionId);
|
|
306
|
+
const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
|
|
307
|
+
if (captured !== null) {
|
|
308
|
+
whole.captured.push(captured);
|
|
309
|
+
} else if (everyPartition || !aWorkerCanSerialise) {
|
|
310
|
+
whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
|
|
545
311
|
}
|
|
546
312
|
}
|
|
547
|
-
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
// src/persistence/durability/checkpoint.ts
|
|
551
|
-
function walPrefix(indexName, partitionId) {
|
|
552
|
-
return `${indexName}/wal/${partitionId}/`;
|
|
553
|
-
}
|
|
554
|
-
async function truncateCoveredSegments(directory, indexName, checkpoint) {
|
|
555
|
-
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
556
|
-
const prefix = walPrefix(indexName, partitionId);
|
|
557
|
-
const markerBytes = await directory.read(`${prefix}commit`);
|
|
558
|
-
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
559
|
-
if (marker === null) {
|
|
560
|
-
continue;
|
|
561
|
-
}
|
|
562
|
-
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
563
|
-
const segments = await directory.list(prefix);
|
|
564
|
-
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
565
|
-
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
566
|
-
const start = startSeqNos[i];
|
|
567
|
-
if (start >= activeSegmentSeqNo) {
|
|
568
|
-
break;
|
|
569
|
-
}
|
|
570
|
-
const nextStart = startSeqNos[i + 1];
|
|
571
|
-
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
572
|
-
if (segmentCoversBeyondCheckpoint) {
|
|
573
|
-
break;
|
|
574
|
-
}
|
|
575
|
-
await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
580
|
-
for (const [partitionId, partition] of [...partitions]) {
|
|
581
|
-
if (partitionId < partitionCount) continue;
|
|
582
|
-
try {
|
|
583
|
-
await partition.walWriter.close();
|
|
584
|
-
} catch (err) {
|
|
585
|
-
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
586
|
-
}
|
|
587
|
-
partitions.delete(partitionId);
|
|
588
|
-
}
|
|
589
|
-
const walRoot = `${indexName}/wal/`;
|
|
590
|
-
for (const key of await directory.list(walRoot)) {
|
|
591
|
-
const partitionId = walKeyPartitionId(key, walRoot);
|
|
592
|
-
if (partitionId === null || partitionId < partitionCount) continue;
|
|
593
|
-
await directory.remove(key);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
function walKeyPartitionId(key, walRoot) {
|
|
597
|
-
const tail = key.slice(walRoot.length);
|
|
598
|
-
const separator = tail.indexOf("/");
|
|
599
|
-
if (separator <= 0) {
|
|
600
|
-
return null;
|
|
601
|
-
}
|
|
602
|
-
const segment = tail.slice(0, separator);
|
|
603
|
-
if (!/^\d+$/.test(segment)) {
|
|
604
|
-
return null;
|
|
605
|
-
}
|
|
606
|
-
const value = Number.parseInt(segment, 10);
|
|
607
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
608
|
-
}
|
|
609
|
-
function parseSegmentStartSeqNo(key, prefix) {
|
|
610
|
-
if (!key.startsWith(prefix)) {
|
|
611
|
-
return null;
|
|
612
|
-
}
|
|
613
|
-
const tail = key.slice(prefix.length);
|
|
614
|
-
if (!/^\d{16}$/.test(tail)) {
|
|
615
|
-
return null;
|
|
616
|
-
}
|
|
617
|
-
const value = Number.parseInt(tail, 10);
|
|
618
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
619
|
-
}
|
|
620
|
-
function formatSegmentStart(startSeqNo) {
|
|
621
|
-
return startSeqNo.toString().padStart(16, "0");
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
// src/persistence/durability/checkpoint-capture.ts
|
|
625
|
-
function whileNoMutationApplies(partitions, capture2) {
|
|
626
|
-
return new Promise((resolve, reject) => {
|
|
627
|
-
let release = () => void 0;
|
|
628
|
-
const released = new Promise((settle) => {
|
|
629
|
-
release = settle;
|
|
630
|
-
});
|
|
631
|
-
function captureNow() {
|
|
632
|
-
try {
|
|
633
|
-
resolve(capture2());
|
|
634
|
-
} catch (error) {
|
|
635
|
-
reject(error);
|
|
636
|
-
} finally {
|
|
637
|
-
release();
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
let waiting = partitions.length;
|
|
641
|
-
if (waiting === 0) {
|
|
642
|
-
captureNow();
|
|
643
|
-
return;
|
|
644
|
-
}
|
|
645
|
-
for (const partition of partitions) {
|
|
646
|
-
partition.appendChain = partition.appendChain.then(() => {
|
|
647
|
-
waiting -= 1;
|
|
648
|
-
if (waiting === 0) captureNow();
|
|
649
|
-
return released;
|
|
650
|
-
});
|
|
651
|
-
}
|
|
652
|
-
});
|
|
653
|
-
}
|
|
654
|
-
async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
|
|
655
|
-
for (const [partitionId, partition] of [...indexState.partitions]) {
|
|
656
|
-
await partition.appendChain;
|
|
657
|
-
if (partition.failed !== null) continue;
|
|
658
|
-
try {
|
|
659
|
-
await partition.walWriter.commit();
|
|
660
|
-
} catch (error) {
|
|
661
|
-
failPartition2(partitionId, partition, error);
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
async function openNewLogSegments(indexState, failPartition2) {
|
|
666
|
-
const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
|
|
667
|
-
try {
|
|
668
|
-
if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
|
|
669
|
-
} catch (error) {
|
|
670
|
-
failPartition2(partitionId, partition, error);
|
|
671
|
-
return;
|
|
672
|
-
}
|
|
673
|
-
const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
|
|
674
|
-
failPartition2(partitionId, partition, error);
|
|
675
|
-
});
|
|
676
|
-
partition.appendChain = rolled;
|
|
677
|
-
await rolled;
|
|
678
|
-
});
|
|
679
|
-
await Promise.all(rolls);
|
|
680
|
-
}
|
|
681
|
-
function aTargetFailed(indexState, targets) {
|
|
682
|
-
return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
|
|
683
|
-
}
|
|
684
|
-
function noWholePartitions() {
|
|
685
|
-
return { contents: [], captured: [] };
|
|
686
|
-
}
|
|
687
|
-
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
688
|
-
const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise, captureRecords } = rules;
|
|
689
|
-
return (targets) => {
|
|
690
|
-
const whole = { contents: [], captured: [] };
|
|
691
|
-
const changed = targets.filter(
|
|
692
|
-
(target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
693
|
-
);
|
|
694
|
-
const mostOfEachChanged = changed.every((target) => {
|
|
695
|
-
const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
|
|
696
|
-
const documents = manager.getPartition(target.partitionId).count();
|
|
697
|
-
return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
|
|
698
|
-
});
|
|
699
|
-
const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
|
|
700
|
-
if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
|
|
701
|
-
for (const { partitionId } of changed) {
|
|
702
|
-
const capture2 = captureRecords?.(partitionId);
|
|
703
|
-
const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId, capture2) : null;
|
|
704
|
-
if (captured !== null) {
|
|
705
|
-
whole.captured.push(captured);
|
|
706
|
-
} else if (everyPartition || !aWorkerCanSerialise) {
|
|
707
|
-
whole.contents.push(partitionContentsOf(manager, partitionId, capture2));
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
return whole;
|
|
313
|
+
return whole;
|
|
711
314
|
};
|
|
712
315
|
}
|
|
713
316
|
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured, pinsRetiredVectors = false) {
|
|
@@ -1117,7 +720,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1117
720
|
}
|
|
1118
721
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1119
722
|
const { indexName, indexState, hooks } = input;
|
|
1120
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
723
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-DWKPBSMN.mjs');
|
|
1121
724
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1122
725
|
if (adoption === null) return [];
|
|
1123
726
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -1172,10 +775,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1172
775
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1173
776
|
if (!input.holdsCheckpointFiles()) {
|
|
1174
777
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1175
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
778
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-DWKPBSMN.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
779
|
+
const keptForSync = await input.keepsCheckpointKey();
|
|
1176
780
|
await removeCheckpointGarbage(
|
|
1177
781
|
store,
|
|
1178
|
-
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
782
|
+
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
|
|
1179
783
|
);
|
|
1180
784
|
}
|
|
1181
785
|
const checkpointDocumentCount = documentCount;
|
|
@@ -1184,222 +788,1036 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1184
788
|
if (checkpointMetadata === void 0) {
|
|
1185
789
|
return;
|
|
1186
790
|
}
|
|
1187
|
-
const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
1188
|
-
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
1189
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
|
|
1190
|
-
});
|
|
1191
|
-
|
|
1192
|
-
await
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
input.
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
const
|
|
1202
|
-
|
|
791
|
+
const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
792
|
+
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
793
|
+
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
|
|
794
|
+
});
|
|
795
|
+
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
796
|
+
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
797
|
+
if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
|
|
798
|
+
await reclaimWalBeyondCount(
|
|
799
|
+
input.directory,
|
|
800
|
+
input.indexName,
|
|
801
|
+
manager.partitionCount,
|
|
802
|
+
input.indexState.partitions,
|
|
803
|
+
input.markFatal
|
|
804
|
+
);
|
|
805
|
+
const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
|
|
806
|
+
noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
|
|
807
|
+
const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
|
|
808
|
+
return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
|
|
809
|
+
}
|
|
810
|
+
function recordsBeyond(indexState, targets) {
|
|
811
|
+
let records = 0;
|
|
812
|
+
for (const [partitionId, partition] of indexState.partitions) {
|
|
813
|
+
if (partition.failed !== null) continue;
|
|
814
|
+
records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
|
|
815
|
+
}
|
|
816
|
+
return records;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
// src/persistence/durability/checkpoint-turn.ts
|
|
820
|
+
function createIndexLoads(vectorIndexesOf2) {
|
|
821
|
+
const loads = /* @__PURE__ */ new Map();
|
|
822
|
+
const startGraphRepairs = (indexName) => {
|
|
823
|
+
for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
|
|
824
|
+
vectorIndex.scheduleBuild();
|
|
825
|
+
vectorIndex.startShuffledRebuildWhenDue();
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
return {
|
|
829
|
+
track(indexName, load, startsGraphRepairs = true) {
|
|
830
|
+
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
831
|
+
loads.set(
|
|
832
|
+
indexName,
|
|
833
|
+
loaded.catch(() => void 0)
|
|
834
|
+
);
|
|
835
|
+
return loaded;
|
|
836
|
+
},
|
|
837
|
+
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
838
|
+
startGraphRepairs
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
async function switchInTurn(turns, turn) {
|
|
842
|
+
let swapped = false;
|
|
843
|
+
await checkpointInTurn(turns, {
|
|
844
|
+
joinsInFlight: false,
|
|
845
|
+
mayStart: turn.mayStart,
|
|
846
|
+
perform: async () => {
|
|
847
|
+
swapped = turn.swap();
|
|
848
|
+
if (swapped) await turn.checkpoint();
|
|
849
|
+
},
|
|
850
|
+
afterSettling: turn.afterSettling
|
|
851
|
+
});
|
|
852
|
+
return swapped;
|
|
853
|
+
}
|
|
854
|
+
async function checkpointInTurn(turns, turn) {
|
|
855
|
+
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
856
|
+
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
857
|
+
if (!turn.mayStart()) return;
|
|
858
|
+
const run = turn.perform().finally(() => {
|
|
859
|
+
turns.checkpointInFlight = null;
|
|
860
|
+
turn.afterSettling();
|
|
861
|
+
});
|
|
862
|
+
turns.checkpointInFlight = run;
|
|
863
|
+
return run;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// src/persistence/durability/held-checkpoints.ts
|
|
867
|
+
function failedPartitionIds(indexState) {
|
|
868
|
+
const failed = [];
|
|
869
|
+
for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
|
|
870
|
+
return failed;
|
|
871
|
+
}
|
|
872
|
+
function createHeldCheckpoints(hooks) {
|
|
873
|
+
const reported = /* @__PURE__ */ new WeakSet();
|
|
874
|
+
function heldReport(indexName, indexState) {
|
|
875
|
+
const partitionIds = failedPartitionIds(indexState);
|
|
876
|
+
if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
|
|
877
|
+
return {
|
|
878
|
+
indexName,
|
|
879
|
+
partitionIds,
|
|
880
|
+
recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
|
|
881
|
+
documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
return {
|
|
885
|
+
refuse(indexName, indexState) {
|
|
886
|
+
const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
|
|
887
|
+
if (partitionIds.length === 0) return;
|
|
888
|
+
throw new NarsilError(
|
|
889
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
890
|
+
`Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
|
|
891
|
+
{ indexName, partitionIds }
|
|
892
|
+
);
|
|
893
|
+
},
|
|
894
|
+
holds(indexName, indexState) {
|
|
895
|
+
const held = heldReport(indexName, indexState);
|
|
896
|
+
if (held === null) return false;
|
|
897
|
+
const unreported = held.partitionIds.some((partitionId) => {
|
|
898
|
+
const partition = indexState.partitions.get(partitionId);
|
|
899
|
+
return partition !== void 0 && !reported.has(partition);
|
|
900
|
+
});
|
|
901
|
+
if (unreported) {
|
|
902
|
+
for (const partitionId of held.partitionIds) {
|
|
903
|
+
const partition = indexState.partitions.get(partitionId);
|
|
904
|
+
if (partition !== void 0) reported.add(partition);
|
|
905
|
+
}
|
|
906
|
+
hooks.onCheckpointHeld?.(held);
|
|
907
|
+
}
|
|
908
|
+
return true;
|
|
909
|
+
},
|
|
910
|
+
report(indexes) {
|
|
911
|
+
const held = [];
|
|
912
|
+
for (const [indexName, indexState] of indexes) {
|
|
913
|
+
const entry = heldReport(indexName, indexState);
|
|
914
|
+
if (entry !== null) held.push(entry);
|
|
915
|
+
}
|
|
916
|
+
return held;
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// src/persistence/durability/restore-journal.ts
|
|
922
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
923
|
+
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
924
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
925
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
926
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
927
|
+
function digestOf(indexName) {
|
|
928
|
+
let hex = "";
|
|
929
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
930
|
+
return hex;
|
|
931
|
+
}
|
|
932
|
+
function journalKey(indexName) {
|
|
933
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
934
|
+
}
|
|
935
|
+
function randomSuffix() {
|
|
936
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
937
|
+
}
|
|
938
|
+
function newStagingPrefix() {
|
|
939
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
940
|
+
}
|
|
941
|
+
function newRetiredPrefix() {
|
|
942
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
943
|
+
}
|
|
944
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
945
|
+
return new TextEncoder().encode(`${indexName}
|
|
946
|
+
${stagingPrefix}`);
|
|
947
|
+
}
|
|
948
|
+
function isStagingPrefix(prefix) {
|
|
949
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
950
|
+
}
|
|
951
|
+
function corruptJournal(key, reason) {
|
|
952
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
953
|
+
}
|
|
954
|
+
function readJournal(key, bytes) {
|
|
955
|
+
const text = new TextDecoder().decode(bytes);
|
|
956
|
+
const lineEnd = text.indexOf("\n");
|
|
957
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
958
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
959
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
960
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
961
|
+
}
|
|
962
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
963
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
964
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
965
|
+
}
|
|
966
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
967
|
+
}
|
|
968
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
969
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
970
|
+
if (manifestBytes === null) {
|
|
971
|
+
throw new NarsilError(
|
|
972
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
973
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
974
|
+
{ indexName, stagingPrefix }
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
978
|
+
if (decoded.kind !== "current") {
|
|
979
|
+
throw new NarsilError(
|
|
980
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
981
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
982
|
+
{ indexName }
|
|
983
|
+
);
|
|
984
|
+
}
|
|
985
|
+
const root = `${indexName}/`;
|
|
986
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
987
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
988
|
+
return keys;
|
|
989
|
+
}
|
|
990
|
+
function withoutTrailingSlash(prefix) {
|
|
991
|
+
return prefix.slice(0, -1);
|
|
992
|
+
}
|
|
993
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
994
|
+
const root = `${indexName}/`;
|
|
995
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
996
|
+
const staged = await storage.list(stagingPrefix);
|
|
997
|
+
let retiredPrefix = null;
|
|
998
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
999
|
+
retiredPrefix = newRetiredPrefix();
|
|
1000
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
1001
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
1002
|
+
} else {
|
|
1003
|
+
const stagedKeys = new Set(staged);
|
|
1004
|
+
for (const key of listed) {
|
|
1005
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
for (const key of await storage.list(root)) {
|
|
1009
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
1010
|
+
}
|
|
1011
|
+
await storage.remove(journalKey(indexName));
|
|
1012
|
+
return retiredPrefix;
|
|
1013
|
+
}
|
|
1014
|
+
async function removePrefix(storage, prefix) {
|
|
1015
|
+
if (storage.removeTree !== void 0) {
|
|
1016
|
+
await storage.removeTree(prefix);
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
1020
|
+
}
|
|
1021
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
1022
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
1023
|
+
}
|
|
1024
|
+
function isTopLevelJournal(key) {
|
|
1025
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
1026
|
+
}
|
|
1027
|
+
async function removeStagingPrefixes(storage) {
|
|
1028
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
1029
|
+
for (const key of await storage.list(".")) {
|
|
1030
|
+
const slash = key.indexOf("/");
|
|
1031
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
1032
|
+
}
|
|
1033
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
1034
|
+
}
|
|
1035
|
+
async function recoverRestoreJournals(storage) {
|
|
1036
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
1037
|
+
if (!isTopLevelJournal(key)) continue;
|
|
1038
|
+
const bytes = await storage.read(key);
|
|
1039
|
+
if (bytes === null) continue;
|
|
1040
|
+
const reading = readJournal(key, bytes);
|
|
1041
|
+
if (reading.kind === "leftover") {
|
|
1042
|
+
await storage.remove(key);
|
|
1043
|
+
continue;
|
|
1044
|
+
}
|
|
1045
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
1046
|
+
}
|
|
1047
|
+
await removeStagingPrefixes(storage);
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
// src/persistence/durability/restore-directory.ts
|
|
1051
|
+
function missing(error) {
|
|
1052
|
+
return error.code === "ENOENT";
|
|
1053
|
+
}
|
|
1054
|
+
function storageFailed(code, message, key, error) {
|
|
1055
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
1056
|
+
}
|
|
1057
|
+
function directoryRestoreStorage(directory) {
|
|
1058
|
+
return {
|
|
1059
|
+
directory,
|
|
1060
|
+
read: (key) => directory.read(key),
|
|
1061
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
1062
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
1063
|
+
remove: (key) => directory.remove(key),
|
|
1064
|
+
list: (prefix) => directory.list(prefix),
|
|
1065
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
1066
|
+
pathOf: (key) => directory.pathOf(key),
|
|
1067
|
+
async moveTree(from, to) {
|
|
1068
|
+
const source = await directory.pathOf(from);
|
|
1069
|
+
const target = await directory.pathOf(to);
|
|
1070
|
+
const fs = await getFs();
|
|
1071
|
+
try {
|
|
1072
|
+
await fs.rename(source, target);
|
|
1073
|
+
} catch (error) {
|
|
1074
|
+
if (missing(error)) return;
|
|
1075
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
1076
|
+
}
|
|
1077
|
+
await directory.syncDirectoryOf(to);
|
|
1078
|
+
},
|
|
1079
|
+
async removeTree(prefix) {
|
|
1080
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
1081
|
+
const fs = await getFs();
|
|
1082
|
+
try {
|
|
1083
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
1084
|
+
} catch (error) {
|
|
1085
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
1086
|
+
}
|
|
1087
|
+
await directory.syncDirectoryOf(key);
|
|
1088
|
+
}
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
function recoverDirectoryRestores(directory) {
|
|
1092
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
// src/persistence/durability/restore-staging.ts
|
|
1096
|
+
function toError2(error) {
|
|
1097
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1098
|
+
}
|
|
1099
|
+
function pathUnder(path, root) {
|
|
1100
|
+
if (!path.startsWith(root)) return false;
|
|
1101
|
+
const next = path.charAt(root.length);
|
|
1102
|
+
return next === "" || next === "/" || next === "\\";
|
|
1103
|
+
}
|
|
1104
|
+
function stagedDirectory(directory, staged, unstaged) {
|
|
1105
|
+
const { readRange, byteLength } = directory;
|
|
1106
|
+
return {
|
|
1107
|
+
root: directory.root,
|
|
1108
|
+
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
1109
|
+
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
1110
|
+
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
1111
|
+
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
1112
|
+
read: (key) => directory.read(staged(key)),
|
|
1113
|
+
remove: (key) => directory.remove(staged(key)),
|
|
1114
|
+
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
1115
|
+
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
1116
|
+
pathOf: (key) => directory.pathOf(staged(key)),
|
|
1117
|
+
...readRange === void 0 ? {} : {
|
|
1118
|
+
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
1119
|
+
},
|
|
1120
|
+
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
1124
|
+
const stagingPrefix = newStagingPrefix();
|
|
1125
|
+
const indexRoot = `${indexName}/`;
|
|
1126
|
+
let committed = false;
|
|
1127
|
+
const staged = (key) => {
|
|
1128
|
+
if (!key.startsWith(indexRoot)) {
|
|
1129
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
1130
|
+
key
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
1134
|
+
};
|
|
1135
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
1136
|
+
const store = {
|
|
1137
|
+
read: (key) => storage.read(staged(key)),
|
|
1138
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
1139
|
+
remove: (key) => storage.remove(staged(key)),
|
|
1140
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
1141
|
+
};
|
|
1142
|
+
const pathOf = storage.pathOf;
|
|
1143
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
1144
|
+
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
1145
|
+
let retiredPrefix = null;
|
|
1146
|
+
async function finishOnStorage() {
|
|
1147
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
1148
|
+
}
|
|
1149
|
+
return {
|
|
1150
|
+
store,
|
|
1151
|
+
async relocation() {
|
|
1152
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
1153
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
1154
|
+
const to = await pathOf.call(storage, indexName);
|
|
1155
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
1156
|
+
},
|
|
1157
|
+
async commit() {
|
|
1158
|
+
try {
|
|
1159
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
1160
|
+
} catch (error) {
|
|
1161
|
+
try {
|
|
1162
|
+
await storage.remove(journalKey(indexName));
|
|
1163
|
+
} catch (removal) {
|
|
1164
|
+
committed = true;
|
|
1165
|
+
options.onStorageLost?.(toError2(removal));
|
|
1166
|
+
}
|
|
1167
|
+
throw error;
|
|
1168
|
+
}
|
|
1169
|
+
committed = true;
|
|
1170
|
+
},
|
|
1171
|
+
async finish() {
|
|
1172
|
+
try {
|
|
1173
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
1174
|
+
} catch (error) {
|
|
1175
|
+
options.onStorageLost?.(toError2(error));
|
|
1176
|
+
}
|
|
1177
|
+
},
|
|
1178
|
+
async removeRetired() {
|
|
1179
|
+
const prefix = retiredPrefix;
|
|
1180
|
+
retiredPrefix = null;
|
|
1181
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
1182
|
+
},
|
|
1183
|
+
async discard() {
|
|
1184
|
+
if (committed) return;
|
|
1185
|
+
await removePrefix(storage, stagingPrefix);
|
|
1186
|
+
}
|
|
1187
|
+
};
|
|
1188
|
+
}
|
|
1189
|
+
function adapterRestoreStorage(adapter) {
|
|
1190
|
+
const store = adapterCheckpointStore(adapter);
|
|
1191
|
+
return {
|
|
1192
|
+
read: (key) => store.read(key),
|
|
1193
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
1194
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
1195
|
+
remove: (key) => store.remove(key),
|
|
1196
|
+
list: (prefix) => store.list(prefix),
|
|
1197
|
+
async moveKey(from, to) {
|
|
1198
|
+
const bytes = await store.read(from);
|
|
1199
|
+
if (bytes === null) return;
|
|
1200
|
+
await store.write(to, [bytes]);
|
|
1201
|
+
await store.remove(from);
|
|
1202
|
+
}
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
1207
|
+
function toError3(error) {
|
|
1208
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1209
|
+
}
|
|
1210
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
1211
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
1212
|
+
async finish(run) {
|
|
1213
|
+
try {
|
|
1214
|
+
await run();
|
|
1215
|
+
} catch (error) {
|
|
1216
|
+
deps.markFatal(toError3(error));
|
|
1217
|
+
throw error;
|
|
1218
|
+
}
|
|
1219
|
+
},
|
|
1220
|
+
onStorageLost: deps.markFatal
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
function releaser(held, indexName) {
|
|
1224
|
+
let released = false;
|
|
1225
|
+
return () => {
|
|
1226
|
+
if (released) return;
|
|
1227
|
+
released = true;
|
|
1228
|
+
const count = (held.get(indexName) ?? 1) - 1;
|
|
1229
|
+
if (count <= 0) held.delete(indexName);
|
|
1230
|
+
else held.set(indexName, count);
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
function createArchiveCheckpointAccess(deps) {
|
|
1234
|
+
const store = directoryCheckpointStore(deps.directory);
|
|
1235
|
+
return {
|
|
1236
|
+
async openArchiveCheckpoint(indexName) {
|
|
1237
|
+
deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
|
|
1238
|
+
const release = releaser(deps.held, indexName);
|
|
1239
|
+
try {
|
|
1240
|
+
await deps.checkpoint(indexName, false);
|
|
1241
|
+
await deps.checkpoint(indexName, true);
|
|
1242
|
+
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
1243
|
+
if (manifestBytes === null) {
|
|
1244
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
1245
|
+
indexName
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1249
|
+
if (decoded.kind !== "current") {
|
|
1250
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
1251
|
+
indexName
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
return {
|
|
1255
|
+
manifestBytes,
|
|
1256
|
+
manifest: decoded.manifest,
|
|
1257
|
+
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
1258
|
+
readRange: async (ref, offset, length) => {
|
|
1259
|
+
const readRange = deps.directory.readRange;
|
|
1260
|
+
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
1261
|
+
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
1262
|
+
return whole.slice(offset, offset + length);
|
|
1263
|
+
},
|
|
1264
|
+
release,
|
|
1265
|
+
directory: deps.directory
|
|
1266
|
+
};
|
|
1267
|
+
} catch (error) {
|
|
1268
|
+
release();
|
|
1269
|
+
throw error;
|
|
1270
|
+
}
|
|
1271
|
+
},
|
|
1272
|
+
stageRestore(indexName) {
|
|
1273
|
+
return stageDirectoryRestore(deps, indexName);
|
|
1274
|
+
},
|
|
1275
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
1276
|
+
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
1277
|
+
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
};
|
|
1203
1281
|
}
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1282
|
+
|
|
1283
|
+
// src/persistence/durability/query-history.ts
|
|
1284
|
+
async function removeQueryHistoryOf(storage, indexName) {
|
|
1285
|
+
const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
|
|
1286
|
+
if (storage.kind === "directory") {
|
|
1287
|
+
for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
|
|
1288
|
+
return;
|
|
1209
1289
|
}
|
|
1210
|
-
|
|
1290
|
+
for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
|
|
1211
1291
|
}
|
|
1212
|
-
|
|
1213
|
-
// src/persistence/durability/checkpoint-turn.ts
|
|
1214
|
-
function createIndexLoads(vectorIndexesOf2) {
|
|
1215
|
-
const loads = /* @__PURE__ */ new Map();
|
|
1216
|
-
const startGraphRepairs = (indexName) => {
|
|
1217
|
-
for (const vectorIndex of vectorIndexesOf2(indexName).values()) {
|
|
1218
|
-
vectorIndex.scheduleBuild();
|
|
1219
|
-
vectorIndex.startShuffledRebuildWhenDue();
|
|
1220
|
-
}
|
|
1221
|
-
};
|
|
1292
|
+
function queryHistoryAccess(storage) {
|
|
1222
1293
|
return {
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
loads.set(
|
|
1226
|
-
indexName,
|
|
1227
|
-
loaded.catch(() => void 0)
|
|
1228
|
-
);
|
|
1229
|
-
return loaded;
|
|
1230
|
-
},
|
|
1231
|
-
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1232
|
-
startGraphRepairs
|
|
1294
|
+
historyStorage: () => storage,
|
|
1295
|
+
removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
|
|
1233
1296
|
};
|
|
1234
1297
|
}
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1298
|
+
|
|
1299
|
+
// src/persistence/durability/manager-unload.ts
|
|
1300
|
+
function createIndexUnloading(deps) {
|
|
1301
|
+
const { directory, hooks, indexes, metadataWrites } = deps;
|
|
1302
|
+
return {
|
|
1303
|
+
async removeIndex(indexName) {
|
|
1304
|
+
hooks.writeRecords?.dropIndex(indexName);
|
|
1305
|
+
const indexState = indexes.get(indexName);
|
|
1306
|
+
if (indexState !== void 0) {
|
|
1307
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
|
|
1308
|
+
indexes.delete(indexName);
|
|
1309
|
+
metadataWrites.forget(indexName);
|
|
1310
|
+
}
|
|
1311
|
+
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
1312
|
+
for (const key of await directory.list(`${indexName}/`)) {
|
|
1313
|
+
await directory.remove(key);
|
|
1314
|
+
}
|
|
1315
|
+
await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
|
|
1316
|
+
deps.forgetSyncState(indexName);
|
|
1243
1317
|
},
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
});
|
|
1256
|
-
turns.checkpointInFlight = run;
|
|
1257
|
-
return run;
|
|
1318
|
+
async unloadIndex(indexName) {
|
|
1319
|
+
const indexState = indexes.get(indexName);
|
|
1320
|
+
if (indexState === void 0) {
|
|
1321
|
+
return;
|
|
1322
|
+
}
|
|
1323
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
1324
|
+
indexes.delete(indexName);
|
|
1325
|
+
metadataWrites.forget(indexName);
|
|
1326
|
+
deps.forgetSyncState(indexName);
|
|
1327
|
+
}
|
|
1328
|
+
};
|
|
1258
1329
|
}
|
|
1259
1330
|
|
|
1260
|
-
// src/persistence/durability/
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
}
|
|
1266
|
-
function
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1331
|
+
// src/persistence/durability/sync/frame-cursor.ts
|
|
1332
|
+
var SegmentFileMissing = class extends Error {
|
|
1333
|
+
};
|
|
1334
|
+
function corrupt(key, message, details) {
|
|
1335
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
|
|
1336
|
+
}
|
|
1337
|
+
function isMissingFile(error) {
|
|
1338
|
+
return error?.code === "ENOENT";
|
|
1339
|
+
}
|
|
1340
|
+
var FrameCursor = class {
|
|
1341
|
+
constructor(directory, key, limit) {
|
|
1342
|
+
this.directory = directory;
|
|
1343
|
+
this.key = key;
|
|
1344
|
+
this.limit = limit;
|
|
1345
|
+
}
|
|
1346
|
+
buffer = new Uint8Array(0);
|
|
1347
|
+
bufferStart = 0;
|
|
1348
|
+
whole = null;
|
|
1349
|
+
async load(offset, length) {
|
|
1350
|
+
const readRange = this.directory.readRange;
|
|
1351
|
+
if (readRange === void 0) {
|
|
1352
|
+
if (this.whole === null) {
|
|
1353
|
+
const bytes = await this.directory.read(this.key);
|
|
1354
|
+
if (bytes === null) throw new SegmentFileMissing(this.key);
|
|
1355
|
+
this.whole = bytes;
|
|
1356
|
+
}
|
|
1357
|
+
this.buffer = this.whole;
|
|
1358
|
+
this.bufferStart = 0;
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
|
|
1362
|
+
try {
|
|
1363
|
+
this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
|
|
1364
|
+
} catch (error) {
|
|
1365
|
+
if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
|
|
1366
|
+
throw error;
|
|
1367
|
+
}
|
|
1368
|
+
this.bufferStart = offset;
|
|
1276
1369
|
}
|
|
1277
|
-
async
|
|
1370
|
+
async bytesAt(offset, length) {
|
|
1371
|
+
const end = offset + length;
|
|
1372
|
+
if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
|
|
1373
|
+
if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
|
|
1374
|
+
return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
|
|
1375
|
+
}
|
|
1376
|
+
async checkHeader() {
|
|
1377
|
+
if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
|
|
1378
|
+
const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
|
|
1379
|
+
if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
|
|
1380
|
+
}
|
|
1381
|
+
async frameLengthAt(offset) {
|
|
1382
|
+
const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
|
|
1383
|
+
const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
|
|
1384
|
+
0,
|
|
1385
|
+
false
|
|
1386
|
+
);
|
|
1387
|
+
if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
|
|
1388
|
+
corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
|
|
1389
|
+
}
|
|
1390
|
+
return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
|
|
1391
|
+
}
|
|
1392
|
+
async frameAt(offset, frameBytes) {
|
|
1393
|
+
const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
|
|
1394
|
+
const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
|
|
1395
|
+
const payload = body.subarray(0, recordLength);
|
|
1396
|
+
const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
|
|
1397
|
+
if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
|
|
1398
|
+
let seqNo;
|
|
1278
1399
|
try {
|
|
1279
|
-
|
|
1280
|
-
} catch (
|
|
1281
|
-
|
|
1400
|
+
seqNo = decodeWalEntry(payload).seqNo;
|
|
1401
|
+
} catch (error) {
|
|
1402
|
+
corrupt(this.key, "A log record fails to decode", {
|
|
1403
|
+
offset,
|
|
1404
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
1405
|
+
});
|
|
1282
1406
|
}
|
|
1407
|
+
return { seqNo, payload: payload.slice(), frameBytes };
|
|
1408
|
+
}
|
|
1409
|
+
};
|
|
1410
|
+
|
|
1411
|
+
// src/persistence/durability/sync/log-reader.ts
|
|
1412
|
+
async function walSources(directory, indexName, partitionId) {
|
|
1413
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
1414
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
1415
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
1416
|
+
if (marker === null) return { sources: [], head: 0 };
|
|
1417
|
+
const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
|
|
1418
|
+
const sources = [];
|
|
1419
|
+
for (const key of await directory.list(prefix)) {
|
|
1420
|
+
const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
|
|
1421
|
+
if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
|
|
1422
|
+
const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
|
|
1423
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
1424
|
+
sources.push({ key, startSeqNo, readableBytes, inHistory: false });
|
|
1425
|
+
}
|
|
1426
|
+
return { sources, head: highestDurableSeqNo };
|
|
1427
|
+
}
|
|
1428
|
+
async function openPartitionLog(directory, indexName, partitionId) {
|
|
1429
|
+
const { sources, head } = await walSources(directory, indexName, partitionId);
|
|
1430
|
+
const byStart = /* @__PURE__ */ new Map();
|
|
1431
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
1432
|
+
const readableBytes = await directory.byteLength?.(segment.key);
|
|
1433
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
1434
|
+
byStart.set(segment.startSeqNo, {
|
|
1435
|
+
key: segment.key,
|
|
1436
|
+
startSeqNo: segment.startSeqNo,
|
|
1437
|
+
readableBytes,
|
|
1438
|
+
inHistory: true
|
|
1439
|
+
});
|
|
1283
1440
|
}
|
|
1284
|
-
|
|
1285
|
-
|
|
1441
|
+
for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
|
|
1442
|
+
const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
|
|
1443
|
+
return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
|
|
1444
|
+
}
|
|
1445
|
+
function sourceHolding(log, seqNo) {
|
|
1446
|
+
let found = -1;
|
|
1447
|
+
for (let i = 0; i < log.sources.length; i++) {
|
|
1448
|
+
if (log.sources[i].startSeqNo > seqNo) break;
|
|
1449
|
+
found = i;
|
|
1450
|
+
}
|
|
1451
|
+
return found;
|
|
1452
|
+
}
|
|
1453
|
+
async function offsetAfter(cursor, source, afterSeqNo) {
|
|
1454
|
+
let offset = SEGMENT_HEADER_SIZE;
|
|
1455
|
+
for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
|
|
1456
|
+
offset += await cursor.frameLengthAt(offset);
|
|
1457
|
+
}
|
|
1458
|
+
return offset;
|
|
1459
|
+
}
|
|
1460
|
+
async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
|
|
1461
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
1462
|
+
await cursor.checkHeader();
|
|
1463
|
+
let offset = await offsetAfter(cursor, source, afterSeqNo);
|
|
1464
|
+
let checksNumbering = true;
|
|
1465
|
+
while (offset < cursor.limit) {
|
|
1466
|
+
const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
|
|
1467
|
+
if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
|
|
1468
|
+
checksNumbering = false;
|
|
1469
|
+
if (frame.seqNo > afterSeqNo) {
|
|
1470
|
+
offset = SEGMENT_HEADER_SIZE;
|
|
1471
|
+
continue;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
checksNumbering = false;
|
|
1475
|
+
offset += frame.frameBytes;
|
|
1476
|
+
if (frame.seqNo <= afterSeqNo) continue;
|
|
1477
|
+
if (frame.seqNo > upToSeqNo) return true;
|
|
1478
|
+
if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
|
|
1479
|
+
read.frames.push(frame);
|
|
1480
|
+
read.bytes += frame.payload.byteLength;
|
|
1481
|
+
if (frame.seqNo === upToSeqNo) return true;
|
|
1482
|
+
}
|
|
1483
|
+
return false;
|
|
1484
|
+
}
|
|
1485
|
+
async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
1486
|
+
const read = { frames: [], bytes: 0, gone: false };
|
|
1487
|
+
if (afterSeqNo >= upToSeqNo) return read;
|
|
1488
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1489
|
+
read.frames = [];
|
|
1490
|
+
read.bytes = 0;
|
|
1491
|
+
const log = await openPartitionLog(directory, indexName, partitionId);
|
|
1492
|
+
let index = sourceHolding(log, afterSeqNo + 1);
|
|
1493
|
+
if (index < 0) {
|
|
1494
|
+
read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
|
|
1495
|
+
return read;
|
|
1496
|
+
}
|
|
1286
1497
|
try {
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1498
|
+
for (; index < log.sources.length; index++) {
|
|
1499
|
+
const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
|
|
1500
|
+
if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
|
|
1501
|
+
}
|
|
1502
|
+
return read;
|
|
1503
|
+
} catch (error) {
|
|
1504
|
+
if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
|
|
1290
1505
|
}
|
|
1291
|
-
return new NarsilError(
|
|
1292
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1293
|
-
`The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
|
|
1294
|
-
{ directory: directory.root }
|
|
1295
|
-
);
|
|
1296
1506
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
if (
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
)
|
|
1313
|
-
);
|
|
1314
|
-
return false;
|
|
1507
|
+
return read;
|
|
1508
|
+
}
|
|
1509
|
+
async function logBytesAfter(directory, log, afterSeqNo) {
|
|
1510
|
+
const first = sourceHolding(log, afterSeqNo + 1);
|
|
1511
|
+
let bytes = 0;
|
|
1512
|
+
for (let i = Math.max(first, 0); i < log.sources.length; i++) {
|
|
1513
|
+
const source = log.sources[i];
|
|
1514
|
+
if (i === first) {
|
|
1515
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
1516
|
+
try {
|
|
1517
|
+
bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
|
|
1518
|
+
} catch (error) {
|
|
1519
|
+
if (!(error instanceof SegmentFileMissing)) throw error;
|
|
1520
|
+
}
|
|
1521
|
+
continue;
|
|
1315
1522
|
}
|
|
1316
|
-
|
|
1317
|
-
if (takenOver === null) return true;
|
|
1318
|
-
onLost(takenOver);
|
|
1319
|
-
return false;
|
|
1523
|
+
bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
|
|
1320
1524
|
}
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1525
|
+
return bytes;
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
// src/persistence/durability/sync/pins.ts
|
|
1529
|
+
function createFilePins(remove) {
|
|
1530
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1531
|
+
const awaitingRemoval = /* @__PURE__ */ new Set();
|
|
1532
|
+
function unpin(key) {
|
|
1533
|
+
const count = (counts.get(key) ?? 1) - 1;
|
|
1534
|
+
if (count > 0) {
|
|
1535
|
+
counts.set(key, count);
|
|
1536
|
+
return;
|
|
1325
1537
|
}
|
|
1538
|
+
counts.delete(key);
|
|
1539
|
+
if (!awaitingRemoval.delete(key)) return;
|
|
1540
|
+
void remove(key).catch(() => void 0);
|
|
1326
1541
|
}
|
|
1327
1542
|
return {
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
async verifyOnce() {
|
|
1338
|
-
if (expected === null && directory.identity !== void 0) await verify();
|
|
1339
|
-
},
|
|
1340
|
-
start() {
|
|
1341
|
-
if (directory.identity === void 0 || timer !== null || isStopped()) return;
|
|
1342
|
-
timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
|
|
1543
|
+
pin(keys) {
|
|
1544
|
+
const pinned = [...keys];
|
|
1545
|
+
for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1546
|
+
let released = false;
|
|
1547
|
+
return () => {
|
|
1548
|
+
if (released) return;
|
|
1549
|
+
released = true;
|
|
1550
|
+
for (const key of pinned) unpin(key);
|
|
1551
|
+
};
|
|
1343
1552
|
},
|
|
1344
|
-
|
|
1553
|
+
isPinned: (key) => counts.has(key),
|
|
1554
|
+
removeWhenUnpinned(key) {
|
|
1555
|
+
if (!counts.has(key)) return false;
|
|
1556
|
+
awaitingRemoval.add(key);
|
|
1557
|
+
return true;
|
|
1558
|
+
}
|
|
1559
|
+
};
|
|
1560
|
+
}
|
|
1561
|
+
var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
|
|
1562
|
+
var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
|
|
1563
|
+
function hexOf(bytes) {
|
|
1564
|
+
let hex = "";
|
|
1565
|
+
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
1566
|
+
return hex;
|
|
1567
|
+
}
|
|
1568
|
+
function isBytes(value) {
|
|
1569
|
+
return value instanceof Uint8Array;
|
|
1570
|
+
}
|
|
1571
|
+
async function publishedFrom(indexName, payload, config, manifestBytes) {
|
|
1572
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1573
|
+
if (decoded.kind !== "current") {
|
|
1574
|
+
throw new NarsilError(
|
|
1575
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1576
|
+
`The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
|
|
1577
|
+
{ indexName }
|
|
1578
|
+
);
|
|
1579
|
+
}
|
|
1580
|
+
return {
|
|
1581
|
+
config,
|
|
1582
|
+
manifestBytes,
|
|
1583
|
+
manifest: decoded.manifest,
|
|
1584
|
+
entityTag: hexOf(sha256(payload)),
|
|
1585
|
+
keys: manifestReferencedKeys(decoded.manifest)
|
|
1345
1586
|
};
|
|
1346
1587
|
}
|
|
1588
|
+
async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
|
|
1589
|
+
const payload = encode({ config, manifest: manifestBytes });
|
|
1590
|
+
const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
|
|
1591
|
+
const published = await publishedFrom(indexName, payload, config, manifestBytes);
|
|
1592
|
+
await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
|
|
1593
|
+
return published;
|
|
1594
|
+
}
|
|
1595
|
+
async function readPublishedSnapshot(directory, indexName) {
|
|
1596
|
+
const bytes = await directory.read(publishedSnapshotKey(indexName));
|
|
1597
|
+
if (bytes === null) return null;
|
|
1598
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
1599
|
+
envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
|
|
1600
|
+
what: PUBLISHED_SNAPSHOT_WHAT,
|
|
1601
|
+
allowsCompression: false
|
|
1602
|
+
});
|
|
1603
|
+
const raw = decode(payload);
|
|
1604
|
+
if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
|
|
1605
|
+
throw new NarsilError(
|
|
1606
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1607
|
+
`The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
|
|
1608
|
+
{ indexName }
|
|
1609
|
+
);
|
|
1610
|
+
}
|
|
1611
|
+
return publishedFrom(indexName, payload, raw.config, raw.manifest);
|
|
1612
|
+
}
|
|
1347
1613
|
|
|
1348
|
-
// src/persistence/durability/
|
|
1349
|
-
function
|
|
1350
|
-
|
|
1351
|
-
for (const
|
|
1352
|
-
|
|
1614
|
+
// src/persistence/durability/sync/retention.ts
|
|
1615
|
+
async function firstLogStart(directory, indexName, partitionId) {
|
|
1616
|
+
let first = Number.POSITIVE_INFINITY;
|
|
1617
|
+
for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
|
|
1618
|
+
const start = parseWalSegmentStart(indexName, partitionId, key);
|
|
1619
|
+
if (start !== null && start < first) first = start;
|
|
1620
|
+
}
|
|
1621
|
+
return first;
|
|
1622
|
+
}
|
|
1623
|
+
async function sizedQueues(retention) {
|
|
1624
|
+
const { directory, indexName } = retention;
|
|
1625
|
+
const queues = /* @__PURE__ */ new Map();
|
|
1626
|
+
for (const segment of await listHistorySegments(directory, indexName)) {
|
|
1627
|
+
const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
|
|
1628
|
+
if (bytes === void 0 || bytes === null) continue;
|
|
1629
|
+
const queue = queues.get(segment.partitionId) ?? [];
|
|
1630
|
+
queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
|
|
1631
|
+
queues.set(segment.partitionId, queue);
|
|
1632
|
+
}
|
|
1633
|
+
for (const [partitionId, queue] of queues) {
|
|
1634
|
+
const logStart = await firstLogStart(directory, indexName, partitionId);
|
|
1635
|
+
for (let i = 0; i < queue.length; i++) {
|
|
1636
|
+
const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
|
|
1637
|
+
queue[i].lastSeqNo = nextStart - 1;
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
return queues;
|
|
1641
|
+
}
|
|
1642
|
+
function nextToDelete(queues, totalBytes, retention) {
|
|
1643
|
+
const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
|
|
1644
|
+
let oldest = null;
|
|
1645
|
+
for (const queue of queues.values()) {
|
|
1646
|
+
const head = queue[0];
|
|
1647
|
+
if (head === void 0) continue;
|
|
1648
|
+
if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
|
|
1649
|
+
}
|
|
1650
|
+
if (oldest === null) return null;
|
|
1651
|
+
if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
|
|
1652
|
+
return null;
|
|
1653
|
+
}
|
|
1654
|
+
async function enforceHistoryLimits(retention) {
|
|
1655
|
+
const queues = await sizedQueues(retention);
|
|
1656
|
+
let totalBytes = 0;
|
|
1657
|
+
for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
|
|
1658
|
+
let published = false;
|
|
1659
|
+
let deleted = 0;
|
|
1660
|
+
let segment = nextToDelete(queues, totalBytes, retention);
|
|
1661
|
+
while (segment !== null) {
|
|
1662
|
+
const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
|
|
1663
|
+
if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
|
|
1664
|
+
published = true;
|
|
1665
|
+
await retention.publish();
|
|
1666
|
+
}
|
|
1667
|
+
await retention.directory.remove(segment.key);
|
|
1668
|
+
queues.get(segment.partitionId)?.shift();
|
|
1669
|
+
totalBytes -= segment.bytes;
|
|
1670
|
+
deleted += 1;
|
|
1671
|
+
segment = nextToDelete(queues, totalBytes, retention);
|
|
1672
|
+
}
|
|
1673
|
+
return deleted;
|
|
1353
1674
|
}
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1675
|
+
|
|
1676
|
+
// src/persistence/durability/sync/storage.ts
|
|
1677
|
+
function createSyncStorage(deps) {
|
|
1678
|
+
const { directory } = deps;
|
|
1679
|
+
const store = directoryCheckpointStore(directory);
|
|
1680
|
+
const cache = /* @__PURE__ */ new Map();
|
|
1681
|
+
const publishing = /* @__PURE__ */ new Map();
|
|
1682
|
+
const pins = createFilePins((key) => directory.remove(key));
|
|
1683
|
+
async function published(indexName) {
|
|
1684
|
+
const cached = cache.get(indexName);
|
|
1685
|
+
if (cached !== void 0) return cached;
|
|
1686
|
+
const read = await readPublishedSnapshot(directory, indexName);
|
|
1687
|
+
cache.set(indexName, read);
|
|
1688
|
+
return read;
|
|
1689
|
+
}
|
|
1690
|
+
async function removeUnlisted(indexName, previous, kept) {
|
|
1691
|
+
const manifest = await readCurrentManifest(store, indexName);
|
|
1692
|
+
const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
|
|
1693
|
+
for (const key of previous) {
|
|
1694
|
+
if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
|
|
1695
|
+
await directory.remove(key);
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
async function publishNow(indexName, configurationOf) {
|
|
1699
|
+
const archive = await deps.openArchiveCheckpoint(indexName);
|
|
1700
|
+
try {
|
|
1701
|
+
const config = await configurationOf(archive.manifest);
|
|
1702
|
+
const previous = await published(indexName);
|
|
1703
|
+
const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
|
|
1704
|
+
cache.set(indexName, next);
|
|
1705
|
+
if (previous !== null) {
|
|
1706
|
+
await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
|
|
1707
|
+
}
|
|
1708
|
+
return next;
|
|
1709
|
+
} finally {
|
|
1710
|
+
archive.release();
|
|
1711
|
+
}
|
|
1365
1712
|
}
|
|
1366
1713
|
return {
|
|
1367
|
-
|
|
1368
|
-
const
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1714
|
+
publish(indexName, configurationOf) {
|
|
1715
|
+
const inFlight = publishing.get(indexName);
|
|
1716
|
+
const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
|
|
1717
|
+
const tracked = run.finally(() => {
|
|
1718
|
+
if (publishing.get(indexName) === tracked) publishing.delete(indexName);
|
|
1719
|
+
});
|
|
1720
|
+
publishing.set(indexName, tracked);
|
|
1721
|
+
return tracked;
|
|
1722
|
+
},
|
|
1723
|
+
published,
|
|
1724
|
+
pin: (snapshot) => pins.pin(snapshot.keys),
|
|
1725
|
+
async readFileRange(key, offset, length) {
|
|
1726
|
+
const readRange = directory.readRange;
|
|
1727
|
+
if (readRange !== void 0) return readRange.call(directory, key, offset, length);
|
|
1728
|
+
const whole = await directory.read(key);
|
|
1729
|
+
return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
|
|
1730
|
+
},
|
|
1731
|
+
async release(indexName) {
|
|
1732
|
+
await deps.inCheckpointTurn(indexName, async () => {
|
|
1733
|
+
const previous = await published(indexName);
|
|
1734
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
1735
|
+
cache.set(indexName, null);
|
|
1736
|
+
if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
|
|
1737
|
+
await removeHistory(directory, indexName);
|
|
1738
|
+
});
|
|
1375
1739
|
},
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1740
|
+
removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
|
|
1741
|
+
async enforceHistoryLimits(indexName, limits, publish, nowMs) {
|
|
1742
|
+
const current = await published(indexName);
|
|
1743
|
+
return enforceHistoryLimits({
|
|
1744
|
+
directory,
|
|
1745
|
+
indexName,
|
|
1746
|
+
limits,
|
|
1747
|
+
nowMs,
|
|
1748
|
+
publish,
|
|
1749
|
+
publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
|
|
1382
1750
|
});
|
|
1383
|
-
if (unreported) {
|
|
1384
|
-
for (const partitionId of held.partitionIds) {
|
|
1385
|
-
const partition = indexState.partitions.get(partitionId);
|
|
1386
|
-
if (partition !== void 0) reported.add(partition);
|
|
1387
|
-
}
|
|
1388
|
-
hooks.onCheckpointHeld?.(held);
|
|
1389
|
-
}
|
|
1390
|
-
return true;
|
|
1391
1751
|
},
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1752
|
+
openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
|
|
1753
|
+
readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
|
|
1754
|
+
logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
|
|
1755
|
+
currentManifest: (indexName) => readCurrentManifest(store, indexName),
|
|
1756
|
+
async keepsCheckpointKey(indexName) {
|
|
1757
|
+
const current = await published(indexName);
|
|
1758
|
+
return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
|
|
1759
|
+
},
|
|
1760
|
+
async settleAtRecovery(indexName, active) {
|
|
1761
|
+
cache.delete(indexName);
|
|
1762
|
+
const publishedKey = publishedSnapshotKey(indexName);
|
|
1763
|
+
for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
|
|
1764
|
+
if (key !== publishedKey) await directory.remove(key);
|
|
1397
1765
|
}
|
|
1398
|
-
|
|
1766
|
+
if (!active) {
|
|
1767
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
1768
|
+
await removeHistory(directory, indexName);
|
|
1769
|
+
cache.set(indexName, null);
|
|
1770
|
+
return /* @__PURE__ */ new Set();
|
|
1771
|
+
}
|
|
1772
|
+
try {
|
|
1773
|
+
return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
|
|
1774
|
+
} catch {
|
|
1775
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
1776
|
+
cache.set(indexName, null);
|
|
1777
|
+
return /* @__PURE__ */ new Set();
|
|
1778
|
+
}
|
|
1779
|
+
},
|
|
1780
|
+
forget(indexName) {
|
|
1781
|
+
cache.delete(indexName);
|
|
1399
1782
|
}
|
|
1400
1783
|
};
|
|
1401
1784
|
}
|
|
1402
1785
|
|
|
1786
|
+
// src/persistence/durability/manager-storage.ts
|
|
1787
|
+
function createIndexStorageAccess(deps) {
|
|
1788
|
+
const { directory, indexes } = deps;
|
|
1789
|
+
const archiveAccess = createArchiveCheckpointAccess({
|
|
1790
|
+
directory,
|
|
1791
|
+
held: deps.heldCheckpointFiles,
|
|
1792
|
+
checkpoint: deps.checkpoint,
|
|
1793
|
+
registerPartition: deps.registerPartition,
|
|
1794
|
+
markFatal: deps.markFatal
|
|
1795
|
+
});
|
|
1796
|
+
const syncStorage = createSyncStorage({
|
|
1797
|
+
directory,
|
|
1798
|
+
openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
|
|
1799
|
+
inCheckpointTurn: (indexName, work) => {
|
|
1800
|
+
const indexState = indexes.get(indexName);
|
|
1801
|
+
if (indexState === void 0) return work();
|
|
1802
|
+
return checkpointInTurn(indexState, {
|
|
1803
|
+
joinsInFlight: false,
|
|
1804
|
+
mayStart: () => true,
|
|
1805
|
+
perform: work,
|
|
1806
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
});
|
|
1810
|
+
const unloading = createIndexUnloading({
|
|
1811
|
+
directory,
|
|
1812
|
+
hooks: deps.hooks,
|
|
1813
|
+
indexes,
|
|
1814
|
+
metadataWrites: deps.metadataWrites,
|
|
1815
|
+
closeWriterReportingFailure: deps.closeWriterReportingFailure,
|
|
1816
|
+
forgetSyncState: syncStorage.forget
|
|
1817
|
+
});
|
|
1818
|
+
return { archiveAccess, syncStorage, unloading };
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1403
1821
|
// src/persistence/durability/metadata-write-queue.ts
|
|
1404
1822
|
function createMetadataWriteQueue() {
|
|
1405
1823
|
const writes = /* @__PURE__ */ new Map();
|
|
@@ -1645,8 +2063,8 @@ function createGroupCommitCoordinator(sync) {
|
|
|
1645
2063
|
// src/persistence/durability/wal-writer.ts
|
|
1646
2064
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
1647
2065
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
1648
|
-
const
|
|
1649
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2066
|
+
const padded = startSeqNo.toString().padStart(16, "0");
|
|
2067
|
+
return `${indexName}/wal/${partitionId}/${padded}`;
|
|
1650
2068
|
}
|
|
1651
2069
|
function parseSegmentStartSeqNo2(key) {
|
|
1652
2070
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -2370,7 +2788,7 @@ var decoder = new TextDecoder();
|
|
|
2370
2788
|
async function countPagedPartition(store, partition) {
|
|
2371
2789
|
const { directory } = store;
|
|
2372
2790
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2373
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
2791
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-DWKPBSMN.mjs');
|
|
2374
2792
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2375
2793
|
const scans = [];
|
|
2376
2794
|
try {
|
|
@@ -2653,12 +3071,14 @@ async function recoverCheckpoint(deps) {
|
|
|
2653
3071
|
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
2654
3072
|
}
|
|
2655
3073
|
if (decoded === null || decoded.kind !== "current") {
|
|
2656
|
-
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName,
|
|
3074
|
+
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
|
|
2657
3075
|
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
2658
3076
|
}
|
|
2659
3077
|
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
2660
3078
|
if (!deps.keepsUnlistedFiles) {
|
|
2661
|
-
|
|
3079
|
+
const referenced = manifestReferencedKeys(decoded.manifest);
|
|
3080
|
+
for (const key of deps.keptForSync) referenced.add(key);
|
|
3081
|
+
await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
|
|
2662
3082
|
}
|
|
2663
3083
|
const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
|
|
2664
3084
|
manager: deps.manager,
|
|
@@ -2779,6 +3199,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
2779
3199
|
}
|
|
2780
3200
|
return replayed;
|
|
2781
3201
|
};
|
|
3202
|
+
const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
|
|
3203
|
+
const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
|
|
2782
3204
|
const recovered = await recoverCheckpoint({
|
|
2783
3205
|
store,
|
|
2784
3206
|
indexName,
|
|
@@ -2788,6 +3210,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
2788
3210
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
2789
3211
|
compactionThreshold: deps.compactionThreshold,
|
|
2790
3212
|
keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
|
|
3213
|
+
keptForSync,
|
|
2791
3214
|
replay,
|
|
2792
3215
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
2793
3216
|
writeRecords
|
|
@@ -2903,6 +3326,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2903
3326
|
mutationThreshold: checkpointMutationThreshold,
|
|
2904
3327
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
2905
3328
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
3329
|
+
keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
|
|
2906
3330
|
queueMetadataWrite,
|
|
2907
3331
|
markFatal,
|
|
2908
3332
|
failPartition: partitionFailureFor(indexName, partitionBatchDeps)
|
|
@@ -2956,6 +3380,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2956
3380
|
startAsyncFlushTimer();
|
|
2957
3381
|
return outcomes;
|
|
2958
3382
|
}
|
|
3383
|
+
const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
|
|
3384
|
+
directory,
|
|
3385
|
+
hooks,
|
|
3386
|
+
indexes,
|
|
3387
|
+
metadataWrites,
|
|
3388
|
+
heldCheckpointFiles,
|
|
3389
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3390
|
+
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3391
|
+
markFatal,
|
|
3392
|
+
closeWriterReportingFailure
|
|
3393
|
+
});
|
|
2959
3394
|
const recoveryDeps = {
|
|
2960
3395
|
directory,
|
|
2961
3396
|
hooks,
|
|
@@ -2964,7 +3399,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
2964
3399
|
getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
|
|
2965
3400
|
},
|
|
2966
3401
|
holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
|
|
2967
|
-
queueMetadataWrite
|
|
3402
|
+
queueMetadataWrite,
|
|
3403
|
+
syncStorage
|
|
2968
3404
|
};
|
|
2969
3405
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
2970
3406
|
return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
|
|
@@ -3052,35 +3488,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3052
3488
|
await checkpointIndex(indexName);
|
|
3053
3489
|
}
|
|
3054
3490
|
},
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
|
|
3060
|
-
indexes.delete(indexName);
|
|
3061
|
-
metadataWrites.forget(indexName);
|
|
3062
|
-
}
|
|
3063
|
-
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
3064
|
-
for (const key of await directory.list(`${indexName}/`)) {
|
|
3065
|
-
await directory.remove(key);
|
|
3066
|
-
}
|
|
3067
|
-
},
|
|
3068
|
-
async unloadIndex(indexName) {
|
|
3069
|
-
const indexState = indexes.get(indexName);
|
|
3070
|
-
if (indexState === void 0) {
|
|
3071
|
-
return;
|
|
3072
|
-
}
|
|
3073
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
3074
|
-
indexes.delete(indexName);
|
|
3075
|
-
metadataWrites.forget(indexName);
|
|
3076
|
-
},
|
|
3077
|
-
...createArchiveCheckpointAccess({
|
|
3078
|
-
directory,
|
|
3079
|
-
held: heldCheckpointFiles,
|
|
3080
|
-
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3081
|
-
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3082
|
-
markFatal
|
|
3083
|
-
}),
|
|
3491
|
+
...unloading,
|
|
3492
|
+
syncStorage: () => syncStorage,
|
|
3493
|
+
...queryHistoryAccess({ kind: "directory", directory }),
|
|
3494
|
+
...archiveAccess,
|
|
3084
3495
|
async shutdown() {
|
|
3085
3496
|
shuttingDown = true;
|
|
3086
3497
|
stopTimers();
|
|
@@ -3347,6 +3758,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3347
3758
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
3348
3759
|
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
3349
3760
|
keepsUnlistedFiles: false,
|
|
3761
|
+
keptForSync: /* @__PURE__ */ new Set(),
|
|
3350
3762
|
replay: async (checkpoint) => checkpoint,
|
|
3351
3763
|
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
3352
3764
|
});
|
|
@@ -3482,6 +3894,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3482
3894
|
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
3483
3895
|
}
|
|
3484
3896
|
},
|
|
3897
|
+
...queryHistoryAccess({ kind: "adapter", adapter }),
|
|
3485
3898
|
async removeIndex(indexName) {
|
|
3486
3899
|
await settleRestore(indexName).catch(() => void 0);
|
|
3487
3900
|
restores.forget(indexName);
|
|
@@ -3490,6 +3903,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3490
3903
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
3491
3904
|
await adapter.delete(key);
|
|
3492
3905
|
}
|
|
3906
|
+
await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
|
|
3493
3907
|
},
|
|
3494
3908
|
async unloadIndex(indexName) {
|
|
3495
3909
|
await settleRestore(indexName);
|
|
@@ -3544,7 +3958,9 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
3544
3958
|
recordCheckpoint: hooks.recordCheckpoint,
|
|
3545
3959
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
3546
3960
|
onCheckpointHeld: hooks.onCheckpointHeld,
|
|
3547
|
-
onFatalError: hooks.onFatalError
|
|
3961
|
+
onFatalError: hooks.onFatalError,
|
|
3962
|
+
...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
|
|
3963
|
+
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
3548
3964
|
};
|
|
3549
3965
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
3550
3966
|
function partitionFor(indexName, docId) {
|
|
@@ -3629,7 +4045,12 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
3629
4045
|
recordCheckpoint: wiring.recordCheckpoint,
|
|
3630
4046
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
3631
4047
|
onCheckpointHeld: wiring.emitCheckpointHeld,
|
|
3632
|
-
onFatalError: wiring.emitFatalError
|
|
4048
|
+
onFatalError: wiring.emitFatalError,
|
|
4049
|
+
keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
|
|
4050
|
+
onHistoryMoved(indexName) {
|
|
4051
|
+
if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
|
|
4052
|
+
import('./retention-HROPV2QU.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
|
|
4053
|
+
}
|
|
3633
4054
|
});
|
|
3634
4055
|
}
|
|
3635
4056
|
|