@delali/narsil 0.3.1-next.153 → 0.3.1-next.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
- package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
- package/dist/archive-history-E4R73RWD.mjs +29 -0
- package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
- package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
- package/dist/browser/chunk-23XRXIBW.mjs +85 -0
- package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
- package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
- package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
- package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
- package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
- package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
- package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
- package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
- package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
- package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
- package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
- package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
- package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
- package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
- package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
- package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
- package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
- package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
- package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
- package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
- package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
- package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
- package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
- package/dist/browser/chunk-KKAMCABR.mjs +432 -0
- package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
- package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
- package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
- package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
- package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
- package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
- package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
- package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
- package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
- package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
- package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
- package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
- package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
- package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
- package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
- package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
- package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
- package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
- package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
- package/dist/browser/import-OM6QAW45.mjs +18 -0
- package/dist/browser/methods-KTQCSIN4.mjs +30 -0
- package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
- package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
- package/dist/browser/operations-OJ3BXS55.mjs +5 -0
- package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
- package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
- package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
- package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
- package/dist/browser/retention-ASWUHAYI.mjs +209 -0
- package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
- package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
- package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
- package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
- package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
- package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
- package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
- package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
- package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
- package/dist/chunk-3HB3PEC6.mjs +240 -0
- package/dist/chunk-3HHQLEEQ.mjs +1092 -0
- package/dist/chunk-3OH67G7R.mjs +214 -0
- package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
- package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
- package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
- package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
- package/dist/chunk-5FVU47TB.mjs +20 -0
- package/dist/chunk-5YNRKI2C.mjs +81 -0
- package/dist/chunk-6L3XJXGU.mjs +470 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
- package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
- package/dist/chunk-7GXJUJ24.mjs +49 -0
- package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
- package/dist/chunk-BDRIVYOK.mjs +72 -0
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6ZDGYS.mjs +50 -0
- package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
- package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
- package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-GH4WQ22V.mjs +34 -0
- package/dist/chunk-GQPTDQCL.mjs +54 -0
- package/dist/chunk-H4KHPLHJ.mjs +382 -0
- package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
- package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
- package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
- package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
- package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
- package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
- package/dist/chunk-IPN7S4KX.mjs +130 -0
- package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
- package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
- package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
- package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
- package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
- package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
- package/dist/chunk-KJ3RTZDJ.mjs +101 -0
- package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
- package/dist/chunk-KP7TYHWT.mjs +92 -0
- package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
- package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
- package/dist/chunk-KYYABCSW.mjs +72 -0
- package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
- package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
- package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
- package/dist/chunk-MT2NRTA6.mjs +216 -0
- package/dist/chunk-NF4DT3IH.mjs +85 -0
- package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
- package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
- package/dist/chunk-PHQNACDT.mjs +63 -0
- package/dist/chunk-Q622CTMC.mjs +74 -0
- package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
- package/dist/chunk-QDZJM7NI.mjs +23 -0
- package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
- package/dist/chunk-QJVOLKU6.mjs +111 -0
- package/dist/chunk-QUJHMPTP.mjs +30 -0
- package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
- package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
- package/dist/chunk-RZWWB2XR.mjs +96 -0
- package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
- package/dist/chunk-S5LN746F.mjs +10 -0
- package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
- package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
- package/dist/chunk-SY3EWKQY.mjs +242 -0
- package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
- package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
- package/dist/chunk-UHICS3JI.mjs +83 -0
- package/dist/chunk-UZRZKNNZ.mjs +101 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
- package/dist/chunk-VHBNF4MA.mjs +543 -0
- package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
- package/dist/chunk-WQRN64Q6.mjs +196 -0
- package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
- package/dist/chunk-Y4WPNHMC.mjs +50 -0
- package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
- package/dist/chunk-YEKL4JBK.mjs +96 -0
- package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
- package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
- package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
- package/dist/chunk-ZTW5WTGG.mjs +167 -0
- package/dist/chunk-ZYAY26KG.mjs +29 -0
- package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
- package/dist/client.d.ts +9 -7
- package/dist/client.mjs +120 -7
- package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
- package/dist/config-CoSig5Ox.d.ts +482 -0
- package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
- package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +41 -9
- package/dist/distribution.mjs +9304 -7271
- package/dist/durability-config-GE4U7MCZ.mjs +89 -0
- package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
- package/dist/engine-hooks-DHCMOADY.mjs +47 -0
- package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
- package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
- package/dist/index.browser.mjs +9 -4
- package/dist/index.d.ts +7 -6
- package/dist/index.mjs +29 -21
- package/dist/index.react-native.mjs +19 -12
- package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
- package/dist/methods-7IOINERF.mjs +30 -0
- package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
- package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
- package/dist/operations-G5EP6JPX.mjs +5 -0
- package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
- package/dist/partial-words.mjs +7 -5
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
- package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
- package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +81 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +246 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +62 -0
- package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
- package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
- package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
- package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
- package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
- package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
- package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
- package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
- package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
- package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
- package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
- package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
- package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
- package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
- package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
- package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
- package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
- package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
- package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
- package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
- package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
- package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
- package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
- package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
- package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
- package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
- package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
- package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
- package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
- package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
- package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
- package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
- package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
- package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
- package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
- package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
- package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
- package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
- package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
- package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
- package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
- package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
- package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
- package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
- package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
- package/dist/react-native/partial-words.mjs +8 -5
- package/dist/react-native/pattern.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +258 -0
- package/dist/react-native/sync.mjs +2299 -0
- package/dist/react-native/synonyms.mjs +7 -4
- package/dist/react.d.ts +11 -10
- package/dist/react.mjs +8 -1
- package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
- package/dist/restore-queue-JVGENGGE.mjs +55 -0
- package/dist/retention-HROPV2QU.mjs +59 -0
- package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
- package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
- package/dist/server.d.ts +119 -20
- package/dist/server.mjs +1123 -58
- package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
- package/dist/start-UTWHGH42.mjs +59 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +271 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +237 -0
- package/dist/sync-state-ZWHLAFLE.mjs +15 -0
- package/dist/sync.browser.mjs +2294 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2268 -0
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
- package/dist/thread-6MOXY2KD.mjs +45 -0
- package/dist/types-BZCGjfLn.d.ts +227 -0
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +7 -7
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
- package/openapi.json +912 -5
- package/package.json +45 -8
- package/dist/browser/import-QOY6ZH6X.mjs +0 -12
- package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
- package/dist/chunk-7LDMWKP6.mjs +0 -209
- package/dist/engine-CJRUHXgf.d.ts +0 -664
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
- package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
- package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
- package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
- package/dist/restore-queue-AOF5OTQV.mjs +0 -41
- package/dist/thread-I7DVHHW7.mjs +0 -31
|
@@ -1,20 +1,34 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-G64Q6OJF.mjs';
|
|
2
|
+
import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-RND5CLX3.mjs';
|
|
3
|
+
import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-A5KUEGDY.mjs';
|
|
4
|
+
import './chunk-YJHGOJAC.mjs';
|
|
5
|
+
import { sha256 } from './chunk-M3MQE2X7.mjs';
|
|
6
|
+
import './chunk-CJSYBQ4J.mjs';
|
|
7
|
+
import './chunk-KKAMCABR.mjs';
|
|
8
|
+
import { QUERY_HISTORY_FOLDER } from './chunk-7IGKGWSQ.mjs';
|
|
2
9
|
import './chunk-JNOAKDUM.mjs';
|
|
3
|
-
import {
|
|
4
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
5
|
-
import {
|
|
6
|
-
import './chunk-
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
9
|
-
import {
|
|
10
|
+
import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-56W3HQVJ.mjs';
|
|
11
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-XY4VVBBO.mjs';
|
|
12
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-ZI2V4VD7.mjs';
|
|
13
|
+
import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-23XRXIBW.mjs';
|
|
14
|
+
import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-SMEQ36NT.mjs';
|
|
15
|
+
import './chunk-ML7THVLJ.mjs';
|
|
16
|
+
import { createRequestId } from './chunk-VI7BQR4U.mjs';
|
|
10
17
|
import './chunk-JPTQZWJQ.mjs';
|
|
18
|
+
import { encodeDocument, openChecksummedEnvelope, crc32, requireValidFieldTypes, HEADER_SIZE } from './chunk-JIJXMI42.mjs';
|
|
19
|
+
import { fnv1a } from './chunk-RA6LSBCM.mjs';
|
|
11
20
|
import './chunk-26A5JFT2.mjs';
|
|
12
|
-
import
|
|
21
|
+
import './chunk-3I27KXQD.mjs';
|
|
22
|
+
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
23
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-5TCXK5SB.mjs';
|
|
24
|
+
import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime, resolveWorkerEntry } from './chunk-4WCVMOU3.mjs';
|
|
13
25
|
import './chunk-TY7L6OXO.mjs';
|
|
14
26
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-4RD3AFSB.mjs';
|
|
27
|
+
import { compareCodePoints, MIN_M } from './chunk-LFIWRCEA.mjs';
|
|
28
|
+
import { MAX_FILE_BYTES } from './chunk-GPPOG6PT.mjs';
|
|
15
29
|
import './chunk-WXLBZJMT.mjs';
|
|
16
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
17
|
-
import { decode } from '@msgpack/msgpack';
|
|
30
|
+
import { NarsilError, ErrorCodes } from './chunk-SRMZXJSL.mjs';
|
|
31
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
18
32
|
|
|
19
33
|
// src/persistence/durability/node-fs.browser.ts
|
|
20
34
|
function filesystemUnavailable() {
|
|
@@ -516,7 +530,9 @@ function createDurableDirectory(root) {
|
|
|
516
530
|
const fs = await getFs();
|
|
517
531
|
const pathMod = await getPath();
|
|
518
532
|
const resolvedBase = pathMod.resolve(root);
|
|
519
|
-
const
|
|
533
|
+
const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
|
|
534
|
+
const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
|
|
535
|
+
const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
|
|
520
536
|
return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
|
|
521
537
|
},
|
|
522
538
|
async pathOf(key) {
|
|
@@ -526,714 +542,298 @@ function createDurableDirectory(root) {
|
|
|
526
542
|
};
|
|
527
543
|
}
|
|
528
544
|
|
|
529
|
-
// src/persistence/durability/
|
|
530
|
-
var
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
3049323471,
|
|
534
|
-
3921009573,
|
|
535
|
-
961987163,
|
|
536
|
-
1508970993,
|
|
537
|
-
2453635748,
|
|
538
|
-
2870763221,
|
|
539
|
-
3624381080,
|
|
540
|
-
310598401,
|
|
541
|
-
607225278,
|
|
542
|
-
1426881987,
|
|
543
|
-
1925078388,
|
|
544
|
-
2162078206,
|
|
545
|
-
2614888103,
|
|
546
|
-
3248222580,
|
|
547
|
-
3835390401,
|
|
548
|
-
4022224774,
|
|
549
|
-
264347078,
|
|
550
|
-
604807628,
|
|
551
|
-
770255983,
|
|
552
|
-
1249150122,
|
|
553
|
-
1555081692,
|
|
554
|
-
1996064986,
|
|
555
|
-
2554220882,
|
|
556
|
-
2821834349,
|
|
557
|
-
2952996808,
|
|
558
|
-
3210313671,
|
|
559
|
-
3336571891,
|
|
560
|
-
3584528711,
|
|
561
|
-
113926993,
|
|
562
|
-
338241895,
|
|
563
|
-
666307205,
|
|
564
|
-
773529912,
|
|
565
|
-
1294757372,
|
|
566
|
-
1396182291,
|
|
567
|
-
1695183700,
|
|
568
|
-
1986661051,
|
|
569
|
-
2177026350,
|
|
570
|
-
2456956037,
|
|
571
|
-
2730485921,
|
|
572
|
-
2820302411,
|
|
573
|
-
3259730800,
|
|
574
|
-
3345764771,
|
|
575
|
-
3516065817,
|
|
576
|
-
3600352804,
|
|
577
|
-
4094571909,
|
|
578
|
-
275423344,
|
|
579
|
-
430227734,
|
|
580
|
-
506948616,
|
|
581
|
-
659060556,
|
|
582
|
-
883997877,
|
|
583
|
-
958139571,
|
|
584
|
-
1322822218,
|
|
585
|
-
1537002063,
|
|
586
|
-
1747873779,
|
|
587
|
-
1955562222,
|
|
588
|
-
2024104815,
|
|
589
|
-
2227730452,
|
|
590
|
-
2361852424,
|
|
591
|
-
2428436474,
|
|
592
|
-
2756734187,
|
|
593
|
-
3204031479,
|
|
594
|
-
3329325298
|
|
595
|
-
]);
|
|
596
|
-
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
597
|
-
var BLOCK_BYTES = 64;
|
|
598
|
-
var LENGTH_BYTES = 8;
|
|
599
|
-
function rotateRight(value, bits) {
|
|
600
|
-
return value >>> bits | value << 32 - bits;
|
|
601
|
-
}
|
|
602
|
-
function padded(bytes) {
|
|
603
|
-
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
604
|
-
const message = new Uint8Array(total);
|
|
605
|
-
message.set(bytes);
|
|
606
|
-
message[bytes.byteLength] = 128;
|
|
607
|
-
const view = new DataView(message.buffer);
|
|
608
|
-
const bitLength = bytes.byteLength * 8;
|
|
609
|
-
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
610
|
-
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
611
|
-
return view;
|
|
612
|
-
}
|
|
613
|
-
function sha256(bytes) {
|
|
614
|
-
const view = padded(bytes);
|
|
615
|
-
const state = INITIAL_STATE.slice();
|
|
616
|
-
const words = new Uint32Array(64);
|
|
617
|
-
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
618
|
-
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
619
|
-
for (let i = 16; i < 64; i++) {
|
|
620
|
-
const w15 = words[i - 15];
|
|
621
|
-
const w2 = words[i - 2];
|
|
622
|
-
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
623
|
-
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
624
|
-
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
625
|
-
}
|
|
626
|
-
let [a, b, c, d, e, f, g, h] = state;
|
|
627
|
-
for (let i = 0; i < 64; i++) {
|
|
628
|
-
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
629
|
-
const choice = e & f ^ ~e & g;
|
|
630
|
-
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
631
|
-
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
632
|
-
const majority = a & b ^ a & c ^ b & c;
|
|
633
|
-
const t2 = s0 + majority >>> 0;
|
|
634
|
-
h = g;
|
|
635
|
-
g = f;
|
|
636
|
-
f = e;
|
|
637
|
-
e = d + t1 >>> 0;
|
|
638
|
-
d = c;
|
|
639
|
-
c = b;
|
|
640
|
-
b = a;
|
|
641
|
-
a = t1 + t2 >>> 0;
|
|
642
|
-
}
|
|
643
|
-
state[0] = state[0] + a >>> 0;
|
|
644
|
-
state[1] = state[1] + b >>> 0;
|
|
645
|
-
state[2] = state[2] + c >>> 0;
|
|
646
|
-
state[3] = state[3] + d >>> 0;
|
|
647
|
-
state[4] = state[4] + e >>> 0;
|
|
648
|
-
state[5] = state[5] + f >>> 0;
|
|
649
|
-
state[6] = state[6] + g >>> 0;
|
|
650
|
-
state[7] = state[7] + h >>> 0;
|
|
651
|
-
}
|
|
652
|
-
const digest = new Uint8Array(32);
|
|
653
|
-
const out = new DataView(digest.buffer);
|
|
654
|
-
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
655
|
-
return digest;
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
// src/persistence/durability/restore-journal.ts
|
|
659
|
-
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
660
|
-
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
661
|
-
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
662
|
-
var LOCK_FILE_KEY = ".narsil.lock";
|
|
663
|
-
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
664
|
-
function digestOf(indexName) {
|
|
665
|
-
let hex = "";
|
|
666
|
-
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
667
|
-
return hex;
|
|
545
|
+
// src/persistence/durability/sync/layout.ts
|
|
546
|
+
var SYNC_KEY_NUMBER_WIDTH = 16;
|
|
547
|
+
function syncDirectoryPrefix(indexName) {
|
|
548
|
+
return `${indexName}/sync/`;
|
|
668
549
|
}
|
|
669
|
-
function
|
|
670
|
-
return `${
|
|
550
|
+
function publishedSnapshotKey(indexName) {
|
|
551
|
+
return `${syncDirectoryPrefix(indexName)}published`;
|
|
671
552
|
}
|
|
672
|
-
function
|
|
673
|
-
return `${
|
|
553
|
+
function historyRoot(indexName) {
|
|
554
|
+
return `${indexName}/history/`;
|
|
674
555
|
}
|
|
675
|
-
function
|
|
676
|
-
return `${
|
|
556
|
+
function historyPrefix(indexName, partitionId) {
|
|
557
|
+
return `${historyRoot(indexName)}${partitionId}/`;
|
|
677
558
|
}
|
|
678
|
-
function
|
|
679
|
-
return `${
|
|
559
|
+
function walSegmentPrefix(indexName, partitionId) {
|
|
560
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
680
561
|
}
|
|
681
|
-
function
|
|
682
|
-
return
|
|
683
|
-
|
|
562
|
+
function formatSyncKeyNumber(value) {
|
|
563
|
+
return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
|
|
564
|
+
}
|
|
565
|
+
function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
|
|
566
|
+
return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
|
|
567
|
+
}
|
|
568
|
+
function walSegmentKey(indexName, partitionId, startSeqNo) {
|
|
569
|
+
return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
|
|
570
|
+
}
|
|
571
|
+
var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
|
|
572
|
+
function parseHistoryKey(indexName, key) {
|
|
573
|
+
const root = historyRoot(indexName);
|
|
574
|
+
if (!key.startsWith(root)) return null;
|
|
575
|
+
const match = HISTORY_TAIL.exec(key.slice(root.length));
|
|
576
|
+
if (match === null) return null;
|
|
577
|
+
const partitionId = Number.parseInt(match[1], 10);
|
|
578
|
+
const startSeqNo = Number.parseInt(match[2], 10);
|
|
579
|
+
const movedAtMs = Number.parseInt(match[3], 10);
|
|
580
|
+
if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
|
|
581
|
+
return null;
|
|
582
|
+
}
|
|
583
|
+
return { key, partitionId, startSeqNo, movedAtMs };
|
|
684
584
|
}
|
|
685
|
-
function
|
|
686
|
-
|
|
585
|
+
function parseWalSegmentStart(indexName, partitionId, key) {
|
|
586
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
587
|
+
if (!key.startsWith(prefix)) return null;
|
|
588
|
+
const tail = key.slice(prefix.length);
|
|
589
|
+
if (!/^\d{16}$/.test(tail)) return null;
|
|
590
|
+
const value = Number.parseInt(tail, 10);
|
|
591
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
687
592
|
}
|
|
688
|
-
|
|
689
|
-
|
|
593
|
+
|
|
594
|
+
// src/persistence/durability/sync/history.ts
|
|
595
|
+
function compareHistorySegments(a, b) {
|
|
596
|
+
return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
|
|
597
|
+
}
|
|
598
|
+
async function listHistorySegments(directory, indexName, partitionId) {
|
|
599
|
+
const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
|
|
600
|
+
const segments = [];
|
|
601
|
+
for (const key of await directory.list(prefix)) {
|
|
602
|
+
const segment = parseHistoryKey(indexName, key);
|
|
603
|
+
if (segment !== null) segments.push(segment);
|
|
604
|
+
}
|
|
605
|
+
return segments.sort(compareHistorySegments);
|
|
606
|
+
}
|
|
607
|
+
async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
|
|
608
|
+
await directory.move(
|
|
609
|
+
walSegmentKey(indexName, partitionId, startSeqNo),
|
|
610
|
+
historyKey(indexName, partitionId, startSeqNo, movedAtMs)
|
|
611
|
+
);
|
|
690
612
|
}
|
|
691
|
-
function
|
|
692
|
-
const
|
|
693
|
-
|
|
694
|
-
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
695
|
-
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
696
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
697
|
-
corruptJournal(key, "names an index that no index name can match");
|
|
698
|
-
}
|
|
699
|
-
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
700
|
-
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
701
|
-
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
613
|
+
async function removeHistory(directory, indexName, partitionId) {
|
|
614
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
615
|
+
await directory.remove(segment.key);
|
|
702
616
|
}
|
|
703
|
-
return { kind: "journal", indexName, stagingPrefix };
|
|
704
617
|
}
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
710
|
-
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
711
|
-
{ indexName, stagingPrefix }
|
|
712
|
-
);
|
|
713
|
-
}
|
|
714
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
715
|
-
if (decoded.kind !== "current") {
|
|
716
|
-
throw new NarsilError(
|
|
717
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
718
|
-
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
719
|
-
{ indexName }
|
|
720
|
-
);
|
|
721
|
-
}
|
|
722
|
-
const root = `${indexName}/`;
|
|
723
|
-
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
724
|
-
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
725
|
-
return keys;
|
|
618
|
+
|
|
619
|
+
// src/persistence/durability/checkpoint.ts
|
|
620
|
+
function walPrefix(indexName, partitionId) {
|
|
621
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
726
622
|
}
|
|
727
|
-
function
|
|
728
|
-
|
|
623
|
+
async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
|
|
624
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
625
|
+
const prefix = walPrefix(indexName, partitionId);
|
|
626
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
627
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
628
|
+
if (marker === null) {
|
|
629
|
+
continue;
|
|
630
|
+
}
|
|
631
|
+
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
632
|
+
const segments = await directory.list(prefix);
|
|
633
|
+
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
634
|
+
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
635
|
+
const start = startSeqNos[i];
|
|
636
|
+
if (start >= activeSegmentSeqNo) {
|
|
637
|
+
break;
|
|
638
|
+
}
|
|
639
|
+
const nextStart = startSeqNos[i + 1];
|
|
640
|
+
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
641
|
+
if (segmentCoversBeyondCheckpoint) {
|
|
642
|
+
break;
|
|
643
|
+
}
|
|
644
|
+
if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
645
|
+
else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
729
648
|
}
|
|
730
|
-
async function
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
738
|
-
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
739
|
-
} else {
|
|
740
|
-
const stagedKeys = new Set(staged);
|
|
741
|
-
for (const key of listed) {
|
|
742
|
-
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
649
|
+
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
650
|
+
for (const [partitionId, partition] of [...partitions]) {
|
|
651
|
+
if (partitionId < partitionCount) continue;
|
|
652
|
+
try {
|
|
653
|
+
await partition.walWriter.close();
|
|
654
|
+
} catch (err) {
|
|
655
|
+
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
743
656
|
}
|
|
657
|
+
partitions.delete(partitionId);
|
|
744
658
|
}
|
|
745
|
-
|
|
746
|
-
|
|
659
|
+
const walRoot = `${indexName}/wal/`;
|
|
660
|
+
for (const key of await directory.list(walRoot)) {
|
|
661
|
+
const partitionId = walKeyPartitionId(key, walRoot);
|
|
662
|
+
if (partitionId === null || partitionId < partitionCount) continue;
|
|
663
|
+
await directory.remove(key);
|
|
747
664
|
}
|
|
748
|
-
await storage.remove(journalKey(indexName));
|
|
749
|
-
return retiredPrefix;
|
|
750
665
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
666
|
+
function walKeyPartitionId(key, walRoot) {
|
|
667
|
+
const tail = key.slice(walRoot.length);
|
|
668
|
+
const separator = tail.indexOf("/");
|
|
669
|
+
if (separator <= 0) {
|
|
670
|
+
return null;
|
|
755
671
|
}
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
760
|
-
}
|
|
761
|
-
function isTopLevelJournal(key) {
|
|
762
|
-
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
763
|
-
}
|
|
764
|
-
async function removeStagingPrefixes(storage) {
|
|
765
|
-
const prefixes = /* @__PURE__ */ new Set();
|
|
766
|
-
for (const key of await storage.list(".")) {
|
|
767
|
-
const slash = key.indexOf("/");
|
|
768
|
-
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
672
|
+
const segment = tail.slice(0, separator);
|
|
673
|
+
if (!/^\d+$/.test(segment)) {
|
|
674
|
+
return null;
|
|
769
675
|
}
|
|
770
|
-
|
|
676
|
+
const value = Number.parseInt(segment, 10);
|
|
677
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
771
678
|
}
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
const bytes = await storage.read(key);
|
|
776
|
-
if (bytes === null) continue;
|
|
777
|
-
const reading = readJournal(key, bytes);
|
|
778
|
-
if (reading.kind === "leftover") {
|
|
779
|
-
await storage.remove(key);
|
|
780
|
-
continue;
|
|
781
|
-
}
|
|
782
|
-
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
679
|
+
function parseSegmentStartSeqNo(key, prefix) {
|
|
680
|
+
if (!key.startsWith(prefix)) {
|
|
681
|
+
return null;
|
|
783
682
|
}
|
|
784
|
-
|
|
683
|
+
const tail = key.slice(prefix.length);
|
|
684
|
+
if (!/^\d{16}$/.test(tail)) {
|
|
685
|
+
return null;
|
|
686
|
+
}
|
|
687
|
+
const value = Number.parseInt(tail, 10);
|
|
688
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
785
689
|
}
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
function missing(error) {
|
|
789
|
-
return error.code === "ENOENT";
|
|
690
|
+
function formatSegmentStart(startSeqNo) {
|
|
691
|
+
return startSeqNo.toString().padStart(16, "0");
|
|
790
692
|
}
|
|
791
|
-
|
|
792
|
-
|
|
693
|
+
|
|
694
|
+
// src/persistence/durability/marker-writer.ts
|
|
695
|
+
function markerKey(indexName, partitionId) {
|
|
696
|
+
return `${indexName}/wal/${partitionId}/commit`;
|
|
793
697
|
}
|
|
794
|
-
function
|
|
698
|
+
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
699
|
+
const key = markerKey(indexName, partitionId);
|
|
700
|
+
const handle = await directory.markerHandle(key);
|
|
701
|
+
const existing = await handle.read();
|
|
702
|
+
const current = readCommitMarker(existing);
|
|
703
|
+
const createdFresh = existing.length === 0;
|
|
704
|
+
let lastSlot = current?.slotIndex ?? null;
|
|
705
|
+
let writeSeq = current?.state.writeSeq ?? 0;
|
|
795
706
|
return {
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
async moveTree(from, to) {
|
|
805
|
-
const source = await directory.pathOf(from);
|
|
806
|
-
const target = await directory.pathOf(to);
|
|
807
|
-
const fs = await getFs();
|
|
808
|
-
try {
|
|
809
|
-
await fs.rename(source, target);
|
|
810
|
-
} catch (error) {
|
|
811
|
-
if (missing(error)) return;
|
|
812
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
813
|
-
}
|
|
814
|
-
await directory.syncDirectoryOf(to);
|
|
707
|
+
created: createdFresh,
|
|
708
|
+
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
709
|
+
async commit(state) {
|
|
710
|
+
const slotIndex = nextMarkerSlot(lastSlot);
|
|
711
|
+
writeSeq += 1;
|
|
712
|
+
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
713
|
+
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
714
|
+
lastSlot = slotIndex;
|
|
815
715
|
},
|
|
816
|
-
async
|
|
817
|
-
|
|
818
|
-
const fs = await getFs();
|
|
819
|
-
try {
|
|
820
|
-
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
821
|
-
} catch (error) {
|
|
822
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
823
|
-
}
|
|
824
|
-
await directory.syncDirectoryOf(key);
|
|
716
|
+
async close() {
|
|
717
|
+
await handle.close();
|
|
825
718
|
}
|
|
826
719
|
};
|
|
827
720
|
}
|
|
828
|
-
function recoverDirectoryRestores(directory) {
|
|
829
|
-
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
830
|
-
}
|
|
831
721
|
|
|
832
|
-
// src/persistence/durability/
|
|
833
|
-
function
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
return next === "" || next === "/" || next === "\\";
|
|
722
|
+
// src/persistence/durability/recovery.ts
|
|
723
|
+
async function removeLeftoverRewriteFiles(directory, names) {
|
|
724
|
+
for (const indexName of names) {
|
|
725
|
+
for (const place of ["rebuilt", "received"]) {
|
|
726
|
+
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
840
729
|
}
|
|
841
|
-
function
|
|
842
|
-
const
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
848
|
-
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
849
|
-
read: (key) => directory.read(staged(key)),
|
|
850
|
-
remove: (key) => directory.remove(staged(key)),
|
|
851
|
-
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
852
|
-
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
853
|
-
pathOf: (key) => directory.pathOf(staged(key)),
|
|
854
|
-
...readRange === void 0 ? {} : {
|
|
855
|
-
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
856
|
-
},
|
|
857
|
-
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
858
|
-
};
|
|
859
|
-
}
|
|
860
|
-
function createRestoreStaging(storage, indexName, options = {}) {
|
|
861
|
-
const stagingPrefix = newStagingPrefix();
|
|
862
|
-
const indexRoot = `${indexName}/`;
|
|
863
|
-
let committed = false;
|
|
864
|
-
const staged = (key) => {
|
|
865
|
-
if (!key.startsWith(indexRoot)) {
|
|
866
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
867
|
-
key
|
|
868
|
-
});
|
|
730
|
+
async function listPersistedIndexes(directory) {
|
|
731
|
+
const metaKeys = await directory.list("");
|
|
732
|
+
const names = [];
|
|
733
|
+
for (const key of metaKeys) {
|
|
734
|
+
if (key.endsWith("/meta")) {
|
|
735
|
+
names.push(key.slice(0, -"/meta".length));
|
|
869
736
|
}
|
|
870
|
-
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
871
|
-
};
|
|
872
|
-
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
873
|
-
const store = {
|
|
874
|
-
read: (key) => storage.read(staged(key)),
|
|
875
|
-
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
876
|
-
remove: (key) => storage.remove(staged(key)),
|
|
877
|
-
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
878
|
-
};
|
|
879
|
-
const pathOf = storage.pathOf;
|
|
880
|
-
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
881
|
-
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
882
|
-
let retiredPrefix = null;
|
|
883
|
-
async function finishOnStorage() {
|
|
884
|
-
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
885
737
|
}
|
|
886
|
-
return
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
738
|
+
return names;
|
|
739
|
+
}
|
|
740
|
+
async function loadMetadata(directory, indexName) {
|
|
741
|
+
const bytes = await directory.read(`${indexName}/meta`);
|
|
742
|
+
if (bytes === null) {
|
|
743
|
+
return null;
|
|
744
|
+
}
|
|
745
|
+
const { metadata } = await readMetadataEnvelope(bytes);
|
|
746
|
+
return metadata;
|
|
747
|
+
}
|
|
748
|
+
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
749
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
750
|
+
const read = await readWalSegments(directory, indexName, partitionId);
|
|
751
|
+
if (read === null) {
|
|
752
|
+
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
753
|
+
}
|
|
754
|
+
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
755
|
+
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
756
|
+
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
757
|
+
}
|
|
758
|
+
let highestSeqNo = fromSeqNoExclusive;
|
|
759
|
+
let highestPrimaryTerm = 0;
|
|
760
|
+
const replay = async (entries) => {
|
|
761
|
+
for (const entry of entries) {
|
|
762
|
+
if (entry.seqNo <= highestSeqNo) {
|
|
763
|
+
continue;
|
|
905
764
|
}
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
} catch (error) {
|
|
912
|
-
options.onStorageLost?.(toError2(error));
|
|
765
|
+
if (turnIsSpent()) await yieldTurn();
|
|
766
|
+
applyEntry(entry, deps);
|
|
767
|
+
highestSeqNo = entry.seqNo;
|
|
768
|
+
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
769
|
+
highestPrimaryTerm = entry.primaryTerm;
|
|
913
770
|
}
|
|
914
|
-
},
|
|
915
|
-
async removeRetired() {
|
|
916
|
-
const prefix = retiredPrefix;
|
|
917
|
-
retiredPrefix = null;
|
|
918
|
-
if (prefix !== null) await removePrefix(storage, prefix);
|
|
919
|
-
},
|
|
920
|
-
async discard() {
|
|
921
|
-
if (committed) return;
|
|
922
|
-
await removePrefix(storage, stagingPrefix);
|
|
923
771
|
}
|
|
924
772
|
};
|
|
773
|
+
await replay(durableEntries);
|
|
774
|
+
if (activeTail !== null) {
|
|
775
|
+
await replay(activeTail.entries);
|
|
776
|
+
}
|
|
777
|
+
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
778
|
+
if (activeTail !== null) {
|
|
779
|
+
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
780
|
+
}
|
|
781
|
+
return { highestSeqNo, highestPrimaryTerm };
|
|
925
782
|
}
|
|
926
|
-
function
|
|
927
|
-
const
|
|
928
|
-
return
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
783
|
+
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
784
|
+
const lastReplayed = tail.entries.at(-1);
|
|
785
|
+
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
786
|
+
const handle = await directory.appendHandle(tail.key);
|
|
787
|
+
try {
|
|
788
|
+
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
789
|
+
else await handle.sync();
|
|
790
|
+
} finally {
|
|
791
|
+
await handle.close();
|
|
792
|
+
}
|
|
793
|
+
if (lastReplayed === void 0) return;
|
|
794
|
+
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
795
|
+
try {
|
|
796
|
+
await writer.commit({
|
|
797
|
+
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
798
|
+
durableByteLength: tail.cleanEnd,
|
|
799
|
+
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
800
|
+
});
|
|
801
|
+
} finally {
|
|
802
|
+
await writer.close();
|
|
803
|
+
}
|
|
941
804
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
805
|
+
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
806
|
+
for (const { key, startSeqNo } of segments) {
|
|
807
|
+
if (startSeqNo > activeSegmentSeqNo) {
|
|
808
|
+
await directory.remove(key);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
946
811
|
}
|
|
947
|
-
function
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
throw error;
|
|
955
|
-
}
|
|
956
|
-
},
|
|
957
|
-
onStorageLost: deps.markFatal
|
|
958
|
-
});
|
|
812
|
+
function applyEntry(entry, deps) {
|
|
813
|
+
deps.writeRecords?.replayed(entry);
|
|
814
|
+
if (entry.operation === "DELETE") {
|
|
815
|
+
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
959
819
|
}
|
|
960
|
-
function
|
|
961
|
-
|
|
962
|
-
return () => {
|
|
963
|
-
if (released) return;
|
|
964
|
-
released = true;
|
|
965
|
-
const count = (held.get(indexName) ?? 1) - 1;
|
|
966
|
-
if (count <= 0) held.delete(indexName);
|
|
967
|
-
else held.set(indexName, count);
|
|
968
|
-
};
|
|
820
|
+
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
821
|
+
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
969
822
|
}
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
823
|
+
|
|
824
|
+
// src/persistence/durability/checkpoint-capture.ts
|
|
825
|
+
function whileNoMutationApplies(partitions, capture2) {
|
|
826
|
+
return new Promise((resolve, reject) => {
|
|
827
|
+
let release = () => void 0;
|
|
828
|
+
const released = new Promise((settle) => {
|
|
829
|
+
release = settle;
|
|
830
|
+
});
|
|
831
|
+
function captureNow() {
|
|
976
832
|
try {
|
|
977
|
-
|
|
978
|
-
await deps.checkpoint(indexName, true);
|
|
979
|
-
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
980
|
-
if (manifestBytes === null) {
|
|
981
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
982
|
-
indexName
|
|
983
|
-
});
|
|
984
|
-
}
|
|
985
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
986
|
-
if (decoded.kind !== "current") {
|
|
987
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
988
|
-
indexName
|
|
989
|
-
});
|
|
990
|
-
}
|
|
991
|
-
return {
|
|
992
|
-
manifestBytes,
|
|
993
|
-
manifest: decoded.manifest,
|
|
994
|
-
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
995
|
-
readRange: async (ref, offset, length) => {
|
|
996
|
-
const readRange = deps.directory.readRange;
|
|
997
|
-
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
998
|
-
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
999
|
-
return whole.slice(offset, offset + length);
|
|
1000
|
-
},
|
|
1001
|
-
release,
|
|
1002
|
-
directory: deps.directory
|
|
1003
|
-
};
|
|
833
|
+
resolve(capture2());
|
|
1004
834
|
} catch (error) {
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
}
|
|
1008
|
-
},
|
|
1009
|
-
stageRestore(indexName) {
|
|
1010
|
-
return stageDirectoryRestore(deps, indexName);
|
|
1011
|
-
},
|
|
1012
|
-
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
1013
|
-
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
1014
|
-
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
1015
|
-
}
|
|
1016
|
-
}
|
|
1017
|
-
};
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
// src/persistence/durability/checkpoint.ts
|
|
1021
|
-
function walPrefix(indexName, partitionId) {
|
|
1022
|
-
return `${indexName}/wal/${partitionId}/`;
|
|
1023
|
-
}
|
|
1024
|
-
async function truncateCoveredSegments(directory, indexName, checkpoint) {
|
|
1025
|
-
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
1026
|
-
const prefix = walPrefix(indexName, partitionId);
|
|
1027
|
-
const markerBytes = await directory.read(`${prefix}commit`);
|
|
1028
|
-
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
1029
|
-
if (marker === null) {
|
|
1030
|
-
continue;
|
|
1031
|
-
}
|
|
1032
|
-
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
1033
|
-
const segments = await directory.list(prefix);
|
|
1034
|
-
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
1035
|
-
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
1036
|
-
const start = startSeqNos[i];
|
|
1037
|
-
if (start >= activeSegmentSeqNo) {
|
|
1038
|
-
break;
|
|
1039
|
-
}
|
|
1040
|
-
const nextStart = startSeqNos[i + 1];
|
|
1041
|
-
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
1042
|
-
if (segmentCoversBeyondCheckpoint) {
|
|
1043
|
-
break;
|
|
1044
|
-
}
|
|
1045
|
-
await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
1046
|
-
}
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
1050
|
-
for (const [partitionId, partition] of [...partitions]) {
|
|
1051
|
-
if (partitionId < partitionCount) continue;
|
|
1052
|
-
try {
|
|
1053
|
-
await partition.walWriter.close();
|
|
1054
|
-
} catch (err) {
|
|
1055
|
-
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
1056
|
-
}
|
|
1057
|
-
partitions.delete(partitionId);
|
|
1058
|
-
}
|
|
1059
|
-
const walRoot = `${indexName}/wal/`;
|
|
1060
|
-
for (const key of await directory.list(walRoot)) {
|
|
1061
|
-
const partitionId = walKeyPartitionId(key, walRoot);
|
|
1062
|
-
if (partitionId === null || partitionId < partitionCount) continue;
|
|
1063
|
-
await directory.remove(key);
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
function walKeyPartitionId(key, walRoot) {
|
|
1067
|
-
const tail = key.slice(walRoot.length);
|
|
1068
|
-
const separator = tail.indexOf("/");
|
|
1069
|
-
if (separator <= 0) {
|
|
1070
|
-
return null;
|
|
1071
|
-
}
|
|
1072
|
-
const segment = tail.slice(0, separator);
|
|
1073
|
-
if (!/^\d+$/.test(segment)) {
|
|
1074
|
-
return null;
|
|
1075
|
-
}
|
|
1076
|
-
const value = Number.parseInt(segment, 10);
|
|
1077
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
1078
|
-
}
|
|
1079
|
-
function parseSegmentStartSeqNo(key, prefix) {
|
|
1080
|
-
if (!key.startsWith(prefix)) {
|
|
1081
|
-
return null;
|
|
1082
|
-
}
|
|
1083
|
-
const tail = key.slice(prefix.length);
|
|
1084
|
-
if (!/^\d{16}$/.test(tail)) {
|
|
1085
|
-
return null;
|
|
1086
|
-
}
|
|
1087
|
-
const value = Number.parseInt(tail, 10);
|
|
1088
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
1089
|
-
}
|
|
1090
|
-
function formatSegmentStart(startSeqNo) {
|
|
1091
|
-
return startSeqNo.toString().padStart(16, "0");
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
// src/persistence/durability/marker-writer.ts
|
|
1095
|
-
function markerKey(indexName, partitionId) {
|
|
1096
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
1097
|
-
}
|
|
1098
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
1099
|
-
const key = markerKey(indexName, partitionId);
|
|
1100
|
-
const handle = await directory.markerHandle(key);
|
|
1101
|
-
const existing = await handle.read();
|
|
1102
|
-
const current = readCommitMarker(existing);
|
|
1103
|
-
const createdFresh = existing.length === 0;
|
|
1104
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
1105
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
1106
|
-
return {
|
|
1107
|
-
created: createdFresh,
|
|
1108
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
1109
|
-
async commit(state) {
|
|
1110
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
1111
|
-
writeSeq += 1;
|
|
1112
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
1113
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
1114
|
-
lastSlot = slotIndex;
|
|
1115
|
-
},
|
|
1116
|
-
async close() {
|
|
1117
|
-
await handle.close();
|
|
1118
|
-
}
|
|
1119
|
-
};
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
// src/persistence/durability/recovery.ts
|
|
1123
|
-
async function removeLeftoverRewriteFiles(directory, names) {
|
|
1124
|
-
for (const indexName of names) {
|
|
1125
|
-
for (const place of ["rebuilt", "received"]) {
|
|
1126
|
-
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1130
|
-
async function listPersistedIndexes(directory) {
|
|
1131
|
-
const metaKeys = await directory.list("");
|
|
1132
|
-
const names = [];
|
|
1133
|
-
for (const key of metaKeys) {
|
|
1134
|
-
if (key.endsWith("/meta")) {
|
|
1135
|
-
names.push(key.slice(0, -"/meta".length));
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
|
-
return names;
|
|
1139
|
-
}
|
|
1140
|
-
async function loadMetadata(directory, indexName) {
|
|
1141
|
-
const bytes = await directory.read(`${indexName}/meta`);
|
|
1142
|
-
if (bytes === null) {
|
|
1143
|
-
return null;
|
|
1144
|
-
}
|
|
1145
|
-
const { metadata } = await readMetadataEnvelope(bytes);
|
|
1146
|
-
return metadata;
|
|
1147
|
-
}
|
|
1148
|
-
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
1149
|
-
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
1150
|
-
const read = await readWalSegments(directory, indexName, partitionId);
|
|
1151
|
-
if (read === null) {
|
|
1152
|
-
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
1153
|
-
}
|
|
1154
|
-
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
1155
|
-
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
1156
|
-
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
1157
|
-
}
|
|
1158
|
-
let highestSeqNo = fromSeqNoExclusive;
|
|
1159
|
-
let highestPrimaryTerm = 0;
|
|
1160
|
-
const replay = async (entries) => {
|
|
1161
|
-
for (const entry of entries) {
|
|
1162
|
-
if (entry.seqNo <= highestSeqNo) {
|
|
1163
|
-
continue;
|
|
1164
|
-
}
|
|
1165
|
-
if (turnIsSpent()) await yieldTurn();
|
|
1166
|
-
applyEntry(entry, deps);
|
|
1167
|
-
highestSeqNo = entry.seqNo;
|
|
1168
|
-
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
1169
|
-
highestPrimaryTerm = entry.primaryTerm;
|
|
1170
|
-
}
|
|
1171
|
-
}
|
|
1172
|
-
};
|
|
1173
|
-
await replay(durableEntries);
|
|
1174
|
-
if (activeTail !== null) {
|
|
1175
|
-
await replay(activeTail.entries);
|
|
1176
|
-
}
|
|
1177
|
-
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
1178
|
-
if (activeTail !== null) {
|
|
1179
|
-
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
1180
|
-
}
|
|
1181
|
-
return { highestSeqNo, highestPrimaryTerm };
|
|
1182
|
-
}
|
|
1183
|
-
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
1184
|
-
const lastReplayed = tail.entries.at(-1);
|
|
1185
|
-
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
1186
|
-
const handle = await directory.appendHandle(tail.key);
|
|
1187
|
-
try {
|
|
1188
|
-
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
1189
|
-
else await handle.sync();
|
|
1190
|
-
} finally {
|
|
1191
|
-
await handle.close();
|
|
1192
|
-
}
|
|
1193
|
-
if (lastReplayed === void 0) return;
|
|
1194
|
-
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
1195
|
-
try {
|
|
1196
|
-
await writer.commit({
|
|
1197
|
-
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
1198
|
-
durableByteLength: tail.cleanEnd,
|
|
1199
|
-
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
1200
|
-
});
|
|
1201
|
-
} finally {
|
|
1202
|
-
await writer.close();
|
|
1203
|
-
}
|
|
1204
|
-
}
|
|
1205
|
-
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
1206
|
-
for (const { key, startSeqNo } of segments) {
|
|
1207
|
-
if (startSeqNo > activeSegmentSeqNo) {
|
|
1208
|
-
await directory.remove(key);
|
|
1209
|
-
}
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
|
-
function applyEntry(entry, deps) {
|
|
1213
|
-
deps.writeRecords?.replayed(entry);
|
|
1214
|
-
if (entry.operation === "DELETE") {
|
|
1215
|
-
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
1216
|
-
return;
|
|
1217
|
-
}
|
|
1218
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
1219
|
-
}
|
|
1220
|
-
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
1221
|
-
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
1222
|
-
}
|
|
1223
|
-
|
|
1224
|
-
// src/persistence/durability/checkpoint-capture.ts
|
|
1225
|
-
function whileNoMutationApplies(partitions, capture2) {
|
|
1226
|
-
return new Promise((resolve, reject) => {
|
|
1227
|
-
let release = () => void 0;
|
|
1228
|
-
const released = new Promise((settle) => {
|
|
1229
|
-
release = settle;
|
|
1230
|
-
});
|
|
1231
|
-
function captureNow() {
|
|
1232
|
-
try {
|
|
1233
|
-
resolve(capture2());
|
|
1234
|
-
} catch (error) {
|
|
1235
|
-
reject(error);
|
|
1236
|
-
} finally {
|
|
835
|
+
reject(error);
|
|
836
|
+
} finally {
|
|
1237
837
|
release();
|
|
1238
838
|
}
|
|
1239
839
|
}
|
|
@@ -1717,7 +1317,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1717
1317
|
}
|
|
1718
1318
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1719
1319
|
const { indexName, indexState, hooks } = input;
|
|
1720
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
1320
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-E33X4AOK.mjs');
|
|
1721
1321
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1722
1322
|
if (adoption === null) return [];
|
|
1723
1323
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -1772,10 +1372,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1772
1372
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
1773
1373
|
if (!input.holdsCheckpointFiles()) {
|
|
1774
1374
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
1775
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
1375
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-E33X4AOK.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1376
|
+
const keptForSync = await input.keepsCheckpointKey();
|
|
1776
1377
|
await removeCheckpointGarbage(
|
|
1777
1378
|
store,
|
|
1778
|
-
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
1379
|
+
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
|
|
1779
1380
|
);
|
|
1780
1381
|
}
|
|
1781
1382
|
const checkpointDocumentCount = documentCount;
|
|
@@ -1788,7 +1389,9 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
1788
1389
|
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
1789
1390
|
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
|
|
1790
1391
|
});
|
|
1791
|
-
|
|
1392
|
+
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
1393
|
+
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
1394
|
+
if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
|
|
1792
1395
|
await reclaimWalBeyondCount(
|
|
1793
1396
|
input.directory,
|
|
1794
1397
|
input.indexName,
|
|
@@ -1820,202 +1423,1014 @@ function createIndexLoads(vectorIndexesOf2) {
|
|
|
1820
1423
|
}
|
|
1821
1424
|
};
|
|
1822
1425
|
return {
|
|
1823
|
-
track(indexName, load, startsGraphRepairs = true) {
|
|
1824
|
-
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
1825
|
-
loads.set(
|
|
1826
|
-
indexName,
|
|
1827
|
-
loaded.catch(() => void 0)
|
|
1828
|
-
);
|
|
1829
|
-
return loaded;
|
|
1830
|
-
},
|
|
1831
|
-
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1832
|
-
startGraphRepairs
|
|
1426
|
+
track(indexName, load, startsGraphRepairs = true) {
|
|
1427
|
+
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
1428
|
+
loads.set(
|
|
1429
|
+
indexName,
|
|
1430
|
+
loaded.catch(() => void 0)
|
|
1431
|
+
);
|
|
1432
|
+
return loaded;
|
|
1433
|
+
},
|
|
1434
|
+
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1435
|
+
startGraphRepairs
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
async function switchInTurn(turns, turn) {
|
|
1439
|
+
let swapped = false;
|
|
1440
|
+
await checkpointInTurn(turns, {
|
|
1441
|
+
joinsInFlight: false,
|
|
1442
|
+
mayStart: turn.mayStart,
|
|
1443
|
+
perform: async () => {
|
|
1444
|
+
swapped = turn.swap();
|
|
1445
|
+
if (swapped) await turn.checkpoint();
|
|
1446
|
+
},
|
|
1447
|
+
afterSettling: turn.afterSettling
|
|
1448
|
+
});
|
|
1449
|
+
return swapped;
|
|
1450
|
+
}
|
|
1451
|
+
async function checkpointInTurn(turns, turn) {
|
|
1452
|
+
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
1453
|
+
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
1454
|
+
if (!turn.mayStart()) return;
|
|
1455
|
+
const run = turn.perform().finally(() => {
|
|
1456
|
+
turns.checkpointInFlight = null;
|
|
1457
|
+
turn.afterSettling();
|
|
1458
|
+
});
|
|
1459
|
+
turns.checkpointInFlight = run;
|
|
1460
|
+
return run;
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
// src/persistence/durability/held-checkpoints.ts
|
|
1464
|
+
function failedPartitionIds(indexState) {
|
|
1465
|
+
const failed = [];
|
|
1466
|
+
for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
|
|
1467
|
+
return failed;
|
|
1468
|
+
}
|
|
1469
|
+
function createHeldCheckpoints(hooks) {
|
|
1470
|
+
const reported = /* @__PURE__ */ new WeakSet();
|
|
1471
|
+
function heldReport(indexName, indexState) {
|
|
1472
|
+
const partitionIds = failedPartitionIds(indexState);
|
|
1473
|
+
if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
|
|
1474
|
+
return {
|
|
1475
|
+
indexName,
|
|
1476
|
+
partitionIds,
|
|
1477
|
+
recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
|
|
1478
|
+
documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
return {
|
|
1482
|
+
refuse(indexName, indexState) {
|
|
1483
|
+
const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
|
|
1484
|
+
if (partitionIds.length === 0) return;
|
|
1485
|
+
throw new NarsilError(
|
|
1486
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1487
|
+
`Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
|
|
1488
|
+
{ indexName, partitionIds }
|
|
1489
|
+
);
|
|
1490
|
+
},
|
|
1491
|
+
holds(indexName, indexState) {
|
|
1492
|
+
const held = heldReport(indexName, indexState);
|
|
1493
|
+
if (held === null) return false;
|
|
1494
|
+
const unreported = held.partitionIds.some((partitionId) => {
|
|
1495
|
+
const partition = indexState.partitions.get(partitionId);
|
|
1496
|
+
return partition !== void 0 && !reported.has(partition);
|
|
1497
|
+
});
|
|
1498
|
+
if (unreported) {
|
|
1499
|
+
for (const partitionId of held.partitionIds) {
|
|
1500
|
+
const partition = indexState.partitions.get(partitionId);
|
|
1501
|
+
if (partition !== void 0) reported.add(partition);
|
|
1502
|
+
}
|
|
1503
|
+
hooks.onCheckpointHeld?.(held);
|
|
1504
|
+
}
|
|
1505
|
+
return true;
|
|
1506
|
+
},
|
|
1507
|
+
report(indexes) {
|
|
1508
|
+
const held = [];
|
|
1509
|
+
for (const [indexName, indexState] of indexes) {
|
|
1510
|
+
const entry = heldReport(indexName, indexState);
|
|
1511
|
+
if (entry !== null) held.push(entry);
|
|
1512
|
+
}
|
|
1513
|
+
return held;
|
|
1514
|
+
}
|
|
1515
|
+
};
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
// src/persistence/durability/manager-state.ts
|
|
1519
|
+
function closePartitionWriter(partition) {
|
|
1520
|
+
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
1521
|
+
}
|
|
1522
|
+
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
1523
|
+
indexState.unloading = true;
|
|
1524
|
+
try {
|
|
1525
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
1526
|
+
await pendingMetadataWrite;
|
|
1527
|
+
for (const partition of indexState.partitions.values()) {
|
|
1528
|
+
await closeWalWriter(closePartitionWriter(partition));
|
|
1529
|
+
}
|
|
1530
|
+
} catch (error) {
|
|
1531
|
+
indexState.unloading = false;
|
|
1532
|
+
throw error;
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
// src/persistence/durability/restore-journal.ts
|
|
1537
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
1538
|
+
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
1539
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
1540
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
1541
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
1542
|
+
function digestOf(indexName) {
|
|
1543
|
+
let hex = "";
|
|
1544
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
1545
|
+
return hex;
|
|
1546
|
+
}
|
|
1547
|
+
function journalKey(indexName) {
|
|
1548
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
1549
|
+
}
|
|
1550
|
+
function randomSuffix() {
|
|
1551
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
1552
|
+
}
|
|
1553
|
+
function newStagingPrefix() {
|
|
1554
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
1555
|
+
}
|
|
1556
|
+
function newRetiredPrefix() {
|
|
1557
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
1558
|
+
}
|
|
1559
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
1560
|
+
return new TextEncoder().encode(`${indexName}
|
|
1561
|
+
${stagingPrefix}`);
|
|
1562
|
+
}
|
|
1563
|
+
function isStagingPrefix(prefix) {
|
|
1564
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
1565
|
+
}
|
|
1566
|
+
function corruptJournal(key, reason) {
|
|
1567
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
1568
|
+
}
|
|
1569
|
+
function readJournal(key, bytes) {
|
|
1570
|
+
const text = new TextDecoder().decode(bytes);
|
|
1571
|
+
const lineEnd = text.indexOf("\n");
|
|
1572
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
1573
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
1574
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
1575
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
1576
|
+
}
|
|
1577
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
1578
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
1579
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
1580
|
+
}
|
|
1581
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
1582
|
+
}
|
|
1583
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
1584
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
1585
|
+
if (manifestBytes === null) {
|
|
1586
|
+
throw new NarsilError(
|
|
1587
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1588
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
1589
|
+
{ indexName, stagingPrefix }
|
|
1590
|
+
);
|
|
1591
|
+
}
|
|
1592
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1593
|
+
if (decoded.kind !== "current") {
|
|
1594
|
+
throw new NarsilError(
|
|
1595
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1596
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
1597
|
+
{ indexName }
|
|
1598
|
+
);
|
|
1599
|
+
}
|
|
1600
|
+
const root = `${indexName}/`;
|
|
1601
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
1602
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
1603
|
+
return keys;
|
|
1604
|
+
}
|
|
1605
|
+
function withoutTrailingSlash(prefix) {
|
|
1606
|
+
return prefix.slice(0, -1);
|
|
1607
|
+
}
|
|
1608
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
1609
|
+
const root = `${indexName}/`;
|
|
1610
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
1611
|
+
const staged = await storage.list(stagingPrefix);
|
|
1612
|
+
let retiredPrefix = null;
|
|
1613
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
1614
|
+
retiredPrefix = newRetiredPrefix();
|
|
1615
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
1616
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
1617
|
+
} else {
|
|
1618
|
+
const stagedKeys = new Set(staged);
|
|
1619
|
+
for (const key of listed) {
|
|
1620
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
for (const key of await storage.list(root)) {
|
|
1624
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
1625
|
+
}
|
|
1626
|
+
await storage.remove(journalKey(indexName));
|
|
1627
|
+
return retiredPrefix;
|
|
1628
|
+
}
|
|
1629
|
+
async function removePrefix(storage, prefix) {
|
|
1630
|
+
if (storage.removeTree !== void 0) {
|
|
1631
|
+
await storage.removeTree(prefix);
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
1635
|
+
}
|
|
1636
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
1637
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
1638
|
+
}
|
|
1639
|
+
function isTopLevelJournal(key) {
|
|
1640
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
1641
|
+
}
|
|
1642
|
+
async function removeStagingPrefixes(storage) {
|
|
1643
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
1644
|
+
for (const key of await storage.list(".")) {
|
|
1645
|
+
const slash = key.indexOf("/");
|
|
1646
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
1647
|
+
}
|
|
1648
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
1649
|
+
}
|
|
1650
|
+
async function recoverRestoreJournals(storage) {
|
|
1651
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
1652
|
+
if (!isTopLevelJournal(key)) continue;
|
|
1653
|
+
const bytes = await storage.read(key);
|
|
1654
|
+
if (bytes === null) continue;
|
|
1655
|
+
const reading = readJournal(key, bytes);
|
|
1656
|
+
if (reading.kind === "leftover") {
|
|
1657
|
+
await storage.remove(key);
|
|
1658
|
+
continue;
|
|
1659
|
+
}
|
|
1660
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
1661
|
+
}
|
|
1662
|
+
await removeStagingPrefixes(storage);
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
// src/persistence/durability/restore-directory.ts
|
|
1666
|
+
function missing(error) {
|
|
1667
|
+
return error.code === "ENOENT";
|
|
1668
|
+
}
|
|
1669
|
+
function storageFailed(code, message, key, error) {
|
|
1670
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
1671
|
+
}
|
|
1672
|
+
function directoryRestoreStorage(directory) {
|
|
1673
|
+
return {
|
|
1674
|
+
directory,
|
|
1675
|
+
read: (key) => directory.read(key),
|
|
1676
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
1677
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
1678
|
+
remove: (key) => directory.remove(key),
|
|
1679
|
+
list: (prefix) => directory.list(prefix),
|
|
1680
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
1681
|
+
pathOf: (key) => directory.pathOf(key),
|
|
1682
|
+
async moveTree(from, to) {
|
|
1683
|
+
const source = await directory.pathOf(from);
|
|
1684
|
+
const target = await directory.pathOf(to);
|
|
1685
|
+
const fs = await getFs();
|
|
1686
|
+
try {
|
|
1687
|
+
await fs.rename(source, target);
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
if (missing(error)) return;
|
|
1690
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
1691
|
+
}
|
|
1692
|
+
await directory.syncDirectoryOf(to);
|
|
1693
|
+
},
|
|
1694
|
+
async removeTree(prefix) {
|
|
1695
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
1696
|
+
const fs = await getFs();
|
|
1697
|
+
try {
|
|
1698
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
1699
|
+
} catch (error) {
|
|
1700
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
1701
|
+
}
|
|
1702
|
+
await directory.syncDirectoryOf(key);
|
|
1703
|
+
}
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
function recoverDirectoryRestores(directory) {
|
|
1707
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
// src/persistence/durability/restore-staging.ts
|
|
1711
|
+
function toError2(error) {
|
|
1712
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1713
|
+
}
|
|
1714
|
+
function pathUnder(path, root) {
|
|
1715
|
+
if (!path.startsWith(root)) return false;
|
|
1716
|
+
const next = path.charAt(root.length);
|
|
1717
|
+
return next === "" || next === "/" || next === "\\";
|
|
1718
|
+
}
|
|
1719
|
+
function stagedDirectory(directory, staged, unstaged) {
|
|
1720
|
+
const { readRange, byteLength } = directory;
|
|
1721
|
+
return {
|
|
1722
|
+
root: directory.root,
|
|
1723
|
+
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
1724
|
+
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
1725
|
+
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
1726
|
+
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
1727
|
+
read: (key) => directory.read(staged(key)),
|
|
1728
|
+
remove: (key) => directory.remove(staged(key)),
|
|
1729
|
+
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
1730
|
+
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
1731
|
+
pathOf: (key) => directory.pathOf(staged(key)),
|
|
1732
|
+
...readRange === void 0 ? {} : {
|
|
1733
|
+
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
1734
|
+
},
|
|
1735
|
+
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
1736
|
+
};
|
|
1737
|
+
}
|
|
1738
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
1739
|
+
const stagingPrefix = newStagingPrefix();
|
|
1740
|
+
const indexRoot = `${indexName}/`;
|
|
1741
|
+
let committed = false;
|
|
1742
|
+
const staged = (key) => {
|
|
1743
|
+
if (!key.startsWith(indexRoot)) {
|
|
1744
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
1745
|
+
key
|
|
1746
|
+
});
|
|
1747
|
+
}
|
|
1748
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
1749
|
+
};
|
|
1750
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
1751
|
+
const store = {
|
|
1752
|
+
read: (key) => storage.read(staged(key)),
|
|
1753
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
1754
|
+
remove: (key) => storage.remove(staged(key)),
|
|
1755
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
1756
|
+
};
|
|
1757
|
+
const pathOf = storage.pathOf;
|
|
1758
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
1759
|
+
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
1760
|
+
let retiredPrefix = null;
|
|
1761
|
+
async function finishOnStorage() {
|
|
1762
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
1763
|
+
}
|
|
1764
|
+
return {
|
|
1765
|
+
store,
|
|
1766
|
+
async relocation() {
|
|
1767
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
1768
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
1769
|
+
const to = await pathOf.call(storage, indexName);
|
|
1770
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
1771
|
+
},
|
|
1772
|
+
async commit() {
|
|
1773
|
+
try {
|
|
1774
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
1775
|
+
} catch (error) {
|
|
1776
|
+
try {
|
|
1777
|
+
await storage.remove(journalKey(indexName));
|
|
1778
|
+
} catch (removal) {
|
|
1779
|
+
committed = true;
|
|
1780
|
+
options.onStorageLost?.(toError2(removal));
|
|
1781
|
+
}
|
|
1782
|
+
throw error;
|
|
1783
|
+
}
|
|
1784
|
+
committed = true;
|
|
1785
|
+
},
|
|
1786
|
+
async finish() {
|
|
1787
|
+
try {
|
|
1788
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
1789
|
+
} catch (error) {
|
|
1790
|
+
options.onStorageLost?.(toError2(error));
|
|
1791
|
+
}
|
|
1792
|
+
},
|
|
1793
|
+
async removeRetired() {
|
|
1794
|
+
const prefix = retiredPrefix;
|
|
1795
|
+
retiredPrefix = null;
|
|
1796
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
1797
|
+
},
|
|
1798
|
+
async discard() {
|
|
1799
|
+
if (committed) return;
|
|
1800
|
+
await removePrefix(storage, stagingPrefix);
|
|
1801
|
+
}
|
|
1802
|
+
};
|
|
1803
|
+
}
|
|
1804
|
+
function adapterRestoreStorage(adapter) {
|
|
1805
|
+
const store = adapterCheckpointStore(adapter);
|
|
1806
|
+
return {
|
|
1807
|
+
read: (key) => store.read(key),
|
|
1808
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
1809
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
1810
|
+
remove: (key) => store.remove(key),
|
|
1811
|
+
list: (prefix) => store.list(prefix),
|
|
1812
|
+
async moveKey(from, to) {
|
|
1813
|
+
const bytes = await store.read(from);
|
|
1814
|
+
if (bytes === null) return;
|
|
1815
|
+
await store.write(to, [bytes]);
|
|
1816
|
+
await store.remove(from);
|
|
1817
|
+
}
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
1822
|
+
function toError3(error) {
|
|
1823
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1824
|
+
}
|
|
1825
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
1826
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
1827
|
+
async finish(run) {
|
|
1828
|
+
try {
|
|
1829
|
+
await run();
|
|
1830
|
+
} catch (error) {
|
|
1831
|
+
deps.markFatal(toError3(error));
|
|
1832
|
+
throw error;
|
|
1833
|
+
}
|
|
1834
|
+
},
|
|
1835
|
+
onStorageLost: deps.markFatal
|
|
1836
|
+
});
|
|
1837
|
+
}
|
|
1838
|
+
function releaser(held, indexName) {
|
|
1839
|
+
let released = false;
|
|
1840
|
+
return () => {
|
|
1841
|
+
if (released) return;
|
|
1842
|
+
released = true;
|
|
1843
|
+
const count = (held.get(indexName) ?? 1) - 1;
|
|
1844
|
+
if (count <= 0) held.delete(indexName);
|
|
1845
|
+
else held.set(indexName, count);
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
function createArchiveCheckpointAccess(deps) {
|
|
1849
|
+
const store = directoryCheckpointStore(deps.directory);
|
|
1850
|
+
return {
|
|
1851
|
+
async openArchiveCheckpoint(indexName) {
|
|
1852
|
+
deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
|
|
1853
|
+
const release = releaser(deps.held, indexName);
|
|
1854
|
+
try {
|
|
1855
|
+
await deps.checkpoint(indexName, false);
|
|
1856
|
+
await deps.checkpoint(indexName, true);
|
|
1857
|
+
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
1858
|
+
if (manifestBytes === null) {
|
|
1859
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
1860
|
+
indexName
|
|
1861
|
+
});
|
|
1862
|
+
}
|
|
1863
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1864
|
+
if (decoded.kind !== "current") {
|
|
1865
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
1866
|
+
indexName
|
|
1867
|
+
});
|
|
1868
|
+
}
|
|
1869
|
+
return {
|
|
1870
|
+
manifestBytes,
|
|
1871
|
+
manifest: decoded.manifest,
|
|
1872
|
+
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
1873
|
+
readRange: async (ref, offset, length) => {
|
|
1874
|
+
const readRange = deps.directory.readRange;
|
|
1875
|
+
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
1876
|
+
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
1877
|
+
return whole.slice(offset, offset + length);
|
|
1878
|
+
},
|
|
1879
|
+
release,
|
|
1880
|
+
directory: deps.directory
|
|
1881
|
+
};
|
|
1882
|
+
} catch (error) {
|
|
1883
|
+
release();
|
|
1884
|
+
throw error;
|
|
1885
|
+
}
|
|
1886
|
+
},
|
|
1887
|
+
stageRestore(indexName) {
|
|
1888
|
+
return stageDirectoryRestore(deps, indexName);
|
|
1889
|
+
},
|
|
1890
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
1891
|
+
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
1892
|
+
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
};
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
// src/persistence/durability/query-history.ts
|
|
1899
|
+
async function removeQueryHistoryOf(storage, indexName) {
|
|
1900
|
+
const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
|
|
1901
|
+
if (storage.kind === "directory") {
|
|
1902
|
+
for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1905
|
+
for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
|
|
1906
|
+
}
|
|
1907
|
+
function queryHistoryAccess(storage) {
|
|
1908
|
+
return {
|
|
1909
|
+
historyStorage: () => storage,
|
|
1910
|
+
removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
|
|
1833
1911
|
};
|
|
1834
1912
|
}
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1913
|
+
|
|
1914
|
+
// src/persistence/durability/manager-unload.ts
|
|
1915
|
+
function createIndexUnloading(deps) {
|
|
1916
|
+
const { directory, hooks, indexes, metadataWrites } = deps;
|
|
1917
|
+
return {
|
|
1918
|
+
async removeIndex(indexName) {
|
|
1919
|
+
hooks.writeRecords?.dropIndex(indexName);
|
|
1920
|
+
const indexState = indexes.get(indexName);
|
|
1921
|
+
if (indexState !== void 0) {
|
|
1922
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
|
|
1923
|
+
indexes.delete(indexName);
|
|
1924
|
+
metadataWrites.forget(indexName);
|
|
1925
|
+
}
|
|
1926
|
+
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
1927
|
+
for (const key of await directory.list(`${indexName}/`)) {
|
|
1928
|
+
await directory.remove(key);
|
|
1929
|
+
}
|
|
1930
|
+
await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
|
|
1931
|
+
deps.forgetSyncState(indexName);
|
|
1843
1932
|
},
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
});
|
|
1856
|
-
turns.checkpointInFlight = run;
|
|
1857
|
-
return run;
|
|
1933
|
+
async unloadIndex(indexName) {
|
|
1934
|
+
const indexState = indexes.get(indexName);
|
|
1935
|
+
if (indexState === void 0) {
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
1939
|
+
indexes.delete(indexName);
|
|
1940
|
+
metadataWrites.forget(indexName);
|
|
1941
|
+
deps.forgetSyncState(indexName);
|
|
1942
|
+
}
|
|
1943
|
+
};
|
|
1858
1944
|
}
|
|
1859
1945
|
|
|
1860
|
-
// src/persistence/durability/
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
}
|
|
1866
|
-
function
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1946
|
+
// src/persistence/durability/sync/frame-cursor.ts
|
|
1947
|
+
var SegmentFileMissing = class extends Error {
|
|
1948
|
+
};
|
|
1949
|
+
function corrupt(key, message, details) {
|
|
1950
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
|
|
1951
|
+
}
|
|
1952
|
+
function isMissingFile(error) {
|
|
1953
|
+
return error?.code === "ENOENT";
|
|
1954
|
+
}
|
|
1955
|
+
var FrameCursor = class {
|
|
1956
|
+
constructor(directory, key, limit) {
|
|
1957
|
+
this.directory = directory;
|
|
1958
|
+
this.key = key;
|
|
1959
|
+
this.limit = limit;
|
|
1960
|
+
}
|
|
1961
|
+
buffer = new Uint8Array(0);
|
|
1962
|
+
bufferStart = 0;
|
|
1963
|
+
whole = null;
|
|
1964
|
+
async load(offset, length) {
|
|
1965
|
+
const readRange = this.directory.readRange;
|
|
1966
|
+
if (readRange === void 0) {
|
|
1967
|
+
if (this.whole === null) {
|
|
1968
|
+
const bytes = await this.directory.read(this.key);
|
|
1969
|
+
if (bytes === null) throw new SegmentFileMissing(this.key);
|
|
1970
|
+
this.whole = bytes;
|
|
1971
|
+
}
|
|
1972
|
+
this.buffer = this.whole;
|
|
1973
|
+
this.bufferStart = 0;
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
|
|
1977
|
+
try {
|
|
1978
|
+
this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
|
|
1979
|
+
} catch (error) {
|
|
1980
|
+
if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
|
|
1981
|
+
throw error;
|
|
1982
|
+
}
|
|
1983
|
+
this.bufferStart = offset;
|
|
1876
1984
|
}
|
|
1877
|
-
async
|
|
1985
|
+
async bytesAt(offset, length) {
|
|
1986
|
+
const end = offset + length;
|
|
1987
|
+
if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
|
|
1988
|
+
if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
|
|
1989
|
+
return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
|
|
1990
|
+
}
|
|
1991
|
+
async checkHeader() {
|
|
1992
|
+
if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
|
|
1993
|
+
const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
|
|
1994
|
+
if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
|
|
1995
|
+
}
|
|
1996
|
+
async frameLengthAt(offset) {
|
|
1997
|
+
const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
|
|
1998
|
+
const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
|
|
1999
|
+
0,
|
|
2000
|
+
false
|
|
2001
|
+
);
|
|
2002
|
+
if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
|
|
2003
|
+
corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
|
|
2004
|
+
}
|
|
2005
|
+
return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
|
|
2006
|
+
}
|
|
2007
|
+
async frameAt(offset, frameBytes) {
|
|
2008
|
+
const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
|
|
2009
|
+
const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
|
|
2010
|
+
const payload = body.subarray(0, recordLength);
|
|
2011
|
+
const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
|
|
2012
|
+
if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
|
|
2013
|
+
let seqNo;
|
|
1878
2014
|
try {
|
|
1879
|
-
|
|
1880
|
-
} catch (
|
|
1881
|
-
|
|
2015
|
+
seqNo = decodeWalEntry(payload).seqNo;
|
|
2016
|
+
} catch (error) {
|
|
2017
|
+
corrupt(this.key, "A log record fails to decode", {
|
|
2018
|
+
offset,
|
|
2019
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
2020
|
+
});
|
|
1882
2021
|
}
|
|
2022
|
+
return { seqNo, payload: payload.slice(), frameBytes };
|
|
2023
|
+
}
|
|
2024
|
+
};
|
|
2025
|
+
|
|
2026
|
+
// src/persistence/durability/sync/log-reader.ts
|
|
2027
|
+
async function walSources(directory, indexName, partitionId) {
|
|
2028
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
2029
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
2030
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
2031
|
+
if (marker === null) return { sources: [], head: 0 };
|
|
2032
|
+
const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
|
|
2033
|
+
const sources = [];
|
|
2034
|
+
for (const key of await directory.list(prefix)) {
|
|
2035
|
+
const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
|
|
2036
|
+
if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
|
|
2037
|
+
const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
|
|
2038
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
2039
|
+
sources.push({ key, startSeqNo, readableBytes, inHistory: false });
|
|
2040
|
+
}
|
|
2041
|
+
return { sources, head: highestDurableSeqNo };
|
|
2042
|
+
}
|
|
2043
|
+
async function openPartitionLog(directory, indexName, partitionId) {
|
|
2044
|
+
const { sources, head } = await walSources(directory, indexName, partitionId);
|
|
2045
|
+
const byStart = /* @__PURE__ */ new Map();
|
|
2046
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
2047
|
+
const readableBytes = await directory.byteLength?.(segment.key);
|
|
2048
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
2049
|
+
byStart.set(segment.startSeqNo, {
|
|
2050
|
+
key: segment.key,
|
|
2051
|
+
startSeqNo: segment.startSeqNo,
|
|
2052
|
+
readableBytes,
|
|
2053
|
+
inHistory: true
|
|
2054
|
+
});
|
|
1883
2055
|
}
|
|
1884
|
-
|
|
1885
|
-
|
|
2056
|
+
for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
|
|
2057
|
+
const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
|
|
2058
|
+
return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
|
|
2059
|
+
}
|
|
2060
|
+
function sourceHolding(log, seqNo) {
|
|
2061
|
+
let found = -1;
|
|
2062
|
+
for (let i = 0; i < log.sources.length; i++) {
|
|
2063
|
+
if (log.sources[i].startSeqNo > seqNo) break;
|
|
2064
|
+
found = i;
|
|
2065
|
+
}
|
|
2066
|
+
return found;
|
|
2067
|
+
}
|
|
2068
|
+
async function offsetAfter(cursor, source, afterSeqNo) {
|
|
2069
|
+
let offset = SEGMENT_HEADER_SIZE;
|
|
2070
|
+
for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
|
|
2071
|
+
offset += await cursor.frameLengthAt(offset);
|
|
2072
|
+
}
|
|
2073
|
+
return offset;
|
|
2074
|
+
}
|
|
2075
|
+
async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
|
|
2076
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
2077
|
+
await cursor.checkHeader();
|
|
2078
|
+
let offset = await offsetAfter(cursor, source, afterSeqNo);
|
|
2079
|
+
let checksNumbering = true;
|
|
2080
|
+
while (offset < cursor.limit) {
|
|
2081
|
+
const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
|
|
2082
|
+
if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
|
|
2083
|
+
checksNumbering = false;
|
|
2084
|
+
if (frame.seqNo > afterSeqNo) {
|
|
2085
|
+
offset = SEGMENT_HEADER_SIZE;
|
|
2086
|
+
continue;
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
checksNumbering = false;
|
|
2090
|
+
offset += frame.frameBytes;
|
|
2091
|
+
if (frame.seqNo <= afterSeqNo) continue;
|
|
2092
|
+
if (frame.seqNo > upToSeqNo) return true;
|
|
2093
|
+
if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
|
|
2094
|
+
read.frames.push(frame);
|
|
2095
|
+
read.bytes += frame.payload.byteLength;
|
|
2096
|
+
if (frame.seqNo === upToSeqNo) return true;
|
|
2097
|
+
}
|
|
2098
|
+
return false;
|
|
2099
|
+
}
|
|
2100
|
+
async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
2101
|
+
const read = { frames: [], bytes: 0, gone: false };
|
|
2102
|
+
if (afterSeqNo >= upToSeqNo) return read;
|
|
2103
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
2104
|
+
read.frames = [];
|
|
2105
|
+
read.bytes = 0;
|
|
2106
|
+
const log = await openPartitionLog(directory, indexName, partitionId);
|
|
2107
|
+
let index = sourceHolding(log, afterSeqNo + 1);
|
|
2108
|
+
if (index < 0) {
|
|
2109
|
+
read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
|
|
2110
|
+
return read;
|
|
2111
|
+
}
|
|
1886
2112
|
try {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
2113
|
+
for (; index < log.sources.length; index++) {
|
|
2114
|
+
const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
|
|
2115
|
+
if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
|
|
2116
|
+
}
|
|
2117
|
+
return read;
|
|
2118
|
+
} catch (error) {
|
|
2119
|
+
if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
|
|
1890
2120
|
}
|
|
1891
|
-
return new NarsilError(
|
|
1892
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1893
|
-
`The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
|
|
1894
|
-
{ directory: directory.root }
|
|
1895
|
-
);
|
|
1896
2121
|
}
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
if (
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
)
|
|
1913
|
-
);
|
|
1914
|
-
return false;
|
|
2122
|
+
return read;
|
|
2123
|
+
}
|
|
2124
|
+
async function logBytesAfter(directory, log, afterSeqNo) {
|
|
2125
|
+
const first = sourceHolding(log, afterSeqNo + 1);
|
|
2126
|
+
let bytes = 0;
|
|
2127
|
+
for (let i = Math.max(first, 0); i < log.sources.length; i++) {
|
|
2128
|
+
const source = log.sources[i];
|
|
2129
|
+
if (i === first) {
|
|
2130
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
2131
|
+
try {
|
|
2132
|
+
bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
|
|
2133
|
+
} catch (error) {
|
|
2134
|
+
if (!(error instanceof SegmentFileMissing)) throw error;
|
|
2135
|
+
}
|
|
2136
|
+
continue;
|
|
1915
2137
|
}
|
|
1916
|
-
|
|
1917
|
-
if (takenOver === null) return true;
|
|
1918
|
-
onLost(takenOver);
|
|
1919
|
-
return false;
|
|
2138
|
+
bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
|
|
1920
2139
|
}
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
2140
|
+
return bytes;
|
|
2141
|
+
}
|
|
2142
|
+
|
|
2143
|
+
// src/persistence/durability/sync/pins.ts
|
|
2144
|
+
function createFilePins(remove) {
|
|
2145
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2146
|
+
const awaitingRemoval = /* @__PURE__ */ new Set();
|
|
2147
|
+
function unpin(key) {
|
|
2148
|
+
const count = (counts.get(key) ?? 1) - 1;
|
|
2149
|
+
if (count > 0) {
|
|
2150
|
+
counts.set(key, count);
|
|
2151
|
+
return;
|
|
1925
2152
|
}
|
|
2153
|
+
counts.delete(key);
|
|
2154
|
+
if (!awaitingRemoval.delete(key)) return;
|
|
2155
|
+
void remove(key).catch(() => void 0);
|
|
1926
2156
|
}
|
|
1927
2157
|
return {
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
async verifyOnce() {
|
|
1938
|
-
if (expected === null && directory.identity !== void 0) await verify();
|
|
1939
|
-
},
|
|
1940
|
-
start() {
|
|
1941
|
-
if (directory.identity === void 0 || timer !== null || isStopped()) return;
|
|
1942
|
-
timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
|
|
2158
|
+
pin(keys) {
|
|
2159
|
+
const pinned = [...keys];
|
|
2160
|
+
for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2161
|
+
let released = false;
|
|
2162
|
+
return () => {
|
|
2163
|
+
if (released) return;
|
|
2164
|
+
released = true;
|
|
2165
|
+
for (const key of pinned) unpin(key);
|
|
2166
|
+
};
|
|
1943
2167
|
},
|
|
1944
|
-
|
|
2168
|
+
isPinned: (key) => counts.has(key),
|
|
2169
|
+
removeWhenUnpinned(key) {
|
|
2170
|
+
if (!counts.has(key)) return false;
|
|
2171
|
+
awaitingRemoval.add(key);
|
|
2172
|
+
return true;
|
|
2173
|
+
}
|
|
2174
|
+
};
|
|
2175
|
+
}
|
|
2176
|
+
var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
|
|
2177
|
+
var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
|
|
2178
|
+
function hexOf(bytes) {
|
|
2179
|
+
let hex = "";
|
|
2180
|
+
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
2181
|
+
return hex;
|
|
2182
|
+
}
|
|
2183
|
+
function isBytes(value) {
|
|
2184
|
+
return value instanceof Uint8Array;
|
|
2185
|
+
}
|
|
2186
|
+
async function publishedFrom(indexName, payload, config, manifestBytes) {
|
|
2187
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
2188
|
+
if (decoded.kind !== "current") {
|
|
2189
|
+
throw new NarsilError(
|
|
2190
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2191
|
+
`The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
|
|
2192
|
+
{ indexName }
|
|
2193
|
+
);
|
|
2194
|
+
}
|
|
2195
|
+
return {
|
|
2196
|
+
config,
|
|
2197
|
+
manifestBytes,
|
|
2198
|
+
manifest: decoded.manifest,
|
|
2199
|
+
entityTag: hexOf(sha256(payload)),
|
|
2200
|
+
keys: manifestReferencedKeys(decoded.manifest)
|
|
1945
2201
|
};
|
|
1946
2202
|
}
|
|
2203
|
+
async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
|
|
2204
|
+
const payload = encode({ config, manifest: manifestBytes });
|
|
2205
|
+
const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
|
|
2206
|
+
const published = await publishedFrom(indexName, payload, config, manifestBytes);
|
|
2207
|
+
await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
|
|
2208
|
+
return published;
|
|
2209
|
+
}
|
|
2210
|
+
async function readPublishedSnapshot(directory, indexName) {
|
|
2211
|
+
const bytes = await directory.read(publishedSnapshotKey(indexName));
|
|
2212
|
+
if (bytes === null) return null;
|
|
2213
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
2214
|
+
envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
|
|
2215
|
+
what: PUBLISHED_SNAPSHOT_WHAT,
|
|
2216
|
+
allowsCompression: false
|
|
2217
|
+
});
|
|
2218
|
+
const raw = decode(payload);
|
|
2219
|
+
if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
|
|
2220
|
+
throw new NarsilError(
|
|
2221
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2222
|
+
`The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
|
|
2223
|
+
{ indexName }
|
|
2224
|
+
);
|
|
2225
|
+
}
|
|
2226
|
+
return publishedFrom(indexName, payload, raw.config, raw.manifest);
|
|
2227
|
+
}
|
|
1947
2228
|
|
|
1948
|
-
// src/persistence/durability/
|
|
1949
|
-
function
|
|
1950
|
-
|
|
1951
|
-
for (const
|
|
1952
|
-
|
|
2229
|
+
// src/persistence/durability/sync/retention.ts
|
|
2230
|
+
async function firstLogStart(directory, indexName, partitionId) {
|
|
2231
|
+
let first = Number.POSITIVE_INFINITY;
|
|
2232
|
+
for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
|
|
2233
|
+
const start = parseWalSegmentStart(indexName, partitionId, key);
|
|
2234
|
+
if (start !== null && start < first) first = start;
|
|
2235
|
+
}
|
|
2236
|
+
return first;
|
|
2237
|
+
}
|
|
2238
|
+
async function sizedQueues(retention) {
|
|
2239
|
+
const { directory, indexName } = retention;
|
|
2240
|
+
const queues = /* @__PURE__ */ new Map();
|
|
2241
|
+
for (const segment of await listHistorySegments(directory, indexName)) {
|
|
2242
|
+
const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
|
|
2243
|
+
if (bytes === void 0 || bytes === null) continue;
|
|
2244
|
+
const queue = queues.get(segment.partitionId) ?? [];
|
|
2245
|
+
queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
|
|
2246
|
+
queues.set(segment.partitionId, queue);
|
|
2247
|
+
}
|
|
2248
|
+
for (const [partitionId, queue] of queues) {
|
|
2249
|
+
const logStart = await firstLogStart(directory, indexName, partitionId);
|
|
2250
|
+
for (let i = 0; i < queue.length; i++) {
|
|
2251
|
+
const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
|
|
2252
|
+
queue[i].lastSeqNo = nextStart - 1;
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
return queues;
|
|
2256
|
+
}
|
|
2257
|
+
function nextToDelete(queues, totalBytes, retention) {
|
|
2258
|
+
const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
|
|
2259
|
+
let oldest = null;
|
|
2260
|
+
for (const queue of queues.values()) {
|
|
2261
|
+
const head = queue[0];
|
|
2262
|
+
if (head === void 0) continue;
|
|
2263
|
+
if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
|
|
2264
|
+
}
|
|
2265
|
+
if (oldest === null) return null;
|
|
2266
|
+
if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
|
|
2267
|
+
return null;
|
|
2268
|
+
}
|
|
2269
|
+
async function enforceHistoryLimits(retention) {
|
|
2270
|
+
const queues = await sizedQueues(retention);
|
|
2271
|
+
let totalBytes = 0;
|
|
2272
|
+
for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
|
|
2273
|
+
let published = false;
|
|
2274
|
+
let deleted = 0;
|
|
2275
|
+
let segment = nextToDelete(queues, totalBytes, retention);
|
|
2276
|
+
while (segment !== null) {
|
|
2277
|
+
const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
|
|
2278
|
+
if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
|
|
2279
|
+
published = true;
|
|
2280
|
+
await retention.publish();
|
|
2281
|
+
}
|
|
2282
|
+
await retention.directory.remove(segment.key);
|
|
2283
|
+
queues.get(segment.partitionId)?.shift();
|
|
2284
|
+
totalBytes -= segment.bytes;
|
|
2285
|
+
deleted += 1;
|
|
2286
|
+
segment = nextToDelete(queues, totalBytes, retention);
|
|
2287
|
+
}
|
|
2288
|
+
return deleted;
|
|
1953
2289
|
}
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
2290
|
+
|
|
2291
|
+
// src/persistence/durability/sync/storage.ts
|
|
2292
|
+
function createSyncStorage(deps) {
|
|
2293
|
+
const { directory } = deps;
|
|
2294
|
+
const store = directoryCheckpointStore(directory);
|
|
2295
|
+
const cache = /* @__PURE__ */ new Map();
|
|
2296
|
+
const publishing = /* @__PURE__ */ new Map();
|
|
2297
|
+
const pins = createFilePins((key) => directory.remove(key));
|
|
2298
|
+
async function published(indexName) {
|
|
2299
|
+
const cached = cache.get(indexName);
|
|
2300
|
+
if (cached !== void 0) return cached;
|
|
2301
|
+
const read = await readPublishedSnapshot(directory, indexName);
|
|
2302
|
+
cache.set(indexName, read);
|
|
2303
|
+
return read;
|
|
2304
|
+
}
|
|
2305
|
+
async function removeUnlisted(indexName, previous, kept) {
|
|
2306
|
+
const manifest = await readCurrentManifest(store, indexName);
|
|
2307
|
+
const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
|
|
2308
|
+
for (const key of previous) {
|
|
2309
|
+
if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
|
|
2310
|
+
await directory.remove(key);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
async function publishNow(indexName, configurationOf) {
|
|
2314
|
+
const archive = await deps.openArchiveCheckpoint(indexName);
|
|
2315
|
+
try {
|
|
2316
|
+
const config = await configurationOf(archive.manifest);
|
|
2317
|
+
const previous = await published(indexName);
|
|
2318
|
+
const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
|
|
2319
|
+
cache.set(indexName, next);
|
|
2320
|
+
if (previous !== null) {
|
|
2321
|
+
await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
|
|
2322
|
+
}
|
|
2323
|
+
return next;
|
|
2324
|
+
} finally {
|
|
2325
|
+
archive.release();
|
|
2326
|
+
}
|
|
1965
2327
|
}
|
|
1966
2328
|
return {
|
|
1967
|
-
|
|
1968
|
-
const
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
2329
|
+
publish(indexName, configurationOf) {
|
|
2330
|
+
const inFlight = publishing.get(indexName);
|
|
2331
|
+
const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
|
|
2332
|
+
const tracked = run.finally(() => {
|
|
2333
|
+
if (publishing.get(indexName) === tracked) publishing.delete(indexName);
|
|
2334
|
+
});
|
|
2335
|
+
publishing.set(indexName, tracked);
|
|
2336
|
+
return tracked;
|
|
1975
2337
|
},
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
const
|
|
1980
|
-
|
|
1981
|
-
|
|
2338
|
+
published,
|
|
2339
|
+
pin: (snapshot) => pins.pin(snapshot.keys),
|
|
2340
|
+
async readFileRange(key, offset, length) {
|
|
2341
|
+
const readRange = directory.readRange;
|
|
2342
|
+
if (readRange !== void 0) return readRange.call(directory, key, offset, length);
|
|
2343
|
+
const whole = await directory.read(key);
|
|
2344
|
+
return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
|
|
2345
|
+
},
|
|
2346
|
+
async release(indexName) {
|
|
2347
|
+
await deps.inCheckpointTurn(indexName, async () => {
|
|
2348
|
+
const previous = await published(indexName);
|
|
2349
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2350
|
+
cache.set(indexName, null);
|
|
2351
|
+
if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
|
|
2352
|
+
await removeHistory(directory, indexName);
|
|
1982
2353
|
});
|
|
1983
|
-
if (unreported) {
|
|
1984
|
-
for (const partitionId of held.partitionIds) {
|
|
1985
|
-
const partition = indexState.partitions.get(partitionId);
|
|
1986
|
-
if (partition !== void 0) reported.add(partition);
|
|
1987
|
-
}
|
|
1988
|
-
hooks.onCheckpointHeld?.(held);
|
|
1989
|
-
}
|
|
1990
|
-
return true;
|
|
1991
2354
|
},
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
2355
|
+
removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
|
|
2356
|
+
async enforceHistoryLimits(indexName, limits, publish, nowMs) {
|
|
2357
|
+
const current = await published(indexName);
|
|
2358
|
+
return enforceHistoryLimits({
|
|
2359
|
+
directory,
|
|
2360
|
+
indexName,
|
|
2361
|
+
limits,
|
|
2362
|
+
nowMs,
|
|
2363
|
+
publish,
|
|
2364
|
+
publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
|
|
2365
|
+
});
|
|
2366
|
+
},
|
|
2367
|
+
openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
|
|
2368
|
+
readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
|
|
2369
|
+
logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
|
|
2370
|
+
currentManifest: (indexName) => readCurrentManifest(store, indexName),
|
|
2371
|
+
async keepsCheckpointKey(indexName) {
|
|
2372
|
+
const current = await published(indexName);
|
|
2373
|
+
return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
|
|
2374
|
+
},
|
|
2375
|
+
async settleAtRecovery(indexName, active) {
|
|
2376
|
+
cache.delete(indexName);
|
|
2377
|
+
const publishedKey = publishedSnapshotKey(indexName);
|
|
2378
|
+
for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
|
|
2379
|
+
if (key !== publishedKey) await directory.remove(key);
|
|
1997
2380
|
}
|
|
1998
|
-
|
|
2381
|
+
if (!active) {
|
|
2382
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2383
|
+
await removeHistory(directory, indexName);
|
|
2384
|
+
cache.set(indexName, null);
|
|
2385
|
+
return /* @__PURE__ */ new Set();
|
|
2386
|
+
}
|
|
2387
|
+
try {
|
|
2388
|
+
return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
|
|
2389
|
+
} catch {
|
|
2390
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2391
|
+
cache.set(indexName, null);
|
|
2392
|
+
return /* @__PURE__ */ new Set();
|
|
2393
|
+
}
|
|
2394
|
+
},
|
|
2395
|
+
forget(indexName) {
|
|
2396
|
+
cache.delete(indexName);
|
|
1999
2397
|
}
|
|
2000
2398
|
};
|
|
2001
2399
|
}
|
|
2002
2400
|
|
|
2003
|
-
// src/persistence/durability/manager-
|
|
2004
|
-
function
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2401
|
+
// src/persistence/durability/manager-storage.ts
|
|
2402
|
+
function createIndexStorageAccess(deps) {
|
|
2403
|
+
const { directory, indexes } = deps;
|
|
2404
|
+
const archiveAccess = createArchiveCheckpointAccess({
|
|
2405
|
+
directory,
|
|
2406
|
+
held: deps.heldCheckpointFiles,
|
|
2407
|
+
checkpoint: deps.checkpoint,
|
|
2408
|
+
registerPartition: deps.registerPartition,
|
|
2409
|
+
markFatal: deps.markFatal
|
|
2410
|
+
});
|
|
2411
|
+
const syncStorage = createSyncStorage({
|
|
2412
|
+
directory,
|
|
2413
|
+
openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
|
|
2414
|
+
inCheckpointTurn: (indexName, work) => {
|
|
2415
|
+
const indexState = indexes.get(indexName);
|
|
2416
|
+
if (indexState === void 0) return work();
|
|
2417
|
+
return checkpointInTurn(indexState, {
|
|
2418
|
+
joinsInFlight: false,
|
|
2419
|
+
mayStart: () => true,
|
|
2420
|
+
perform: work,
|
|
2421
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
2422
|
+
});
|
|
2014
2423
|
}
|
|
2015
|
-
}
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2424
|
+
});
|
|
2425
|
+
const unloading = createIndexUnloading({
|
|
2426
|
+
directory,
|
|
2427
|
+
hooks: deps.hooks,
|
|
2428
|
+
indexes,
|
|
2429
|
+
metadataWrites: deps.metadataWrites,
|
|
2430
|
+
closeWriterReportingFailure: deps.closeWriterReportingFailure,
|
|
2431
|
+
forgetSyncState: syncStorage.forget
|
|
2432
|
+
});
|
|
2433
|
+
return { archiveAccess, syncStorage, unloading };
|
|
2019
2434
|
}
|
|
2020
2435
|
|
|
2021
2436
|
// src/persistence/durability/metadata-write-queue.ts
|
|
@@ -2263,8 +2678,8 @@ function createGroupCommitCoordinator(sync) {
|
|
|
2263
2678
|
// src/persistence/durability/wal-writer.ts
|
|
2264
2679
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
2265
2680
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
2266
|
-
const
|
|
2267
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2681
|
+
const padded = startSeqNo.toString().padStart(16, "0");
|
|
2682
|
+
return `${indexName}/wal/${partitionId}/${padded}`;
|
|
2268
2683
|
}
|
|
2269
2684
|
function parseSegmentStartSeqNo2(key) {
|
|
2270
2685
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -2988,7 +3403,7 @@ var decoder = new TextDecoder();
|
|
|
2988
3403
|
async function countPagedPartition(store, partition) {
|
|
2989
3404
|
const { directory } = store;
|
|
2990
3405
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
2991
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
3406
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-E33X4AOK.mjs');
|
|
2992
3407
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
2993
3408
|
const scans = [];
|
|
2994
3409
|
try {
|
|
@@ -3271,12 +3686,14 @@ async function recoverCheckpoint(deps) {
|
|
|
3271
3686
|
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
3272
3687
|
}
|
|
3273
3688
|
if (decoded === null || decoded.kind !== "current") {
|
|
3274
|
-
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName,
|
|
3689
|
+
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
|
|
3275
3690
|
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
3276
3691
|
}
|
|
3277
3692
|
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
3278
3693
|
if (!deps.keepsUnlistedFiles) {
|
|
3279
|
-
|
|
3694
|
+
const referenced = manifestReferencedKeys(decoded.manifest);
|
|
3695
|
+
for (const key of deps.keptForSync) referenced.add(key);
|
|
3696
|
+
await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
|
|
3280
3697
|
}
|
|
3281
3698
|
const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
|
|
3282
3699
|
manager: deps.manager,
|
|
@@ -3397,6 +3814,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3397
3814
|
}
|
|
3398
3815
|
return replayed;
|
|
3399
3816
|
};
|
|
3817
|
+
const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
|
|
3818
|
+
const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
|
|
3400
3819
|
const recovered = await recoverCheckpoint({
|
|
3401
3820
|
store,
|
|
3402
3821
|
indexName,
|
|
@@ -3406,6 +3825,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3406
3825
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
3407
3826
|
compactionThreshold: deps.compactionThreshold,
|
|
3408
3827
|
keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
|
|
3828
|
+
keptForSync,
|
|
3409
3829
|
replay,
|
|
3410
3830
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
3411
3831
|
writeRecords
|
|
@@ -3521,6 +3941,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3521
3941
|
mutationThreshold: checkpointMutationThreshold,
|
|
3522
3942
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3523
3943
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
3944
|
+
keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
|
|
3524
3945
|
queueMetadataWrite,
|
|
3525
3946
|
markFatal,
|
|
3526
3947
|
failPartition: partitionFailureFor(indexName, partitionBatchDeps)
|
|
@@ -3574,6 +3995,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3574
3995
|
startAsyncFlushTimer();
|
|
3575
3996
|
return outcomes;
|
|
3576
3997
|
}
|
|
3998
|
+
const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
|
|
3999
|
+
directory,
|
|
4000
|
+
hooks,
|
|
4001
|
+
indexes,
|
|
4002
|
+
metadataWrites,
|
|
4003
|
+
heldCheckpointFiles,
|
|
4004
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
4005
|
+
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
4006
|
+
markFatal,
|
|
4007
|
+
closeWriterReportingFailure
|
|
4008
|
+
});
|
|
3577
4009
|
const recoveryDeps = {
|
|
3578
4010
|
directory,
|
|
3579
4011
|
hooks,
|
|
@@ -3582,7 +4014,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3582
4014
|
getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
|
|
3583
4015
|
},
|
|
3584
4016
|
holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
|
|
3585
|
-
queueMetadataWrite
|
|
4017
|
+
queueMetadataWrite,
|
|
4018
|
+
syncStorage
|
|
3586
4019
|
};
|
|
3587
4020
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
3588
4021
|
return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
|
|
@@ -3670,35 +4103,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3670
4103
|
await checkpointIndex(indexName);
|
|
3671
4104
|
}
|
|
3672
4105
|
},
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
|
|
3678
|
-
indexes.delete(indexName);
|
|
3679
|
-
metadataWrites.forget(indexName);
|
|
3680
|
-
}
|
|
3681
|
-
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
3682
|
-
for (const key of await directory.list(`${indexName}/`)) {
|
|
3683
|
-
await directory.remove(key);
|
|
3684
|
-
}
|
|
3685
|
-
},
|
|
3686
|
-
async unloadIndex(indexName) {
|
|
3687
|
-
const indexState = indexes.get(indexName);
|
|
3688
|
-
if (indexState === void 0) {
|
|
3689
|
-
return;
|
|
3690
|
-
}
|
|
3691
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
3692
|
-
indexes.delete(indexName);
|
|
3693
|
-
metadataWrites.forget(indexName);
|
|
3694
|
-
},
|
|
3695
|
-
...createArchiveCheckpointAccess({
|
|
3696
|
-
directory,
|
|
3697
|
-
held: heldCheckpointFiles,
|
|
3698
|
-
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3699
|
-
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3700
|
-
markFatal
|
|
3701
|
-
}),
|
|
4106
|
+
...unloading,
|
|
4107
|
+
syncStorage: () => syncStorage,
|
|
4108
|
+
...queryHistoryAccess({ kind: "directory", directory }),
|
|
4109
|
+
...archiveAccess,
|
|
3702
4110
|
async shutdown() {
|
|
3703
4111
|
shuttingDown = true;
|
|
3704
4112
|
stopTimers();
|
|
@@ -3965,6 +4373,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
3965
4373
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
3966
4374
|
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
3967
4375
|
keepsUnlistedFiles: false,
|
|
4376
|
+
keptForSync: /* @__PURE__ */ new Set(),
|
|
3968
4377
|
replay: async (checkpoint) => checkpoint,
|
|
3969
4378
|
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
3970
4379
|
});
|
|
@@ -4100,6 +4509,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4100
4509
|
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
4101
4510
|
}
|
|
4102
4511
|
},
|
|
4512
|
+
...queryHistoryAccess({ kind: "adapter", adapter }),
|
|
4103
4513
|
async removeIndex(indexName) {
|
|
4104
4514
|
await settleRestore(indexName).catch(() => void 0);
|
|
4105
4515
|
restores.forget(indexName);
|
|
@@ -4108,6 +4518,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4108
4518
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
4109
4519
|
await adapter.delete(key);
|
|
4110
4520
|
}
|
|
4521
|
+
await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
|
|
4111
4522
|
},
|
|
4112
4523
|
async unloadIndex(indexName) {
|
|
4113
4524
|
await settleRestore(indexName);
|
|
@@ -4162,7 +4573,9 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4162
4573
|
recordCheckpoint: hooks.recordCheckpoint,
|
|
4163
4574
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
4164
4575
|
onCheckpointHeld: hooks.onCheckpointHeld,
|
|
4165
|
-
onFatalError: hooks.onFatalError
|
|
4576
|
+
onFatalError: hooks.onFatalError,
|
|
4577
|
+
...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
|
|
4578
|
+
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
4166
4579
|
};
|
|
4167
4580
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
4168
4581
|
function partitionFor(indexName, docId) {
|
|
@@ -4247,7 +4660,12 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4247
4660
|
recordCheckpoint: wiring.recordCheckpoint,
|
|
4248
4661
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
4249
4662
|
onCheckpointHeld: wiring.emitCheckpointHeld,
|
|
4250
|
-
onFatalError: wiring.emitFatalError
|
|
4663
|
+
onFatalError: wiring.emitFatalError,
|
|
4664
|
+
keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
|
|
4665
|
+
onHistoryMoved(indexName) {
|
|
4666
|
+
if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
|
|
4667
|
+
import('./retention-ASWUHAYI.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
|
|
4668
|
+
}
|
|
4251
4669
|
});
|
|
4252
4670
|
}
|
|
4253
4671
|
|