@delali/narsil 0.3.1-next.153 → 0.3.1-next.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/{analysis-change-2OYM7ARW.mjs → analysis-change-3TFIHT5E.mjs} +32 -23
- package/dist/{react-native/chunks/analysis-change-entry-CXNRW324.mjs → analysis-change-entry-KA6JAO4X.mjs} +1 -1
- package/dist/archive-history-E4R73RWD.mjs +29 -0
- package/dist/browser/{analysis-change-2V2HZ2NY.mjs → analysis-change-TC36BU47.mjs} +39 -10
- package/dist/{analysis-change-entry-QD7YNDEE.mjs → browser/analysis-change-entry-4MH3CZXH.mjs} +1 -1
- package/dist/browser/archive-history-4IP5XQZY.mjs +29 -0
- package/dist/browser/chunk-23XRXIBW.mjs +85 -0
- package/dist/browser/chunk-3UIEB4QF.mjs +63 -0
- package/dist/browser/{chunk-CSCKZBVJ.mjs → chunk-4EWUGZYZ.mjs} +4 -4
- package/dist/browser/chunk-4WCVMOU3.mjs +151 -0
- package/dist/browser/{chunk-QJCO3TJU.mjs → chunk-56W3HQVJ.mjs} +9 -150
- package/dist/{react-native/chunks/chunk-VGUWIZFQ.mjs → browser/chunk-5TCXK5SB.mjs} +2 -2
- package/dist/browser/chunk-5W3MMTLG.mjs +19 -0
- package/dist/browser/chunk-7IGKGWSQ.mjs +79 -0
- package/dist/browser/chunk-A5KUEGDY.mjs +241 -0
- package/dist/browser/{chunk-HT7J73QL.mjs → chunk-ABCXENU5.mjs} +2 -2
- package/dist/browser/chunk-ATEBYXP6.mjs +10 -0
- package/dist/browser/chunk-BXM5XOCA.mjs +315 -0
- package/dist/browser/chunk-CJSYBQ4J.mjs +100 -0
- package/dist/browser/chunk-DSXIBE6N.mjs +78 -0
- package/dist/browser/chunk-ELV4OZOI.mjs +29 -0
- package/dist/{react-native/chunks/chunk-QC4ZRBSX.mjs → browser/chunk-EXQZGUQH.mjs} +2 -79
- package/dist/browser/{chunk-2A2IPRC3.mjs → chunk-FNIFPFB7.mjs} +5 -31
- package/dist/browser/chunk-G64Q6OJF.mjs +96 -0
- package/dist/browser/chunk-GPPOG6PT.mjs +9 -0
- package/dist/browser/chunk-IKDKX5VQ.mjs +1007 -0
- package/dist/browser/chunk-JIJXMI42.mjs +1829 -0
- package/dist/browser/chunk-JVBP5EYO.mjs +1106 -0
- package/dist/browser/{chunk-TQ5UMFG5.mjs → chunk-K4GCSZLI.mjs} +15 -6
- package/dist/browser/{chunk-PM4MUM5N.mjs → chunk-K5AXJBLL.mjs} +6 -5
- package/dist/browser/chunk-KKAMCABR.mjs +432 -0
- package/dist/browser/{chunk-IAXZAVYL.mjs → chunk-LFIWRCEA.mjs} +1 -159
- package/dist/browser/chunk-M3MQE2X7.mjs +151 -0
- package/dist/browser/chunk-ML7THVLJ.mjs +38 -0
- package/dist/browser/chunk-NLWTBAGB.mjs +111 -0
- package/dist/browser/chunk-OEDCEZXJ.mjs +92 -0
- package/dist/browser/{chunk-MPK76GVS.mjs → chunk-RA6LSBCM.mjs} +1 -21
- package/dist/browser/{chunk-GJKXW3SF.mjs → chunk-RND5CLX3.mjs} +147 -253
- package/dist/{react-native/chunks/chunk-DFQJ4IJB.mjs → browser/chunk-RQBUNZQW.mjs} +2 -2
- package/dist/browser/{chunk-VD7JEUIR.mjs → chunk-RRO3EV4C.mjs} +9 -8
- package/dist/browser/{chunk-KWYQ6KX4.mjs → chunk-SMEQ36NT.mjs} +162 -2019
- package/dist/{chunk-MDF6PDIA.mjs → browser/chunk-SRMZXJSL.mjs} +8 -1
- package/dist/browser/{chunk-7ZJTRUVW.mjs → chunk-SUDNZKPL.mjs} +2 -2
- package/dist/browser/{chunk-LURXVCHG.mjs → chunk-U3JOQGWX.mjs} +1 -1
- package/dist/browser/chunk-VI7BQR4U.mjs +21 -0
- package/dist/browser/{chunk-B3JM6Z74.mjs → chunk-XY4VVBBO.mjs} +3 -2
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/chunk-ZI2V4VD7.mjs +21 -0
- package/dist/browser/{cold-module-YECAK4MN.mjs → cold-module-NEUGHUPA.mjs} +21 -14
- package/dist/browser/durability-config-TIYKW6XX.mjs +89 -0
- package/dist/browser/{durability-wiring-LM7WFL6X.mjs → durability-wiring-CVGPRO6D.mjs} +1299 -881
- package/dist/browser/engine-hooks-YXMPHZKZ.mjs +16 -0
- package/dist/browser/import-OM6QAW45.mjs +18 -0
- package/dist/browser/methods-KTQCSIN4.mjs +30 -0
- package/dist/browser/{module-XQLEOYNM.mjs → module-3HNFCBJ5.mjs} +10 -8
- package/dist/browser/{module-KAMRAJLE.mjs → module-E33X4AOK.mjs} +17 -10
- package/dist/browser/operations-OJ3BXS55.mjs +5 -0
- package/dist/browser/{phrase-module-CNKMUY4U.mjs → phrase-module-ZXXU7S3B.mjs} +1 -1
- package/dist/browser/{pool-DTYSPBMQ.mjs → pool-3JLET5RO.mjs} +3 -2
- package/dist/browser/restore-hooks-IF2ZROUP.mjs +113 -0
- package/dist/browser/restore-queue-ZBS7CJA6.mjs +55 -0
- package/dist/browser/retention-ASWUHAYI.mjs +209 -0
- package/dist/browser/{runtime-D2ULZGCN.mjs → runtime-OTAGEFQW.mjs} +1 -1
- package/dist/browser/{snapshot-UGZVMP4H.mjs → snapshot-PN7ZW6Q4.mjs} +80 -315
- package/dist/browser/start-ZIRB7MQ2.mjs +24 -0
- package/dist/browser/sync-state-P5XG6LTQ.mjs +15 -0
- package/dist/browser/{write-I5P5MW4O.mjs → write-HMDYCTV2.mjs} +17 -11
- package/dist/{capabilities-BT-HLDC8.d.ts → capabilities-Da4ywVVl.d.ts} +94 -7
- package/dist/{chunk-LDGGR7YT.mjs → chunk-2MANFZJ3.mjs} +2 -2
- package/dist/{chunk-IZT3G24Q.mjs → chunk-2NE7VCYE.mjs} +2 -2
- package/dist/{chunk-3TVDTACC.mjs → chunk-3C7MTCCI.mjs} +1 -1
- package/dist/chunk-3HB3PEC6.mjs +240 -0
- package/dist/chunk-3HHQLEEQ.mjs +1092 -0
- package/dist/chunk-3OH67G7R.mjs +214 -0
- package/dist/{chunk-ZS6CMXAE.mjs → chunk-3R4UZWQG.mjs} +98 -250
- package/dist/{chunk-65WCV5JO.mjs → chunk-4BYWRYPM.mjs} +1 -1
- package/dist/{chunk-SOUZMPME.mjs → chunk-4KRC2APY.mjs} +7 -194
- package/dist/{chunk-5QMHMELB.mjs → chunk-5FLRXF73.mjs} +8 -1
- package/dist/chunk-5FVU47TB.mjs +20 -0
- package/dist/chunk-5YNRKI2C.mjs +81 -0
- package/dist/chunk-6L3XJXGU.mjs +470 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/{chunk-T4CJDBBE.mjs → chunk-6OEGXLBX.mjs} +1 -1
- package/dist/{chunk-3SBJVQLL.mjs → chunk-6RC4K46J.mjs} +2 -2
- package/dist/chunk-7GXJUJ24.mjs +49 -0
- package/dist/{chunk-FNBNZCGP.mjs → chunk-AILVAOLX.mjs} +2 -2
- package/dist/chunk-BDRIVYOK.mjs +72 -0
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6ZDGYS.mjs +50 -0
- package/dist/{chunk-5WB2A3XU.mjs → chunk-DCNKVEDB.mjs} +511 -27
- package/dist/{chunk-NITCPDFL.mjs → chunk-DIBHI6W5.mjs} +2 -2
- package/dist/{chunk-4QCSHLNM.mjs → chunk-DUOLGBTE.mjs} +3 -3
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-GH4WQ22V.mjs +34 -0
- package/dist/chunk-GQPTDQCL.mjs +54 -0
- package/dist/chunk-H4KHPLHJ.mjs +382 -0
- package/dist/{chunk-7TE2AJIH.mjs → chunk-H6VIGE62.mjs} +13 -10
- package/dist/{chunk-J7M73VY7.mjs → chunk-HJJI72X7.mjs} +1 -1
- package/dist/{chunk-DGRM4EZZ.mjs → chunk-HNVVEQP7.mjs} +1 -1
- package/dist/{chunk-RBEHLIKX.mjs → chunk-HW5TNXBO.mjs} +35 -11
- package/dist/{chunk-KYVRZAYP.mjs → chunk-I54T6CQP.mjs} +2 -69
- package/dist/{chunk-IRWP524D.mjs → chunk-INJYO3BU.mjs} +1 -1
- package/dist/chunk-IPN7S4KX.mjs +130 -0
- package/dist/{chunk-CSHHU7PA.mjs → chunk-J4IWBIVY.mjs} +11 -30
- package/dist/{chunk-G5KHSM6E.mjs → chunk-K4ZIGL2O.mjs} +1 -1
- package/dist/{chunk-6RKSY3BX.mjs → chunk-KBDBPVLN.mjs} +5 -5
- package/dist/{chunk-J4BDAH2W.mjs → chunk-KBKEXQZA.mjs} +2 -2
- package/dist/{chunk-IJSFWGVW.mjs → chunk-KFLK5NA2.mjs} +3 -3
- package/dist/{chunk-YGQOPB2Z.mjs → chunk-KHKIRS5D.mjs} +1 -1
- package/dist/chunk-KJ3RTZDJ.mjs +101 -0
- package/dist/{chunk-2PNYKG2D.mjs → chunk-KMUNHGBQ.mjs} +139 -432
- package/dist/chunk-KP7TYHWT.mjs +92 -0
- package/dist/{chunk-VP6XRMWR.mjs → chunk-KPIV4RUC.mjs} +6 -32
- package/dist/{chunk-NPXVQS33.mjs → chunk-KXCVQWOR.mjs} +4 -3
- package/dist/chunk-KYYABCSW.mjs +72 -0
- package/dist/{chunk-6F6USYII.mjs → chunk-L7ZD4L3K.mjs} +267 -265
- package/dist/{chunk-U46IZNCW.mjs → chunk-LC3FDWTX.mjs} +3 -2
- package/dist/{chunk-6BWLDLS3.mjs → chunk-LHKOSI2J.mjs} +1 -1
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/{react-native/chunks/chunk-YH7W556M.mjs → chunk-MP7ABPTD.mjs} +12 -2
- package/dist/chunk-MT2NRTA6.mjs +216 -0
- package/dist/chunk-NF4DT3IH.mjs +85 -0
- package/dist/{chunk-POUYICOU.mjs → chunk-OIJ64LGT.mjs} +1 -1
- package/dist/{chunk-SEAJS664.mjs → chunk-PCRA7HTI.mjs} +5 -5
- package/dist/chunk-PHQNACDT.mjs +63 -0
- package/dist/chunk-Q622CTMC.mjs +74 -0
- package/dist/{chunk-NVTFQFUL.mjs → chunk-Q7G3BV3Y.mjs} +1 -1
- package/dist/chunk-QDZJM7NI.mjs +23 -0
- package/dist/{chunk-52QQWEBS.mjs → chunk-QH3YIQP4.mjs} +36 -7
- package/dist/chunk-QJVOLKU6.mjs +111 -0
- package/dist/chunk-QUJHMPTP.mjs +30 -0
- package/dist/{chunk-3E4RRF5J.mjs → chunk-RSVULW4Q.mjs} +100 -8
- package/dist/{chunk-IJPK5PGM.mjs → chunk-RVGRFOGN.mjs} +1 -1
- package/dist/chunk-RZWWB2XR.mjs +96 -0
- package/dist/{chunk-KQJVNPRM.mjs → chunk-S2EQU2GE.mjs} +19 -16
- package/dist/chunk-S5LN746F.mjs +10 -0
- package/dist/{chunk-LJ4ENXOD.mjs → chunk-SG3JBNHY.mjs} +2 -2
- package/dist/{chunk-B2FTZNNF.mjs → chunk-SMVH6JBE.mjs} +34 -14
- package/dist/chunk-SY3EWKQY.mjs +242 -0
- package/dist/{chunk-NYSONUB7.mjs → chunk-TQXHPFWT.mjs} +1 -1
- package/dist/{chunk-ZHAXP7SD.mjs → chunk-U4BS74YN.mjs} +26 -238
- package/dist/chunk-UHICS3JI.mjs +83 -0
- package/dist/chunk-UZRZKNNZ.mjs +101 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/{chunk-TTCKT2RT.mjs → chunk-VGF7CFRP.mjs} +134 -11
- package/dist/chunk-VHBNF4MA.mjs +543 -0
- package/dist/{chunk-AWNHDBKV.mjs → chunk-VQEUR6T6.mjs} +16 -8
- package/dist/chunk-WQRN64Q6.mjs +196 -0
- package/dist/{chunk-DMNPYF54.mjs → chunk-WZ7YV54N.mjs} +2 -2
- package/dist/chunk-Y4WPNHMC.mjs +50 -0
- package/dist/{chunk-2HJWQPR6.mjs → chunk-YEDIDCR7.mjs} +6 -122
- package/dist/chunk-YEKL4JBK.mjs +96 -0
- package/dist/{chunk-UGVUW27S.mjs → chunk-Z2BE2MAM.mjs} +14 -10
- package/dist/{chunk-3HV7M5OJ.mjs → chunk-ZAUJFLEN.mjs} +11 -135
- package/dist/{chunk-FDG4KXX5.mjs → chunk-ZHCUZZE5.mjs} +6 -4
- package/dist/chunk-ZTW5WTGG.mjs +167 -0
- package/dist/chunk-ZYAY26KG.mjs +29 -0
- package/dist/{client-Cn3gNxUU.d.ts → client-DRv7UVeY.d.ts} +193 -11
- package/dist/client.d.ts +9 -7
- package/dist/client.mjs +120 -7
- package/dist/{cold-module-PYSSBIL7.mjs → cold-module-7XXCEGTZ.mjs} +43 -33
- package/dist/config-CoSig5Ox.d.ts +482 -0
- package/dist/{config-CfhH1IWQ.d.ts → config-DMwffLKn.d.ts} +20 -5
- package/dist/{direct-executor-WYALQSMD.mjs → direct-executor-G5TK7NLA.mjs} +18 -13
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +41 -9
- package/dist/distribution.mjs +9304 -7271
- package/dist/durability-config-GE4U7MCZ.mjs +89 -0
- package/dist/{durability-wiring-7VBZEJLG.mjs → durability-wiring-NCKUC5EO.mjs} +1293 -872
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{config-aIsy7mcL.d.ts → engine-CFxPYqEk.d.ts} +716 -407
- package/dist/engine-hooks-DHCMOADY.mjs +47 -0
- package/dist/{import-GJOMB2HC.mjs → import-NBFXFCSG.mjs} +32 -24
- package/dist/{index-MsF7e5hL.d.ts → index-C4AmUnl1.d.ts} +1 -1
- package/dist/index.browser.mjs +9 -4
- package/dist/index.d.ts +7 -6
- package/dist/index.mjs +29 -21
- package/dist/index.react-native.mjs +19 -12
- package/dist/{long-query-thread-SCUE62TO.mjs → long-query-thread-J4DVRM4A.mjs} +33 -19
- package/dist/methods-7IOINERF.mjs +30 -0
- package/dist/{module-6RLIKSZQ.mjs → module-DWKPBSMN.mjs} +38 -28
- package/dist/{module-AVQEIB7O.mjs → module-UN76TF7I.mjs} +14 -14
- package/dist/operations-G5EP6JPX.mjs +5 -0
- package/dist/{paged-partition-archive-KGRW366Y.mjs → paged-partition-archive-DPNWVSK2.mjs} +46 -31
- package/dist/partial-words.mjs +7 -5
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +40 -29
- package/dist/{phrase-module-OUQIRUQU.mjs → phrase-module-ZI4WSQVH.mjs} +1 -1
- package/dist/{pool-UPV5PVPY.mjs → pool-Y4A46T36.mjs} +2 -2
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +81 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +246 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +62 -0
- package/dist/react-native/chunks/{analysis-change-DXPZRGWF.mjs → analysis-change-6BPDPH6K.mjs} +50 -19
- package/dist/{browser/analysis-change-entry-JL5U2N3Q.mjs → react-native/chunks/analysis-change-entry-JLL42PH6.mjs} +1 -1
- package/dist/react-native/chunks/archive-history-MJKUQLSU.mjs +29 -0
- package/dist/react-native/chunks/chunk-3JIBF3XT.mjs +191 -0
- package/dist/react-native/chunks/chunk-3NQOVO3K.mjs +1 -0
- package/dist/react-native/chunks/{chunk-UFNCLEQM.mjs → chunk-4DII3ZSM.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-BRNAVOKP.mjs → chunk-4J74WHQM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-4LCEN2IU.mjs +315 -0
- package/dist/react-native/chunks/{chunk-PU55IOG4.mjs → chunk-4Y53T4YE.mjs} +6 -190
- package/dist/react-native/chunks/chunk-5CQX3XXV.mjs +17 -0
- package/dist/react-native/chunks/chunk-6VBXNFLE.mjs +9 -0
- package/dist/react-native/chunks/chunk-6WA3C3DV.mjs +85 -0
- package/dist/react-native/chunks/chunk-7GRCBBME.mjs +63 -0
- package/dist/react-native/chunks/{chunk-2J2727V4.mjs → chunk-7PXRWYUU.mjs} +1 -1
- package/dist/{browser/chunk-VBX2TI5V.mjs → react-native/chunks/chunk-ALOIBXWN.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-FS7BYE23.mjs → chunk-C3PTWYKT.mjs} +10 -153
- package/dist/react-native/chunks/{chunk-DKJTZ625.mjs → chunk-C5G6TJ7A.mjs} +2 -69
- package/dist/react-native/chunks/chunk-CHPTFFJM.mjs +43 -0
- package/dist/react-native/chunks/{chunk-LRGWK367.mjs → chunk-CKNRFDTZ.mjs} +1 -1
- package/dist/react-native/chunks/chunk-CUH5F57W.mjs +242 -0
- package/dist/react-native/chunks/chunk-DIYDOIM4.mjs +10 -0
- package/dist/{browser/chunk-55WYRLDZ.mjs → react-native/chunks/chunk-DVDX4SDR.mjs} +2 -2
- package/dist/{browser/chunk-PIS7ZAP7.mjs → react-native/chunks/chunk-DZ7H7D2C.mjs} +12 -2
- package/dist/react-native/chunks/{chunk-KOCTCUXW.mjs → chunk-EIOYM5TO.mjs} +17 -9
- package/dist/react-native/chunks/chunk-F753DA54.mjs +23 -0
- package/dist/react-native/chunks/chunk-GIZTJXYW.mjs +1007 -0
- package/dist/react-native/chunks/{chunk-CLE3LZ77.mjs → chunk-GJFXPHXI.mjs} +5 -4
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/{chunk-IDFMNRIC.mjs → chunk-HUOYZJ3Q.mjs} +140 -430
- package/dist/react-native/chunks/chunk-IYNT75CE.mjs +111 -0
- package/dist/react-native/chunks/chunk-JDOEEMEF.mjs +38 -0
- package/dist/react-native/chunks/{chunk-LQAKO5QV.mjs → chunk-JLIIWJUV.mjs} +2 -2
- package/dist/react-native/chunks/chunk-JWUCMVRK.mjs +15 -0
- package/dist/react-native/chunks/chunk-KJIG57UD.mjs +1106 -0
- package/dist/react-native/chunks/{chunk-KLZGAOLT.mjs → chunk-KTTAU63H.mjs} +5 -5
- package/dist/react-native/chunks/chunk-LU3Y3P63.mjs +68 -0
- package/dist/{browser/chunk-NVEJJP4R.mjs → react-native/chunks/chunk-LZSLNSUN.mjs} +2 -79
- package/dist/react-native/chunks/{chunk-3CTWSQRW.mjs → chunk-MJDQU5MW.mjs} +4 -30
- package/dist/react-native/chunks/{chunk-NFGBOI46.mjs → chunk-MVXELMJX.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-D7LFYPYI.mjs → chunk-N2WUOWLL.mjs} +12 -10
- package/dist/react-native/chunks/chunk-NVVRSRX5.mjs +78 -0
- package/dist/react-native/chunks/{chunk-JYL35A4A.mjs → chunk-OL3WDV53.mjs} +7 -33
- package/dist/react-native/chunks/chunk-PB2YXVQW.mjs +29 -0
- package/dist/react-native/chunks/{chunk-UCJU6WNH.mjs → chunk-PEHS2UO3.mjs} +7 -6
- package/dist/react-native/chunks/chunk-PJHAB5ZY.mjs +79 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-QJXTNPLQ.mjs +100 -0
- package/dist/react-native/chunks/{chunk-ZCKNBPQ4.mjs → chunk-RDWG33MM.mjs} +2 -2
- package/dist/react-native/chunks/chunk-SUPXRTMD.mjs +96 -0
- package/dist/react-native/chunks/chunk-TYIRMPD3.mjs +432 -0
- package/dist/react-native/chunks/{chunk-TLYD7IJ2.mjs → chunk-U4KSKA4Z.mjs} +1 -1
- package/dist/react-native/chunks/chunk-UNUFZ2QY.mjs +68 -0
- package/dist/react-native/chunks/{chunk-7IQR3SZO.mjs → chunk-VXGE4ZXL.mjs} +151 -257
- package/dist/react-native/chunks/{chunk-R4OPUZQ5.mjs → chunk-WBYQRHDY.mjs} +2 -2
- package/dist/react-native/chunks/chunk-YH7QY657.mjs +288 -0
- package/dist/react-native/chunks/chunk-ZZT7CEN7.mjs +92 -0
- package/dist/react-native/chunks/{cold-module-KFGEWYEL.mjs → cold-module-5PHF6DZR.mjs} +31 -22
- package/dist/react-native/chunks/durability-config-QXVZEO7T.mjs +89 -0
- package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs} +1309 -889
- package/dist/react-native/chunks/engine-hooks-MVRRHH6B.mjs +28 -0
- package/dist/react-native/chunks/gzip.react-native-HLEPDKZ3.mjs +3 -0
- package/dist/react-native/chunks/import-YVZA4V4V.mjs +30 -0
- package/dist/react-native/chunks/methods-6W5U4C3H.mjs +30 -0
- package/dist/react-native/chunks/{module-UJ5VUARS.mjs → module-H5MVPSP2.mjs} +13 -11
- package/dist/react-native/chunks/module-ZL5ZG6YB.mjs +34 -0
- package/dist/react-native/chunks/operations-GNRMD6EX.mjs +5 -0
- package/dist/react-native/chunks/{phrase-module-76UAPPSG.mjs → phrase-module-RTVKEWJT.mjs} +1 -1
- package/dist/react-native/chunks/{pool-CD4ANYUI.mjs → pool-JR3NJCAF.mjs} +3 -2
- package/dist/react-native/chunks/restore-hooks-SILFPU6B.mjs +125 -0
- package/dist/react-native/chunks/restore-queue-HSYP2AEC.mjs +55 -0
- package/dist/react-native/chunks/retention-7I7UAWYK.mjs +221 -0
- package/dist/react-native/chunks/{runtime-3MCDLJAF.mjs → runtime-R5E3TB3J.mjs} +1 -1
- package/dist/react-native/chunks/{snapshot-KVBD4LRC.mjs → snapshot-EZYB6SLF.mjs} +90 -323
- package/dist/react-native/chunks/start-SHVZ72I5.mjs +36 -0
- package/dist/react-native/chunks/sync-state-PASCIHHT.mjs +15 -0
- package/dist/react-native/chunks/{write-ES5A3WUA.mjs → write-JUESXHDL.mjs} +27 -19
- package/dist/react-native/partial-words.mjs +8 -5
- package/dist/react-native/pattern.mjs +4 -3
- package/dist/react-native/query-suggestions.mjs +258 -0
- package/dist/react-native/sync.mjs +2299 -0
- package/dist/react-native/synonyms.mjs +7 -4
- package/dist/react.d.ts +11 -10
- package/dist/react.mjs +8 -1
- package/dist/{rerank-3W54N6P2.mjs → rerank-MQ37IYRV.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-hooks-HZ3ILAMO.mjs +146 -0
- package/dist/restore-queue-JVGENGGE.mjs +55 -0
- package/dist/retention-HROPV2QU.mjs +59 -0
- package/dist/{runtime-L2DZYY6N.mjs → runtime-COIUORCN.mjs} +5 -5
- package/dist/{schema-D33CyXCy.d.ts → schema-Jwt928nJ.d.ts} +1 -1
- package/dist/server.d.ts +119 -20
- package/dist/server.mjs +1123 -58
- package/dist/{snapshot-WMX6JCLP.mjs → snapshot-4HS2G2UM.mjs} +106 -49
- package/dist/start-UTWHGH42.mjs +59 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +271 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +237 -0
- package/dist/sync-state-ZWHLAFLE.mjs +15 -0
- package/dist/sync.browser.mjs +2294 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2268 -0
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-C4ElXr4P.d.ts → tasks-qrJojgST.d.ts} +13 -5
- package/dist/thread-6MOXY2KD.mjs +45 -0
- package/dist/types-BZCGjfLn.d.ts +227 -0
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +7 -7
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-SKJ6HW7O.mjs → write-GTTWWRRG.mjs} +36 -28
- package/openapi.json +912 -5
- package/package.json +45 -8
- package/dist/browser/import-QOY6ZH6X.mjs +0 -12
- package/dist/browser/restore-queue-UQ6AAUVX.mjs +0 -41
- package/dist/chunk-7LDMWKP6.mjs +0 -209
- package/dist/engine-CJRUHXgf.d.ts +0 -664
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +0 -110
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +0 -25
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +0 -3
- package/dist/react-native/chunks/import-XFBCCENV.mjs +0 -22
- package/dist/react-native/chunks/module-FHJ2GOBI.mjs +0 -25
- package/dist/react-native/chunks/restore-queue-H5VCN453.mjs +0 -41
- package/dist/restore-queue-AOF5OTQV.mjs +0 -41
- package/dist/thread-I7DVHHW7.mjs +0 -31
package/dist/react-native/chunks/{durability-wiring-QKJB2QBT.mjs → durability-wiring-KO565FZC.mjs}
RENAMED
|
@@ -1,30 +1,46 @@
|
|
|
1
|
-
import { writeMetadataEnvelope,
|
|
2
|
-
import {
|
|
1
|
+
import { writeMetadataEnvelope, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readCurrentManifest, readMetadataEnvelope, createSeqOwner, SEGMENT_HEADER_SIZE, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, checkSegmentHeader, RECORD_LENGTH_SIZE, MAX_WAL_RECORD_BYTES, FRAME_CRC_SIZE, decodeWalEntry, readCommitMarker, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, documentsOfPayloadV2, metadataConfigOf, durableMetadataConfigOf, buildEntry, captureWholePartition } from './chunk-VXGE4ZXL.mjs';
|
|
2
|
+
import { createDirectoryWatch, unrefInterval, LOG_READ_CHUNK_BYTES } from './chunk-SUPXRTMD.mjs';
|
|
3
|
+
import './chunk-CKNRFDTZ.mjs';
|
|
3
4
|
import './chunk-HJBJBH26.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import './chunk-
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
5
|
+
import { decodeSegmentManifest, isRecord, invalidManifest, isId, isUint32, isSequenceNumber } from './chunk-CUH5F57W.mjs';
|
|
6
|
+
import './chunk-JWUCMVRK.mjs';
|
|
7
|
+
import './chunk-QJXTNPLQ.mjs';
|
|
8
|
+
import { sha256 } from './chunk-H3MDVPBO.mjs';
|
|
9
|
+
import './chunk-TYIRMPD3.mjs';
|
|
10
|
+
import { nodeThreadId } from './chunk-PSWKQVVT.mjs';
|
|
11
|
+
import { QUERY_HISTORY_FOLDER } from './chunk-PJHAB5ZY.mjs';
|
|
12
|
+
import { adapterCheckpointStore, directoryCheckpointStore, readListedFile, readSegmentFile } from './chunk-C3PTWYKT.mjs';
|
|
13
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-GJFXPHXI.mjs';
|
|
14
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, DIRECTORY_LOCK_ATTEMPTS, PHONE_FILE_CHUNK_BYTES, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-F753DA54.mjs';
|
|
15
|
+
import { manifestKey, manifestReferencedKeys, snapshotBundleKey, manifestFileRefs, MAX_SEGMENTS_PER_PARTITION } from './chunk-6WA3C3DV.mjs';
|
|
16
|
+
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-DVDX4SDR.mjs';
|
|
17
|
+
import { toError, packChecksummedEnvelope, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-HUOYZJ3Q.mjs';
|
|
12
18
|
import './chunk-IO2SMUQD.mjs';
|
|
19
|
+
import { turnsAreBudgeted, turnIsSpent, yieldTurn, waitOutsideCountedWrite, detectRuntime } from './chunk-JLIIWJUV.mjs';
|
|
13
20
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-AM7RSBPF.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import {
|
|
16
|
-
import
|
|
17
|
-
import './chunk-
|
|
18
|
-
import './chunk-
|
|
19
|
-
import './chunk-
|
|
21
|
+
import { reactNativeHost } from './chunk-MVXELMJX.mjs';
|
|
22
|
+
import { createRequestId } from './chunk-NNE4VDMM.mjs';
|
|
23
|
+
import './chunk-LU3Y3P63.mjs';
|
|
24
|
+
import './chunk-RDWG33MM.mjs';
|
|
25
|
+
import './chunk-4Y53T4YE.mjs';
|
|
26
|
+
import './chunk-UNUFZ2QY.mjs';
|
|
27
|
+
import './chunk-3NQOVO3K.mjs';
|
|
28
|
+
import './chunk-JDOEEMEF.mjs';
|
|
29
|
+
import './chunk-7PXRWYUU.mjs';
|
|
30
|
+
import { openChecksummedEnvelope, crc32, HEADER_SIZE } from './chunk-YH7QY657.mjs';
|
|
31
|
+
import { fnv1a } from './chunk-CHPTFFJM.mjs';
|
|
32
|
+
import { encodeDocument, requireValidFieldTypes } from './chunk-3JIBF3XT.mjs';
|
|
33
|
+
import { MIN_M } from './chunk-5CQX3XXV.mjs';
|
|
34
|
+
import { MAX_FILE_BYTES } from './chunk-6VBXNFLE.mjs';
|
|
35
|
+
import './chunk-C5G6TJ7A.mjs';
|
|
36
|
+
import './chunk-MJDQU5MW.mjs';
|
|
20
37
|
import './chunk-KWRW5U65.mjs';
|
|
21
38
|
import { compareCodePoints } from './chunk-PPEPTUVV.mjs';
|
|
22
39
|
import './chunk-LVIKKAA6.mjs';
|
|
23
40
|
import './chunk-AAAETEJK.mjs';
|
|
24
41
|
import './chunk-JV6VQY2S.mjs';
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import { decode } from '@msgpack/msgpack';
|
|
42
|
+
import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
|
|
43
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
28
44
|
|
|
29
45
|
// src/runtime/posix-path.ts
|
|
30
46
|
function normalizeParts(path, absolute) {
|
|
@@ -808,7 +824,9 @@ function createDurableDirectory(root) {
|
|
|
808
824
|
const fs = await getFs();
|
|
809
825
|
const pathMod = await getPath();
|
|
810
826
|
const resolvedBase = pathMod.resolve(root);
|
|
811
|
-
const
|
|
827
|
+
const directoryOfPrefix = prefix.slice(0, prefix.lastIndexOf("/") + 1);
|
|
828
|
+
const walkFrom = directoryOfPrefix.length === 0 ? resolvedBase : (await resolve(directoryOfPrefix)).path;
|
|
829
|
+
const all = await listRecursive(walkFrom, resolvedBase, pathMod, fs);
|
|
812
830
|
return all.filter((entry) => entry.startsWith(prefix)).sort(compareCodePoints);
|
|
813
831
|
},
|
|
814
832
|
async pathOf(key) {
|
|
@@ -818,714 +836,298 @@ function createDurableDirectory(root) {
|
|
|
818
836
|
};
|
|
819
837
|
}
|
|
820
838
|
|
|
821
|
-
// src/persistence/durability/
|
|
822
|
-
var
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
3049323471,
|
|
826
|
-
3921009573,
|
|
827
|
-
961987163,
|
|
828
|
-
1508970993,
|
|
829
|
-
2453635748,
|
|
830
|
-
2870763221,
|
|
831
|
-
3624381080,
|
|
832
|
-
310598401,
|
|
833
|
-
607225278,
|
|
834
|
-
1426881987,
|
|
835
|
-
1925078388,
|
|
836
|
-
2162078206,
|
|
837
|
-
2614888103,
|
|
838
|
-
3248222580,
|
|
839
|
-
3835390401,
|
|
840
|
-
4022224774,
|
|
841
|
-
264347078,
|
|
842
|
-
604807628,
|
|
843
|
-
770255983,
|
|
844
|
-
1249150122,
|
|
845
|
-
1555081692,
|
|
846
|
-
1996064986,
|
|
847
|
-
2554220882,
|
|
848
|
-
2821834349,
|
|
849
|
-
2952996808,
|
|
850
|
-
3210313671,
|
|
851
|
-
3336571891,
|
|
852
|
-
3584528711,
|
|
853
|
-
113926993,
|
|
854
|
-
338241895,
|
|
855
|
-
666307205,
|
|
856
|
-
773529912,
|
|
857
|
-
1294757372,
|
|
858
|
-
1396182291,
|
|
859
|
-
1695183700,
|
|
860
|
-
1986661051,
|
|
861
|
-
2177026350,
|
|
862
|
-
2456956037,
|
|
863
|
-
2730485921,
|
|
864
|
-
2820302411,
|
|
865
|
-
3259730800,
|
|
866
|
-
3345764771,
|
|
867
|
-
3516065817,
|
|
868
|
-
3600352804,
|
|
869
|
-
4094571909,
|
|
870
|
-
275423344,
|
|
871
|
-
430227734,
|
|
872
|
-
506948616,
|
|
873
|
-
659060556,
|
|
874
|
-
883997877,
|
|
875
|
-
958139571,
|
|
876
|
-
1322822218,
|
|
877
|
-
1537002063,
|
|
878
|
-
1747873779,
|
|
879
|
-
1955562222,
|
|
880
|
-
2024104815,
|
|
881
|
-
2227730452,
|
|
882
|
-
2361852424,
|
|
883
|
-
2428436474,
|
|
884
|
-
2756734187,
|
|
885
|
-
3204031479,
|
|
886
|
-
3329325298
|
|
887
|
-
]);
|
|
888
|
-
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
889
|
-
var BLOCK_BYTES = 64;
|
|
890
|
-
var LENGTH_BYTES = 8;
|
|
891
|
-
function rotateRight(value, bits) {
|
|
892
|
-
return value >>> bits | value << 32 - bits;
|
|
893
|
-
}
|
|
894
|
-
function padded(bytes) {
|
|
895
|
-
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
896
|
-
const message = new Uint8Array(total);
|
|
897
|
-
message.set(bytes);
|
|
898
|
-
message[bytes.byteLength] = 128;
|
|
899
|
-
const view = new DataView(message.buffer);
|
|
900
|
-
const bitLength = bytes.byteLength * 8;
|
|
901
|
-
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
902
|
-
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
903
|
-
return view;
|
|
904
|
-
}
|
|
905
|
-
function sha256(bytes) {
|
|
906
|
-
const view = padded(bytes);
|
|
907
|
-
const state = INITIAL_STATE.slice();
|
|
908
|
-
const words = new Uint32Array(64);
|
|
909
|
-
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
910
|
-
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
911
|
-
for (let i = 16; i < 64; i++) {
|
|
912
|
-
const w15 = words[i - 15];
|
|
913
|
-
const w2 = words[i - 2];
|
|
914
|
-
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
915
|
-
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
916
|
-
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
917
|
-
}
|
|
918
|
-
let [a, b, c, d, e, f, g, h] = state;
|
|
919
|
-
for (let i = 0; i < 64; i++) {
|
|
920
|
-
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
921
|
-
const choice = e & f ^ ~e & g;
|
|
922
|
-
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
923
|
-
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
924
|
-
const majority = a & b ^ a & c ^ b & c;
|
|
925
|
-
const t2 = s0 + majority >>> 0;
|
|
926
|
-
h = g;
|
|
927
|
-
g = f;
|
|
928
|
-
f = e;
|
|
929
|
-
e = d + t1 >>> 0;
|
|
930
|
-
d = c;
|
|
931
|
-
c = b;
|
|
932
|
-
b = a;
|
|
933
|
-
a = t1 + t2 >>> 0;
|
|
934
|
-
}
|
|
935
|
-
state[0] = state[0] + a >>> 0;
|
|
936
|
-
state[1] = state[1] + b >>> 0;
|
|
937
|
-
state[2] = state[2] + c >>> 0;
|
|
938
|
-
state[3] = state[3] + d >>> 0;
|
|
939
|
-
state[4] = state[4] + e >>> 0;
|
|
940
|
-
state[5] = state[5] + f >>> 0;
|
|
941
|
-
state[6] = state[6] + g >>> 0;
|
|
942
|
-
state[7] = state[7] + h >>> 0;
|
|
943
|
-
}
|
|
944
|
-
const digest = new Uint8Array(32);
|
|
945
|
-
const out = new DataView(digest.buffer);
|
|
946
|
-
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
947
|
-
return digest;
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
// src/persistence/durability/restore-journal.ts
|
|
951
|
-
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
952
|
-
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
953
|
-
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
954
|
-
var LOCK_FILE_KEY = ".narsil.lock";
|
|
955
|
-
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
956
|
-
function digestOf(indexName) {
|
|
957
|
-
let hex = "";
|
|
958
|
-
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
959
|
-
return hex;
|
|
839
|
+
// src/persistence/durability/sync/layout.ts
|
|
840
|
+
var SYNC_KEY_NUMBER_WIDTH = 16;
|
|
841
|
+
function syncDirectoryPrefix(indexName) {
|
|
842
|
+
return `${indexName}/sync/`;
|
|
960
843
|
}
|
|
961
|
-
function
|
|
962
|
-
return `${
|
|
844
|
+
function publishedSnapshotKey(indexName) {
|
|
845
|
+
return `${syncDirectoryPrefix(indexName)}published`;
|
|
963
846
|
}
|
|
964
|
-
function
|
|
965
|
-
return `${
|
|
847
|
+
function historyRoot(indexName) {
|
|
848
|
+
return `${indexName}/history/`;
|
|
966
849
|
}
|
|
967
|
-
function
|
|
968
|
-
return `${
|
|
850
|
+
function historyPrefix(indexName, partitionId) {
|
|
851
|
+
return `${historyRoot(indexName)}${partitionId}/`;
|
|
969
852
|
}
|
|
970
|
-
function
|
|
971
|
-
return `${
|
|
853
|
+
function walSegmentPrefix(indexName, partitionId) {
|
|
854
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
972
855
|
}
|
|
973
|
-
function
|
|
974
|
-
return
|
|
975
|
-
|
|
856
|
+
function formatSyncKeyNumber(value) {
|
|
857
|
+
return value.toString().padStart(SYNC_KEY_NUMBER_WIDTH, "0");
|
|
858
|
+
}
|
|
859
|
+
function historyKey(indexName, partitionId, startSeqNo, movedAtMs) {
|
|
860
|
+
return `${historyPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}-${formatSyncKeyNumber(movedAtMs)}`;
|
|
861
|
+
}
|
|
862
|
+
function walSegmentKey(indexName, partitionId, startSeqNo) {
|
|
863
|
+
return `${walSegmentPrefix(indexName, partitionId)}${formatSyncKeyNumber(startSeqNo)}`;
|
|
864
|
+
}
|
|
865
|
+
var HISTORY_TAIL = /^(\d+)\/(\d{16})-(\d{16})$/;
|
|
866
|
+
function parseHistoryKey(indexName, key) {
|
|
867
|
+
const root = historyRoot(indexName);
|
|
868
|
+
if (!key.startsWith(root)) return null;
|
|
869
|
+
const match = HISTORY_TAIL.exec(key.slice(root.length));
|
|
870
|
+
if (match === null) return null;
|
|
871
|
+
const partitionId = Number.parseInt(match[1], 10);
|
|
872
|
+
const startSeqNo = Number.parseInt(match[2], 10);
|
|
873
|
+
const movedAtMs = Number.parseInt(match[3], 10);
|
|
874
|
+
if (!Number.isSafeInteger(partitionId) || !Number.isSafeInteger(startSeqNo) || !Number.isSafeInteger(movedAtMs)) {
|
|
875
|
+
return null;
|
|
876
|
+
}
|
|
877
|
+
return { key, partitionId, startSeqNo, movedAtMs };
|
|
976
878
|
}
|
|
977
|
-
function
|
|
978
|
-
|
|
879
|
+
function parseWalSegmentStart(indexName, partitionId, key) {
|
|
880
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
881
|
+
if (!key.startsWith(prefix)) return null;
|
|
882
|
+
const tail = key.slice(prefix.length);
|
|
883
|
+
if (!/^\d{16}$/.test(tail)) return null;
|
|
884
|
+
const value = Number.parseInt(tail, 10);
|
|
885
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
979
886
|
}
|
|
980
|
-
|
|
981
|
-
|
|
887
|
+
|
|
888
|
+
// src/persistence/durability/sync/history.ts
|
|
889
|
+
function compareHistorySegments(a, b) {
|
|
890
|
+
return a.partitionId - b.partitionId || a.startSeqNo - b.startSeqNo;
|
|
891
|
+
}
|
|
892
|
+
async function listHistorySegments(directory, indexName, partitionId) {
|
|
893
|
+
const prefix = partitionId === void 0 ? historyRoot(indexName) : historyPrefix(indexName, partitionId);
|
|
894
|
+
const segments = [];
|
|
895
|
+
for (const key of await directory.list(prefix)) {
|
|
896
|
+
const segment = parseHistoryKey(indexName, key);
|
|
897
|
+
if (segment !== null) segments.push(segment);
|
|
898
|
+
}
|
|
899
|
+
return segments.sort(compareHistorySegments);
|
|
900
|
+
}
|
|
901
|
+
async function moveSegmentIntoHistory(directory, indexName, partitionId, startSeqNo, movedAtMs) {
|
|
902
|
+
await directory.move(
|
|
903
|
+
walSegmentKey(indexName, partitionId, startSeqNo),
|
|
904
|
+
historyKey(indexName, partitionId, startSeqNo, movedAtMs)
|
|
905
|
+
);
|
|
982
906
|
}
|
|
983
|
-
function
|
|
984
|
-
const
|
|
985
|
-
|
|
986
|
-
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
987
|
-
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
988
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
989
|
-
corruptJournal(key, "names an index that no index name can match");
|
|
990
|
-
}
|
|
991
|
-
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
992
|
-
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
993
|
-
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
907
|
+
async function removeHistory(directory, indexName, partitionId) {
|
|
908
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
909
|
+
await directory.remove(segment.key);
|
|
994
910
|
}
|
|
995
|
-
return { kind: "journal", indexName, stagingPrefix };
|
|
996
911
|
}
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1002
|
-
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
1003
|
-
{ indexName, stagingPrefix }
|
|
1004
|
-
);
|
|
1005
|
-
}
|
|
1006
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1007
|
-
if (decoded.kind !== "current") {
|
|
1008
|
-
throw new NarsilError(
|
|
1009
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1010
|
-
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
1011
|
-
{ indexName }
|
|
1012
|
-
);
|
|
1013
|
-
}
|
|
1014
|
-
const root = `${indexName}/`;
|
|
1015
|
-
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
1016
|
-
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
1017
|
-
return keys;
|
|
912
|
+
|
|
913
|
+
// src/persistence/durability/checkpoint.ts
|
|
914
|
+
function walPrefix(indexName, partitionId) {
|
|
915
|
+
return `${indexName}/wal/${partitionId}/`;
|
|
1018
916
|
}
|
|
1019
|
-
function
|
|
1020
|
-
|
|
917
|
+
async function truncateCoveredSegments(directory, indexName, checkpoint, movedIntoHistoryAtMs) {
|
|
918
|
+
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
919
|
+
const prefix = walPrefix(indexName, partitionId);
|
|
920
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
921
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
922
|
+
if (marker === null) {
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
926
|
+
const segments = await directory.list(prefix);
|
|
927
|
+
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
928
|
+
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
929
|
+
const start = startSeqNos[i];
|
|
930
|
+
if (start >= activeSegmentSeqNo) {
|
|
931
|
+
break;
|
|
932
|
+
}
|
|
933
|
+
const nextStart = startSeqNos[i + 1];
|
|
934
|
+
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
935
|
+
if (segmentCoversBeyondCheckpoint) {
|
|
936
|
+
break;
|
|
937
|
+
}
|
|
938
|
+
if (movedIntoHistoryAtMs === null) await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
939
|
+
else await moveSegmentIntoHistory(directory, indexName, partitionId, start, movedIntoHistoryAtMs);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
1021
942
|
}
|
|
1022
|
-
async function
|
|
1023
|
-
const
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
1030
|
-
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
1031
|
-
} else {
|
|
1032
|
-
const stagedKeys = new Set(staged);
|
|
1033
|
-
for (const key of listed) {
|
|
1034
|
-
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
943
|
+
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
944
|
+
for (const [partitionId, partition] of [...partitions]) {
|
|
945
|
+
if (partitionId < partitionCount) continue;
|
|
946
|
+
try {
|
|
947
|
+
await partition.walWriter.close();
|
|
948
|
+
} catch (err) {
|
|
949
|
+
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
1035
950
|
}
|
|
951
|
+
partitions.delete(partitionId);
|
|
1036
952
|
}
|
|
1037
|
-
|
|
1038
|
-
|
|
953
|
+
const walRoot = `${indexName}/wal/`;
|
|
954
|
+
for (const key of await directory.list(walRoot)) {
|
|
955
|
+
const partitionId = walKeyPartitionId(key, walRoot);
|
|
956
|
+
if (partitionId === null || partitionId < partitionCount) continue;
|
|
957
|
+
await directory.remove(key);
|
|
1039
958
|
}
|
|
1040
|
-
await storage.remove(journalKey(indexName));
|
|
1041
|
-
return retiredPrefix;
|
|
1042
959
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
960
|
+
function walKeyPartitionId(key, walRoot) {
|
|
961
|
+
const tail = key.slice(walRoot.length);
|
|
962
|
+
const separator = tail.indexOf("/");
|
|
963
|
+
if (separator <= 0) {
|
|
964
|
+
return null;
|
|
1047
965
|
}
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
1052
|
-
}
|
|
1053
|
-
function isTopLevelJournal(key) {
|
|
1054
|
-
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
1055
|
-
}
|
|
1056
|
-
async function removeStagingPrefixes(storage) {
|
|
1057
|
-
const prefixes = /* @__PURE__ */ new Set();
|
|
1058
|
-
for (const key of await storage.list(".")) {
|
|
1059
|
-
const slash = key.indexOf("/");
|
|
1060
|
-
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
966
|
+
const segment = tail.slice(0, separator);
|
|
967
|
+
if (!/^\d+$/.test(segment)) {
|
|
968
|
+
return null;
|
|
1061
969
|
}
|
|
1062
|
-
|
|
970
|
+
const value = Number.parseInt(segment, 10);
|
|
971
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
1063
972
|
}
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
const bytes = await storage.read(key);
|
|
1068
|
-
if (bytes === null) continue;
|
|
1069
|
-
const reading = readJournal(key, bytes);
|
|
1070
|
-
if (reading.kind === "leftover") {
|
|
1071
|
-
await storage.remove(key);
|
|
1072
|
-
continue;
|
|
1073
|
-
}
|
|
1074
|
-
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
973
|
+
function parseSegmentStartSeqNo(key, prefix) {
|
|
974
|
+
if (!key.startsWith(prefix)) {
|
|
975
|
+
return null;
|
|
1075
976
|
}
|
|
1076
|
-
|
|
977
|
+
const tail = key.slice(prefix.length);
|
|
978
|
+
if (!/^\d{16}$/.test(tail)) {
|
|
979
|
+
return null;
|
|
980
|
+
}
|
|
981
|
+
const value = Number.parseInt(tail, 10);
|
|
982
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
1077
983
|
}
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
function missing(error) {
|
|
1081
|
-
return error.code === "ENOENT";
|
|
984
|
+
function formatSegmentStart(startSeqNo) {
|
|
985
|
+
return startSeqNo.toString().padStart(16, "0");
|
|
1082
986
|
}
|
|
1083
|
-
|
|
1084
|
-
|
|
987
|
+
|
|
988
|
+
// src/persistence/durability/marker-writer.ts
|
|
989
|
+
function markerKey(indexName, partitionId) {
|
|
990
|
+
return `${indexName}/wal/${partitionId}/commit`;
|
|
1085
991
|
}
|
|
1086
|
-
function
|
|
992
|
+
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
993
|
+
const key = markerKey(indexName, partitionId);
|
|
994
|
+
const handle = await directory.markerHandle(key);
|
|
995
|
+
const existing = await handle.read();
|
|
996
|
+
const current = readCommitMarker(existing);
|
|
997
|
+
const createdFresh = existing.length === 0;
|
|
998
|
+
let lastSlot = current?.slotIndex ?? null;
|
|
999
|
+
let writeSeq = current?.state.writeSeq ?? 0;
|
|
1087
1000
|
return {
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
async moveTree(from, to) {
|
|
1097
|
-
const source = await directory.pathOf(from);
|
|
1098
|
-
const target = await directory.pathOf(to);
|
|
1099
|
-
const fs = await getFs();
|
|
1100
|
-
try {
|
|
1101
|
-
await fs.rename(source, target);
|
|
1102
|
-
} catch (error) {
|
|
1103
|
-
if (missing(error)) return;
|
|
1104
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
1105
|
-
}
|
|
1106
|
-
await directory.syncDirectoryOf(to);
|
|
1001
|
+
created: createdFresh,
|
|
1002
|
+
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
1003
|
+
async commit(state) {
|
|
1004
|
+
const slotIndex = nextMarkerSlot(lastSlot);
|
|
1005
|
+
writeSeq += 1;
|
|
1006
|
+
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
1007
|
+
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
1008
|
+
lastSlot = slotIndex;
|
|
1107
1009
|
},
|
|
1108
|
-
async
|
|
1109
|
-
|
|
1110
|
-
const fs = await getFs();
|
|
1111
|
-
try {
|
|
1112
|
-
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
1113
|
-
} catch (error) {
|
|
1114
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
1115
|
-
}
|
|
1116
|
-
await directory.syncDirectoryOf(key);
|
|
1010
|
+
async close() {
|
|
1011
|
+
await handle.close();
|
|
1117
1012
|
}
|
|
1118
1013
|
};
|
|
1119
1014
|
}
|
|
1120
|
-
function recoverDirectoryRestores(directory) {
|
|
1121
|
-
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
1122
|
-
}
|
|
1123
1015
|
|
|
1124
|
-
// src/persistence/durability/
|
|
1125
|
-
function
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
return next === "" || next === "/" || next === "\\";
|
|
1016
|
+
// src/persistence/durability/recovery.ts
|
|
1017
|
+
async function removeLeftoverRewriteFiles(directory, names) {
|
|
1018
|
+
for (const indexName of names) {
|
|
1019
|
+
for (const place of ["rebuilt", "received"]) {
|
|
1020
|
+
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1132
1023
|
}
|
|
1133
|
-
function
|
|
1134
|
-
const
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
1140
|
-
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
1141
|
-
read: (key) => directory.read(staged(key)),
|
|
1142
|
-
remove: (key) => directory.remove(staged(key)),
|
|
1143
|
-
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
1144
|
-
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
1145
|
-
pathOf: (key) => directory.pathOf(staged(key)),
|
|
1146
|
-
...readRange === void 0 ? {} : {
|
|
1147
|
-
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
1148
|
-
},
|
|
1149
|
-
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
1150
|
-
};
|
|
1151
|
-
}
|
|
1152
|
-
function createRestoreStaging(storage, indexName, options = {}) {
|
|
1153
|
-
const stagingPrefix = newStagingPrefix();
|
|
1154
|
-
const indexRoot = `${indexName}/`;
|
|
1155
|
-
let committed = false;
|
|
1156
|
-
const staged = (key) => {
|
|
1157
|
-
if (!key.startsWith(indexRoot)) {
|
|
1158
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
1159
|
-
key
|
|
1160
|
-
});
|
|
1024
|
+
async function listPersistedIndexes(directory) {
|
|
1025
|
+
const metaKeys = await directory.list("");
|
|
1026
|
+
const names = [];
|
|
1027
|
+
for (const key of metaKeys) {
|
|
1028
|
+
if (key.endsWith("/meta")) {
|
|
1029
|
+
names.push(key.slice(0, -"/meta".length));
|
|
1161
1030
|
}
|
|
1162
|
-
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
1163
|
-
};
|
|
1164
|
-
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
1165
|
-
const store = {
|
|
1166
|
-
read: (key) => storage.read(staged(key)),
|
|
1167
|
-
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
1168
|
-
remove: (key) => storage.remove(staged(key)),
|
|
1169
|
-
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
1170
|
-
};
|
|
1171
|
-
const pathOf = storage.pathOf;
|
|
1172
|
-
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
1173
|
-
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
1174
|
-
let retiredPrefix = null;
|
|
1175
|
-
async function finishOnStorage() {
|
|
1176
|
-
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
1177
1031
|
}
|
|
1178
|
-
return
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1032
|
+
return names;
|
|
1033
|
+
}
|
|
1034
|
+
async function loadMetadata(directory, indexName) {
|
|
1035
|
+
const bytes = await directory.read(`${indexName}/meta`);
|
|
1036
|
+
if (bytes === null) {
|
|
1037
|
+
return null;
|
|
1038
|
+
}
|
|
1039
|
+
const { metadata } = await readMetadataEnvelope(bytes);
|
|
1040
|
+
return metadata;
|
|
1041
|
+
}
|
|
1042
|
+
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
1043
|
+
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
1044
|
+
const read = await readWalSegments(directory, indexName, partitionId);
|
|
1045
|
+
if (read === null) {
|
|
1046
|
+
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
1047
|
+
}
|
|
1048
|
+
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
1049
|
+
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
1050
|
+
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
1051
|
+
}
|
|
1052
|
+
let highestSeqNo = fromSeqNoExclusive;
|
|
1053
|
+
let highestPrimaryTerm = 0;
|
|
1054
|
+
const replay = async (entries) => {
|
|
1055
|
+
for (const entry of entries) {
|
|
1056
|
+
if (entry.seqNo <= highestSeqNo) {
|
|
1057
|
+
continue;
|
|
1197
1058
|
}
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
} catch (error) {
|
|
1204
|
-
options.onStorageLost?.(toError2(error));
|
|
1059
|
+
if (turnIsSpent()) await yieldTurn();
|
|
1060
|
+
applyEntry(entry, deps);
|
|
1061
|
+
highestSeqNo = entry.seqNo;
|
|
1062
|
+
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
1063
|
+
highestPrimaryTerm = entry.primaryTerm;
|
|
1205
1064
|
}
|
|
1206
|
-
},
|
|
1207
|
-
async removeRetired() {
|
|
1208
|
-
const prefix = retiredPrefix;
|
|
1209
|
-
retiredPrefix = null;
|
|
1210
|
-
if (prefix !== null) await removePrefix(storage, prefix);
|
|
1211
|
-
},
|
|
1212
|
-
async discard() {
|
|
1213
|
-
if (committed) return;
|
|
1214
|
-
await removePrefix(storage, stagingPrefix);
|
|
1215
1065
|
}
|
|
1216
1066
|
};
|
|
1067
|
+
await replay(durableEntries);
|
|
1068
|
+
if (activeTail !== null) {
|
|
1069
|
+
await replay(activeTail.entries);
|
|
1070
|
+
}
|
|
1071
|
+
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
1072
|
+
if (activeTail !== null) {
|
|
1073
|
+
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
1074
|
+
}
|
|
1075
|
+
return { highestSeqNo, highestPrimaryTerm };
|
|
1217
1076
|
}
|
|
1218
|
-
function
|
|
1219
|
-
const
|
|
1220
|
-
return
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1077
|
+
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
1078
|
+
const lastReplayed = tail.entries.at(-1);
|
|
1079
|
+
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
1080
|
+
const handle = await directory.appendHandle(tail.key);
|
|
1081
|
+
try {
|
|
1082
|
+
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
1083
|
+
else await handle.sync();
|
|
1084
|
+
} finally {
|
|
1085
|
+
await handle.close();
|
|
1086
|
+
}
|
|
1087
|
+
if (lastReplayed === void 0) return;
|
|
1088
|
+
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
1089
|
+
try {
|
|
1090
|
+
await writer.commit({
|
|
1091
|
+
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
1092
|
+
durableByteLength: tail.cleanEnd,
|
|
1093
|
+
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
1094
|
+
});
|
|
1095
|
+
} finally {
|
|
1096
|
+
await writer.close();
|
|
1097
|
+
}
|
|
1233
1098
|
}
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1099
|
+
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
1100
|
+
for (const { key, startSeqNo } of segments) {
|
|
1101
|
+
if (startSeqNo > activeSegmentSeqNo) {
|
|
1102
|
+
await directory.remove(key);
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1238
1105
|
}
|
|
1239
|
-
function
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
throw error;
|
|
1247
|
-
}
|
|
1248
|
-
},
|
|
1249
|
-
onStorageLost: deps.markFatal
|
|
1250
|
-
});
|
|
1106
|
+
function applyEntry(entry, deps) {
|
|
1107
|
+
deps.writeRecords?.replayed(entry);
|
|
1108
|
+
if (entry.operation === "DELETE") {
|
|
1109
|
+
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
1251
1113
|
}
|
|
1252
|
-
function
|
|
1253
|
-
|
|
1254
|
-
return () => {
|
|
1255
|
-
if (released) return;
|
|
1256
|
-
released = true;
|
|
1257
|
-
const count = (held.get(indexName) ?? 1) - 1;
|
|
1258
|
-
if (count <= 0) held.delete(indexName);
|
|
1259
|
-
else held.set(indexName, count);
|
|
1260
|
-
};
|
|
1114
|
+
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
1115
|
+
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
1261
1116
|
}
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1117
|
+
|
|
1118
|
+
// src/persistence/durability/checkpoint-capture.ts
|
|
1119
|
+
function whileNoMutationApplies(partitions, capture2) {
|
|
1120
|
+
return new Promise((resolve, reject) => {
|
|
1121
|
+
let release = () => void 0;
|
|
1122
|
+
const released = new Promise((settle) => {
|
|
1123
|
+
release = settle;
|
|
1124
|
+
});
|
|
1125
|
+
function captureNow() {
|
|
1268
1126
|
try {
|
|
1269
|
-
|
|
1270
|
-
await deps.checkpoint(indexName, true);
|
|
1271
|
-
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
1272
|
-
if (manifestBytes === null) {
|
|
1273
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
1274
|
-
indexName
|
|
1275
|
-
});
|
|
1276
|
-
}
|
|
1277
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1278
|
-
if (decoded.kind !== "current") {
|
|
1279
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
1280
|
-
indexName
|
|
1281
|
-
});
|
|
1282
|
-
}
|
|
1283
|
-
return {
|
|
1284
|
-
manifestBytes,
|
|
1285
|
-
manifest: decoded.manifest,
|
|
1286
|
-
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
1287
|
-
readRange: async (ref, offset, length) => {
|
|
1288
|
-
const readRange = deps.directory.readRange;
|
|
1289
|
-
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
1290
|
-
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
1291
|
-
return whole.slice(offset, offset + length);
|
|
1292
|
-
},
|
|
1293
|
-
release,
|
|
1294
|
-
directory: deps.directory
|
|
1295
|
-
};
|
|
1127
|
+
resolve(capture2());
|
|
1296
1128
|
} catch (error) {
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
}
|
|
1300
|
-
},
|
|
1301
|
-
stageRestore(indexName) {
|
|
1302
|
-
return stageDirectoryRestore(deps, indexName);
|
|
1303
|
-
},
|
|
1304
|
-
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
1305
|
-
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
1306
|
-
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
};
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
// src/persistence/durability/checkpoint.ts
|
|
1313
|
-
function walPrefix(indexName, partitionId) {
|
|
1314
|
-
return `${indexName}/wal/${partitionId}/`;
|
|
1315
|
-
}
|
|
1316
|
-
async function truncateCoveredSegments(directory, indexName, checkpoint) {
|
|
1317
|
-
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
1318
|
-
const prefix = walPrefix(indexName, partitionId);
|
|
1319
|
-
const markerBytes = await directory.read(`${prefix}commit`);
|
|
1320
|
-
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
1321
|
-
if (marker === null) {
|
|
1322
|
-
continue;
|
|
1323
|
-
}
|
|
1324
|
-
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
1325
|
-
const segments = await directory.list(prefix);
|
|
1326
|
-
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
1327
|
-
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
1328
|
-
const start = startSeqNos[i];
|
|
1329
|
-
if (start >= activeSegmentSeqNo) {
|
|
1330
|
-
break;
|
|
1331
|
-
}
|
|
1332
|
-
const nextStart = startSeqNos[i + 1];
|
|
1333
|
-
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
1334
|
-
if (segmentCoversBeyondCheckpoint) {
|
|
1335
|
-
break;
|
|
1336
|
-
}
|
|
1337
|
-
await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
1338
|
-
}
|
|
1339
|
-
}
|
|
1340
|
-
}
|
|
1341
|
-
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
1342
|
-
for (const [partitionId, partition] of [...partitions]) {
|
|
1343
|
-
if (partitionId < partitionCount) continue;
|
|
1344
|
-
try {
|
|
1345
|
-
await partition.walWriter.close();
|
|
1346
|
-
} catch (err) {
|
|
1347
|
-
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
1348
|
-
}
|
|
1349
|
-
partitions.delete(partitionId);
|
|
1350
|
-
}
|
|
1351
|
-
const walRoot = `${indexName}/wal/`;
|
|
1352
|
-
for (const key of await directory.list(walRoot)) {
|
|
1353
|
-
const partitionId = walKeyPartitionId(key, walRoot);
|
|
1354
|
-
if (partitionId === null || partitionId < partitionCount) continue;
|
|
1355
|
-
await directory.remove(key);
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
function walKeyPartitionId(key, walRoot) {
|
|
1359
|
-
const tail = key.slice(walRoot.length);
|
|
1360
|
-
const separator = tail.indexOf("/");
|
|
1361
|
-
if (separator <= 0) {
|
|
1362
|
-
return null;
|
|
1363
|
-
}
|
|
1364
|
-
const segment = tail.slice(0, separator);
|
|
1365
|
-
if (!/^\d+$/.test(segment)) {
|
|
1366
|
-
return null;
|
|
1367
|
-
}
|
|
1368
|
-
const value = Number.parseInt(segment, 10);
|
|
1369
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
1370
|
-
}
|
|
1371
|
-
function parseSegmentStartSeqNo(key, prefix) {
|
|
1372
|
-
if (!key.startsWith(prefix)) {
|
|
1373
|
-
return null;
|
|
1374
|
-
}
|
|
1375
|
-
const tail = key.slice(prefix.length);
|
|
1376
|
-
if (!/^\d{16}$/.test(tail)) {
|
|
1377
|
-
return null;
|
|
1378
|
-
}
|
|
1379
|
-
const value = Number.parseInt(tail, 10);
|
|
1380
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
1381
|
-
}
|
|
1382
|
-
function formatSegmentStart(startSeqNo) {
|
|
1383
|
-
return startSeqNo.toString().padStart(16, "0");
|
|
1384
|
-
}
|
|
1385
|
-
|
|
1386
|
-
// src/persistence/durability/marker-writer.ts
|
|
1387
|
-
function markerKey(indexName, partitionId) {
|
|
1388
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
1389
|
-
}
|
|
1390
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
1391
|
-
const key = markerKey(indexName, partitionId);
|
|
1392
|
-
const handle = await directory.markerHandle(key);
|
|
1393
|
-
const existing = await handle.read();
|
|
1394
|
-
const current = readCommitMarker(existing);
|
|
1395
|
-
const createdFresh = existing.length === 0;
|
|
1396
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
1397
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
1398
|
-
return {
|
|
1399
|
-
created: createdFresh,
|
|
1400
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
1401
|
-
async commit(state) {
|
|
1402
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
1403
|
-
writeSeq += 1;
|
|
1404
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
1405
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
1406
|
-
lastSlot = slotIndex;
|
|
1407
|
-
},
|
|
1408
|
-
async close() {
|
|
1409
|
-
await handle.close();
|
|
1410
|
-
}
|
|
1411
|
-
};
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
// src/persistence/durability/recovery.ts
|
|
1415
|
-
async function removeLeftoverRewriteFiles(directory, names) {
|
|
1416
|
-
for (const indexName of names) {
|
|
1417
|
-
for (const place of ["rebuilt", "received"]) {
|
|
1418
|
-
for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
|
|
1419
|
-
}
|
|
1420
|
-
}
|
|
1421
|
-
}
|
|
1422
|
-
async function listPersistedIndexes(directory) {
|
|
1423
|
-
const metaKeys = await directory.list("");
|
|
1424
|
-
const names = [];
|
|
1425
|
-
for (const key of metaKeys) {
|
|
1426
|
-
if (key.endsWith("/meta")) {
|
|
1427
|
-
names.push(key.slice(0, -"/meta".length));
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
return names;
|
|
1431
|
-
}
|
|
1432
|
-
async function loadMetadata(directory, indexName) {
|
|
1433
|
-
const bytes = await directory.read(`${indexName}/meta`);
|
|
1434
|
-
if (bytes === null) {
|
|
1435
|
-
return null;
|
|
1436
|
-
}
|
|
1437
|
-
const { metadata } = await readMetadataEnvelope(bytes);
|
|
1438
|
-
return metadata;
|
|
1439
|
-
}
|
|
1440
|
-
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
1441
|
-
await directory.remove(nextSegmentKey(indexName, partitionId));
|
|
1442
|
-
const read = await readWalSegments(directory, indexName, partitionId);
|
|
1443
|
-
if (read === null) {
|
|
1444
|
-
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
1445
|
-
}
|
|
1446
|
-
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
1447
|
-
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
1448
|
-
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
1449
|
-
}
|
|
1450
|
-
let highestSeqNo = fromSeqNoExclusive;
|
|
1451
|
-
let highestPrimaryTerm = 0;
|
|
1452
|
-
const replay = async (entries) => {
|
|
1453
|
-
for (const entry of entries) {
|
|
1454
|
-
if (entry.seqNo <= highestSeqNo) {
|
|
1455
|
-
continue;
|
|
1456
|
-
}
|
|
1457
|
-
if (turnIsSpent()) await yieldTurn();
|
|
1458
|
-
applyEntry(entry, deps);
|
|
1459
|
-
highestSeqNo = entry.seqNo;
|
|
1460
|
-
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
1461
|
-
highestPrimaryTerm = entry.primaryTerm;
|
|
1462
|
-
}
|
|
1463
|
-
}
|
|
1464
|
-
};
|
|
1465
|
-
await replay(durableEntries);
|
|
1466
|
-
if (activeTail !== null) {
|
|
1467
|
-
await replay(activeTail.entries);
|
|
1468
|
-
}
|
|
1469
|
-
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
1470
|
-
if (activeTail !== null) {
|
|
1471
|
-
await makeReplayedTailDurable(directory, indexName, partitionId, marker, activeTail);
|
|
1472
|
-
}
|
|
1473
|
-
return { highestSeqNo, highestPrimaryTerm };
|
|
1474
|
-
}
|
|
1475
|
-
async function makeReplayedTailDurable(directory, indexName, partitionId, marker, tail) {
|
|
1476
|
-
const lastReplayed = tail.entries.at(-1);
|
|
1477
|
-
if (lastReplayed === void 0 && tail.cleanEnd >= tail.segmentLength) return;
|
|
1478
|
-
const handle = await directory.appendHandle(tail.key);
|
|
1479
|
-
try {
|
|
1480
|
-
if (tail.cleanEnd < await handle.size()) await handle.truncate(tail.cleanEnd);
|
|
1481
|
-
else await handle.sync();
|
|
1482
|
-
} finally {
|
|
1483
|
-
await handle.close();
|
|
1484
|
-
}
|
|
1485
|
-
if (lastReplayed === void 0) return;
|
|
1486
|
-
const writer = await createMarkerWriter(directory, indexName, partitionId);
|
|
1487
|
-
try {
|
|
1488
|
-
await writer.commit({
|
|
1489
|
-
activeSegmentSeqNo: marker.state.activeSegmentSeqNo,
|
|
1490
|
-
durableByteLength: tail.cleanEnd,
|
|
1491
|
-
highestDurableSeqNo: Math.max(marker.state.highestDurableSeqNo, lastReplayed.seqNo)
|
|
1492
|
-
});
|
|
1493
|
-
} finally {
|
|
1494
|
-
await writer.close();
|
|
1495
|
-
}
|
|
1496
|
-
}
|
|
1497
|
-
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
1498
|
-
for (const { key, startSeqNo } of segments) {
|
|
1499
|
-
if (startSeqNo > activeSegmentSeqNo) {
|
|
1500
|
-
await directory.remove(key);
|
|
1501
|
-
}
|
|
1502
|
-
}
|
|
1503
|
-
}
|
|
1504
|
-
function applyEntry(entry, deps) {
|
|
1505
|
-
deps.writeRecords?.replayed(entry);
|
|
1506
|
-
if (entry.operation === "DELETE") {
|
|
1507
|
-
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
1508
|
-
return;
|
|
1509
|
-
}
|
|
1510
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes, deps.defersGraphBuilds !== true);
|
|
1511
|
-
}
|
|
1512
|
-
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
1513
|
-
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
// src/persistence/durability/checkpoint-capture.ts
|
|
1517
|
-
function whileNoMutationApplies(partitions, capture2) {
|
|
1518
|
-
return new Promise((resolve, reject) => {
|
|
1519
|
-
let release = () => void 0;
|
|
1520
|
-
const released = new Promise((settle) => {
|
|
1521
|
-
release = settle;
|
|
1522
|
-
});
|
|
1523
|
-
function captureNow() {
|
|
1524
|
-
try {
|
|
1525
|
-
resolve(capture2());
|
|
1526
|
-
} catch (error) {
|
|
1527
|
-
reject(error);
|
|
1528
|
-
} finally {
|
|
1129
|
+
reject(error);
|
|
1130
|
+
} finally {
|
|
1529
1131
|
release();
|
|
1530
1132
|
}
|
|
1531
1133
|
}
|
|
@@ -1983,7 +1585,7 @@ async function writeOneBoundedCheckpoint(input) {
|
|
|
1983
1585
|
}
|
|
1984
1586
|
async function adoptWrittenSegments(input, manager, partitions, changes) {
|
|
1985
1587
|
const { indexName, indexState, hooks } = input;
|
|
1986
|
-
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-
|
|
1588
|
+
const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-ZL5ZG6YB.mjs');
|
|
1987
1589
|
const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
|
|
1988
1590
|
if (adoption === null) return [];
|
|
1989
1591
|
const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
@@ -2038,10 +1640,11 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
2038
1640
|
const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
|
|
2039
1641
|
if (!input.holdsCheckpointFiles()) {
|
|
2040
1642
|
const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
|
|
2041
|
-
const stillRead = metadata.textStorage === "disk" ? (await import('./module-
|
|
1643
|
+
const stillRead = metadata.textStorage === "disk" ? (await import('./module-ZL5ZG6YB.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
|
|
1644
|
+
const keptForSync = await input.keepsCheckpointKey();
|
|
2042
1645
|
await removeCheckpointGarbage(
|
|
2043
1646
|
store,
|
|
2044
|
-
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
|
|
1647
|
+
removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key) && !keptForSync(key))
|
|
2045
1648
|
);
|
|
2046
1649
|
}
|
|
2047
1650
|
const checkpointDocumentCount = documentCount;
|
|
@@ -2054,7 +1657,9 @@ async function finishBoundedCheckpoint(input, manager, store, metadata, priorMan
|
|
|
2054
1657
|
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
2055
1658
|
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
|
|
2056
1659
|
});
|
|
2057
|
-
|
|
1660
|
+
const movedIntoHistoryAtMs = input.hooks.keepsSyncHistory?.(input.indexName) === true ? Date.now() : null;
|
|
1661
|
+
await truncateCoveredSegments(input.directory, input.indexName, targets, movedIntoHistoryAtMs);
|
|
1662
|
+
if (movedIntoHistoryAtMs !== null) input.hooks.onHistoryMoved?.(input.indexName);
|
|
2058
1663
|
await reclaimWalBeyondCount(
|
|
2059
1664
|
input.directory,
|
|
2060
1665
|
input.indexName,
|
|
@@ -2086,202 +1691,1014 @@ function createIndexLoads(vectorIndexesOf2) {
|
|
|
2086
1691
|
}
|
|
2087
1692
|
};
|
|
2088
1693
|
return {
|
|
2089
|
-
track(indexName, load, startsGraphRepairs = true) {
|
|
2090
|
-
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
2091
|
-
loads.set(
|
|
2092
|
-
indexName,
|
|
2093
|
-
loaded.catch(() => void 0)
|
|
2094
|
-
);
|
|
2095
|
-
return loaded;
|
|
2096
|
-
},
|
|
2097
|
-
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
2098
|
-
startGraphRepairs
|
|
1694
|
+
track(indexName, load, startsGraphRepairs = true) {
|
|
1695
|
+
const loaded = startsGraphRepairs ? load.then(() => startGraphRepairs(indexName)) : load;
|
|
1696
|
+
loads.set(
|
|
1697
|
+
indexName,
|
|
1698
|
+
loaded.catch(() => void 0)
|
|
1699
|
+
);
|
|
1700
|
+
return loaded;
|
|
1701
|
+
},
|
|
1702
|
+
settled: (indexName) => loads.get(indexName) ?? Promise.resolve(),
|
|
1703
|
+
startGraphRepairs
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
async function switchInTurn(turns, turn) {
|
|
1707
|
+
let swapped = false;
|
|
1708
|
+
await checkpointInTurn(turns, {
|
|
1709
|
+
joinsInFlight: false,
|
|
1710
|
+
mayStart: turn.mayStart,
|
|
1711
|
+
perform: async () => {
|
|
1712
|
+
swapped = turn.swap();
|
|
1713
|
+
if (swapped) await turn.checkpoint();
|
|
1714
|
+
},
|
|
1715
|
+
afterSettling: turn.afterSettling
|
|
1716
|
+
});
|
|
1717
|
+
return swapped;
|
|
1718
|
+
}
|
|
1719
|
+
async function checkpointInTurn(turns, turn) {
|
|
1720
|
+
if (turn.joinsInFlight && turns.checkpointInFlight !== null) return turns.checkpointInFlight;
|
|
1721
|
+
while (turns.checkpointInFlight !== null) await turns.checkpointInFlight.catch(() => void 0);
|
|
1722
|
+
if (!turn.mayStart()) return;
|
|
1723
|
+
const run = turn.perform().finally(() => {
|
|
1724
|
+
turns.checkpointInFlight = null;
|
|
1725
|
+
turn.afterSettling();
|
|
1726
|
+
});
|
|
1727
|
+
turns.checkpointInFlight = run;
|
|
1728
|
+
return run;
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
// src/persistence/durability/held-checkpoints.ts
|
|
1732
|
+
function failedPartitionIds(indexState) {
|
|
1733
|
+
const failed = [];
|
|
1734
|
+
for (const [partitionId, partition] of indexState.partitions) if (partition.failed !== null) failed.push(partitionId);
|
|
1735
|
+
return failed;
|
|
1736
|
+
}
|
|
1737
|
+
function createHeldCheckpoints(hooks) {
|
|
1738
|
+
const reported = /* @__PURE__ */ new WeakSet();
|
|
1739
|
+
function heldReport(indexName, indexState) {
|
|
1740
|
+
const partitionIds = failedPartitionIds(indexState);
|
|
1741
|
+
if (partitionIds.length === 0 || hooks.getVectorIndexes(indexName).size === 0) return null;
|
|
1742
|
+
return {
|
|
1743
|
+
indexName,
|
|
1744
|
+
partitionIds,
|
|
1745
|
+
recordsSinceCheckpoint: indexState.mutationsSinceCheckpoint,
|
|
1746
|
+
documentBytesSinceCheckpoint: indexState.documentBytesSinceCheckpoint
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
return {
|
|
1750
|
+
refuse(indexName, indexState) {
|
|
1751
|
+
const partitionIds = indexState === void 0 ? [] : failedPartitionIds(indexState);
|
|
1752
|
+
if (partitionIds.length === 0) return;
|
|
1753
|
+
throw new NarsilError(
|
|
1754
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1755
|
+
`Index "${indexName}" cannot take a checkpoint that covers every acknowledged write, because partitions ${partitionIds.join(", ")} failed and have not recovered`,
|
|
1756
|
+
{ indexName, partitionIds }
|
|
1757
|
+
);
|
|
1758
|
+
},
|
|
1759
|
+
holds(indexName, indexState) {
|
|
1760
|
+
const held = heldReport(indexName, indexState);
|
|
1761
|
+
if (held === null) return false;
|
|
1762
|
+
const unreported = held.partitionIds.some((partitionId) => {
|
|
1763
|
+
const partition = indexState.partitions.get(partitionId);
|
|
1764
|
+
return partition !== void 0 && !reported.has(partition);
|
|
1765
|
+
});
|
|
1766
|
+
if (unreported) {
|
|
1767
|
+
for (const partitionId of held.partitionIds) {
|
|
1768
|
+
const partition = indexState.partitions.get(partitionId);
|
|
1769
|
+
if (partition !== void 0) reported.add(partition);
|
|
1770
|
+
}
|
|
1771
|
+
hooks.onCheckpointHeld?.(held);
|
|
1772
|
+
}
|
|
1773
|
+
return true;
|
|
1774
|
+
},
|
|
1775
|
+
report(indexes) {
|
|
1776
|
+
const held = [];
|
|
1777
|
+
for (const [indexName, indexState] of indexes) {
|
|
1778
|
+
const entry = heldReport(indexName, indexState);
|
|
1779
|
+
if (entry !== null) held.push(entry);
|
|
1780
|
+
}
|
|
1781
|
+
return held;
|
|
1782
|
+
}
|
|
1783
|
+
};
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
// src/persistence/durability/manager-state.ts
|
|
1787
|
+
function closePartitionWriter(partition) {
|
|
1788
|
+
return partition.failed === null ? partition.walWriter.close() : partition.walWriter.abandon();
|
|
1789
|
+
}
|
|
1790
|
+
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
1791
|
+
indexState.unloading = true;
|
|
1792
|
+
try {
|
|
1793
|
+
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
1794
|
+
await pendingMetadataWrite;
|
|
1795
|
+
for (const partition of indexState.partitions.values()) {
|
|
1796
|
+
await closeWalWriter(closePartitionWriter(partition));
|
|
1797
|
+
}
|
|
1798
|
+
} catch (error) {
|
|
1799
|
+
indexState.unloading = false;
|
|
1800
|
+
throw error;
|
|
1801
|
+
}
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
// src/persistence/durability/restore-journal.ts
|
|
1805
|
+
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
1806
|
+
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
1807
|
+
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
1808
|
+
var LOCK_FILE_KEY = ".narsil.lock";
|
|
1809
|
+
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
1810
|
+
function digestOf(indexName) {
|
|
1811
|
+
let hex = "";
|
|
1812
|
+
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
1813
|
+
return hex;
|
|
1814
|
+
}
|
|
1815
|
+
function journalKey(indexName) {
|
|
1816
|
+
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
1817
|
+
}
|
|
1818
|
+
function randomSuffix() {
|
|
1819
|
+
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
1820
|
+
}
|
|
1821
|
+
function newStagingPrefix() {
|
|
1822
|
+
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
1823
|
+
}
|
|
1824
|
+
function newRetiredPrefix() {
|
|
1825
|
+
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
1826
|
+
}
|
|
1827
|
+
function encodeJournal(indexName, stagingPrefix) {
|
|
1828
|
+
return new TextEncoder().encode(`${indexName}
|
|
1829
|
+
${stagingPrefix}`);
|
|
1830
|
+
}
|
|
1831
|
+
function isStagingPrefix(prefix) {
|
|
1832
|
+
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
1833
|
+
}
|
|
1834
|
+
function corruptJournal(key, reason) {
|
|
1835
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
1836
|
+
}
|
|
1837
|
+
function readJournal(key, bytes) {
|
|
1838
|
+
const text = new TextDecoder().decode(bytes);
|
|
1839
|
+
const lineEnd = text.indexOf("\n");
|
|
1840
|
+
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
1841
|
+
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
1842
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
1843
|
+
corruptJournal(key, "names an index that no index name can match");
|
|
1844
|
+
}
|
|
1845
|
+
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
1846
|
+
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
1847
|
+
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
1848
|
+
}
|
|
1849
|
+
return { kind: "journal", indexName, stagingPrefix };
|
|
1850
|
+
}
|
|
1851
|
+
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
1852
|
+
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
1853
|
+
if (manifestBytes === null) {
|
|
1854
|
+
throw new NarsilError(
|
|
1855
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1856
|
+
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
1857
|
+
{ indexName, stagingPrefix }
|
|
1858
|
+
);
|
|
1859
|
+
}
|
|
1860
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
1861
|
+
if (decoded.kind !== "current") {
|
|
1862
|
+
throw new NarsilError(
|
|
1863
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1864
|
+
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
1865
|
+
{ indexName }
|
|
1866
|
+
);
|
|
1867
|
+
}
|
|
1868
|
+
const root = `${indexName}/`;
|
|
1869
|
+
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
1870
|
+
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
1871
|
+
return keys;
|
|
1872
|
+
}
|
|
1873
|
+
function withoutTrailingSlash(prefix) {
|
|
1874
|
+
return prefix.slice(0, -1);
|
|
1875
|
+
}
|
|
1876
|
+
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
1877
|
+
const root = `${indexName}/`;
|
|
1878
|
+
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
1879
|
+
const staged = await storage.list(stagingPrefix);
|
|
1880
|
+
let retiredPrefix = null;
|
|
1881
|
+
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
1882
|
+
retiredPrefix = newRetiredPrefix();
|
|
1883
|
+
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
1884
|
+
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
1885
|
+
} else {
|
|
1886
|
+
const stagedKeys = new Set(staged);
|
|
1887
|
+
for (const key of listed) {
|
|
1888
|
+
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
for (const key of await storage.list(root)) {
|
|
1892
|
+
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
1893
|
+
}
|
|
1894
|
+
await storage.remove(journalKey(indexName));
|
|
1895
|
+
return retiredPrefix;
|
|
1896
|
+
}
|
|
1897
|
+
async function removePrefix(storage, prefix) {
|
|
1898
|
+
if (storage.removeTree !== void 0) {
|
|
1899
|
+
await storage.removeTree(prefix);
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
1903
|
+
}
|
|
1904
|
+
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
1905
|
+
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
1906
|
+
}
|
|
1907
|
+
function isTopLevelJournal(key) {
|
|
1908
|
+
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
1909
|
+
}
|
|
1910
|
+
async function removeStagingPrefixes(storage) {
|
|
1911
|
+
const prefixes = /* @__PURE__ */ new Set();
|
|
1912
|
+
for (const key of await storage.list(".")) {
|
|
1913
|
+
const slash = key.indexOf("/");
|
|
1914
|
+
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
1915
|
+
}
|
|
1916
|
+
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
1917
|
+
}
|
|
1918
|
+
async function recoverRestoreJournals(storage) {
|
|
1919
|
+
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
1920
|
+
if (!isTopLevelJournal(key)) continue;
|
|
1921
|
+
const bytes = await storage.read(key);
|
|
1922
|
+
if (bytes === null) continue;
|
|
1923
|
+
const reading = readJournal(key, bytes);
|
|
1924
|
+
if (reading.kind === "leftover") {
|
|
1925
|
+
await storage.remove(key);
|
|
1926
|
+
continue;
|
|
1927
|
+
}
|
|
1928
|
+
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
1929
|
+
}
|
|
1930
|
+
await removeStagingPrefixes(storage);
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
// src/persistence/durability/restore-directory.ts
|
|
1934
|
+
function missing(error) {
|
|
1935
|
+
return error.code === "ENOENT";
|
|
1936
|
+
}
|
|
1937
|
+
function storageFailed(code, message, key, error) {
|
|
1938
|
+
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
1939
|
+
}
|
|
1940
|
+
function directoryRestoreStorage(directory) {
|
|
1941
|
+
return {
|
|
1942
|
+
directory,
|
|
1943
|
+
read: (key) => directory.read(key),
|
|
1944
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
1945
|
+
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
1946
|
+
remove: (key) => directory.remove(key),
|
|
1947
|
+
list: (prefix) => directory.list(prefix),
|
|
1948
|
+
moveKey: (from, to) => directory.move(from, to),
|
|
1949
|
+
pathOf: (key) => directory.pathOf(key),
|
|
1950
|
+
async moveTree(from, to) {
|
|
1951
|
+
const source = await directory.pathOf(from);
|
|
1952
|
+
const target = await directory.pathOf(to);
|
|
1953
|
+
const fs = await getFs();
|
|
1954
|
+
try {
|
|
1955
|
+
await fs.rename(source, target);
|
|
1956
|
+
} catch (error) {
|
|
1957
|
+
if (missing(error)) return;
|
|
1958
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
1959
|
+
}
|
|
1960
|
+
await directory.syncDirectoryOf(to);
|
|
1961
|
+
},
|
|
1962
|
+
async removeTree(prefix) {
|
|
1963
|
+
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
1964
|
+
const fs = await getFs();
|
|
1965
|
+
try {
|
|
1966
|
+
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
1967
|
+
} catch (error) {
|
|
1968
|
+
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
1969
|
+
}
|
|
1970
|
+
await directory.syncDirectoryOf(key);
|
|
1971
|
+
}
|
|
1972
|
+
};
|
|
1973
|
+
}
|
|
1974
|
+
function recoverDirectoryRestores(directory) {
|
|
1975
|
+
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// src/persistence/durability/restore-staging.ts
|
|
1979
|
+
function toError2(error) {
|
|
1980
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1981
|
+
}
|
|
1982
|
+
function pathUnder(path, root) {
|
|
1983
|
+
if (!path.startsWith(root)) return false;
|
|
1984
|
+
const next = path.charAt(root.length);
|
|
1985
|
+
return next === "" || next === "/" || next === "\\";
|
|
1986
|
+
}
|
|
1987
|
+
function stagedDirectory(directory, staged, unstaged) {
|
|
1988
|
+
const { readRange, byteLength } = directory;
|
|
1989
|
+
return {
|
|
1990
|
+
root: directory.root,
|
|
1991
|
+
appendHandle: (key) => directory.appendHandle(staged(key)),
|
|
1992
|
+
markerHandle: (key) => directory.markerHandle(staged(key)),
|
|
1993
|
+
atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
|
|
1994
|
+
syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
|
|
1995
|
+
read: (key) => directory.read(staged(key)),
|
|
1996
|
+
remove: (key) => directory.remove(staged(key)),
|
|
1997
|
+
move: (from, to) => directory.move(staged(from), staged(to)),
|
|
1998
|
+
list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
|
|
1999
|
+
pathOf: (key) => directory.pathOf(staged(key)),
|
|
2000
|
+
...readRange === void 0 ? {} : {
|
|
2001
|
+
readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
|
|
2002
|
+
},
|
|
2003
|
+
...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
|
|
2004
|
+
};
|
|
2005
|
+
}
|
|
2006
|
+
function createRestoreStaging(storage, indexName, options = {}) {
|
|
2007
|
+
const stagingPrefix = newStagingPrefix();
|
|
2008
|
+
const indexRoot = `${indexName}/`;
|
|
2009
|
+
let committed = false;
|
|
2010
|
+
const staged = (key) => {
|
|
2011
|
+
if (!key.startsWith(indexRoot)) {
|
|
2012
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
2013
|
+
key
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
2017
|
+
};
|
|
2018
|
+
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
2019
|
+
const store = {
|
|
2020
|
+
read: (key) => storage.read(staged(key)),
|
|
2021
|
+
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
2022
|
+
remove: (key) => storage.remove(staged(key)),
|
|
2023
|
+
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
2024
|
+
};
|
|
2025
|
+
const pathOf = storage.pathOf;
|
|
2026
|
+
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
2027
|
+
if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
|
|
2028
|
+
let retiredPrefix = null;
|
|
2029
|
+
async function finishOnStorage() {
|
|
2030
|
+
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
2031
|
+
}
|
|
2032
|
+
return {
|
|
2033
|
+
store,
|
|
2034
|
+
async relocation() {
|
|
2035
|
+
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
2036
|
+
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
2037
|
+
const to = await pathOf.call(storage, indexName);
|
|
2038
|
+
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
2039
|
+
},
|
|
2040
|
+
async commit() {
|
|
2041
|
+
try {
|
|
2042
|
+
await commitJournal(storage, indexName, stagingPrefix);
|
|
2043
|
+
} catch (error) {
|
|
2044
|
+
try {
|
|
2045
|
+
await storage.remove(journalKey(indexName));
|
|
2046
|
+
} catch (removal) {
|
|
2047
|
+
committed = true;
|
|
2048
|
+
options.onStorageLost?.(toError2(removal));
|
|
2049
|
+
}
|
|
2050
|
+
throw error;
|
|
2051
|
+
}
|
|
2052
|
+
committed = true;
|
|
2053
|
+
},
|
|
2054
|
+
async finish() {
|
|
2055
|
+
try {
|
|
2056
|
+
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
2057
|
+
} catch (error) {
|
|
2058
|
+
options.onStorageLost?.(toError2(error));
|
|
2059
|
+
}
|
|
2060
|
+
},
|
|
2061
|
+
async removeRetired() {
|
|
2062
|
+
const prefix = retiredPrefix;
|
|
2063
|
+
retiredPrefix = null;
|
|
2064
|
+
if (prefix !== null) await removePrefix(storage, prefix);
|
|
2065
|
+
},
|
|
2066
|
+
async discard() {
|
|
2067
|
+
if (committed) return;
|
|
2068
|
+
await removePrefix(storage, stagingPrefix);
|
|
2069
|
+
}
|
|
2070
|
+
};
|
|
2071
|
+
}
|
|
2072
|
+
function adapterRestoreStorage(adapter) {
|
|
2073
|
+
const store = adapterCheckpointStore(adapter);
|
|
2074
|
+
return {
|
|
2075
|
+
read: (key) => store.read(key),
|
|
2076
|
+
write: (key, chunks) => store.write(key, chunks),
|
|
2077
|
+
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
2078
|
+
remove: (key) => store.remove(key),
|
|
2079
|
+
list: (prefix) => store.list(prefix),
|
|
2080
|
+
async moveKey(from, to) {
|
|
2081
|
+
const bytes = await store.read(from);
|
|
2082
|
+
if (bytes === null) return;
|
|
2083
|
+
await store.write(to, [bytes]);
|
|
2084
|
+
await store.remove(from);
|
|
2085
|
+
}
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
// src/persistence/durability/archive-checkpoint.ts
|
|
2090
|
+
function toError3(error) {
|
|
2091
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
2092
|
+
}
|
|
2093
|
+
function stageDirectoryRestore(deps, indexName) {
|
|
2094
|
+
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
2095
|
+
async finish(run) {
|
|
2096
|
+
try {
|
|
2097
|
+
await run();
|
|
2098
|
+
} catch (error) {
|
|
2099
|
+
deps.markFatal(toError3(error));
|
|
2100
|
+
throw error;
|
|
2101
|
+
}
|
|
2102
|
+
},
|
|
2103
|
+
onStorageLost: deps.markFatal
|
|
2104
|
+
});
|
|
2105
|
+
}
|
|
2106
|
+
function releaser(held, indexName) {
|
|
2107
|
+
let released = false;
|
|
2108
|
+
return () => {
|
|
2109
|
+
if (released) return;
|
|
2110
|
+
released = true;
|
|
2111
|
+
const count = (held.get(indexName) ?? 1) - 1;
|
|
2112
|
+
if (count <= 0) held.delete(indexName);
|
|
2113
|
+
else held.set(indexName, count);
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
function createArchiveCheckpointAccess(deps) {
|
|
2117
|
+
const store = directoryCheckpointStore(deps.directory);
|
|
2118
|
+
return {
|
|
2119
|
+
async openArchiveCheckpoint(indexName) {
|
|
2120
|
+
deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
|
|
2121
|
+
const release = releaser(deps.held, indexName);
|
|
2122
|
+
try {
|
|
2123
|
+
await deps.checkpoint(indexName, false);
|
|
2124
|
+
await deps.checkpoint(indexName, true);
|
|
2125
|
+
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
2126
|
+
if (manifestBytes === null) {
|
|
2127
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
2128
|
+
indexName
|
|
2129
|
+
});
|
|
2130
|
+
}
|
|
2131
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
2132
|
+
if (decoded.kind !== "current") {
|
|
2133
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
2134
|
+
indexName
|
|
2135
|
+
});
|
|
2136
|
+
}
|
|
2137
|
+
return {
|
|
2138
|
+
manifestBytes,
|
|
2139
|
+
manifest: decoded.manifest,
|
|
2140
|
+
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
2141
|
+
readRange: async (ref, offset, length) => {
|
|
2142
|
+
const readRange = deps.directory.readRange;
|
|
2143
|
+
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
2144
|
+
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
2145
|
+
return whole.slice(offset, offset + length);
|
|
2146
|
+
},
|
|
2147
|
+
release,
|
|
2148
|
+
directory: deps.directory
|
|
2149
|
+
};
|
|
2150
|
+
} catch (error) {
|
|
2151
|
+
release();
|
|
2152
|
+
throw error;
|
|
2153
|
+
}
|
|
2154
|
+
},
|
|
2155
|
+
stageRestore(indexName) {
|
|
2156
|
+
return stageDirectoryRestore(deps, indexName);
|
|
2157
|
+
},
|
|
2158
|
+
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
2159
|
+
for (const { partitionId, lastSeqNo, primaryTerm } of checkpoint) {
|
|
2160
|
+
deps.registerPartition(indexName, partitionId, lastSeqNo, primaryTerm);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
};
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
// src/persistence/durability/query-history.ts
|
|
2167
|
+
async function removeQueryHistoryOf(storage, indexName) {
|
|
2168
|
+
const folder = `${QUERY_HISTORY_FOLDER}/${indexName}/`;
|
|
2169
|
+
if (storage.kind === "directory") {
|
|
2170
|
+
for (const key of await storage.directory.list(folder)) await storage.directory.remove(key);
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
for (const key of await storage.adapter.list(folder)) await storage.adapter.delete(key);
|
|
2174
|
+
}
|
|
2175
|
+
function queryHistoryAccess(storage) {
|
|
2176
|
+
return {
|
|
2177
|
+
historyStorage: () => storage,
|
|
2178
|
+
removeQueryHistory: (indexName) => removeQueryHistoryOf(storage, indexName)
|
|
2099
2179
|
};
|
|
2100
2180
|
}
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2181
|
+
|
|
2182
|
+
// src/persistence/durability/manager-unload.ts
|
|
2183
|
+
function createIndexUnloading(deps) {
|
|
2184
|
+
const { directory, hooks, indexes, metadataWrites } = deps;
|
|
2185
|
+
return {
|
|
2186
|
+
async removeIndex(indexName) {
|
|
2187
|
+
hooks.writeRecords?.dropIndex(indexName);
|
|
2188
|
+
const indexState = indexes.get(indexName);
|
|
2189
|
+
if (indexState !== void 0) {
|
|
2190
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), deps.closeWriterReportingFailure);
|
|
2191
|
+
indexes.delete(indexName);
|
|
2192
|
+
metadataWrites.forget(indexName);
|
|
2193
|
+
}
|
|
2194
|
+
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
2195
|
+
for (const key of await directory.list(`${indexName}/`)) {
|
|
2196
|
+
await directory.remove(key);
|
|
2197
|
+
}
|
|
2198
|
+
await removeQueryHistoryOf({ kind: "directory", directory }, indexName);
|
|
2199
|
+
deps.forgetSyncState(indexName);
|
|
2109
2200
|
},
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
});
|
|
2122
|
-
turns.checkpointInFlight = run;
|
|
2123
|
-
return run;
|
|
2201
|
+
async unloadIndex(indexName) {
|
|
2202
|
+
const indexState = indexes.get(indexName);
|
|
2203
|
+
if (indexState === void 0) {
|
|
2204
|
+
return;
|
|
2205
|
+
}
|
|
2206
|
+
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
2207
|
+
indexes.delete(indexName);
|
|
2208
|
+
metadataWrites.forget(indexName);
|
|
2209
|
+
deps.forgetSyncState(indexName);
|
|
2210
|
+
}
|
|
2211
|
+
};
|
|
2124
2212
|
}
|
|
2125
2213
|
|
|
2126
|
-
// src/persistence/durability/
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
}
|
|
2132
|
-
function
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2214
|
+
// src/persistence/durability/sync/frame-cursor.ts
|
|
2215
|
+
var SegmentFileMissing = class extends Error {
|
|
2216
|
+
};
|
|
2217
|
+
function corrupt(key, message, details) {
|
|
2218
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_WAL_CORRUPT, `${message} in "${key}"`, { key, ...details });
|
|
2219
|
+
}
|
|
2220
|
+
function isMissingFile(error) {
|
|
2221
|
+
return error?.code === "ENOENT";
|
|
2222
|
+
}
|
|
2223
|
+
var FrameCursor = class {
|
|
2224
|
+
constructor(directory, key, limit) {
|
|
2225
|
+
this.directory = directory;
|
|
2226
|
+
this.key = key;
|
|
2227
|
+
this.limit = limit;
|
|
2228
|
+
}
|
|
2229
|
+
buffer = new Uint8Array(0);
|
|
2230
|
+
bufferStart = 0;
|
|
2231
|
+
whole = null;
|
|
2232
|
+
async load(offset, length) {
|
|
2233
|
+
const readRange = this.directory.readRange;
|
|
2234
|
+
if (readRange === void 0) {
|
|
2235
|
+
if (this.whole === null) {
|
|
2236
|
+
const bytes = await this.directory.read(this.key);
|
|
2237
|
+
if (bytes === null) throw new SegmentFileMissing(this.key);
|
|
2238
|
+
this.whole = bytes;
|
|
2239
|
+
}
|
|
2240
|
+
this.buffer = this.whole;
|
|
2241
|
+
this.bufferStart = 0;
|
|
2242
|
+
return;
|
|
2243
|
+
}
|
|
2244
|
+
const wanted = Math.min(Math.max(length, LOG_READ_CHUNK_BYTES), this.limit - offset);
|
|
2245
|
+
try {
|
|
2246
|
+
this.buffer = await readRange.call(this.directory, this.key, offset, wanted);
|
|
2247
|
+
} catch (error) {
|
|
2248
|
+
if (isMissingFile(error)) throw new SegmentFileMissing(this.key);
|
|
2249
|
+
throw error;
|
|
2250
|
+
}
|
|
2251
|
+
this.bufferStart = offset;
|
|
2252
|
+
}
|
|
2253
|
+
async bytesAt(offset, length) {
|
|
2254
|
+
const end = offset + length;
|
|
2255
|
+
if (end > this.limit) corrupt(this.key, "A log frame overruns the readable region", { offset, length });
|
|
2256
|
+
if (offset < this.bufferStart || end > this.bufferStart + this.buffer.byteLength) await this.load(offset, length);
|
|
2257
|
+
return this.buffer.subarray(offset - this.bufferStart, end - this.bufferStart);
|
|
2258
|
+
}
|
|
2259
|
+
async checkHeader() {
|
|
2260
|
+
if (this.limit < SEGMENT_HEADER_SIZE) corrupt(this.key, "A log segment is shorter than its header", {});
|
|
2261
|
+
const header = checkSegmentHeader(await this.bytesAt(0, SEGMENT_HEADER_SIZE));
|
|
2262
|
+
if (!header.ok) corrupt(this.key, `A log segment header is invalid: ${header.reason}`, {});
|
|
2142
2263
|
}
|
|
2143
|
-
async
|
|
2264
|
+
async frameLengthAt(offset) {
|
|
2265
|
+
const lengthBytes = await this.bytesAt(offset, RECORD_LENGTH_SIZE);
|
|
2266
|
+
const recordLength = new DataView(lengthBytes.buffer, lengthBytes.byteOffset, RECORD_LENGTH_SIZE).getUint32(
|
|
2267
|
+
0,
|
|
2268
|
+
false
|
|
2269
|
+
);
|
|
2270
|
+
if (recordLength === 0 || recordLength > MAX_WAL_RECORD_BYTES) {
|
|
2271
|
+
corrupt(this.key, "A log record length is zero or above the largest record", { offset, recordLength });
|
|
2272
|
+
}
|
|
2273
|
+
return RECORD_LENGTH_SIZE + recordLength + FRAME_CRC_SIZE;
|
|
2274
|
+
}
|
|
2275
|
+
async frameAt(offset, frameBytes) {
|
|
2276
|
+
const recordLength = frameBytes - RECORD_LENGTH_SIZE - FRAME_CRC_SIZE;
|
|
2277
|
+
const body = await this.bytesAt(offset + RECORD_LENGTH_SIZE, recordLength + FRAME_CRC_SIZE);
|
|
2278
|
+
const payload = body.subarray(0, recordLength);
|
|
2279
|
+
const storedCrc = new DataView(body.buffer, body.byteOffset + recordLength, FRAME_CRC_SIZE).getUint32(0, false);
|
|
2280
|
+
if (crc32(payload) !== storedCrc) corrupt(this.key, "A log record fails its frame checksum", { offset });
|
|
2281
|
+
let seqNo;
|
|
2144
2282
|
try {
|
|
2145
|
-
|
|
2146
|
-
} catch (
|
|
2147
|
-
|
|
2283
|
+
seqNo = decodeWalEntry(payload).seqNo;
|
|
2284
|
+
} catch (error) {
|
|
2285
|
+
corrupt(this.key, "A log record fails to decode", {
|
|
2286
|
+
offset,
|
|
2287
|
+
cause: error instanceof Error ? error.message : String(error)
|
|
2288
|
+
});
|
|
2148
2289
|
}
|
|
2290
|
+
return { seqNo, payload: payload.slice(), frameBytes };
|
|
2149
2291
|
}
|
|
2150
|
-
|
|
2151
|
-
|
|
2292
|
+
};
|
|
2293
|
+
|
|
2294
|
+
// src/persistence/durability/sync/log-reader.ts
|
|
2295
|
+
async function walSources(directory, indexName, partitionId) {
|
|
2296
|
+
const prefix = walSegmentPrefix(indexName, partitionId);
|
|
2297
|
+
const markerBytes = await directory.read(`${prefix}commit`);
|
|
2298
|
+
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
2299
|
+
if (marker === null) return { sources: [], head: 0 };
|
|
2300
|
+
const { activeSegmentSeqNo, durableByteLength, highestDurableSeqNo } = marker.state;
|
|
2301
|
+
const sources = [];
|
|
2302
|
+
for (const key of await directory.list(prefix)) {
|
|
2303
|
+
const startSeqNo = parseWalSegmentStart(indexName, partitionId, key);
|
|
2304
|
+
if (startSeqNo === null || startSeqNo > activeSegmentSeqNo) continue;
|
|
2305
|
+
const readableBytes = startSeqNo === activeSegmentSeqNo ? durableByteLength : await directory.byteLength?.(key);
|
|
2306
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
2307
|
+
sources.push({ key, startSeqNo, readableBytes, inHistory: false });
|
|
2308
|
+
}
|
|
2309
|
+
return { sources, head: highestDurableSeqNo };
|
|
2310
|
+
}
|
|
2311
|
+
async function openPartitionLog(directory, indexName, partitionId) {
|
|
2312
|
+
const { sources, head } = await walSources(directory, indexName, partitionId);
|
|
2313
|
+
const byStart = /* @__PURE__ */ new Map();
|
|
2314
|
+
for (const segment of await listHistorySegments(directory, indexName, partitionId)) {
|
|
2315
|
+
const readableBytes = await directory.byteLength?.(segment.key);
|
|
2316
|
+
if (readableBytes === null || readableBytes === void 0) continue;
|
|
2317
|
+
byStart.set(segment.startSeqNo, {
|
|
2318
|
+
key: segment.key,
|
|
2319
|
+
startSeqNo: segment.startSeqNo,
|
|
2320
|
+
readableBytes,
|
|
2321
|
+
inHistory: true
|
|
2322
|
+
});
|
|
2323
|
+
}
|
|
2324
|
+
for (const source of sources) if (!byStart.has(source.startSeqNo)) byStart.set(source.startSeqNo, source);
|
|
2325
|
+
const ordered = [...byStart.values()].sort((a, b) => a.startSeqNo - b.startSeqNo);
|
|
2326
|
+
return { sources: ordered, head, oldestSeqNo: ordered.length > 0 ? ordered[0].startSeqNo : head + 1 };
|
|
2327
|
+
}
|
|
2328
|
+
function sourceHolding(log, seqNo) {
|
|
2329
|
+
let found = -1;
|
|
2330
|
+
for (let i = 0; i < log.sources.length; i++) {
|
|
2331
|
+
if (log.sources[i].startSeqNo > seqNo) break;
|
|
2332
|
+
found = i;
|
|
2333
|
+
}
|
|
2334
|
+
return found;
|
|
2335
|
+
}
|
|
2336
|
+
async function offsetAfter(cursor, source, afterSeqNo) {
|
|
2337
|
+
let offset = SEGMENT_HEADER_SIZE;
|
|
2338
|
+
for (let skip = afterSeqNo + 1 - source.startSeqNo; skip > 0 && offset < cursor.limit; skip--) {
|
|
2339
|
+
offset += await cursor.frameLengthAt(offset);
|
|
2340
|
+
}
|
|
2341
|
+
return offset;
|
|
2342
|
+
}
|
|
2343
|
+
async function readSource(directory, source, afterSeqNo, upToSeqNo, maxBytes, read) {
|
|
2344
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
2345
|
+
await cursor.checkHeader();
|
|
2346
|
+
let offset = await offsetAfter(cursor, source, afterSeqNo);
|
|
2347
|
+
let checksNumbering = true;
|
|
2348
|
+
while (offset < cursor.limit) {
|
|
2349
|
+
const frame = await cursor.frameAt(offset, await cursor.frameLengthAt(offset));
|
|
2350
|
+
if (checksNumbering && frame.seqNo !== Math.max(afterSeqNo + 1, source.startSeqNo)) {
|
|
2351
|
+
checksNumbering = false;
|
|
2352
|
+
if (frame.seqNo > afterSeqNo) {
|
|
2353
|
+
offset = SEGMENT_HEADER_SIZE;
|
|
2354
|
+
continue;
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
checksNumbering = false;
|
|
2358
|
+
offset += frame.frameBytes;
|
|
2359
|
+
if (frame.seqNo <= afterSeqNo) continue;
|
|
2360
|
+
if (frame.seqNo > upToSeqNo) return true;
|
|
2361
|
+
if (read.frames.length > 0 && read.bytes + frame.payload.byteLength > maxBytes) return true;
|
|
2362
|
+
read.frames.push(frame);
|
|
2363
|
+
read.bytes += frame.payload.byteLength;
|
|
2364
|
+
if (frame.seqNo === upToSeqNo) return true;
|
|
2365
|
+
}
|
|
2366
|
+
return false;
|
|
2367
|
+
}
|
|
2368
|
+
async function readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
|
|
2369
|
+
const read = { frames: [], bytes: 0, gone: false };
|
|
2370
|
+
if (afterSeqNo >= upToSeqNo) return read;
|
|
2371
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
2372
|
+
read.frames = [];
|
|
2373
|
+
read.bytes = 0;
|
|
2374
|
+
const log = await openPartitionLog(directory, indexName, partitionId);
|
|
2375
|
+
let index = sourceHolding(log, afterSeqNo + 1);
|
|
2376
|
+
if (index < 0) {
|
|
2377
|
+
read.gone = log.sources.length > 0 || afterSeqNo + 1 <= log.head;
|
|
2378
|
+
return read;
|
|
2379
|
+
}
|
|
2152
2380
|
try {
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2381
|
+
for (; index < log.sources.length; index++) {
|
|
2382
|
+
const from = Math.max(afterSeqNo, read.frames.at(-1)?.seqNo ?? afterSeqNo);
|
|
2383
|
+
if (await readSource(directory, log.sources[index], from, upToSeqNo, maxBytes, read)) break;
|
|
2384
|
+
}
|
|
2385
|
+
return read;
|
|
2386
|
+
} catch (error) {
|
|
2387
|
+
if (!(error instanceof SegmentFileMissing) || attempt > 0) throw error;
|
|
2156
2388
|
}
|
|
2157
|
-
return new NarsilError(
|
|
2158
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2159
|
-
`The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
|
|
2160
|
-
{ directory: directory.root }
|
|
2161
|
-
);
|
|
2162
2389
|
}
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
if (
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
)
|
|
2179
|
-
);
|
|
2180
|
-
return false;
|
|
2390
|
+
return read;
|
|
2391
|
+
}
|
|
2392
|
+
async function logBytesAfter(directory, log, afterSeqNo) {
|
|
2393
|
+
const first = sourceHolding(log, afterSeqNo + 1);
|
|
2394
|
+
let bytes = 0;
|
|
2395
|
+
for (let i = Math.max(first, 0); i < log.sources.length; i++) {
|
|
2396
|
+
const source = log.sources[i];
|
|
2397
|
+
if (i === first) {
|
|
2398
|
+
const cursor = new FrameCursor(directory, source.key, source.readableBytes);
|
|
2399
|
+
try {
|
|
2400
|
+
bytes += Math.max(0, source.readableBytes - await offsetAfter(cursor, source, afterSeqNo));
|
|
2401
|
+
} catch (error) {
|
|
2402
|
+
if (!(error instanceof SegmentFileMissing)) throw error;
|
|
2403
|
+
}
|
|
2404
|
+
continue;
|
|
2181
2405
|
}
|
|
2182
|
-
|
|
2183
|
-
if (takenOver === null) return true;
|
|
2184
|
-
onLost(takenOver);
|
|
2185
|
-
return false;
|
|
2406
|
+
bytes += Math.max(0, source.readableBytes - SEGMENT_HEADER_SIZE);
|
|
2186
2407
|
}
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2408
|
+
return bytes;
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
// src/persistence/durability/sync/pins.ts
|
|
2412
|
+
function createFilePins(remove) {
|
|
2413
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2414
|
+
const awaitingRemoval = /* @__PURE__ */ new Set();
|
|
2415
|
+
function unpin(key) {
|
|
2416
|
+
const count = (counts.get(key) ?? 1) - 1;
|
|
2417
|
+
if (count > 0) {
|
|
2418
|
+
counts.set(key, count);
|
|
2419
|
+
return;
|
|
2191
2420
|
}
|
|
2421
|
+
counts.delete(key);
|
|
2422
|
+
if (!awaitingRemoval.delete(key)) return;
|
|
2423
|
+
void remove(key).catch(() => void 0);
|
|
2192
2424
|
}
|
|
2193
2425
|
return {
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
async verifyOnce() {
|
|
2204
|
-
if (expected === null && directory.identity !== void 0) await verify();
|
|
2205
|
-
},
|
|
2206
|
-
start() {
|
|
2207
|
-
if (directory.identity === void 0 || timer !== null || isStopped()) return;
|
|
2208
|
-
timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
|
|
2426
|
+
pin(keys) {
|
|
2427
|
+
const pinned = [...keys];
|
|
2428
|
+
for (const key of pinned) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
2429
|
+
let released = false;
|
|
2430
|
+
return () => {
|
|
2431
|
+
if (released) return;
|
|
2432
|
+
released = true;
|
|
2433
|
+
for (const key of pinned) unpin(key);
|
|
2434
|
+
};
|
|
2209
2435
|
},
|
|
2210
|
-
|
|
2436
|
+
isPinned: (key) => counts.has(key),
|
|
2437
|
+
removeWhenUnpinned(key) {
|
|
2438
|
+
if (!counts.has(key)) return false;
|
|
2439
|
+
awaitingRemoval.add(key);
|
|
2440
|
+
return true;
|
|
2441
|
+
}
|
|
2211
2442
|
};
|
|
2212
2443
|
}
|
|
2444
|
+
var PUBLISHED_SNAPSHOT_ENVELOPE_VERSION = 1;
|
|
2445
|
+
var PUBLISHED_SNAPSHOT_WHAT = "published snapshot";
|
|
2446
|
+
function hexOf(bytes) {
|
|
2447
|
+
let hex = "";
|
|
2448
|
+
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
2449
|
+
return hex;
|
|
2450
|
+
}
|
|
2451
|
+
function isBytes(value) {
|
|
2452
|
+
return value instanceof Uint8Array;
|
|
2453
|
+
}
|
|
2454
|
+
async function publishedFrom(indexName, payload, config, manifestBytes) {
|
|
2455
|
+
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
2456
|
+
if (decoded.kind !== "current") {
|
|
2457
|
+
throw new NarsilError(
|
|
2458
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2459
|
+
`The published snapshot of index "${indexName}" holds a manifest of an earlier layout`,
|
|
2460
|
+
{ indexName }
|
|
2461
|
+
);
|
|
2462
|
+
}
|
|
2463
|
+
return {
|
|
2464
|
+
config,
|
|
2465
|
+
manifestBytes,
|
|
2466
|
+
manifest: decoded.manifest,
|
|
2467
|
+
entityTag: hexOf(sha256(payload)),
|
|
2468
|
+
keys: manifestReferencedKeys(decoded.manifest)
|
|
2469
|
+
};
|
|
2470
|
+
}
|
|
2471
|
+
async function writePublishedSnapshot(directory, indexName, config, manifestBytes) {
|
|
2472
|
+
const payload = encode({ config, manifest: manifestBytes });
|
|
2473
|
+
const packed = await packChecksummedEnvelope([payload], PUBLISHED_SNAPSHOT_ENVELOPE_VERSION, PUBLISHED_SNAPSHOT_WHAT);
|
|
2474
|
+
const published = await publishedFrom(indexName, payload, config, manifestBytes);
|
|
2475
|
+
await directory.atomicWrite(publishedSnapshotKey(indexName), packed.chunks);
|
|
2476
|
+
return published;
|
|
2477
|
+
}
|
|
2478
|
+
async function readPublishedSnapshot(directory, indexName) {
|
|
2479
|
+
const bytes = await directory.read(publishedSnapshotKey(indexName));
|
|
2480
|
+
if (bytes === null) return null;
|
|
2481
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
2482
|
+
envelopeFormatVersion: PUBLISHED_SNAPSHOT_ENVELOPE_VERSION,
|
|
2483
|
+
what: PUBLISHED_SNAPSHOT_WHAT,
|
|
2484
|
+
allowsCompression: false
|
|
2485
|
+
});
|
|
2486
|
+
const raw = decode(payload);
|
|
2487
|
+
if (raw === null || typeof raw !== "object" || !isBytes(raw.config) || !isBytes(raw.manifest)) {
|
|
2488
|
+
throw new NarsilError(
|
|
2489
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2490
|
+
`The published snapshot of index "${indexName}" lacks its configuration or its manifest`,
|
|
2491
|
+
{ indexName }
|
|
2492
|
+
);
|
|
2493
|
+
}
|
|
2494
|
+
return publishedFrom(indexName, payload, raw.config, raw.manifest);
|
|
2495
|
+
}
|
|
2213
2496
|
|
|
2214
|
-
// src/persistence/durability/
|
|
2215
|
-
function
|
|
2216
|
-
|
|
2217
|
-
for (const
|
|
2218
|
-
|
|
2497
|
+
// src/persistence/durability/sync/retention.ts
|
|
2498
|
+
async function firstLogStart(directory, indexName, partitionId) {
|
|
2499
|
+
let first = Number.POSITIVE_INFINITY;
|
|
2500
|
+
for (const key of await directory.list(walSegmentPrefix(indexName, partitionId))) {
|
|
2501
|
+
const start = parseWalSegmentStart(indexName, partitionId, key);
|
|
2502
|
+
if (start !== null && start < first) first = start;
|
|
2503
|
+
}
|
|
2504
|
+
return first;
|
|
2505
|
+
}
|
|
2506
|
+
async function sizedQueues(retention) {
|
|
2507
|
+
const { directory, indexName } = retention;
|
|
2508
|
+
const queues = /* @__PURE__ */ new Map();
|
|
2509
|
+
for (const segment of await listHistorySegments(directory, indexName)) {
|
|
2510
|
+
const bytes = await directory.byteLength?.(segment.key) ?? (await directory.read(segment.key))?.byteLength;
|
|
2511
|
+
if (bytes === void 0 || bytes === null) continue;
|
|
2512
|
+
const queue = queues.get(segment.partitionId) ?? [];
|
|
2513
|
+
queue.push({ ...segment, bytes, lastSeqNo: Number.POSITIVE_INFINITY });
|
|
2514
|
+
queues.set(segment.partitionId, queue);
|
|
2515
|
+
}
|
|
2516
|
+
for (const [partitionId, queue] of queues) {
|
|
2517
|
+
const logStart = await firstLogStart(directory, indexName, partitionId);
|
|
2518
|
+
for (let i = 0; i < queue.length; i++) {
|
|
2519
|
+
const nextStart = i + 1 < queue.length ? queue[i + 1].startSeqNo : logStart;
|
|
2520
|
+
queue[i].lastSeqNo = nextStart - 1;
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
return queues;
|
|
2524
|
+
}
|
|
2525
|
+
function nextToDelete(queues, totalBytes, retention) {
|
|
2526
|
+
const oldestKept = retention.nowMs - retention.limits.maxAgeMs;
|
|
2527
|
+
let oldest = null;
|
|
2528
|
+
for (const queue of queues.values()) {
|
|
2529
|
+
const head = queue[0];
|
|
2530
|
+
if (head === void 0) continue;
|
|
2531
|
+
if (oldest === null || head.movedAtMs < oldest.movedAtMs) oldest = head;
|
|
2532
|
+
}
|
|
2533
|
+
if (oldest === null) return null;
|
|
2534
|
+
if (oldest.movedAtMs < oldestKept || totalBytes > retention.limits.maxBytes) return oldest;
|
|
2535
|
+
return null;
|
|
2219
2536
|
}
|
|
2220
|
-
function
|
|
2221
|
-
const
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2537
|
+
async function enforceHistoryLimits(retention) {
|
|
2538
|
+
const queues = await sizedQueues(retention);
|
|
2539
|
+
let totalBytes = 0;
|
|
2540
|
+
for (const queue of queues.values()) for (const segment2 of queue) totalBytes += segment2.bytes;
|
|
2541
|
+
let published = false;
|
|
2542
|
+
let deleted = 0;
|
|
2543
|
+
let segment = nextToDelete(queues, totalBytes, retention);
|
|
2544
|
+
while (segment !== null) {
|
|
2545
|
+
const publishedLast = retention.publishedLastSeqNo(segment.partitionId);
|
|
2546
|
+
if (!published && retention.publish !== null && (publishedLast === null || segment.lastSeqNo > publishedLast)) {
|
|
2547
|
+
published = true;
|
|
2548
|
+
await retention.publish();
|
|
2549
|
+
}
|
|
2550
|
+
await retention.directory.remove(segment.key);
|
|
2551
|
+
queues.get(segment.partitionId)?.shift();
|
|
2552
|
+
totalBytes -= segment.bytes;
|
|
2553
|
+
deleted += 1;
|
|
2554
|
+
segment = nextToDelete(queues, totalBytes, retention);
|
|
2555
|
+
}
|
|
2556
|
+
return deleted;
|
|
2557
|
+
}
|
|
2558
|
+
|
|
2559
|
+
// src/persistence/durability/sync/storage.ts
|
|
2560
|
+
function createSyncStorage(deps) {
|
|
2561
|
+
const { directory } = deps;
|
|
2562
|
+
const store = directoryCheckpointStore(directory);
|
|
2563
|
+
const cache = /* @__PURE__ */ new Map();
|
|
2564
|
+
const publishing = /* @__PURE__ */ new Map();
|
|
2565
|
+
const pins = createFilePins((key) => directory.remove(key));
|
|
2566
|
+
async function published(indexName) {
|
|
2567
|
+
const cached = cache.get(indexName);
|
|
2568
|
+
if (cached !== void 0) return cached;
|
|
2569
|
+
const read = await readPublishedSnapshot(directory, indexName);
|
|
2570
|
+
cache.set(indexName, read);
|
|
2571
|
+
return read;
|
|
2572
|
+
}
|
|
2573
|
+
async function removeUnlisted(indexName, previous, kept) {
|
|
2574
|
+
const manifest = await readCurrentManifest(store, indexName);
|
|
2575
|
+
const current = manifest === null ? /* @__PURE__ */ new Set() : manifestReferencedKeys(manifest);
|
|
2576
|
+
for (const key of previous) {
|
|
2577
|
+
if (kept.has(key) || current.has(key) || pins.removeWhenUnpinned(key)) continue;
|
|
2578
|
+
await directory.remove(key);
|
|
2579
|
+
}
|
|
2580
|
+
}
|
|
2581
|
+
async function publishNow(indexName, configurationOf) {
|
|
2582
|
+
const archive = await deps.openArchiveCheckpoint(indexName);
|
|
2583
|
+
try {
|
|
2584
|
+
const config = await configurationOf(archive.manifest);
|
|
2585
|
+
const previous = await published(indexName);
|
|
2586
|
+
const next = await writePublishedSnapshot(directory, indexName, config, archive.manifestBytes);
|
|
2587
|
+
cache.set(indexName, next);
|
|
2588
|
+
if (previous !== null) {
|
|
2589
|
+
await deps.inCheckpointTurn(indexName, () => removeUnlisted(indexName, previous.keys, next.keys));
|
|
2590
|
+
}
|
|
2591
|
+
return next;
|
|
2592
|
+
} finally {
|
|
2593
|
+
archive.release();
|
|
2594
|
+
}
|
|
2231
2595
|
}
|
|
2232
2596
|
return {
|
|
2233
|
-
|
|
2234
|
-
const
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2597
|
+
publish(indexName, configurationOf) {
|
|
2598
|
+
const inFlight = publishing.get(indexName);
|
|
2599
|
+
const run = (inFlight ?? Promise.resolve()).catch(() => void 0).then(() => publishNow(indexName, configurationOf));
|
|
2600
|
+
const tracked = run.finally(() => {
|
|
2601
|
+
if (publishing.get(indexName) === tracked) publishing.delete(indexName);
|
|
2602
|
+
});
|
|
2603
|
+
publishing.set(indexName, tracked);
|
|
2604
|
+
return tracked;
|
|
2241
2605
|
},
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
const
|
|
2246
|
-
|
|
2247
|
-
|
|
2606
|
+
published,
|
|
2607
|
+
pin: (snapshot) => pins.pin(snapshot.keys),
|
|
2608
|
+
async readFileRange(key, offset, length) {
|
|
2609
|
+
const readRange = directory.readRange;
|
|
2610
|
+
if (readRange !== void 0) return readRange.call(directory, key, offset, length);
|
|
2611
|
+
const whole = await directory.read(key);
|
|
2612
|
+
return (whole ?? new Uint8Array(0)).subarray(offset, offset + length);
|
|
2613
|
+
},
|
|
2614
|
+
async release(indexName) {
|
|
2615
|
+
await deps.inCheckpointTurn(indexName, async () => {
|
|
2616
|
+
const previous = await published(indexName);
|
|
2617
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2618
|
+
cache.set(indexName, null);
|
|
2619
|
+
if (previous !== null) await removeUnlisted(indexName, previous.keys, /* @__PURE__ */ new Set());
|
|
2620
|
+
await removeHistory(directory, indexName);
|
|
2248
2621
|
});
|
|
2249
|
-
if (unreported) {
|
|
2250
|
-
for (const partitionId of held.partitionIds) {
|
|
2251
|
-
const partition = indexState.partitions.get(partitionId);
|
|
2252
|
-
if (partition !== void 0) reported.add(partition);
|
|
2253
|
-
}
|
|
2254
|
-
hooks.onCheckpointHeld?.(held);
|
|
2255
|
-
}
|
|
2256
|
-
return true;
|
|
2257
2622
|
},
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2623
|
+
removeHistory: (indexName, partitionId) => removeHistory(directory, indexName, partitionId),
|
|
2624
|
+
async enforceHistoryLimits(indexName, limits, publish, nowMs) {
|
|
2625
|
+
const current = await published(indexName);
|
|
2626
|
+
return enforceHistoryLimits({
|
|
2627
|
+
directory,
|
|
2628
|
+
indexName,
|
|
2629
|
+
limits,
|
|
2630
|
+
nowMs,
|
|
2631
|
+
publish,
|
|
2632
|
+
publishedLastSeqNo: (partitionId) => cache.get(indexName)?.manifest.checkpoint.find((entry) => entry.partitionId === partitionId)?.lastSeqNo ?? (current === null ? null : 0)
|
|
2633
|
+
});
|
|
2634
|
+
},
|
|
2635
|
+
openPartitionLog: (indexName, partitionId) => openPartitionLog(directory, indexName, partitionId),
|
|
2636
|
+
readPartitionLog: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => readPartitionLog(directory, indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
|
|
2637
|
+
logBytesAfter: (log, afterSeqNo) => logBytesAfter(directory, log, afterSeqNo),
|
|
2638
|
+
currentManifest: (indexName) => readCurrentManifest(store, indexName),
|
|
2639
|
+
async keepsCheckpointKey(indexName) {
|
|
2640
|
+
const current = await published(indexName);
|
|
2641
|
+
return (key) => pins.isPinned(key) || (current?.keys.has(key) ?? false);
|
|
2642
|
+
},
|
|
2643
|
+
async settleAtRecovery(indexName, active) {
|
|
2644
|
+
cache.delete(indexName);
|
|
2645
|
+
const publishedKey = publishedSnapshotKey(indexName);
|
|
2646
|
+
for (const key of await directory.list(syncDirectoryPrefix(indexName))) {
|
|
2647
|
+
if (key !== publishedKey) await directory.remove(key);
|
|
2263
2648
|
}
|
|
2264
|
-
|
|
2649
|
+
if (!active) {
|
|
2650
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2651
|
+
await removeHistory(directory, indexName);
|
|
2652
|
+
cache.set(indexName, null);
|
|
2653
|
+
return /* @__PURE__ */ new Set();
|
|
2654
|
+
}
|
|
2655
|
+
try {
|
|
2656
|
+
return (await published(indexName))?.keys ?? /* @__PURE__ */ new Set();
|
|
2657
|
+
} catch {
|
|
2658
|
+
await directory.remove(publishedSnapshotKey(indexName));
|
|
2659
|
+
cache.set(indexName, null);
|
|
2660
|
+
return /* @__PURE__ */ new Set();
|
|
2661
|
+
}
|
|
2662
|
+
},
|
|
2663
|
+
forget(indexName) {
|
|
2664
|
+
cache.delete(indexName);
|
|
2265
2665
|
}
|
|
2266
2666
|
};
|
|
2267
2667
|
}
|
|
2268
2668
|
|
|
2269
|
-
// src/persistence/durability/manager-
|
|
2270
|
-
function
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2669
|
+
// src/persistence/durability/manager-storage.ts
|
|
2670
|
+
function createIndexStorageAccess(deps) {
|
|
2671
|
+
const { directory, indexes } = deps;
|
|
2672
|
+
const archiveAccess = createArchiveCheckpointAccess({
|
|
2673
|
+
directory,
|
|
2674
|
+
held: deps.heldCheckpointFiles,
|
|
2675
|
+
checkpoint: deps.checkpoint,
|
|
2676
|
+
registerPartition: deps.registerPartition,
|
|
2677
|
+
markFatal: deps.markFatal
|
|
2678
|
+
});
|
|
2679
|
+
const syncStorage = createSyncStorage({
|
|
2680
|
+
directory,
|
|
2681
|
+
openArchiveCheckpoint: archiveAccess.openArchiveCheckpoint,
|
|
2682
|
+
inCheckpointTurn: (indexName, work) => {
|
|
2683
|
+
const indexState = indexes.get(indexName);
|
|
2684
|
+
if (indexState === void 0) return work();
|
|
2685
|
+
return checkpointInTurn(indexState, {
|
|
2686
|
+
joinsInFlight: false,
|
|
2687
|
+
mayStart: () => true,
|
|
2688
|
+
perform: work,
|
|
2689
|
+
afterSettling: () => releaseStalledWrites(indexState)
|
|
2690
|
+
});
|
|
2280
2691
|
}
|
|
2281
|
-
}
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2692
|
+
});
|
|
2693
|
+
const unloading = createIndexUnloading({
|
|
2694
|
+
directory,
|
|
2695
|
+
hooks: deps.hooks,
|
|
2696
|
+
indexes,
|
|
2697
|
+
metadataWrites: deps.metadataWrites,
|
|
2698
|
+
closeWriterReportingFailure: deps.closeWriterReportingFailure,
|
|
2699
|
+
forgetSyncState: syncStorage.forget
|
|
2700
|
+
});
|
|
2701
|
+
return { archiveAccess, syncStorage, unloading };
|
|
2285
2702
|
}
|
|
2286
2703
|
|
|
2287
2704
|
// src/persistence/durability/metadata-write-queue.ts
|
|
@@ -2529,8 +2946,8 @@ function createGroupCommitCoordinator(sync) {
|
|
|
2529
2946
|
// src/persistence/durability/wal-writer.ts
|
|
2530
2947
|
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
2531
2948
|
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
2532
|
-
const
|
|
2533
|
-
return `${indexName}/wal/${partitionId}/${
|
|
2949
|
+
const padded = startSeqNo.toString().padStart(16, "0");
|
|
2950
|
+
return `${indexName}/wal/${partitionId}/${padded}`;
|
|
2534
2951
|
}
|
|
2535
2952
|
function parseSegmentStartSeqNo2(key) {
|
|
2536
2953
|
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
@@ -3254,7 +3671,7 @@ var decoder = new TextDecoder();
|
|
|
3254
3671
|
async function countPagedPartition(store, partition) {
|
|
3255
3672
|
const { directory } = store;
|
|
3256
3673
|
if (directory === void 0 || partition.segments.length === 0) return null;
|
|
3257
|
-
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-
|
|
3674
|
+
const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-ZL5ZG6YB.mjs');
|
|
3258
3675
|
for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
|
|
3259
3676
|
const scans = [];
|
|
3260
3677
|
try {
|
|
@@ -3537,12 +3954,14 @@ async function recoverCheckpoint(deps) {
|
|
|
3537
3954
|
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
3538
3955
|
}
|
|
3539
3956
|
if (decoded === null || decoded.kind !== "current") {
|
|
3540
|
-
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName,
|
|
3957
|
+
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, deps.keptForSync);
|
|
3541
3958
|
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
3542
3959
|
}
|
|
3543
3960
|
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
3544
3961
|
if (!deps.keepsUnlistedFiles) {
|
|
3545
|
-
|
|
3962
|
+
const referenced = manifestReferencedKeys(decoded.manifest);
|
|
3963
|
+
for (const key of deps.keptForSync) referenced.add(key);
|
|
3964
|
+
await reclaimUnlistedKeys(deps.store, deps.indexName, referenced);
|
|
3546
3965
|
}
|
|
3547
3966
|
const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
|
|
3548
3967
|
manager: deps.manager,
|
|
@@ -3663,6 +4082,8 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3663
4082
|
}
|
|
3664
4083
|
return replayed;
|
|
3665
4084
|
};
|
|
4085
|
+
const syncActive = metadata.sync?.enabled === true && metadata.syncActive === true;
|
|
4086
|
+
const keptForSync = await deps.syncStorage.settleAtRecovery(indexName, syncActive);
|
|
3666
4087
|
const recovered = await recoverCheckpoint({
|
|
3667
4088
|
store,
|
|
3668
4089
|
indexName,
|
|
@@ -3672,6 +4093,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
|
3672
4093
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
3673
4094
|
compactionThreshold: deps.compactionThreshold,
|
|
3674
4095
|
keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
|
|
4096
|
+
keptForSync,
|
|
3675
4097
|
replay,
|
|
3676
4098
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
3677
4099
|
writeRecords
|
|
@@ -3787,6 +4209,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3787
4209
|
mutationThreshold: checkpointMutationThreshold,
|
|
3788
4210
|
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
3789
4211
|
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
4212
|
+
keepsCheckpointKey: () => syncStorage.keepsCheckpointKey(indexName),
|
|
3790
4213
|
queueMetadataWrite,
|
|
3791
4214
|
markFatal,
|
|
3792
4215
|
failPartition: partitionFailureFor(indexName, partitionBatchDeps)
|
|
@@ -3840,6 +4263,17 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3840
4263
|
startAsyncFlushTimer();
|
|
3841
4264
|
return outcomes;
|
|
3842
4265
|
}
|
|
4266
|
+
const { archiveAccess, syncStorage, unloading } = createIndexStorageAccess({
|
|
4267
|
+
directory,
|
|
4268
|
+
hooks,
|
|
4269
|
+
indexes,
|
|
4270
|
+
metadataWrites,
|
|
4271
|
+
heldCheckpointFiles,
|
|
4272
|
+
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
4273
|
+
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
4274
|
+
markFatal,
|
|
4275
|
+
closeWriterReportingFailure
|
|
4276
|
+
});
|
|
3843
4277
|
const recoveryDeps = {
|
|
3844
4278
|
directory,
|
|
3845
4279
|
hooks,
|
|
@@ -3848,7 +4282,8 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3848
4282
|
getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm);
|
|
3849
4283
|
},
|
|
3850
4284
|
holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
|
|
3851
|
-
queueMetadataWrite
|
|
4285
|
+
queueMetadataWrite,
|
|
4286
|
+
syncStorage
|
|
3852
4287
|
};
|
|
3853
4288
|
function recoverIndex(indexName, metadataOnly = false, startsGraphRepairs = true) {
|
|
3854
4289
|
return loads.track(indexName, recoverPersistedIndex(recoveryDeps, indexName, metadataOnly), startsGraphRepairs);
|
|
@@ -3936,35 +4371,10 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
3936
4371
|
await checkpointIndex(indexName);
|
|
3937
4372
|
}
|
|
3938
4373
|
},
|
|
3939
|
-
|
|
3940
|
-
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), closeWriterReportingFailure);
|
|
3944
|
-
indexes.delete(indexName);
|
|
3945
|
-
metadataWrites.forget(indexName);
|
|
3946
|
-
}
|
|
3947
|
-
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
3948
|
-
for (const key of await directory.list(`${indexName}/`)) {
|
|
3949
|
-
await directory.remove(key);
|
|
3950
|
-
}
|
|
3951
|
-
},
|
|
3952
|
-
async unloadIndex(indexName) {
|
|
3953
|
-
const indexState = indexes.get(indexName);
|
|
3954
|
-
if (indexState === void 0) {
|
|
3955
|
-
return;
|
|
3956
|
-
}
|
|
3957
|
-
await drainIndexStateForUnload(indexState, metadataWrites.pending(indexName), (close) => close);
|
|
3958
|
-
indexes.delete(indexName);
|
|
3959
|
-
metadataWrites.forget(indexName);
|
|
3960
|
-
},
|
|
3961
|
-
...createArchiveCheckpointAccess({
|
|
3962
|
-
directory,
|
|
3963
|
-
held: heldCheckpointFiles,
|
|
3964
|
-
checkpoint: (name, pinsVectors) => requestedCheckpoint(name, false, pinsVectors),
|
|
3965
|
-
registerPartition: (name, partitionId, startSeqNo, primaryTerm) => getOrCreatePartition(name, partitionId, startSeqNo, primaryTerm),
|
|
3966
|
-
markFatal
|
|
3967
|
-
}),
|
|
4374
|
+
...unloading,
|
|
4375
|
+
syncStorage: () => syncStorage,
|
|
4376
|
+
...queryHistoryAccess({ kind: "directory", directory }),
|
|
4377
|
+
...archiveAccess,
|
|
3968
4378
|
async shutdown() {
|
|
3969
4379
|
shuttingDown = true;
|
|
3970
4380
|
stopTimers();
|
|
@@ -4231,6 +4641,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4231
4641
|
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
4232
4642
|
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
4233
4643
|
keepsUnlistedFiles: false,
|
|
4644
|
+
keptForSync: /* @__PURE__ */ new Set(),
|
|
4234
4645
|
replay: async (checkpoint) => checkpoint,
|
|
4235
4646
|
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
4236
4647
|
});
|
|
@@ -4366,6 +4777,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4366
4777
|
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
4367
4778
|
}
|
|
4368
4779
|
},
|
|
4780
|
+
...queryHistoryAccess({ kind: "adapter", adapter }),
|
|
4369
4781
|
async removeIndex(indexName) {
|
|
4370
4782
|
await settleRestore(indexName).catch(() => void 0);
|
|
4371
4783
|
restores.forget(indexName);
|
|
@@ -4374,6 +4786,7 @@ function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
|
4374
4786
|
for (const key of await adapter.list(`${indexName}/`)) {
|
|
4375
4787
|
await adapter.delete(key);
|
|
4376
4788
|
}
|
|
4789
|
+
await removeQueryHistoryOf({ kind: "adapter", adapter }, indexName);
|
|
4377
4790
|
},
|
|
4378
4791
|
async unloadIndex(indexName) {
|
|
4379
4792
|
await settleRestore(indexName);
|
|
@@ -4428,7 +4841,9 @@ function createDurabilityIntegration(tier, hooks) {
|
|
|
4428
4841
|
recordCheckpoint: hooks.recordCheckpoint,
|
|
4429
4842
|
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
4430
4843
|
onCheckpointHeld: hooks.onCheckpointHeld,
|
|
4431
|
-
onFatalError: hooks.onFatalError
|
|
4844
|
+
onFatalError: hooks.onFatalError,
|
|
4845
|
+
...hooks.keepsSyncHistory === void 0 ? {} : { keepsSyncHistory: hooks.keepsSyncHistory },
|
|
4846
|
+
...hooks.onHistoryMoved === void 0 ? {} : { onHistoryMoved: hooks.onHistoryMoved }
|
|
4432
4847
|
};
|
|
4433
4848
|
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
4434
4849
|
function partitionFor(indexName, docId) {
|
|
@@ -4513,7 +4928,12 @@ function createDurabilityFromTier(tier, wiring) {
|
|
|
4513
4928
|
recordCheckpoint: wiring.recordCheckpoint,
|
|
4514
4929
|
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
4515
4930
|
onCheckpointHeld: wiring.emitCheckpointHeld,
|
|
4516
|
-
onFatalError: wiring.emitFatalError
|
|
4931
|
+
onFatalError: wiring.emitFatalError,
|
|
4932
|
+
keepsSyncHistory: (indexName) => wiring.indexRegistry.get(indexName)?.sync?.active === true,
|
|
4933
|
+
onHistoryMoved(indexName) {
|
|
4934
|
+
if (wiring.indexRegistry.get(indexName)?.sync?.active !== true) return;
|
|
4935
|
+
import('./retention-7I7UAWYK.mjs').then((retention) => retention.scheduleSyncRetention(wiring.engineCore(), indexName)).catch((error) => wiring.emitFatalError(error instanceof Error ? error : new Error(String(error))));
|
|
4936
|
+
}
|
|
4517
4937
|
});
|
|
4518
4938
|
}
|
|
4519
4939
|
|