@delali/narsil 0.3.1-next.99 → 0.4.1-next.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/README.md +30 -11
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/chunk-NDYADQ4Y.mjs +23426 -0
- package/dist/browser/chunk-QAOVAYAG.mjs +564 -0
- package/dist/browser/chunk-WC2DEXWU.mjs +29 -0
- package/dist/browser/chunk-YINZFPLT.mjs +1239 -0
- package/dist/browser/chunk-YJHGOJAC.mjs +15 -0
- package/dist/browser/{chunk-Z7QH2XHB.mjs → core-leaf-HWAXCEWX.mjs} +3987 -2103
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/digest-leaf-64IPRPZD.mjs +151 -0
- package/dist/browser/durability-config-75NSFCQN.mjs +97 -0
- package/dist/browser/embed-GG6PS3EA.mjs +264 -0
- package/dist/browser/geo-index-GNHY4PUB.mjs +1484 -0
- package/dist/browser/gzip-JF3S6PPT.mjs +31 -0
- package/dist/browser/module-ZZTNG5JZ.mjs +5762 -0
- package/dist/browser/msgpack-leaf-44XNWSSY.mjs +302 -0
- package/dist/browser/optional-reads-XMVPG6HG.mjs +3230 -0
- package/dist/browser/optional-writes-76DWMRQH.mjs +3603 -0
- package/dist/browser/phrase-module-6UGFVW5J.mjs +617 -0
- package/dist/browser/resolve-vector-text-XRR2OVHE.mjs +85 -0
- package/dist/browser/restore-queue-SG5O735W.mjs +196 -0
- package/dist/browser/{runtime-NYE5TB6O.mjs → runtime-OMKE7DF4.mjs} +83 -25
- package/dist/browser/segment-codec-leaf-A427QJUX.mjs +554 -0
- package/dist/browser/segment-merge-leaf-7BZ54JPI.mjs +1636 -0
- package/dist/browser/storage-leaf-5UTR5K7J.mjs +19829 -0
- package/dist/browser/store-leaf-3L33V35P.mjs +2070 -0
- package/dist/browser/suggestions-leaf-UBNDVKBD.mjs +1430 -0
- package/dist/browser/worker-copies-JXSGYR5X.mjs +2951 -0
- package/dist/{capabilities-BTTIs96E.d.ts → capabilities-DMte5K9_.d.ts} +162 -18
- package/dist/chunk-2AFZE7Y3.mjs +72 -0
- package/dist/chunk-2CKDCC65.mjs +47 -0
- package/dist/chunk-3ECIRFS5.mjs +61 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-4H4RG2FR.mjs +13 -0
- package/dist/chunk-4PY4JYMY.mjs +11 -0
- package/dist/chunk-4ZPVAHWF.mjs +1289 -0
- package/dist/chunk-5EI5CXO6.mjs +197 -0
- package/dist/chunk-5F5MXWS5.mjs +283 -0
- package/dist/{chunk-OHP7QR33.mjs → chunk-5FLRXF73.mjs} +25 -4
- package/dist/chunk-6FOENO4E.mjs +273 -0
- package/dist/chunk-6GHX35ZM.mjs +108 -0
- package/dist/chunk-6MSZ543Y.mjs +32 -0
- package/dist/chunk-6N6HHBCQ.mjs +4596 -0
- package/dist/{chunk-LXPIXDUJ.mjs → chunk-7P5YWDRQ.mjs} +403 -169
- package/dist/{chunk-TGP5M5R7.mjs → chunk-7T4LAYT4.mjs} +1 -1
- package/dist/{chunk-GGSH6KBQ.mjs → chunk-ASZLNVWW.mjs} +23 -3
- package/dist/chunk-AVBPB3MI.mjs +602 -0
- package/dist/{chunk-UCDF35PU.mjs → chunk-AVGUEEQ3.mjs} +494 -245
- package/dist/chunk-BUMREPLU.mjs +15 -0
- package/dist/chunk-BY6NALRK.mjs +31921 -0
- package/dist/chunk-C4ZMXXTL.mjs +92 -0
- package/dist/{chunk-2WYMG3OZ.mjs → chunk-C74KPRR7.mjs} +274 -19
- package/dist/{chunk-RSRVGAIQ.mjs → chunk-D456ZFPQ.mjs} +3 -2
- package/dist/chunk-DXZ6JDWG.mjs +797 -0
- package/dist/chunk-FJXJZUWF.mjs +209 -0
- package/dist/chunk-FS3HPPSC.mjs +212 -0
- package/dist/chunk-FT4KLZXW.mjs +1164 -0
- package/dist/chunk-GR5GMFJJ.mjs +104 -0
- package/dist/chunk-GXCDWLQI.mjs +193 -0
- package/dist/chunk-GXED6R3Q.mjs +271 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-IL5DRCNU.mjs +3041 -0
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/chunk-KYIRXTCA.mjs +100 -0
- package/dist/chunk-LFXS6OF7.mjs +289 -0
- package/dist/chunk-LNGE7SLY.mjs +151 -0
- package/dist/chunk-M2Z5WVWW.mjs +194 -0
- package/dist/{chunk-45AVKHMJ.mjs → chunk-M3B7SGJB.mjs} +63 -50
- package/dist/chunk-M3OW2XSP.mjs +40 -0
- package/dist/chunk-MFPSQZ6P.mjs +127 -0
- package/dist/chunk-MRCJBWJZ.mjs +1292 -0
- package/dist/chunk-MSIN4NM3.mjs +25 -0
- package/dist/chunk-MW3CJVGP.mjs +219 -0
- package/dist/chunk-MYM6CAVK.mjs +76 -0
- package/dist/chunk-NJJN4FMM.mjs +84 -0
- package/dist/chunk-NM7I3BM7.mjs +59 -0
- package/dist/chunk-OIJ64LGT.mjs +19 -0
- package/dist/{chunk-K7D4ZTSW.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/chunk-R2PJKSNL.mjs +54 -0
- package/dist/{chunk-HXJBSHXF.mjs → chunk-R724DJSG.mjs} +5056 -2619
- package/dist/chunk-S2POFTLN.mjs +10 -0
- package/dist/chunk-S5QMWECL.mjs +7 -0
- package/dist/chunk-SLAC5PAB.mjs +5008 -0
- package/dist/chunk-T4HMSZCX.mjs +5 -0
- package/dist/chunk-TACP6VK6.mjs +111 -0
- package/dist/chunk-TGPKS2UM.mjs +125 -0
- package/dist/chunk-TKKXPKEO.mjs +979 -0
- package/dist/chunk-V6EUBAIT.mjs +19 -0
- package/dist/chunk-XL56L4IR.mjs +228 -0
- package/dist/chunk-XQIAMDAG.mjs +3076 -0
- package/dist/chunk-YTUZ4R22.mjs +1063 -0
- package/dist/{chunk-KV3AZFT3.mjs → chunk-ZKOQR7EA.mjs} +45 -1
- package/dist/chunk-ZPJFGGGI.mjs +638 -0
- package/dist/chunk-ZXX5GZEX.mjs +17 -0
- package/dist/{client-i9eLJrw5.d.ts → client-BKqtl0_V.d.ts} +442 -22
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +8 -898
- package/dist/config-Bnl7L-0H.d.ts +608 -0
- package/dist/{config-P__Diiy4.d.ts → config-CJ-jfI0a.d.ts} +119 -9
- package/dist/core-leaf-6JVHEVFB.mjs +8 -0
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/digest-leaf-KXBGKYHZ.mjs +1 -0
- package/dist/direct-executor-V3WOYU2Y.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +63 -7
- package/dist/distribution/coordinator/etcd.mjs +725 -81
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/coordinator/in-memory.mjs +65 -10
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +39 -10
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +9 -6
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +20 -15
- package/dist/distribution.d.ts +267 -56
- package/dist/distribution.mjs +23156 -11362
- package/dist/durability-config-VWS2K24E.mjs +96 -0
- package/dist/embed-4PHHKGBK.mjs +263 -0
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{memory-CUEDwr-F.d.ts → engine-qjAxJSKk.d.ts} +2254 -1123
- package/dist/geo-index-2ZMMWWSK.mjs +1484 -0
- package/dist/gzip-2UCZW2MM.mjs +31 -0
- package/dist/{index-lcdGLyJH.d.ts → index-B_vmxpgW.d.ts} +260 -102
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +2133 -57
- package/dist/index.mjs +16 -32
- package/dist/index.react-native.mjs +12 -0
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -2
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/languages/albanian.mjs +1 -1
- package/dist/languages/amharic.mjs +1 -1
- package/dist/languages/arabic.mjs +1 -1
- package/dist/languages/armenian.mjs +1 -1
- package/dist/languages/azerbaijani.mjs +1 -1
- package/dist/languages/bambara.mjs +1 -1
- package/dist/languages/basque.mjs +1 -1
- package/dist/languages/belarusian.mjs +1 -1
- package/dist/languages/bengali.mjs +1 -1
- package/dist/languages/bosnian.mjs +1 -1
- package/dist/languages/breton.mjs +1 -1
- package/dist/languages/bulgarian.mjs +1 -1
- package/dist/languages/burmese.mjs +1 -1
- package/dist/languages/catalan.mjs +1 -1
- package/dist/languages/chinese.mjs +1 -1
- package/dist/languages/croatian.mjs +1 -1
- package/dist/languages/czech.mjs +1 -1
- package/dist/languages/dagbani.mjs +1 -1
- package/dist/languages/danish.mjs +1 -1
- package/dist/languages/dutch.mjs +1 -1
- package/dist/languages/english.mjs +1 -1
- package/dist/languages/esperanto.mjs +1 -1
- package/dist/languages/estonian.mjs +1 -1
- package/dist/languages/ewe.mjs +1 -1
- package/dist/languages/faroese.mjs +1 -1
- package/dist/languages/fijian.mjs +1 -1
- package/dist/languages/finnish.mjs +1 -1
- package/dist/languages/french.mjs +1 -1
- package/dist/languages/ga.mjs +1 -1
- package/dist/languages/galician.mjs +1 -1
- package/dist/languages/georgian.mjs +1 -1
- package/dist/languages/german.mjs +1 -1
- package/dist/languages/greek.mjs +1 -1
- package/dist/languages/guarani.mjs +1 -1
- package/dist/languages/gujarati.mjs +1 -1
- package/dist/languages/haitian-creole.mjs +1 -1
- package/dist/languages/hausa.mjs +1 -1
- package/dist/languages/hawaiian.mjs +1 -1
- package/dist/languages/hebrew.mjs +1 -1
- package/dist/languages/hindi.mjs +1 -1
- package/dist/languages/hungarian.mjs +1 -1
- package/dist/languages/icelandic.mjs +1 -1
- package/dist/languages/igbo.mjs +1 -1
- package/dist/languages/indonesian.mjs +1 -1
- package/dist/languages/irish.mjs +1 -1
- package/dist/languages/italian.mjs +1 -1
- package/dist/languages/japanese.mjs +1 -1
- package/dist/languages/kannada.mjs +1 -1
- package/dist/languages/kazakh.mjs +1 -1
- package/dist/languages/khmer.mjs +1 -1
- package/dist/languages/kinyarwanda.mjs +1 -1
- package/dist/languages/kirundi.mjs +1 -1
- package/dist/languages/korean.mjs +1 -1
- package/dist/languages/kurmanji.mjs +1 -1
- package/dist/languages/kyrgyz.mjs +1 -1
- package/dist/languages/lao.mjs +1 -1
- package/dist/languages/latin.mjs +1 -1
- package/dist/languages/latvian.mjs +1 -1
- package/dist/languages/lingala.mjs +1 -1
- package/dist/languages/lithuanian.mjs +1 -1
- package/dist/languages/luxembourgish.mjs +1 -1
- package/dist/languages/macedonian.mjs +1 -1
- package/dist/languages/malagasy.mjs +1 -1
- package/dist/languages/malay.mjs +1 -1
- package/dist/languages/malayalam.mjs +1 -1
- package/dist/languages/maltese.mjs +1 -1
- package/dist/languages/maori.mjs +1 -1
- package/dist/languages/marathi.mjs +1 -1
- package/dist/languages/nepali.mjs +1 -1
- package/dist/languages/norwegian.mjs +1 -1
- package/dist/languages/oromo.mjs +1 -1
- package/dist/languages/persian.mjs +1 -1
- package/dist/languages/polish.mjs +1 -1
- package/dist/languages/portuguese.mjs +1 -1
- package/dist/languages/punjabi.mjs +1 -1
- package/dist/languages/romanian.mjs +1 -1
- package/dist/languages/russian.mjs +1 -1
- package/dist/languages/samoan.mjs +1 -1
- package/dist/languages/sanskrit.mjs +1 -1
- package/dist/languages/scottish-gaelic.mjs +1 -1
- package/dist/languages/serbian.mjs +1 -1
- package/dist/languages/shona.mjs +1 -1
- package/dist/languages/sinhala.mjs +1 -1
- package/dist/languages/slovak.mjs +1 -1
- package/dist/languages/slovenian.mjs +1 -1
- package/dist/languages/sorani.mjs +1 -1
- package/dist/languages/spanish.mjs +1 -1
- package/dist/languages/swahili.mjs +1 -1
- package/dist/languages/swedish.mjs +1 -1
- package/dist/languages/tagalog.mjs +1 -1
- package/dist/languages/tamil.mjs +1 -1
- package/dist/languages/tatar.mjs +1 -1
- package/dist/languages/telugu.mjs +1 -1
- package/dist/languages/thai.mjs +1 -1
- package/dist/languages/tibetan.mjs +1 -1
- package/dist/languages/tigrinya.mjs +1 -1
- package/dist/languages/tongan.mjs +1 -1
- package/dist/languages/turkish.mjs +1 -1
- package/dist/languages/twi.mjs +1 -1
- package/dist/languages/ukrainian.mjs +1 -1
- package/dist/languages/urdu.mjs +1 -1
- package/dist/languages/vietnamese.mjs +1 -1
- package/dist/languages/welsh.mjs +1 -1
- package/dist/languages/wolof.mjs +1 -1
- package/dist/languages/xhosa.mjs +1 -1
- package/dist/languages/yoruba.mjs +1 -1
- package/dist/languages/zulu.mjs +1 -1
- package/dist/long-query-thread-UCRK7MSI.mjs +154 -0
- package/dist/module-5ZSLUQII.mjs +3729 -0
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +9 -0
- package/dist/optional-reads-5HFMF2QZ.mjs +3373 -0
- package/dist/optional-writes-4UOQKETU.mjs +247 -0
- package/dist/paged-partition-archive-BBRLZCIF.mjs +253 -0
- package/dist/partial-words.browser.mjs +1650 -0
- package/dist/partial-words.mjs +17 -11
- package/dist/{chunk-ASY5PHYI.mjs → pattern.browser.mjs} +253 -311
- package/dist/pattern.mjs +6 -7
- package/dist/persistence/durability/checkpoint-worker.d.ts +93 -25
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -41
- package/dist/phrase-module-EFESWOCR.mjs +635 -0
- package/dist/query-actions-HMJUVWGT.mjs +119 -0
- package/dist/query-suggestions/merge-worker.d.ts +55 -0
- package/dist/query-suggestions/merge-worker.mjs +53 -0
- package/dist/query-suggestions-CBYkrQ5T.d.ts +311 -0
- package/dist/query-suggestions.browser.mjs +47 -0
- package/dist/query-suggestions.d.ts +20 -0
- package/dist/query-suggestions.mjs +19 -0
- package/dist/react-native/chunks/chunk-3OEFSHVV.mjs +970 -0
- package/dist/{browser/chunk-RELZRJNR.mjs → react-native/chunks/chunk-43R4GCM7.mjs} +49 -50
- package/dist/react-native/chunks/chunk-AKRYHAVG.mjs +6729 -0
- package/dist/react-native/chunks/chunk-CW2ZFXSE.mjs +14 -0
- package/dist/react-native/chunks/chunk-G3WCDTOF.mjs +29640 -0
- package/dist/react-native/chunks/chunk-GXXHHA2T.mjs +212 -0
- package/dist/react-native/chunks/chunk-H3MDVPBO.mjs +151 -0
- package/dist/react-native/chunks/chunk-HQEVQEDJ.mjs +52 -0
- package/dist/react-native/chunks/chunk-HRDMIQ63.mjs +131 -0
- package/dist/react-native/chunks/chunk-HTBY4GON.mjs +565 -0
- package/dist/react-native/chunks/chunk-LX62Y4B5.mjs +61 -0
- package/dist/react-native/chunks/chunk-MRH36RN3.mjs +902 -0
- package/dist/react-native/chunks/chunk-Q32DDFKX.mjs +19 -0
- package/dist/react-native/chunks/chunk-UE4ANHSX.mjs +127 -0
- package/dist/react-native/chunks/chunk-UNZPWLVL.mjs +1063 -0
- package/dist/react-native/chunks/chunk-UO44MA5M.mjs +232 -0
- package/dist/react-native/chunks/chunk-UYKIV2KT.mjs +42 -0
- package/dist/react-native/chunks/chunk-VUJCSQG5.mjs +11 -0
- package/dist/react-native/chunks/chunk-WVHF4ZAK.mjs +802 -0
- package/dist/react-native/chunks/chunk-ZUJCUPVS.mjs +54 -0
- package/dist/react-native/chunks/core-leaf-S2KNBWDY.mjs +7 -0
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/digest-leaf-X6OIFDIU.mjs +1 -0
- package/dist/react-native/chunks/durability-config-FCSD4MXY.mjs +96 -0
- package/dist/react-native/chunks/embed-HA2I7RCF.mjs +263 -0
- package/dist/react-native/chunks/geo-index-QEYKEVPH.mjs +1484 -0
- package/dist/react-native/chunks/gzip.react-native-CVM4NQG2.mjs +12 -0
- package/dist/react-native/chunks/module-FMBX75GP.mjs +5209 -0
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/optional-reads-AFPO2FEB.mjs +3364 -0
- package/dist/react-native/chunks/optional-writes-XOH25XZI.mjs +296 -0
- package/dist/react-native/chunks/phrase-module-IGYBK7XH.mjs +636 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/react-native/chunks/resolve-vector-text-GE2CRJKI.mjs +84 -0
- package/dist/react-native/chunks/restore-queue-LLNMYB4Z.mjs +187 -0
- package/dist/react-native/chunks/runtime-UOHD2XUJ.mjs +615 -0
- package/dist/react-native/chunks/segment-codec-leaf-VR6O5GLC.mjs +13 -0
- package/dist/react-native/chunks/segment-merge-leaf-LLMWU4EW.mjs +14 -0
- package/dist/react-native/chunks/storage-leaf-JD7PIDO4.mjs +19633 -0
- package/dist/react-native/chunks/store-leaf-4UVCMYGN.mjs +15 -0
- package/dist/react-native/chunks/suggestions-leaf-J5VT5RHY.mjs +1321 -0
- package/dist/react-native/chunks/worker-copies-ZYHY7FKJ.mjs +2831 -0
- package/dist/react-native/partial-words.mjs +1658 -0
- package/dist/react-native/pattern.mjs +2570 -0
- package/dist/react-native/query-suggestions.mjs +55 -0
- package/dist/react-native/sync.mjs +2759 -0
- package/dist/react-native/synonyms.mjs +676 -0
- package/dist/react.d.ts +12 -8
- package/dist/react.mjs +45 -1
- package/dist/{rerank-NWTKK636.mjs → rerank-RI2PMNOD.mjs} +22 -12
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/resolve-vector-text-OD4J3WHF.mjs +3 -0
- package/dist/restore-queue-PCKWDAKQ.mjs +192 -0
- package/dist/runtime-USQYKN3I.mjs +615 -0
- package/dist/{schema-4W_7tk1f.d.ts → schema-CzG9OUw5.d.ts} +70 -30
- package/dist/segment-codec-leaf-QJM7I6QG.mjs +18 -0
- package/dist/segment-merge-leaf-4BJGIJHY.mjs +18 -0
- package/dist/server.d.ts +348 -22
- package/dist/server.mjs +2211 -327
- package/dist/storage-leaf-GRRRI7UV.mjs +5962 -0
- package/dist/store-leaf-6VMMSVBE.mjs +20 -0
- package/dist/stored-values-BWIMpiT_.d.ts +25 -0
- package/dist/suggestions-leaf-4KPTMPTL.mjs +54 -0
- package/dist/sync/filesystem.d.ts +30 -0
- package/dist/sync/filesystem.mjs +288 -0
- package/dist/sync/indexeddb.browser.mjs +287 -0
- package/dist/sync/indexeddb.d.ts +30 -0
- package/dist/sync/indexeddb.mjs +242 -0
- package/dist/sync.browser.mjs +1271 -0
- package/dist/sync.d.ts +23 -0
- package/dist/sync.mjs +2712 -0
- package/dist/synonyms.browser.mjs +658 -0
- package/dist/synonyms.mjs +17 -7
- package/dist/tasks-65b6t6lk.d.ts +782 -0
- package/dist/thread-UJ3W5LVI.mjs +31 -0
- package/dist/{types-B3b68nfF.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/types-CpCg-Gq-.d.ts +227 -0
- package/dist/{types-Cb9kiY5P.d.ts → types-DvncmvuQ.d.ts} +12 -1
- package/dist/vector/search-worker.d.ts +10 -2
- package/dist/vector/search-worker.mjs +11 -11
- package/dist/worker-copies-YSXZXIGC.mjs +2232 -0
- package/dist/workers/entry.mjs +75 -15
- package/openapi.json +6840 -0
- package/package.json +404 -19
- package/dist/browser/chunk-CN4I4FZU.mjs +0 -602
- package/dist/browser/chunk-HOZW3PKP.mjs +0 -9
- package/dist/browser/chunk-JPTQZWJQ.mjs +0 -10
- package/dist/browser/chunk-LD55ZGU3.mjs +0 -347
- package/dist/browser/chunk-NAFT3K6X.mjs +0 -24546
- package/dist/browser/chunk-NNG3EEB6.mjs +0 -2728
- package/dist/browser/chunk-QWES44AW.mjs +0 -473
- package/dist/browser/durability-wiring-PY3L7DES.mjs +0 -3679
- package/dist/browser/module-PDXYHT75.mjs +0 -2822
- package/dist/browser/release-76MXRPGV.mjs +0 -33
- package/dist/browser/restore-queue-DVIYJ7RU.mjs +0 -35
- package/dist/browser/snapshot-J57MZK6L.mjs +0 -1207
- package/dist/browser/write-L7FAQA47.mjs +0 -208
- package/dist/chunk-37CD3OKS.mjs +0 -87
- package/dist/chunk-5XNIRLFC.mjs +0 -96
- package/dist/chunk-7X5JST5L.mjs +0 -114
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-AY3AQ3UH.mjs +0 -73
- package/dist/chunk-BKB3XTMI.mjs +0 -707
- package/dist/chunk-C7SLA6CM.mjs +0 -19
- package/dist/chunk-COVCQ73F.mjs +0 -587
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-E74YLAIG.mjs +0 -24
- package/dist/chunk-FMWRCP2C.mjs +0 -57
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-HJSN7UYG.mjs +0 -10
- package/dist/chunk-JFRNNBOM.mjs +0 -35
- package/dist/chunk-K25LZA54.mjs +0 -8484
- package/dist/chunk-KEHK2O6P.mjs +0 -13
- package/dist/chunk-KOKM4PQN.mjs +0 -52
- package/dist/chunk-KZ3NX2P7.mjs +0 -21
- package/dist/chunk-L3L5YFXC.mjs +0 -614
- package/dist/chunk-L5ORED7V.mjs +0 -25
- package/dist/chunk-LH4NZNJJ.mjs +0 -1731
- package/dist/chunk-LU2CE5AE.mjs +0 -507
- package/dist/chunk-LXWDQLTW.mjs +0 -30
- package/dist/chunk-MHOSCQCQ.mjs +0 -948
- package/dist/chunk-NKRBEGJZ.mjs +0 -1705
- package/dist/chunk-NTRMFARG.mjs +0 -462
- package/dist/chunk-NW4XDSP2.mjs +0 -212
- package/dist/chunk-OOEXLUGR.mjs +0 -117
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-OWNZYMRE.mjs +0 -3591
- package/dist/chunk-PPY2GSIB.mjs +0 -44
- package/dist/chunk-QY7GC2MW.mjs +0 -33
- package/dist/chunk-R6A773WK.mjs +0 -226
- package/dist/chunk-R6PR4WYZ.mjs +0 -68
- package/dist/chunk-R7EEKAJE.mjs +0 -593
- package/dist/chunk-RONEOEON.mjs +0 -426
- package/dist/chunk-VWULPWRJ.mjs +0 -7469
- package/dist/chunk-WCJKUGHB.mjs +0 -262
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-X5CV2E2B.mjs +0 -1
- package/dist/chunk-YA7XLE2R.mjs +0 -1256
- package/dist/chunk-YTJCGKM4.mjs +0 -56
- package/dist/chunk-ZG477IPV.mjs +0 -526
- package/dist/chunk-ZYNAMIYX.mjs +0 -113
- package/dist/config-BGB_fDtq.d.ts +0 -237
- package/dist/direct-executor-CXZ66ACP.mjs +0 -22
- package/dist/durability-wiring-QTHOGFMZ.mjs +0 -3229
- package/dist/engine-BGCEm9E_.d.ts +0 -577
- package/dist/module-3NONKBZX.mjs +0 -1860
- package/dist/release-BH3DKW3T.mjs +0 -33
- package/dist/restore-queue-JPRBUYII.mjs +0 -35
- package/dist/runtime-ZKYZFJJC.mjs +0 -177
- package/dist/snapshot-HOGXNO5W.mjs +0 -793
- package/dist/tasks-HG3rlxx6.d.ts +0 -302
- package/dist/thread-B657LQXK.mjs +0 -692
- package/dist/write-XLIGTXBU.mjs +0 -238
|
@@ -1,3229 +0,0 @@
|
|
|
1
|
-
import { readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, loadSegmentedCheckpoint, writeMetadataEnvelope, readCurrentManifest, deserializePayloadV2, unpackEnvelopeBytes, growPartitionsTo, documentsOfPayloadV2 } from './chunk-BKB3XTMI.mjs';
|
|
2
|
-
import { decodeSegmentManifest, metadataFromConfig, createSeqOwner, isRecord, invalidManifest, SINGLE_NODE_PRIMARY_TERM, writeLiveVectors, isId, isUint32, isSequenceNumber, VECTOR_FILE_ENVELOPE_VERSION, metadataConfigOf } from './chunk-ZG477IPV.mjs';
|
|
3
|
-
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-WCJKUGHB.mjs';
|
|
4
|
-
import './chunk-37CD3OKS.mjs';
|
|
5
|
-
import { createDurableDirectory, getFs } from './chunk-45AVKHMJ.mjs';
|
|
6
|
-
import { spawnNodeWorker } from './chunk-C7SLA6CM.mjs';
|
|
7
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, snapshotBundleKey, manifestReferencedKeys, checkpointLastSeqNo, manifestFileRefs, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, checkpointPrimaryTerm, writeCheckpointSegments, partitionContentsOf, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, readWalSegments, durableRecordMissing, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, buildEntry } from './chunk-LH4NZNJJ.mjs';
|
|
8
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-L3L5YFXC.mjs';
|
|
9
|
-
import './chunk-P36MUFEJ.mjs';
|
|
10
|
-
import './chunk-OHP7QR33.mjs';
|
|
11
|
-
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-VWULPWRJ.mjs';
|
|
12
|
-
import './chunk-NKRBEGJZ.mjs';
|
|
13
|
-
import './chunk-R7EEKAJE.mjs';
|
|
14
|
-
import { resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
15
|
-
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
16
|
-
import { buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-OWNZYMRE.mjs';
|
|
17
|
-
import { fnv1a } from './chunk-K25LZA54.mjs';
|
|
18
|
-
import './chunk-JFRNNBOM.mjs';
|
|
19
|
-
import './chunk-HJSN7UYG.mjs';
|
|
20
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LXWDQLTW.mjs';
|
|
21
|
-
import './chunk-TGP5M5R7.mjs';
|
|
22
|
-
import './chunk-U2BKYEC7.mjs';
|
|
23
|
-
import './chunk-KOKM4PQN.mjs';
|
|
24
|
-
import './chunk-QY7GC2MW.mjs';
|
|
25
|
-
import './chunk-GGSH6KBQ.mjs';
|
|
26
|
-
import './chunk-ASY5PHYI.mjs';
|
|
27
|
-
import './chunk-WQBCBKY3.mjs';
|
|
28
|
-
import './chunk-MHOSCQCQ.mjs';
|
|
29
|
-
import { encodeDocument, requireValidFieldTypes } from './chunk-LXPIXDUJ.mjs';
|
|
30
|
-
import './chunk-E74YLAIG.mjs';
|
|
31
|
-
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
32
|
-
import './chunk-GES7U2PM.mjs';
|
|
33
|
-
import './chunk-OOEXLUGR.mjs';
|
|
34
|
-
import './chunk-RONEOEON.mjs';
|
|
35
|
-
import './chunk-X5CV2E2B.mjs';
|
|
36
|
-
import './chunk-7X5JST5L.mjs';
|
|
37
|
-
import './chunk-LU2CE5AE.mjs';
|
|
38
|
-
import './chunk-R6A773WK.mjs';
|
|
39
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
40
|
-
import './chunk-R6PR4WYZ.mjs';
|
|
41
|
-
import './chunk-PPY2GSIB.mjs';
|
|
42
|
-
import './chunk-L5ORED7V.mjs';
|
|
43
|
-
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
44
|
-
import { decode } from '@msgpack/msgpack';
|
|
45
|
-
|
|
46
|
-
// src/persistence/durability/sha256.ts
|
|
47
|
-
var ROUND_CONSTANTS = new Uint32Array([
|
|
48
|
-
1116352408,
|
|
49
|
-
1899447441,
|
|
50
|
-
3049323471,
|
|
51
|
-
3921009573,
|
|
52
|
-
961987163,
|
|
53
|
-
1508970993,
|
|
54
|
-
2453635748,
|
|
55
|
-
2870763221,
|
|
56
|
-
3624381080,
|
|
57
|
-
310598401,
|
|
58
|
-
607225278,
|
|
59
|
-
1426881987,
|
|
60
|
-
1925078388,
|
|
61
|
-
2162078206,
|
|
62
|
-
2614888103,
|
|
63
|
-
3248222580,
|
|
64
|
-
3835390401,
|
|
65
|
-
4022224774,
|
|
66
|
-
264347078,
|
|
67
|
-
604807628,
|
|
68
|
-
770255983,
|
|
69
|
-
1249150122,
|
|
70
|
-
1555081692,
|
|
71
|
-
1996064986,
|
|
72
|
-
2554220882,
|
|
73
|
-
2821834349,
|
|
74
|
-
2952996808,
|
|
75
|
-
3210313671,
|
|
76
|
-
3336571891,
|
|
77
|
-
3584528711,
|
|
78
|
-
113926993,
|
|
79
|
-
338241895,
|
|
80
|
-
666307205,
|
|
81
|
-
773529912,
|
|
82
|
-
1294757372,
|
|
83
|
-
1396182291,
|
|
84
|
-
1695183700,
|
|
85
|
-
1986661051,
|
|
86
|
-
2177026350,
|
|
87
|
-
2456956037,
|
|
88
|
-
2730485921,
|
|
89
|
-
2820302411,
|
|
90
|
-
3259730800,
|
|
91
|
-
3345764771,
|
|
92
|
-
3516065817,
|
|
93
|
-
3600352804,
|
|
94
|
-
4094571909,
|
|
95
|
-
275423344,
|
|
96
|
-
430227734,
|
|
97
|
-
506948616,
|
|
98
|
-
659060556,
|
|
99
|
-
883997877,
|
|
100
|
-
958139571,
|
|
101
|
-
1322822218,
|
|
102
|
-
1537002063,
|
|
103
|
-
1747873779,
|
|
104
|
-
1955562222,
|
|
105
|
-
2024104815,
|
|
106
|
-
2227730452,
|
|
107
|
-
2361852424,
|
|
108
|
-
2428436474,
|
|
109
|
-
2756734187,
|
|
110
|
-
3204031479,
|
|
111
|
-
3329325298
|
|
112
|
-
]);
|
|
113
|
-
var INITIAL_STATE = [1779033703, 3144134277, 1013904242, 2773480762, 1359893119, 2600822924, 528734635, 1541459225];
|
|
114
|
-
var BLOCK_BYTES = 64;
|
|
115
|
-
var LENGTH_BYTES = 8;
|
|
116
|
-
function rotateRight(value, bits) {
|
|
117
|
-
return value >>> bits | value << 32 - bits;
|
|
118
|
-
}
|
|
119
|
-
function padded(bytes) {
|
|
120
|
-
const total = Math.ceil((bytes.byteLength + 1 + LENGTH_BYTES) / BLOCK_BYTES) * BLOCK_BYTES;
|
|
121
|
-
const message = new Uint8Array(total);
|
|
122
|
-
message.set(bytes);
|
|
123
|
-
message[bytes.byteLength] = 128;
|
|
124
|
-
const view = new DataView(message.buffer);
|
|
125
|
-
const bitLength = bytes.byteLength * 8;
|
|
126
|
-
view.setUint32(total - LENGTH_BYTES, Math.floor(bitLength / 4294967296), false);
|
|
127
|
-
view.setUint32(total - 4, bitLength >>> 0, false);
|
|
128
|
-
return view;
|
|
129
|
-
}
|
|
130
|
-
function sha256(bytes) {
|
|
131
|
-
const view = padded(bytes);
|
|
132
|
-
const state = INITIAL_STATE.slice();
|
|
133
|
-
const words = new Uint32Array(64);
|
|
134
|
-
for (let block = 0; block < view.byteLength; block += BLOCK_BYTES) {
|
|
135
|
-
for (let i = 0; i < 16; i++) words[i] = view.getUint32(block + i * 4, false);
|
|
136
|
-
for (let i = 16; i < 64; i++) {
|
|
137
|
-
const w15 = words[i - 15];
|
|
138
|
-
const w2 = words[i - 2];
|
|
139
|
-
const s0 = rotateRight(w15, 7) ^ rotateRight(w15, 18) ^ w15 >>> 3;
|
|
140
|
-
const s1 = rotateRight(w2, 17) ^ rotateRight(w2, 19) ^ w2 >>> 10;
|
|
141
|
-
words[i] = words[i - 16] + s0 + words[i - 7] + s1 >>> 0;
|
|
142
|
-
}
|
|
143
|
-
let [a, b, c, d, e, f, g, h] = state;
|
|
144
|
-
for (let i = 0; i < 64; i++) {
|
|
145
|
-
const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
146
|
-
const choice = e & f ^ ~e & g;
|
|
147
|
-
const t1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i] >>> 0;
|
|
148
|
-
const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
149
|
-
const majority = a & b ^ a & c ^ b & c;
|
|
150
|
-
const t2 = s0 + majority >>> 0;
|
|
151
|
-
h = g;
|
|
152
|
-
g = f;
|
|
153
|
-
f = e;
|
|
154
|
-
e = d + t1 >>> 0;
|
|
155
|
-
d = c;
|
|
156
|
-
c = b;
|
|
157
|
-
b = a;
|
|
158
|
-
a = t1 + t2 >>> 0;
|
|
159
|
-
}
|
|
160
|
-
state[0] = state[0] + a >>> 0;
|
|
161
|
-
state[1] = state[1] + b >>> 0;
|
|
162
|
-
state[2] = state[2] + c >>> 0;
|
|
163
|
-
state[3] = state[3] + d >>> 0;
|
|
164
|
-
state[4] = state[4] + e >>> 0;
|
|
165
|
-
state[5] = state[5] + f >>> 0;
|
|
166
|
-
state[6] = state[6] + g >>> 0;
|
|
167
|
-
state[7] = state[7] + h >>> 0;
|
|
168
|
-
}
|
|
169
|
-
const digest = new Uint8Array(32);
|
|
170
|
-
const out = new DataView(digest.buffer);
|
|
171
|
-
for (let i = 0; i < 8; i++) out.setUint32(i * 4, state[i], false);
|
|
172
|
-
return digest;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// src/persistence/durability/restore-journal.ts
|
|
176
|
-
var RESTORE_JOURNAL_PREFIX = ".swap-";
|
|
177
|
-
var RESTORE_STAGING_PREFIX = ".restore-";
|
|
178
|
-
var RETIRED_INDEX_PREFIX = ".retired-";
|
|
179
|
-
var LOCK_FILE_KEY = ".narsil.lock";
|
|
180
|
-
var STAGING_PREFIX_PATTERN = /^\.[A-Za-z0-9][A-Za-z0-9._-]*\/$/;
|
|
181
|
-
function digestOf(indexName) {
|
|
182
|
-
let hex = "";
|
|
183
|
-
for (const byte of sha256(new TextEncoder().encode(indexName))) hex += byte.toString(16).padStart(2, "0");
|
|
184
|
-
return hex;
|
|
185
|
-
}
|
|
186
|
-
function journalKey(indexName) {
|
|
187
|
-
return `${RESTORE_JOURNAL_PREFIX}${digestOf(indexName)}`;
|
|
188
|
-
}
|
|
189
|
-
function randomSuffix() {
|
|
190
|
-
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
|
|
191
|
-
}
|
|
192
|
-
function newStagingPrefix() {
|
|
193
|
-
return `${RESTORE_STAGING_PREFIX}${randomSuffix()}/`;
|
|
194
|
-
}
|
|
195
|
-
function newRetiredPrefix() {
|
|
196
|
-
return `${RETIRED_INDEX_PREFIX}${randomSuffix()}/`;
|
|
197
|
-
}
|
|
198
|
-
function encodeJournal(indexName, stagingPrefix) {
|
|
199
|
-
return new TextEncoder().encode(`${indexName}
|
|
200
|
-
${stagingPrefix}`);
|
|
201
|
-
}
|
|
202
|
-
function isStagingPrefix(prefix) {
|
|
203
|
-
return STAGING_PREFIX_PATTERN.test(prefix) && !prefix.includes("..");
|
|
204
|
-
}
|
|
205
|
-
function corruptJournal(key, reason) {
|
|
206
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The restore journal "${key}" ${reason}`, { key });
|
|
207
|
-
}
|
|
208
|
-
function readJournal(key, bytes) {
|
|
209
|
-
const text = new TextDecoder().decode(bytes);
|
|
210
|
-
const lineEnd = text.indexOf("\n");
|
|
211
|
-
const indexName = lineEnd === -1 ? text : text.slice(0, lineEnd);
|
|
212
|
-
if (key.slice(RESTORE_JOURNAL_PREFIX.length) !== digestOf(indexName)) return { kind: "leftover" };
|
|
213
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH || !INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
214
|
-
corruptJournal(key, "names an index that no index name can match");
|
|
215
|
-
}
|
|
216
|
-
const stagingPrefix = lineEnd === -1 ? "" : text.slice(lineEnd + 1);
|
|
217
|
-
if (!isStagingPrefix(stagingPrefix) || stagingPrefix === `${LOCK_FILE_KEY}/`) {
|
|
218
|
-
corruptJournal(key, "names no staging prefix that a restore writes");
|
|
219
|
-
}
|
|
220
|
-
return { kind: "journal", indexName, stagingPrefix };
|
|
221
|
-
}
|
|
222
|
-
async function listedKeys(storage, indexName, stagingPrefix) {
|
|
223
|
-
const manifestBytes = await storage.read(`${stagingPrefix}manifest`) ?? await storage.read(manifestKey(indexName));
|
|
224
|
-
if (manifestBytes === null) {
|
|
225
|
-
throw new NarsilError(
|
|
226
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
227
|
-
`The restore of index "${indexName}" lost its manifest, both from staging and from the index`,
|
|
228
|
-
{ indexName, stagingPrefix }
|
|
229
|
-
);
|
|
230
|
-
}
|
|
231
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
232
|
-
if (decoded.kind !== "current") {
|
|
233
|
-
throw new NarsilError(
|
|
234
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
235
|
-
`The restore of index "${indexName}" staged a manifest in the layout of an earlier release`,
|
|
236
|
-
{ indexName }
|
|
237
|
-
);
|
|
238
|
-
}
|
|
239
|
-
const root = `${indexName}/`;
|
|
240
|
-
const keys = /* @__PURE__ */ new Set(["meta", "manifest"]);
|
|
241
|
-
for (const ref of manifestFileRefs(decoded.manifest)) keys.add(ref.key.slice(root.length));
|
|
242
|
-
return keys;
|
|
243
|
-
}
|
|
244
|
-
function withoutTrailingSlash(prefix) {
|
|
245
|
-
return prefix.slice(0, -1);
|
|
246
|
-
}
|
|
247
|
-
async function finishJournal(storage, indexName, stagingPrefix) {
|
|
248
|
-
const root = `${indexName}/`;
|
|
249
|
-
const listed = await listedKeys(storage, indexName, stagingPrefix);
|
|
250
|
-
const staged = await storage.list(stagingPrefix);
|
|
251
|
-
let retiredPrefix = null;
|
|
252
|
-
if (storage.moveTree !== void 0 && staged.length > 0) {
|
|
253
|
-
retiredPrefix = newRetiredPrefix();
|
|
254
|
-
await storage.moveTree(indexName, withoutTrailingSlash(retiredPrefix));
|
|
255
|
-
await storage.moveTree(withoutTrailingSlash(stagingPrefix), indexName);
|
|
256
|
-
} else {
|
|
257
|
-
const stagedKeys = new Set(staged);
|
|
258
|
-
for (const key of listed) {
|
|
259
|
-
if (stagedKeys.has(`${stagingPrefix}${key}`)) await storage.moveKey(`${stagingPrefix}${key}`, `${root}${key}`);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
for (const key of await storage.list(root)) {
|
|
263
|
-
if (!listed.has(key.slice(root.length))) await storage.remove(key);
|
|
264
|
-
}
|
|
265
|
-
await storage.remove(journalKey(indexName));
|
|
266
|
-
return retiredPrefix;
|
|
267
|
-
}
|
|
268
|
-
async function removePrefix(storage, prefix) {
|
|
269
|
-
if (storage.removeTree !== void 0) {
|
|
270
|
-
await storage.removeTree(prefix);
|
|
271
|
-
return;
|
|
272
|
-
}
|
|
273
|
-
for (const key of await storage.list(prefix)) await storage.remove(key);
|
|
274
|
-
}
|
|
275
|
-
async function commitJournal(storage, indexName, stagingPrefix) {
|
|
276
|
-
await storage.writeJournal(journalKey(indexName), encodeJournal(indexName, stagingPrefix));
|
|
277
|
-
}
|
|
278
|
-
function isTopLevelJournal(key) {
|
|
279
|
-
return key.startsWith(RESTORE_JOURNAL_PREFIX) && !key.includes("/");
|
|
280
|
-
}
|
|
281
|
-
async function removeStagingPrefixes(storage) {
|
|
282
|
-
const prefixes = /* @__PURE__ */ new Set();
|
|
283
|
-
for (const key of await storage.list(".")) {
|
|
284
|
-
const slash = key.indexOf("/");
|
|
285
|
-
if (slash !== -1 && isStagingPrefix(key.slice(0, slash + 1))) prefixes.add(key.slice(0, slash + 1));
|
|
286
|
-
}
|
|
287
|
-
for (const prefix of prefixes) await removePrefix(storage, prefix);
|
|
288
|
-
}
|
|
289
|
-
async function recoverRestoreJournals(storage) {
|
|
290
|
-
for (const key of await storage.list(RESTORE_JOURNAL_PREFIX)) {
|
|
291
|
-
if (!isTopLevelJournal(key)) continue;
|
|
292
|
-
const bytes = await storage.read(key);
|
|
293
|
-
if (bytes === null) continue;
|
|
294
|
-
const reading = readJournal(key, bytes);
|
|
295
|
-
if (reading.kind === "leftover") {
|
|
296
|
-
await storage.remove(key);
|
|
297
|
-
continue;
|
|
298
|
-
}
|
|
299
|
-
await finishJournal(storage, reading.indexName, reading.stagingPrefix);
|
|
300
|
-
}
|
|
301
|
-
await removeStagingPrefixes(storage);
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
// src/persistence/durability/restore-directory.ts
|
|
305
|
-
function missing(error) {
|
|
306
|
-
return error.code === "ENOENT";
|
|
307
|
-
}
|
|
308
|
-
function storageFailed(code, message, key, error) {
|
|
309
|
-
return new NarsilError(code, message, { key, cause: error instanceof Error ? error.message : String(error) });
|
|
310
|
-
}
|
|
311
|
-
function directoryRestoreStorage(directory) {
|
|
312
|
-
return {
|
|
313
|
-
read: (key) => directory.read(key),
|
|
314
|
-
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
315
|
-
writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
|
|
316
|
-
remove: (key) => directory.remove(key),
|
|
317
|
-
list: (prefix) => directory.list(prefix),
|
|
318
|
-
moveKey: (from, to) => directory.move(from, to),
|
|
319
|
-
pathOf: (key) => directory.pathOf(key),
|
|
320
|
-
async moveTree(from, to) {
|
|
321
|
-
const source = await directory.pathOf(from);
|
|
322
|
-
const target = await directory.pathOf(to);
|
|
323
|
-
const fs = await getFs();
|
|
324
|
-
try {
|
|
325
|
-
await fs.rename(source, target);
|
|
326
|
-
} catch (error) {
|
|
327
|
-
if (missing(error)) return;
|
|
328
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Failed to move "${from}" to "${to}"`, from, error);
|
|
329
|
-
}
|
|
330
|
-
await directory.syncDirectoryOf(to);
|
|
331
|
-
},
|
|
332
|
-
async removeTree(prefix) {
|
|
333
|
-
const key = prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
334
|
-
const fs = await getFs();
|
|
335
|
-
try {
|
|
336
|
-
await fs.rm(await directory.pathOf(key), { recursive: true, force: true });
|
|
337
|
-
} catch (error) {
|
|
338
|
-
throw storageFailed(ErrorCodes.PERSISTENCE_DELETE_FAILED, `Failed to delete "${key}"`, key, error);
|
|
339
|
-
}
|
|
340
|
-
await directory.syncDirectoryOf(key);
|
|
341
|
-
}
|
|
342
|
-
};
|
|
343
|
-
}
|
|
344
|
-
function recoverDirectoryRestores(directory) {
|
|
345
|
-
return recoverRestoreJournals(directoryRestoreStorage(directory));
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
// src/persistence/durability/restore-staging.ts
|
|
349
|
-
function toError(error) {
|
|
350
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
351
|
-
}
|
|
352
|
-
function pathUnder(path, root) {
|
|
353
|
-
if (!path.startsWith(root)) return false;
|
|
354
|
-
const next = path.charAt(root.length);
|
|
355
|
-
return next === "" || next === "/" || next === "\\";
|
|
356
|
-
}
|
|
357
|
-
function createRestoreStaging(storage, indexName, options = {}) {
|
|
358
|
-
const stagingPrefix = newStagingPrefix();
|
|
359
|
-
const indexRoot = `${indexName}/`;
|
|
360
|
-
let committed = false;
|
|
361
|
-
const staged = (key) => {
|
|
362
|
-
if (!key.startsWith(indexRoot)) {
|
|
363
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `A restore of "${indexName}" cannot stage "${key}"`, {
|
|
364
|
-
key
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
return `${stagingPrefix}${key.slice(indexRoot.length)}`;
|
|
368
|
-
};
|
|
369
|
-
const unstaged = (key) => `${indexRoot}${key.slice(stagingPrefix.length)}`;
|
|
370
|
-
const store = {
|
|
371
|
-
read: (key) => storage.read(staged(key)),
|
|
372
|
-
write: (key, chunks) => storage.write(staged(key), chunks),
|
|
373
|
-
remove: (key) => storage.remove(staged(key)),
|
|
374
|
-
list: async (prefix) => (await storage.list(staged(prefix))).map(unstaged)
|
|
375
|
-
};
|
|
376
|
-
const pathOf = storage.pathOf;
|
|
377
|
-
if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
|
|
378
|
-
let retiredPrefix = null;
|
|
379
|
-
async function finishOnStorage() {
|
|
380
|
-
retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
|
|
381
|
-
}
|
|
382
|
-
return {
|
|
383
|
-
store,
|
|
384
|
-
async relocation() {
|
|
385
|
-
if (pathOf === void 0 || (await storage.list(stagingPrefix)).length > 0) return null;
|
|
386
|
-
const from = await pathOf.call(storage, stagingPrefix.slice(0, -1));
|
|
387
|
-
const to = await pathOf.call(storage, indexName);
|
|
388
|
-
return (path) => pathUnder(path, from) ? `${to}${path.slice(from.length)}` : path;
|
|
389
|
-
},
|
|
390
|
-
async commit() {
|
|
391
|
-
try {
|
|
392
|
-
await commitJournal(storage, indexName, stagingPrefix);
|
|
393
|
-
} catch (error) {
|
|
394
|
-
try {
|
|
395
|
-
await storage.remove(journalKey(indexName));
|
|
396
|
-
} catch (removal) {
|
|
397
|
-
committed = true;
|
|
398
|
-
options.onStorageLost?.(toError(removal));
|
|
399
|
-
}
|
|
400
|
-
throw error;
|
|
401
|
-
}
|
|
402
|
-
committed = true;
|
|
403
|
-
},
|
|
404
|
-
async finish() {
|
|
405
|
-
try {
|
|
406
|
-
await (options.finish === void 0 ? finishOnStorage() : options.finish(finishOnStorage));
|
|
407
|
-
} catch (error) {
|
|
408
|
-
options.onStorageLost?.(toError(error));
|
|
409
|
-
}
|
|
410
|
-
},
|
|
411
|
-
async removeRetired() {
|
|
412
|
-
const prefix = retiredPrefix;
|
|
413
|
-
retiredPrefix = null;
|
|
414
|
-
if (prefix !== null) await removePrefix(storage, prefix);
|
|
415
|
-
},
|
|
416
|
-
async discard() {
|
|
417
|
-
if (committed) return;
|
|
418
|
-
await removePrefix(storage, stagingPrefix);
|
|
419
|
-
}
|
|
420
|
-
};
|
|
421
|
-
}
|
|
422
|
-
function adapterRestoreStorage(adapter) {
|
|
423
|
-
const store = adapterCheckpointStore(adapter);
|
|
424
|
-
return {
|
|
425
|
-
read: (key) => store.read(key),
|
|
426
|
-
write: (key, chunks) => store.write(key, chunks),
|
|
427
|
-
writeJournal: (key, bytes) => store.write(key, [bytes]),
|
|
428
|
-
remove: (key) => store.remove(key),
|
|
429
|
-
list: (prefix) => store.list(prefix),
|
|
430
|
-
async moveKey(from, to) {
|
|
431
|
-
const bytes = await store.read(from);
|
|
432
|
-
if (bytes === null) return;
|
|
433
|
-
await store.write(to, [bytes]);
|
|
434
|
-
await store.remove(from);
|
|
435
|
-
}
|
|
436
|
-
};
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
// src/persistence/durability/archive-checkpoint.ts
|
|
440
|
-
function toError2(error) {
|
|
441
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
442
|
-
}
|
|
443
|
-
function stageDirectoryRestore(deps, indexName) {
|
|
444
|
-
return createRestoreStaging(directoryRestoreStorage(deps.directory), indexName, {
|
|
445
|
-
async finish(run) {
|
|
446
|
-
try {
|
|
447
|
-
await run();
|
|
448
|
-
} catch (error) {
|
|
449
|
-
deps.markFatal(toError2(error));
|
|
450
|
-
throw error;
|
|
451
|
-
}
|
|
452
|
-
},
|
|
453
|
-
onStorageLost: deps.markFatal
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
function releaser(held, indexName) {
|
|
457
|
-
let released = false;
|
|
458
|
-
return () => {
|
|
459
|
-
if (released) return;
|
|
460
|
-
released = true;
|
|
461
|
-
const count = (held.get(indexName) ?? 1) - 1;
|
|
462
|
-
if (count <= 0) held.delete(indexName);
|
|
463
|
-
else held.set(indexName, count);
|
|
464
|
-
};
|
|
465
|
-
}
|
|
466
|
-
function createArchiveCheckpointAccess(deps) {
|
|
467
|
-
const store = directoryCheckpointStore(deps.directory);
|
|
468
|
-
return {
|
|
469
|
-
async openArchiveCheckpoint(indexName) {
|
|
470
|
-
deps.held.set(indexName, (deps.held.get(indexName) ?? 0) + 1);
|
|
471
|
-
const release = releaser(deps.held, indexName);
|
|
472
|
-
try {
|
|
473
|
-
await deps.checkpoint(indexName);
|
|
474
|
-
await deps.checkpoint(indexName);
|
|
475
|
-
const manifestBytes = await deps.directory.read(manifestKey(indexName));
|
|
476
|
-
if (manifestBytes === null) {
|
|
477
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds no checkpoint`, {
|
|
478
|
-
indexName
|
|
479
|
-
});
|
|
480
|
-
}
|
|
481
|
-
const decoded = await decodeSegmentManifest(manifestBytes, indexName);
|
|
482
|
-
if (decoded.kind !== "current") {
|
|
483
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `Index "${indexName}" holds an earlier layout`, {
|
|
484
|
-
indexName
|
|
485
|
-
});
|
|
486
|
-
}
|
|
487
|
-
return {
|
|
488
|
-
manifestBytes,
|
|
489
|
-
manifest: decoded.manifest,
|
|
490
|
-
readFile: (ref) => readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" }),
|
|
491
|
-
readRange: async (ref, offset, length) => {
|
|
492
|
-
const readRange = deps.directory.readRange;
|
|
493
|
-
if (readRange !== void 0) return readRange.call(deps.directory, ref.key, offset, length);
|
|
494
|
-
const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
|
|
495
|
-
return whole.slice(offset, offset + length);
|
|
496
|
-
},
|
|
497
|
-
release
|
|
498
|
-
};
|
|
499
|
-
} catch (error) {
|
|
500
|
-
release();
|
|
501
|
-
throw error;
|
|
502
|
-
}
|
|
503
|
-
},
|
|
504
|
-
stageRestore(indexName) {
|
|
505
|
-
return stageDirectoryRestore(deps, indexName);
|
|
506
|
-
},
|
|
507
|
-
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
508
|
-
for (const { partitionId, lastSeqNo } of checkpoint) deps.registerPartition(indexName, partitionId, lastSeqNo);
|
|
509
|
-
}
|
|
510
|
-
};
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
// src/persistence/durability/checkpoint.ts
|
|
514
|
-
function walPrefix(indexName, partitionId) {
|
|
515
|
-
return `${indexName}/wal/${partitionId}/`;
|
|
516
|
-
}
|
|
517
|
-
async function truncateCoveredSegments(directory, indexName, checkpoint) {
|
|
518
|
-
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
519
|
-
const prefix = walPrefix(indexName, partitionId);
|
|
520
|
-
const markerBytes = await directory.read(`${prefix}commit`);
|
|
521
|
-
const marker = markerBytes === null ? null : readCommitMarker(markerBytes);
|
|
522
|
-
if (marker === null) {
|
|
523
|
-
continue;
|
|
524
|
-
}
|
|
525
|
-
const activeSegmentSeqNo = marker.state.activeSegmentSeqNo;
|
|
526
|
-
const segments = await directory.list(prefix);
|
|
527
|
-
const startSeqNos = segments.map((key) => parseSegmentStartSeqNo(key, prefix)).filter((value) => value !== null).sort((a, b) => a - b);
|
|
528
|
-
for (let i = 0; i < startSeqNos.length; i += 1) {
|
|
529
|
-
const start = startSeqNos[i];
|
|
530
|
-
if (start >= activeSegmentSeqNo) {
|
|
531
|
-
break;
|
|
532
|
-
}
|
|
533
|
-
const nextStart = startSeqNos[i + 1];
|
|
534
|
-
const segmentCoversBeyondCheckpoint = nextStart === void 0 || nextStart > lastSeqNo + 1;
|
|
535
|
-
if (segmentCoversBeyondCheckpoint) {
|
|
536
|
-
break;
|
|
537
|
-
}
|
|
538
|
-
await directory.remove(`${prefix}${formatSegmentStart(start)}`);
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
async function reclaimWalBeyondCount(directory, indexName, partitionCount, partitions, onCloseError) {
|
|
543
|
-
for (const [partitionId, partition] of [...partitions]) {
|
|
544
|
-
if (partitionId < partitionCount) continue;
|
|
545
|
-
try {
|
|
546
|
-
await partition.walWriter.close();
|
|
547
|
-
} catch (err) {
|
|
548
|
-
onCloseError(err instanceof Error ? err : new Error(String(err)));
|
|
549
|
-
}
|
|
550
|
-
partitions.delete(partitionId);
|
|
551
|
-
}
|
|
552
|
-
const walRoot = `${indexName}/wal/`;
|
|
553
|
-
for (const key of await directory.list(walRoot)) {
|
|
554
|
-
const partitionId = walKeyPartitionId(key, walRoot);
|
|
555
|
-
if (partitionId === null || partitionId < partitionCount) continue;
|
|
556
|
-
await directory.remove(key);
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
function walKeyPartitionId(key, walRoot) {
|
|
560
|
-
const tail = key.slice(walRoot.length);
|
|
561
|
-
const separator = tail.indexOf("/");
|
|
562
|
-
if (separator <= 0) {
|
|
563
|
-
return null;
|
|
564
|
-
}
|
|
565
|
-
const segment = tail.slice(0, separator);
|
|
566
|
-
if (!/^\d+$/.test(segment)) {
|
|
567
|
-
return null;
|
|
568
|
-
}
|
|
569
|
-
const value = Number.parseInt(segment, 10);
|
|
570
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
571
|
-
}
|
|
572
|
-
function parseSegmentStartSeqNo(key, prefix) {
|
|
573
|
-
if (!key.startsWith(prefix)) {
|
|
574
|
-
return null;
|
|
575
|
-
}
|
|
576
|
-
const tail = key.slice(prefix.length);
|
|
577
|
-
if (!/^\d{16}$/.test(tail)) {
|
|
578
|
-
return null;
|
|
579
|
-
}
|
|
580
|
-
const value = Number.parseInt(tail, 10);
|
|
581
|
-
return Number.isSafeInteger(value) ? value : null;
|
|
582
|
-
}
|
|
583
|
-
function formatSegmentStart(startSeqNo) {
|
|
584
|
-
return startSeqNo.toString().padStart(16, "0");
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
// src/persistence/durability/recovery.ts
|
|
588
|
-
async function listPersistedIndexes(directory) {
|
|
589
|
-
const metaKeys = await directory.list("");
|
|
590
|
-
const names = [];
|
|
591
|
-
for (const key of metaKeys) {
|
|
592
|
-
if (key.endsWith("/meta")) {
|
|
593
|
-
names.push(key.slice(0, -"/meta".length));
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
return names;
|
|
597
|
-
}
|
|
598
|
-
async function loadMetadata(directory, indexName) {
|
|
599
|
-
const bytes = await directory.read(`${indexName}/meta`);
|
|
600
|
-
if (bytes === null) {
|
|
601
|
-
return null;
|
|
602
|
-
}
|
|
603
|
-
const { metadata } = await readMetadataEnvelope(bytes);
|
|
604
|
-
return metadata;
|
|
605
|
-
}
|
|
606
|
-
async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
|
|
607
|
-
const read = await readWalSegments(directory, indexName, partitionId);
|
|
608
|
-
if (read === null) {
|
|
609
|
-
return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
|
|
610
|
-
}
|
|
611
|
-
const { marker, segments, durableEntries, activeTail, highestReadFromWal } = read;
|
|
612
|
-
if (Math.max(fromSeqNoExclusive, highestReadFromWal) < marker.state.highestDurableSeqNo) {
|
|
613
|
-
throw durableRecordMissing(indexName, partitionId, highestReadFromWal, marker);
|
|
614
|
-
}
|
|
615
|
-
let highestSeqNo = fromSeqNoExclusive;
|
|
616
|
-
let highestPrimaryTerm = 0;
|
|
617
|
-
const replay = (entries) => {
|
|
618
|
-
for (const entry of entries) {
|
|
619
|
-
if (entry.seqNo <= highestSeqNo) {
|
|
620
|
-
continue;
|
|
621
|
-
}
|
|
622
|
-
applyEntry(entry, deps);
|
|
623
|
-
highestSeqNo = entry.seqNo;
|
|
624
|
-
if (entry.primaryTerm > highestPrimaryTerm) {
|
|
625
|
-
highestPrimaryTerm = entry.primaryTerm;
|
|
626
|
-
}
|
|
627
|
-
}
|
|
628
|
-
};
|
|
629
|
-
replay(durableEntries);
|
|
630
|
-
if (activeTail !== null) {
|
|
631
|
-
replay(activeTail.entries);
|
|
632
|
-
}
|
|
633
|
-
await deleteOrphanSegments(directory, segments, marker.state.activeSegmentSeqNo);
|
|
634
|
-
if (activeTail !== null && activeTail.cleanEnd < activeTail.segmentLength) {
|
|
635
|
-
await truncateSegmentTail(directory, activeTail.key, activeTail.cleanEnd);
|
|
636
|
-
}
|
|
637
|
-
return { highestSeqNo, highestPrimaryTerm };
|
|
638
|
-
}
|
|
639
|
-
async function deleteOrphanSegments(directory, segments, activeSegmentSeqNo) {
|
|
640
|
-
for (const { key, startSeqNo } of segments) {
|
|
641
|
-
if (startSeqNo > activeSegmentSeqNo) {
|
|
642
|
-
await directory.remove(key);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
async function truncateSegmentTail(directory, key, durableByteLength) {
|
|
647
|
-
const cleanLength = Math.max(durableByteLength, SEGMENT_HEADER_SIZE);
|
|
648
|
-
const handle = await directory.appendHandle(key);
|
|
649
|
-
try {
|
|
650
|
-
const currentSize = await handle.size();
|
|
651
|
-
if (cleanLength < currentSize) {
|
|
652
|
-
await handle.truncate(cleanLength);
|
|
653
|
-
}
|
|
654
|
-
} finally {
|
|
655
|
-
await handle.close();
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
function applyEntry(entry, deps) {
|
|
659
|
-
if (entry.operation === "DELETE") {
|
|
660
|
-
applyDeleteEntry(entry, deps.manager, deps.vectorIndexes);
|
|
661
|
-
return;
|
|
662
|
-
}
|
|
663
|
-
applyIndexEntry(entry, deps.manager, deps.vectorFieldPaths, deps.vectorIndexes);
|
|
664
|
-
}
|
|
665
|
-
function snapshotCheckpointFor(checkpoint, partitionId) {
|
|
666
|
-
return checkpointLastSeqNo(checkpoint, partitionId);
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
// src/persistence/durability/checkpoint-capture.ts
|
|
670
|
-
function whileNoMutationApplies(partitions, capture2) {
|
|
671
|
-
return new Promise((resolve, reject) => {
|
|
672
|
-
let release = () => void 0;
|
|
673
|
-
const released = new Promise((settle) => {
|
|
674
|
-
release = settle;
|
|
675
|
-
});
|
|
676
|
-
function captureNow() {
|
|
677
|
-
try {
|
|
678
|
-
resolve(capture2());
|
|
679
|
-
} catch (error) {
|
|
680
|
-
reject(error);
|
|
681
|
-
} finally {
|
|
682
|
-
release();
|
|
683
|
-
}
|
|
684
|
-
}
|
|
685
|
-
let waiting = partitions.length;
|
|
686
|
-
if (waiting === 0) {
|
|
687
|
-
captureNow();
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
for (const partition of partitions) {
|
|
691
|
-
partition.appendChain = partition.appendChain.then(() => {
|
|
692
|
-
waiting -= 1;
|
|
693
|
-
if (waiting === 0) captureNow();
|
|
694
|
-
return released;
|
|
695
|
-
});
|
|
696
|
-
}
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
async function makeEveryAppliedMutationDurable(indexState, markFatal) {
|
|
700
|
-
for (const partition of [...indexState.partitions.values()]) {
|
|
701
|
-
await partition.appendChain;
|
|
702
|
-
if (partition.failed !== null) throw partition.failed;
|
|
703
|
-
try {
|
|
704
|
-
await partition.walWriter.commit();
|
|
705
|
-
} catch (error) {
|
|
706
|
-
partition.failed = error instanceof Error ? error : new Error(String(error));
|
|
707
|
-
markFatal(partition.failed);
|
|
708
|
-
throw partition.failed;
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
function noWholePartitions() {
|
|
713
|
-
return { contents: [], captured: [] };
|
|
714
|
-
}
|
|
715
|
-
function wholePartitionsWhereMostChanged(manager, rules) {
|
|
716
|
-
const { priorCheckpoint, priorPartitionIds, everyPartition, aWorkerCanSerialise } = rules;
|
|
717
|
-
return (targets) => {
|
|
718
|
-
const whole = { contents: [], captured: [] };
|
|
719
|
-
const changed = targets.filter(
|
|
720
|
-
(target) => everyPartition || target.lastSeqNo !== snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
721
|
-
);
|
|
722
|
-
const mostOfEachChanged = changed.every((target) => {
|
|
723
|
-
const records = target.lastSeqNo - snapshotCheckpointFor(priorCheckpoint, target.partitionId);
|
|
724
|
-
const documents = manager.getPartition(target.partitionId).count();
|
|
725
|
-
return documents <= WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT && records >= documents * WHOLE_PARTITION_MIN_CHANGED_SHARE;
|
|
726
|
-
});
|
|
727
|
-
const logMatchesMemory = targets.length === 1 && priorPartitionIds.every((partitionId) => partitionId === 0);
|
|
728
|
-
if (!everyPartition && !(mostOfEachChanged && logMatchesMemory)) return whole;
|
|
729
|
-
for (const { partitionId } of changed) {
|
|
730
|
-
const captured = aWorkerCanSerialise ? captureWholePartition(manager, partitionId) : null;
|
|
731
|
-
if (captured !== null) {
|
|
732
|
-
whole.captured.push(captured);
|
|
733
|
-
} else if (everyPartition || !aWorkerCanSerialise) {
|
|
734
|
-
whole.contents.push(partitionContentsOf(manager, partitionId));
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
return whole;
|
|
738
|
-
};
|
|
739
|
-
}
|
|
740
|
-
async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = []) {
|
|
741
|
-
for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
742
|
-
return whileNoMutationApplies([...indexState.partitions.values()], () => {
|
|
743
|
-
const documentCount = manager.countDocuments();
|
|
744
|
-
const targets = [];
|
|
745
|
-
let recordsLeftInLog = 0;
|
|
746
|
-
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
747
|
-
const partition = indexState.partitions.get(partitionId);
|
|
748
|
-
const appliedSeqNo = partition?.appliedSeqNo ?? 0;
|
|
749
|
-
const lastSeqNo = priorCheckpointThatLimitsTheRecords === void 0 ? appliedSeqNo : Math.min(
|
|
750
|
-
appliedSeqNo,
|
|
751
|
-
snapshotCheckpointFor(priorCheckpointThatLimitsTheRecords, partitionId) + LOG_RECORDS_PER_CHECKPOINT
|
|
752
|
-
);
|
|
753
|
-
recordsLeftInLog += appliedSeqNo - lastSeqNo;
|
|
754
|
-
targets.push({
|
|
755
|
-
partitionId,
|
|
756
|
-
lastSeqNo,
|
|
757
|
-
primaryTerm: partition?.seqOwner.primaryTerm ?? SINGLE_NODE_PRIMARY_TERM
|
|
758
|
-
});
|
|
759
|
-
}
|
|
760
|
-
const vectorPlans = /* @__PURE__ */ new Map();
|
|
761
|
-
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
762
|
-
const listed = listedVectorFields.find((ref) => ref.fieldPath === fieldPath);
|
|
763
|
-
const listedKeys2 = listed === void 0 ? null : listed.files.map((file) => file.key);
|
|
764
|
-
vectorPlans.set(fieldPath, vectorIndex.planCheckpoint(listedKeys2));
|
|
765
|
-
}
|
|
766
|
-
const memoryMatchesTheTargets = recordsLeftInLog === 0;
|
|
767
|
-
const wholePartitions = memoryMatchesTheTargets ? wholePartitionsOf(targets) : noWholePartitions();
|
|
768
|
-
return { targets, documentCount, vectorPlans, wholePartitions, recordsLeftInLog };
|
|
769
|
-
});
|
|
770
|
-
}
|
|
771
|
-
async function writeCapturedVectors(store, indexName, capture2, priorManifest, rewriteUnchanged) {
|
|
772
|
-
if (capture2.vectorPlans.size === 0) return { vectors: [], layouts: [], written: [] };
|
|
773
|
-
const priorVectors = priorManifest?.vectors ?? [];
|
|
774
|
-
const priorCheckpoint = priorManifest?.checkpoint ?? [];
|
|
775
|
-
const noPartitionChanged = capture2.targets.every(
|
|
776
|
-
(target) => target.lastSeqNo === snapshotCheckpointFor(priorCheckpoint, target.partitionId)
|
|
777
|
-
);
|
|
778
|
-
const everyFieldWritten = [...capture2.vectorPlans.keys()].every(
|
|
779
|
-
(fieldPath) => priorVectors.some((ref) => ref.fieldPath === fieldPath)
|
|
780
|
-
);
|
|
781
|
-
if (!rewriteUnchanged && everyFieldWritten && noPartitionChanged) {
|
|
782
|
-
return { vectors: priorVectors, layouts: [], written: [] };
|
|
783
|
-
}
|
|
784
|
-
const written = await writeLiveVectors({ store, indexName, plans: capture2.vectorPlans, priorVectors });
|
|
785
|
-
return { vectors: written.refs, layouts: written.layouts, written: written.written };
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
// src/persistence/durability/checkpoint-worker-dispatch.ts
|
|
789
|
-
var workerUsable = true;
|
|
790
|
-
var pooledWorker = null;
|
|
791
|
-
var workerBusy = false;
|
|
792
|
-
var spawnedWorkerCount = 0;
|
|
793
|
-
var workerIdleMs = CHECKPOINT_WORKER_IDLE_MS;
|
|
794
|
-
var idleTimer = null;
|
|
795
|
-
async function spawnWorker() {
|
|
796
|
-
const entryPoint = resolveWorkerEntry(
|
|
797
|
-
import.meta.url,
|
|
798
|
-
/\/src\/persistence\/durability\/[^/]+$/,
|
|
799
|
-
"persistence/durability/checkpoint-worker.mjs"
|
|
800
|
-
);
|
|
801
|
-
if (entryPoint === null) return null;
|
|
802
|
-
try {
|
|
803
|
-
const worker = await spawnNodeWorker(new URL(entryPoint));
|
|
804
|
-
if (worker === null) {
|
|
805
|
-
return null;
|
|
806
|
-
}
|
|
807
|
-
if (typeof worker.unref === "function") {
|
|
808
|
-
worker.unref();
|
|
809
|
-
}
|
|
810
|
-
worker.on("error", () => discardWorker(worker));
|
|
811
|
-
worker.on("exit", () => discardWorker(worker));
|
|
812
|
-
spawnedWorkerCount += 1;
|
|
813
|
-
return worker;
|
|
814
|
-
} catch {
|
|
815
|
-
return null;
|
|
816
|
-
}
|
|
817
|
-
}
|
|
818
|
-
function discardWorker(worker) {
|
|
819
|
-
if (pooledWorker === worker) {
|
|
820
|
-
pooledWorker = null;
|
|
821
|
-
}
|
|
822
|
-
try {
|
|
823
|
-
void worker.terminate();
|
|
824
|
-
} catch {
|
|
825
|
-
}
|
|
826
|
-
}
|
|
827
|
-
function unrefTimer(timer) {
|
|
828
|
-
if (typeof timer.unref === "function") {
|
|
829
|
-
timer.unref();
|
|
830
|
-
}
|
|
831
|
-
return timer;
|
|
832
|
-
}
|
|
833
|
-
function runWorker(worker, request) {
|
|
834
|
-
return new Promise((resolve) => {
|
|
835
|
-
let settled = false;
|
|
836
|
-
let timeoutId = armSilenceTimeout();
|
|
837
|
-
function armSilenceTimeout() {
|
|
838
|
-
return unrefTimer(setTimeout(() => settle({ written: null, timedOut: true }, true), CHECKPOINT_WORKER_TIMEOUT_MS));
|
|
839
|
-
}
|
|
840
|
-
const onMessage = (msg) => {
|
|
841
|
-
const response = msg;
|
|
842
|
-
if (response.type === "heartbeat") {
|
|
843
|
-
clearTimeout(timeoutId);
|
|
844
|
-
timeoutId = armSilenceTimeout();
|
|
845
|
-
return;
|
|
846
|
-
}
|
|
847
|
-
if (response.type === "success") {
|
|
848
|
-
settle({ written: response.segments, timedOut: false }, false);
|
|
849
|
-
} else {
|
|
850
|
-
settle({ written: null, timedOut: false }, true);
|
|
851
|
-
}
|
|
852
|
-
};
|
|
853
|
-
const onError = () => {
|
|
854
|
-
settle({ written: null, timedOut: false }, true);
|
|
855
|
-
};
|
|
856
|
-
const onExit = () => {
|
|
857
|
-
settle({ written: null, timedOut: false }, true);
|
|
858
|
-
};
|
|
859
|
-
function settle(outcome, discard) {
|
|
860
|
-
if (settled) {
|
|
861
|
-
return;
|
|
862
|
-
}
|
|
863
|
-
settled = true;
|
|
864
|
-
clearTimeout(timeoutId);
|
|
865
|
-
worker.off("message", onMessage);
|
|
866
|
-
worker.off("error", onError);
|
|
867
|
-
worker.off("exit", onExit);
|
|
868
|
-
if (discard) {
|
|
869
|
-
discardWorker(worker);
|
|
870
|
-
}
|
|
871
|
-
resolve(outcome);
|
|
872
|
-
}
|
|
873
|
-
worker.on("message", onMessage);
|
|
874
|
-
worker.on("error", onError);
|
|
875
|
-
worker.on("exit", onExit);
|
|
876
|
-
try {
|
|
877
|
-
worker.postMessage(request);
|
|
878
|
-
} catch {
|
|
879
|
-
settle({ written: null, timedOut: false }, true);
|
|
880
|
-
}
|
|
881
|
-
});
|
|
882
|
-
}
|
|
883
|
-
function delay(ms) {
|
|
884
|
-
return new Promise((resolve) => {
|
|
885
|
-
unrefTimer(setTimeout(resolve, ms));
|
|
886
|
-
});
|
|
887
|
-
}
|
|
888
|
-
function checkpointWorkerIsUsable() {
|
|
889
|
-
const runtime = detectRuntime();
|
|
890
|
-
return runtime.supportsWorkerThreads && runtime.supportsFileSystem && workerUsable;
|
|
891
|
-
}
|
|
892
|
-
async function runCheckpointOnWorker(request) {
|
|
893
|
-
const runtime = detectRuntime();
|
|
894
|
-
if (!runtime.supportsWorkerThreads || !runtime.supportsFileSystem || !workerUsable) {
|
|
895
|
-
return null;
|
|
896
|
-
}
|
|
897
|
-
if (workerBusy) {
|
|
898
|
-
return null;
|
|
899
|
-
}
|
|
900
|
-
workerBusy = true;
|
|
901
|
-
cancelIdleRetirement();
|
|
902
|
-
try {
|
|
903
|
-
if (pooledWorker === null) {
|
|
904
|
-
pooledWorker = await spawnWorker();
|
|
905
|
-
}
|
|
906
|
-
const worker = pooledWorker;
|
|
907
|
-
if (worker === null) {
|
|
908
|
-
workerUsable = false;
|
|
909
|
-
return null;
|
|
910
|
-
}
|
|
911
|
-
const outcome = await runWorker(worker, request);
|
|
912
|
-
if (outcome.written !== null) {
|
|
913
|
-
return outcome.written;
|
|
914
|
-
}
|
|
915
|
-
if (outcome.timedOut) {
|
|
916
|
-
await delay(CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS);
|
|
917
|
-
}
|
|
918
|
-
return null;
|
|
919
|
-
} finally {
|
|
920
|
-
workerBusy = false;
|
|
921
|
-
retireWorkerOnceIdle();
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
function cancelIdleRetirement() {
|
|
925
|
-
if (idleTimer !== null) {
|
|
926
|
-
clearTimeout(idleTimer);
|
|
927
|
-
idleTimer = null;
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
function retireWorkerOnceIdle() {
|
|
931
|
-
cancelIdleRetirement();
|
|
932
|
-
idleTimer = unrefTimer(
|
|
933
|
-
setTimeout(() => {
|
|
934
|
-
idleTimer = null;
|
|
935
|
-
if (!workerBusy) terminateCheckpointWorker();
|
|
936
|
-
}, workerIdleMs)
|
|
937
|
-
);
|
|
938
|
-
}
|
|
939
|
-
function terminateCheckpointWorker() {
|
|
940
|
-
cancelIdleRetirement();
|
|
941
|
-
const worker = pooledWorker;
|
|
942
|
-
if (worker === null) {
|
|
943
|
-
return;
|
|
944
|
-
}
|
|
945
|
-
pooledWorker = null;
|
|
946
|
-
try {
|
|
947
|
-
void worker.terminate();
|
|
948
|
-
} catch {
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
// src/persistence/durability/checkpoint-write.ts
|
|
953
|
-
function workerCanWriteEverySegment(input) {
|
|
954
|
-
if (!input.canOffload || input.wholePartitions.length > 0) return false;
|
|
955
|
-
const analysesTextInProcessOnly = input.metadata.tokenizer !== void 0 || input.metadata.stopWords !== void 0;
|
|
956
|
-
if (!analysesTextInProcessOnly) return true;
|
|
957
|
-
return input.targets.every(
|
|
958
|
-
(target) => input.capturedPartitions.some((captured) => captured.partitionId === target.partitionId)
|
|
959
|
-
);
|
|
960
|
-
}
|
|
961
|
-
async function writeIndexCheckpointSegments(input) {
|
|
962
|
-
const { directory, metadata, targets, compactionThreshold, prior, wholePartitions, capturedPartitions } = input;
|
|
963
|
-
const offloaded = workerCanWriteEverySegment(input) ? await runCheckpointOnWorker({
|
|
964
|
-
root: directory.root,
|
|
965
|
-
metadata,
|
|
966
|
-
targets,
|
|
967
|
-
compactionThreshold,
|
|
968
|
-
prior,
|
|
969
|
-
capturedPartitions: [...capturedPartitions]
|
|
970
|
-
}) : null;
|
|
971
|
-
if (offloaded !== null) {
|
|
972
|
-
return offloaded;
|
|
973
|
-
}
|
|
974
|
-
return writeCheckpointSegments({
|
|
975
|
-
store: directoryCheckpointStore(directory),
|
|
976
|
-
directory,
|
|
977
|
-
metadata,
|
|
978
|
-
targets,
|
|
979
|
-
compactionThreshold,
|
|
980
|
-
prior,
|
|
981
|
-
wholePartitions,
|
|
982
|
-
capturedPartitions
|
|
983
|
-
});
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
// src/persistence/durability/segment/vector-adoption.ts
|
|
987
|
-
function recordWrittenVectors(vectorIndexes, written) {
|
|
988
|
-
for (const field of written) {
|
|
989
|
-
vectorIndexes.get(field.fieldPath)?.recordCheckpoint(field.plan, field.files);
|
|
990
|
-
}
|
|
991
|
-
}
|
|
992
|
-
async function adoptVectorLayouts(store, vectorIndexes, layouts) {
|
|
993
|
-
const pathOf = store.pathOf;
|
|
994
|
-
if (pathOf === void 0) return;
|
|
995
|
-
for (const layout of layouts) {
|
|
996
|
-
const vecIndex = vectorIndexes.get(layout.fieldPath);
|
|
997
|
-
if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
|
|
998
|
-
const path = await pathOf.call(store, layout.key);
|
|
999
|
-
await vecIndex.adoptDiskLayout({
|
|
1000
|
-
path,
|
|
1001
|
-
docIds: layout.docIds,
|
|
1002
|
-
dead: layout.dead,
|
|
1003
|
-
vectorsOffset: layout.vectorsOffset
|
|
1004
|
-
});
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
|
|
1008
|
-
const pathOf = store.pathOf;
|
|
1009
|
-
if (pathOf === void 0) return [...keys];
|
|
1010
|
-
const read = /* @__PURE__ */ new Set();
|
|
1011
|
-
for (const vectorIndex of vectorIndexes.values()) {
|
|
1012
|
-
for (const path of vectorIndex.vectorFilesInUse()) read.add(path);
|
|
1013
|
-
}
|
|
1014
|
-
if (read.size === 0) return [...keys];
|
|
1015
|
-
const unread = [];
|
|
1016
|
-
for (const key of keys) {
|
|
1017
|
-
if (!read.has(await pathOf.call(store, key))) unread.push(key);
|
|
1018
|
-
}
|
|
1019
|
-
return unread;
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
// src/persistence/durability/write-stall.ts
|
|
1023
|
-
function checkpointIsDue(indexState, mutationThreshold, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
|
|
1024
|
-
return indexState.mutationsSinceCheckpoint >= mutationThreshold || indexState.documentBytesSinceCheckpoint >= stallAboveBytes;
|
|
1025
|
-
}
|
|
1026
|
-
async function stallWhileCheckpointsFallBehind(indexState, stallAboveBytes = WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES) {
|
|
1027
|
-
while (indexState.documentBytesSinceCheckpoint >= stallAboveBytes && indexState.checkpointInFlight !== null && !indexState.unloading) {
|
|
1028
|
-
await new Promise((resume) => {
|
|
1029
|
-
indexState.stalledWrites.push(resume);
|
|
1030
|
-
});
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
function noteCheckpointCommitted(indexState, recordsBeyondTheCheckpoint) {
|
|
1034
|
-
const counted = indexState.mutationsSinceCheckpoint;
|
|
1035
|
-
const shareLeft = counted <= 0 ? 0 : Math.min(1, recordsBeyondTheCheckpoint / counted);
|
|
1036
|
-
indexState.documentBytesSinceCheckpoint = Math.round(indexState.documentBytesSinceCheckpoint * shareLeft);
|
|
1037
|
-
indexState.mutationsSinceCheckpoint = recordsBeyondTheCheckpoint;
|
|
1038
|
-
releaseStalledWrites(indexState);
|
|
1039
|
-
}
|
|
1040
|
-
function releaseStalledWrites(indexState) {
|
|
1041
|
-
const waiting = indexState.stalledWrites.splice(0);
|
|
1042
|
-
for (const resume of waiting) resume();
|
|
1043
|
-
}
|
|
1044
|
-
|
|
1045
|
-
// src/persistence/durability/checkpoint-run.ts
|
|
1046
|
-
async function bothOrTheFirstFailure(first, second) {
|
|
1047
|
-
const [firstSettled, secondSettled] = await Promise.allSettled([first, second]);
|
|
1048
|
-
if (firstSettled.status === "rejected") throw firstSettled.reason;
|
|
1049
|
-
if (secondSettled.status === "rejected") throw secondSettled.reason;
|
|
1050
|
-
return [firstSettled.value, secondSettled.value];
|
|
1051
|
-
}
|
|
1052
|
-
async function runDurableCheckpoint(input) {
|
|
1053
|
-
try {
|
|
1054
|
-
let recordsLeftInLog = await writeOneBoundedCheckpoint(input);
|
|
1055
|
-
while (recordsLeftInLog > 0 && input.mayContinue()) {
|
|
1056
|
-
recordsLeftInLog = await writeOneBoundedCheckpoint(input);
|
|
1057
|
-
}
|
|
1058
|
-
} catch (err) {
|
|
1059
|
-
if (!isSystemError(err)) throw err;
|
|
1060
|
-
const error = new NarsilError(
|
|
1061
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1062
|
-
`The engine cannot write the checkpoint of index "${input.indexName}" to the durability directory "${input.directory.root}"`,
|
|
1063
|
-
{
|
|
1064
|
-
indexName: input.indexName,
|
|
1065
|
-
directory: input.directory.root,
|
|
1066
|
-
cause: err.message
|
|
1067
|
-
}
|
|
1068
|
-
);
|
|
1069
|
-
input.markFatal(error);
|
|
1070
|
-
throw error;
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
function isSystemError(err) {
|
|
1074
|
-
return err instanceof Error && !(err instanceof NarsilError) && typeof err.code === "string";
|
|
1075
|
-
}
|
|
1076
|
-
async function writeOneBoundedCheckpoint(input) {
|
|
1077
|
-
const manager = input.hooks.getManager(input.indexName);
|
|
1078
|
-
if (manager === void 0) {
|
|
1079
|
-
return 0;
|
|
1080
|
-
}
|
|
1081
|
-
const metadata = input.hooks.buildMetadata(input.indexName);
|
|
1082
|
-
if (metadata === void 0) {
|
|
1083
|
-
return 0;
|
|
1084
|
-
}
|
|
1085
|
-
const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
|
|
1086
|
-
const store = directoryCheckpointStore(input.directory);
|
|
1087
|
-
const priorManifest = await readCurrentManifest(store, input.indexName);
|
|
1088
|
-
const capture2 = await captureCheckpoint(
|
|
1089
|
-
input.indexState,
|
|
1090
|
-
manager,
|
|
1091
|
-
vectorIndexes,
|
|
1092
|
-
wholePartitionsWhereMostChanged(manager, {
|
|
1093
|
-
priorCheckpoint: priorManifest?.checkpoint ?? [],
|
|
1094
|
-
priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
|
|
1095
|
-
everyPartition: input.fromMemory,
|
|
1096
|
-
aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable()
|
|
1097
|
-
}),
|
|
1098
|
-
input.fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
|
|
1099
|
-
priorManifest?.vectors ?? []
|
|
1100
|
-
);
|
|
1101
|
-
const { targets, documentCount } = capture2;
|
|
1102
|
-
await makeEveryAppliedMutationDurable(input.indexState, input.markFatal);
|
|
1103
|
-
const [liveVectors, segments] = await bothOrTheFirstFailure(
|
|
1104
|
-
writeCapturedVectors(store, input.indexName, capture2, priorManifest, input.fromMemory),
|
|
1105
|
-
writeIndexCheckpointSegments({
|
|
1106
|
-
directory: input.directory,
|
|
1107
|
-
metadata,
|
|
1108
|
-
targets,
|
|
1109
|
-
compactionThreshold: input.compactionThreshold,
|
|
1110
|
-
canOffload: input.canOffload,
|
|
1111
|
-
prior: priorManifest,
|
|
1112
|
-
wholePartitions: capture2.wholePartitions.contents,
|
|
1113
|
-
capturedPartitions: capture2.wholePartitions.captured
|
|
1114
|
-
})
|
|
1115
|
-
);
|
|
1116
|
-
await makeEveryAppliedMutationDurable(input.indexState, input.markFatal);
|
|
1117
|
-
const written = await commitCheckpointManifest(store, metadata, segments, liveVectors.vectors);
|
|
1118
|
-
recordWrittenVectors(vectorIndexes, liveVectors.written);
|
|
1119
|
-
await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
|
|
1120
|
-
if (!input.holdsCheckpointFiles()) {
|
|
1121
|
-
await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage));
|
|
1122
|
-
}
|
|
1123
|
-
const checkpointDocumentCount = documentCount;
|
|
1124
|
-
await input.queueMetadataWrite(input.indexName, async () => {
|
|
1125
|
-
const checkpointMetadata = input.hooks.buildMetadata(input.indexName, checkpointDocumentCount);
|
|
1126
|
-
if (checkpointMetadata === void 0) {
|
|
1127
|
-
return;
|
|
1128
|
-
}
|
|
1129
|
-
const bytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
1130
|
-
await input.directory.atomicWrite(`${input.indexName}/meta`, bytes);
|
|
1131
|
-
input.hooks.recordCheckpoint?.(input.indexName, checkpointDocumentCount, manager.partitionCount);
|
|
1132
|
-
});
|
|
1133
|
-
await truncateCoveredSegments(input.directory, input.indexName, targets);
|
|
1134
|
-
await reclaimWalBeyondCount(
|
|
1135
|
-
input.directory,
|
|
1136
|
-
input.indexName,
|
|
1137
|
-
manager.partitionCount,
|
|
1138
|
-
input.indexState.partitions,
|
|
1139
|
-
input.markFatal
|
|
1140
|
-
);
|
|
1141
|
-
const beyondTheCheckpoint = recordsBeyond(input.indexState, targets);
|
|
1142
|
-
noteCheckpointCommitted(input.indexState, beyondTheCheckpoint);
|
|
1143
|
-
const anotherIsDue = input.mutationThreshold > 0 && beyondTheCheckpoint >= input.mutationThreshold;
|
|
1144
|
-
return anotherIsDue ? beyondTheCheckpoint : capture2.recordsLeftInLog;
|
|
1145
|
-
}
|
|
1146
|
-
function recordsBeyond(indexState, targets) {
|
|
1147
|
-
let records = 0;
|
|
1148
|
-
for (const [partitionId, partition] of indexState.partitions) {
|
|
1149
|
-
records += Math.max(0, partition.appliedSeqNo - snapshotCheckpointFor(targets, partitionId));
|
|
1150
|
-
}
|
|
1151
|
-
return records;
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
// src/persistence/durability/directory-watch.ts
|
|
1155
|
-
function unrefInterval(tick, intervalMs) {
|
|
1156
|
-
const timer = setInterval(tick, intervalMs);
|
|
1157
|
-
if (typeof timer.unref === "function") timer.unref();
|
|
1158
|
-
return timer;
|
|
1159
|
-
}
|
|
1160
|
-
function createDirectoryWatch(directory, isStopped, onLost) {
|
|
1161
|
-
let expected = null;
|
|
1162
|
-
let timer = null;
|
|
1163
|
-
let lock = null;
|
|
1164
|
-
function cannotCheck(err) {
|
|
1165
|
-
return new NarsilError(
|
|
1166
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1167
|
-
`The engine cannot check the durability directory "${directory.root}", so it cannot confirm that a write is durable`,
|
|
1168
|
-
{ directory: directory.root, cause: err instanceof Error ? err.message : String(err) }
|
|
1169
|
-
);
|
|
1170
|
-
}
|
|
1171
|
-
async function currentIdentity() {
|
|
1172
|
-
try {
|
|
1173
|
-
return await directory.identity?.() ?? null;
|
|
1174
|
-
} catch (err) {
|
|
1175
|
-
return cannotCheck(err);
|
|
1176
|
-
}
|
|
1177
|
-
}
|
|
1178
|
-
async function lockTakenOver() {
|
|
1179
|
-
if (lock === null) return null;
|
|
1180
|
-
try {
|
|
1181
|
-
if (await lock.holds()) return null;
|
|
1182
|
-
} catch (err) {
|
|
1183
|
-
return cannotCheck(err);
|
|
1184
|
-
}
|
|
1185
|
-
return new NarsilError(
|
|
1186
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1187
|
-
`The lock file of the durability directory "${directory.root}" records another engine, which may be writing the same log`,
|
|
1188
|
-
{ directory: directory.root }
|
|
1189
|
-
);
|
|
1190
|
-
}
|
|
1191
|
-
async function verify() {
|
|
1192
|
-
if (directory.identity === void 0 || isStopped()) return true;
|
|
1193
|
-
const current = await currentIdentity();
|
|
1194
|
-
if (current instanceof Error) {
|
|
1195
|
-
onLost(current);
|
|
1196
|
-
return false;
|
|
1197
|
-
}
|
|
1198
|
-
if (expected === null) {
|
|
1199
|
-
expected = current;
|
|
1200
|
-
} else if (current !== expected) {
|
|
1201
|
-
onLost(
|
|
1202
|
-
new NarsilError(
|
|
1203
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
1204
|
-
`The durability directory "${directory.root}" is missing or is a different directory from the one that this instance opens at start-up, so a write to it is not durable`,
|
|
1205
|
-
{ directory: directory.root }
|
|
1206
|
-
)
|
|
1207
|
-
);
|
|
1208
|
-
return false;
|
|
1209
|
-
}
|
|
1210
|
-
const takenOver = await lockTakenOver();
|
|
1211
|
-
if (takenOver === null) return true;
|
|
1212
|
-
onLost(takenOver);
|
|
1213
|
-
return false;
|
|
1214
|
-
}
|
|
1215
|
-
function stop() {
|
|
1216
|
-
if (timer !== null) {
|
|
1217
|
-
clearInterval(timer);
|
|
1218
|
-
timer = null;
|
|
1219
|
-
}
|
|
1220
|
-
}
|
|
1221
|
-
return {
|
|
1222
|
-
async claim() {
|
|
1223
|
-
if (lock === null && directory.lock !== void 0) lock = await directory.lock();
|
|
1224
|
-
},
|
|
1225
|
-
async release() {
|
|
1226
|
-
const held = lock;
|
|
1227
|
-
lock = null;
|
|
1228
|
-
await held?.release();
|
|
1229
|
-
},
|
|
1230
|
-
verify,
|
|
1231
|
-
async verifyOnce() {
|
|
1232
|
-
if (expected === null && directory.identity !== void 0) await verify();
|
|
1233
|
-
},
|
|
1234
|
-
start() {
|
|
1235
|
-
if (directory.identity === void 0 || timer !== null || isStopped()) return;
|
|
1236
|
-
timer = unrefInterval(() => void verify(), DIRECTORY_IDENTITY_CHECK_INTERVAL_MS);
|
|
1237
|
-
},
|
|
1238
|
-
stop
|
|
1239
|
-
};
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
// src/persistence/durability/manager-state.ts
|
|
1243
|
-
async function drainIndexStateForUnload(indexState, pendingMetadataWrite, closeWalWriter) {
|
|
1244
|
-
indexState.unloading = true;
|
|
1245
|
-
try {
|
|
1246
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
1247
|
-
await pendingMetadataWrite;
|
|
1248
|
-
for (const partition of indexState.partitions.values()) {
|
|
1249
|
-
await closeWalWriter(partition.walWriter.close());
|
|
1250
|
-
}
|
|
1251
|
-
} catch (error) {
|
|
1252
|
-
indexState.unloading = false;
|
|
1253
|
-
throw error;
|
|
1254
|
-
}
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
// src/persistence/durability/mutation-batch.ts
|
|
1258
|
-
function toError3(err) {
|
|
1259
|
-
return err instanceof Error ? err : new Error(String(err));
|
|
1260
|
-
}
|
|
1261
|
-
function failUnlessAlreadyFailed(outcomes, count, error) {
|
|
1262
|
-
for (let i = 0; i < count; i++) {
|
|
1263
|
-
const outcome = outcomes[i];
|
|
1264
|
-
if (outcome === void 0 || outcome.ok) {
|
|
1265
|
-
outcomes[i] = { ok: false, error };
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
}
|
|
1269
|
-
function failPartition(partition, deps, err) {
|
|
1270
|
-
const error = toError3(err);
|
|
1271
|
-
partition.failed = error;
|
|
1272
|
-
deps.markFatal(error);
|
|
1273
|
-
return error;
|
|
1274
|
-
}
|
|
1275
|
-
async function recordBatchesByPartition(records, partitionOf, deps) {
|
|
1276
|
-
const positionsByPartition = /* @__PURE__ */ new Map();
|
|
1277
|
-
for (let i = 0; i < records.length; i++) {
|
|
1278
|
-
const partition = partitionOf(records[i]);
|
|
1279
|
-
const positions = positionsByPartition.get(partition);
|
|
1280
|
-
if (positions === void 0) {
|
|
1281
|
-
positionsByPartition.set(partition, [i]);
|
|
1282
|
-
} else {
|
|
1283
|
-
positions.push(i);
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
|
-
const outcomes = new Array(records.length);
|
|
1287
|
-
await Promise.all(
|
|
1288
|
-
[...positionsByPartition].map(async ([partition, positions]) => {
|
|
1289
|
-
const partitionOutcomes = await recordPartitionBatch(
|
|
1290
|
-
partition,
|
|
1291
|
-
positions.map((position) => records[position]),
|
|
1292
|
-
deps
|
|
1293
|
-
);
|
|
1294
|
-
for (let i = 0; i < positions.length; i++) {
|
|
1295
|
-
outcomes[positions[i]] = partitionOutcomes[i];
|
|
1296
|
-
}
|
|
1297
|
-
})
|
|
1298
|
-
);
|
|
1299
|
-
return outcomes;
|
|
1300
|
-
}
|
|
1301
|
-
async function recordPartitionBatch(partition, records, deps) {
|
|
1302
|
-
const outcomes = new Array(records.length);
|
|
1303
|
-
let appendedCount = 0;
|
|
1304
|
-
const appended = partition.appendChain.then(async () => {
|
|
1305
|
-
const fatal = deps.fatalError();
|
|
1306
|
-
if (fatal !== null) {
|
|
1307
|
-
throw fatal;
|
|
1308
|
-
}
|
|
1309
|
-
if (partition.failed !== null) {
|
|
1310
|
-
throw partition.failed;
|
|
1311
|
-
}
|
|
1312
|
-
const applied = [];
|
|
1313
|
-
for (let i = 0; i < records.length; i++) {
|
|
1314
|
-
try {
|
|
1315
|
-
await records[i].apply();
|
|
1316
|
-
applied.push(i);
|
|
1317
|
-
} catch (err) {
|
|
1318
|
-
outcomes[i] = { ok: false, error: err };
|
|
1319
|
-
}
|
|
1320
|
-
}
|
|
1321
|
-
if (applied.length === 0) {
|
|
1322
|
-
return;
|
|
1323
|
-
}
|
|
1324
|
-
let lastSeqNo = partition.appliedSeqNo;
|
|
1325
|
-
try {
|
|
1326
|
-
const entries = [];
|
|
1327
|
-
for (const i of applied) {
|
|
1328
|
-
lastSeqNo = partition.seqOwner.next();
|
|
1329
|
-
outcomes[i] = { ok: true, seqNo: lastSeqNo };
|
|
1330
|
-
entries.push(deps.buildEntry(records[i], lastSeqNo));
|
|
1331
|
-
}
|
|
1332
|
-
await partition.walWriter.appendAll(entries);
|
|
1333
|
-
} catch (err) {
|
|
1334
|
-
throw failPartition(partition, deps, err);
|
|
1335
|
-
}
|
|
1336
|
-
partition.appliedSeqNo = lastSeqNo;
|
|
1337
|
-
appendedCount = applied.length;
|
|
1338
|
-
});
|
|
1339
|
-
partition.appendChain = appended.catch(() => void 0);
|
|
1340
|
-
try {
|
|
1341
|
-
await appended;
|
|
1342
|
-
if (deps.syncEachBatch && appendedCount > 0) {
|
|
1343
|
-
try {
|
|
1344
|
-
await partition.walWriter.commit();
|
|
1345
|
-
} catch (err) {
|
|
1346
|
-
throw failPartition(partition, deps, err);
|
|
1347
|
-
}
|
|
1348
|
-
}
|
|
1349
|
-
} catch (err) {
|
|
1350
|
-
failUnlessAlreadyFailed(outcomes, records.length, toError3(err));
|
|
1351
|
-
}
|
|
1352
|
-
return outcomes;
|
|
1353
|
-
}
|
|
1354
|
-
var SNAPSHOT_BUNDLE_VERSION = 2;
|
|
1355
|
-
function invalid(message) {
|
|
1356
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Snapshot bundle ${message}`);
|
|
1357
|
-
}
|
|
1358
|
-
function vectorIndexesOf(raw) {
|
|
1359
|
-
if (raw === void 0 || raw === null) return {};
|
|
1360
|
-
if (typeof raw !== "object" || Array.isArray(raw)) invalid("vectorIndexes must be a map");
|
|
1361
|
-
const result = {};
|
|
1362
|
-
for (const [fieldPath, parts] of Object.entries(raw)) {
|
|
1363
|
-
result[fieldPath] = decodeVectorIndexParts(parts);
|
|
1364
|
-
}
|
|
1365
|
-
return result;
|
|
1366
|
-
}
|
|
1367
|
-
function checkpointOf(raw) {
|
|
1368
|
-
if (!Array.isArray(raw)) return [];
|
|
1369
|
-
const result = [];
|
|
1370
|
-
for (const entry of raw) {
|
|
1371
|
-
if (entry === null || typeof entry !== "object") continue;
|
|
1372
|
-
const { partitionId, lastSeqNo, primaryTerm } = entry;
|
|
1373
|
-
if (typeof partitionId !== "number" || typeof lastSeqNo !== "number" || typeof primaryTerm !== "number") continue;
|
|
1374
|
-
result.push({ partitionId, lastSeqNo, primaryTerm });
|
|
1375
|
-
}
|
|
1376
|
-
return result;
|
|
1377
|
-
}
|
|
1378
|
-
async function decodeSnapshotBundle(data) {
|
|
1379
|
-
const { payloadBytes } = await unpackEnvelopeBytes(data);
|
|
1380
|
-
const raw = decode(payloadBytes);
|
|
1381
|
-
if (raw.version !== SNAPSHOT_BUNDLE_VERSION) {
|
|
1382
|
-
throw new NarsilError(
|
|
1383
|
-
ErrorCodes.ENVELOPE_VERSION_MISMATCH,
|
|
1384
|
-
`The snapshot bundle holds layout ${String(raw.version)}, which this release cannot open. Export the index with the release that wrote it and import the export`,
|
|
1385
|
-
{ version: raw.version }
|
|
1386
|
-
);
|
|
1387
|
-
}
|
|
1388
|
-
if (raw.schema === null || typeof raw.schema !== "object" || Array.isArray(raw.schema)) invalid("missing schema");
|
|
1389
|
-
requireValidFieldTypes(raw.schema);
|
|
1390
|
-
if (typeof raw.language !== "string") invalid("missing language");
|
|
1391
|
-
if (!Array.isArray(raw.partitions) || !raw.partitions.every((part) => part instanceof Uint8Array)) {
|
|
1392
|
-
invalid("missing partitions");
|
|
1393
|
-
}
|
|
1394
|
-
return {
|
|
1395
|
-
schema: raw.schema,
|
|
1396
|
-
language: raw.language,
|
|
1397
|
-
partitions: raw.partitions,
|
|
1398
|
-
vectorIndexes: vectorIndexesOf(raw.vectorIndexes),
|
|
1399
|
-
checkpoint: checkpointOf(raw.checkpoint)
|
|
1400
|
-
};
|
|
1401
|
-
}
|
|
1402
|
-
function countSnapshotBundleDocuments(bundle) {
|
|
1403
|
-
let total = 0;
|
|
1404
|
-
for (const partitionBytes of bundle.partitions)
|
|
1405
|
-
total += Object.keys(deserializePayloadV2(partitionBytes).documents).length;
|
|
1406
|
-
return total;
|
|
1407
|
-
}
|
|
1408
|
-
async function loadSnapshotBundle(bytes, deps) {
|
|
1409
|
-
const bundle = await decodeSnapshotBundle(bytes);
|
|
1410
|
-
while (deps.manager.partitionCount < bundle.partitions.length) deps.manager.addPartition();
|
|
1411
|
-
if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
|
|
1412
|
-
for (let i = 0; i < bundle.partitions.length; i += 1) {
|
|
1413
|
-
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]));
|
|
1414
|
-
}
|
|
1415
|
-
for (const [fieldPath, parts] of Object.entries(bundle.vectorIndexes)) {
|
|
1416
|
-
const rebuildsGraph = parts.some((part) => part.graphs.some((graph) => graph.m < MIN_M));
|
|
1417
|
-
deps.vectorIndexes.get(fieldPath)?.deserialize(rebuildsGraph ? parts.map((part) => ({ ...part, graphs: [] })) : parts);
|
|
1418
|
-
}
|
|
1419
|
-
return bundle.checkpoint;
|
|
1420
|
-
}
|
|
1421
|
-
|
|
1422
|
-
// src/persistence/durability/earlier-layout/manifest.ts
|
|
1423
|
-
function isKey(value) {
|
|
1424
|
-
return typeof value === "string" && value.length > 0 && !value.includes("\0");
|
|
1425
|
-
}
|
|
1426
|
-
function checkpointOf2(raw) {
|
|
1427
|
-
if (!Array.isArray(raw)) return [];
|
|
1428
|
-
const result = [];
|
|
1429
|
-
for (const entry of raw) {
|
|
1430
|
-
if (!isRecord(entry) || !isUint32(entry.partitionId)) continue;
|
|
1431
|
-
if (!isSequenceNumber(entry.lastSeqNo) || !isSequenceNumber(entry.primaryTerm)) continue;
|
|
1432
|
-
result.push({ partitionId: entry.partitionId, lastSeqNo: entry.lastSeqNo, primaryTerm: entry.primaryTerm });
|
|
1433
|
-
}
|
|
1434
|
-
return result;
|
|
1435
|
-
}
|
|
1436
|
-
function segmentsOf(raw, partitionId) {
|
|
1437
|
-
if (raw === void 0) return [];
|
|
1438
|
-
if (!Array.isArray(raw) || raw.length > MAX_SEGMENTS_PER_PARTITION) {
|
|
1439
|
-
invalidManifest("has malformed segment references", { partitionId });
|
|
1440
|
-
}
|
|
1441
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1442
|
-
return raw.map((segment) => {
|
|
1443
|
-
if (!isRecord(segment) || !isId(segment.id) || !isUint32(segment.docCount) || !isUint32(segment.tombstoneCount) || !isKey(segment.key) || seen.has(segment.id)) {
|
|
1444
|
-
invalidManifest("has an invalid segment reference", { partitionId });
|
|
1445
|
-
}
|
|
1446
|
-
seen.add(segment.id);
|
|
1447
|
-
return { id: segment.id, key: segment.key, docCount: segment.docCount, tombstoneCount: segment.tombstoneCount };
|
|
1448
|
-
});
|
|
1449
|
-
}
|
|
1450
|
-
function partitionsOf(raw) {
|
|
1451
|
-
if (!Array.isArray(raw)) invalidManifest("is missing partition entries");
|
|
1452
|
-
return raw.map((entry) => {
|
|
1453
|
-
if (!isRecord(entry) || !isUint32(entry.partitionId)) invalidManifest("has a partition without a valid id");
|
|
1454
|
-
const segments = segmentsOf(entry.segments, entry.partitionId);
|
|
1455
|
-
const nextSegmentId = entry.nextSegmentId;
|
|
1456
|
-
if (!isId(nextSegmentId) || segments.some((segment) => segment.id >= nextSegmentId)) {
|
|
1457
|
-
invalidManifest("has an invalid next segment id", { partitionId: entry.partitionId });
|
|
1458
|
-
}
|
|
1459
|
-
return { partitionId: entry.partitionId, nextSegmentId, segments };
|
|
1460
|
-
});
|
|
1461
|
-
}
|
|
1462
|
-
function vectorFileOf(raw, fieldPath) {
|
|
1463
|
-
if (!isRecord(raw) || !isId(raw.id) || !isKey(raw.key)) invalidManifest("has an invalid vector file reference");
|
|
1464
|
-
const count = raw.count;
|
|
1465
|
-
if (!isUint32(count) || count > VECTOR_FILE_MAX_VECTORS) invalidManifest(`gives "${fieldPath}" an invalid count`);
|
|
1466
|
-
const dead = raw.dead ?? null;
|
|
1467
|
-
if (dead !== null && (!(dead instanceof Uint8Array) || dead.byteLength !== deadBytesFor(count))) {
|
|
1468
|
-
invalidManifest(`marks the dead vectors of "${fieldPath}" with the wrong number of bytes`);
|
|
1469
|
-
}
|
|
1470
|
-
return { id: raw.id, key: raw.key, count, dead };
|
|
1471
|
-
}
|
|
1472
|
-
function vectorsOf(raw) {
|
|
1473
|
-
if (raw === void 0 || raw === null) return [];
|
|
1474
|
-
if (!Array.isArray(raw)) invalidManifest("has malformed vector references");
|
|
1475
|
-
return raw.map((field) => {
|
|
1476
|
-
if (!isRecord(field) || typeof field.fieldPath !== "string") invalidManifest("has an invalid vector reference");
|
|
1477
|
-
const fieldPath = field.fieldPath;
|
|
1478
|
-
if (!isId(field.nextFileId) || !Array.isArray(field.files) || !isId(field.graphGeneration)) {
|
|
1479
|
-
invalidManifest(`gives "${fieldPath}" an invalid vector reference`);
|
|
1480
|
-
}
|
|
1481
|
-
const graphKey = field.graphKey ?? null;
|
|
1482
|
-
if (graphKey !== null && (!isKey(graphKey) || field.graphGeneration === 0)) {
|
|
1483
|
-
invalidManifest(`gives "${fieldPath}" an invalid graph key`);
|
|
1484
|
-
}
|
|
1485
|
-
return {
|
|
1486
|
-
fieldPath,
|
|
1487
|
-
nextFileId: field.nextFileId,
|
|
1488
|
-
files: field.files.map((file) => vectorFileOf(file, fieldPath)),
|
|
1489
|
-
graphGeneration: field.graphGeneration,
|
|
1490
|
-
graphKey
|
|
1491
|
-
};
|
|
1492
|
-
});
|
|
1493
|
-
}
|
|
1494
|
-
function decodeEarlierManifest(raw) {
|
|
1495
|
-
const schema = raw.schema;
|
|
1496
|
-
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
1497
|
-
requireValidFieldTypes(schema);
|
|
1498
|
-
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
1499
|
-
return {
|
|
1500
|
-
schema,
|
|
1501
|
-
language: raw.language,
|
|
1502
|
-
checkpoint: checkpointOf2(raw.checkpoint),
|
|
1503
|
-
partitions: partitionsOf(raw.partitions),
|
|
1504
|
-
vectors: vectorsOf(raw.vectors)
|
|
1505
|
-
};
|
|
1506
|
-
}
|
|
1507
|
-
function earlierReferencedKeys(manifest) {
|
|
1508
|
-
const keys = /* @__PURE__ */ new Set();
|
|
1509
|
-
for (const partition of manifest.partitions) {
|
|
1510
|
-
for (const segment of partition.segments) keys.add(segment.key);
|
|
1511
|
-
}
|
|
1512
|
-
for (const vector of manifest.vectors) {
|
|
1513
|
-
for (const file of vector.files) keys.add(file.key);
|
|
1514
|
-
if (vector.graphKey !== null) keys.add(vector.graphKey);
|
|
1515
|
-
}
|
|
1516
|
-
return keys;
|
|
1517
|
-
}
|
|
1518
|
-
|
|
1519
|
-
// src/persistence/durability/earlier-layout/merge.ts
|
|
1520
|
-
function mergeTimeOrderedSegments(ordered, fallback) {
|
|
1521
|
-
const winner = resolveWinners(ordered);
|
|
1522
|
-
const first = ordered[0]?.partition;
|
|
1523
|
-
const merged = {
|
|
1524
|
-
indexName: first?.indexName ?? fallback.indexName,
|
|
1525
|
-
partitionId: first?.partitionId ?? fallback.partitionId,
|
|
1526
|
-
totalPartitions: first?.totalPartitions ?? fallback.totalPartitions,
|
|
1527
|
-
language: first?.language ?? fallback.language,
|
|
1528
|
-
schema: {},
|
|
1529
|
-
docCount: 0,
|
|
1530
|
-
avgDocLength: 0,
|
|
1531
|
-
documents: /* @__PURE__ */ Object.create(null),
|
|
1532
|
-
invertedIndex: /* @__PURE__ */ Object.create(null),
|
|
1533
|
-
fieldIndexes: {
|
|
1534
|
-
numeric: /* @__PURE__ */ Object.create(null),
|
|
1535
|
-
boolean: /* @__PURE__ */ Object.create(null),
|
|
1536
|
-
enum: /* @__PURE__ */ Object.create(null),
|
|
1537
|
-
geopoint: /* @__PURE__ */ Object.create(null)
|
|
1538
|
-
},
|
|
1539
|
-
statistics: {
|
|
1540
|
-
totalDocuments: 0,
|
|
1541
|
-
totalFieldLengths: /* @__PURE__ */ Object.create(null),
|
|
1542
|
-
averageFieldLengths: /* @__PURE__ */ Object.create(null),
|
|
1543
|
-
docFrequencies: /* @__PURE__ */ Object.create(null)
|
|
1544
|
-
}
|
|
1545
|
-
};
|
|
1546
|
-
for (let segIndex = 0; segIndex < ordered.length; segIndex += 1) {
|
|
1547
|
-
const part = ordered[segIndex].partition;
|
|
1548
|
-
addSegmentSchema(merged.schema, part.schema);
|
|
1549
|
-
collectDocuments(merged, part, winner, segIndex);
|
|
1550
|
-
collectInvertedIndex(merged, part, winner, segIndex);
|
|
1551
|
-
collectFieldIndexes(merged, part, winner, segIndex);
|
|
1552
|
-
collectSurfaceForms(merged, part);
|
|
1553
|
-
}
|
|
1554
|
-
recomputeStatistics(merged);
|
|
1555
|
-
validateMergedPartition(merged);
|
|
1556
|
-
return merged;
|
|
1557
|
-
}
|
|
1558
|
-
function countLiveDocuments(ordered) {
|
|
1559
|
-
return resolveWinners(ordered).size;
|
|
1560
|
-
}
|
|
1561
|
-
function resolveWinners(ordered) {
|
|
1562
|
-
const winner = /* @__PURE__ */ new Map();
|
|
1563
|
-
for (let segIndex = 0; segIndex < ordered.length; segIndex += 1) {
|
|
1564
|
-
const segment = ordered[segIndex];
|
|
1565
|
-
for (const docId of segment.tombstones) {
|
|
1566
|
-
winner.delete(docId);
|
|
1567
|
-
}
|
|
1568
|
-
for (const docId of Object.keys(segment.partition.documents)) {
|
|
1569
|
-
winner.set(docId, segIndex);
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
return winner;
|
|
1573
|
-
}
|
|
1574
|
-
function owns(winner, docId, segIndex) {
|
|
1575
|
-
return winner.get(docId) === segIndex;
|
|
1576
|
-
}
|
|
1577
|
-
function addSegmentSchema(agreed, schema) {
|
|
1578
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
1579
|
-
const existing = agreed[field];
|
|
1580
|
-
if (existing !== void 0 && existing !== type) {
|
|
1581
|
-
throw new NarsilError(
|
|
1582
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1583
|
-
`Segment schemas disagree on field "${field}": "${existing}" vs "${type}"`,
|
|
1584
|
-
{ field, existing, found: type }
|
|
1585
|
-
);
|
|
1586
|
-
}
|
|
1587
|
-
agreed[field] = type;
|
|
1588
|
-
}
|
|
1589
|
-
}
|
|
1590
|
-
function collectDocuments(merged, part, winner, segIndex) {
|
|
1591
|
-
for (const [docId, doc] of Object.entries(part.documents)) {
|
|
1592
|
-
if (owns(winner, docId, segIndex)) {
|
|
1593
|
-
merged.documents[docId] = doc;
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
}
|
|
1597
|
-
function collectInvertedIndex(merged, part, winner, segIndex) {
|
|
1598
|
-
for (const [token, list] of Object.entries(part.invertedIndex)) {
|
|
1599
|
-
let target = merged.invertedIndex[token];
|
|
1600
|
-
for (const posting of list.postings) {
|
|
1601
|
-
if (!owns(winner, posting.docId, segIndex)) continue;
|
|
1602
|
-
if (target === void 0) {
|
|
1603
|
-
target = { docFrequency: 0, postings: [] };
|
|
1604
|
-
merged.invertedIndex[token] = target;
|
|
1605
|
-
}
|
|
1606
|
-
target.postings.push(posting);
|
|
1607
|
-
}
|
|
1608
|
-
}
|
|
1609
|
-
}
|
|
1610
|
-
function collectEntries(target, source, winner, segIndex) {
|
|
1611
|
-
for (const [path, entries] of Object.entries(source)) {
|
|
1612
|
-
for (const entry of entries) {
|
|
1613
|
-
if (!owns(winner, entry.docId, segIndex)) continue;
|
|
1614
|
-
let bucket = target[path];
|
|
1615
|
-
if (bucket === void 0) {
|
|
1616
|
-
bucket = [];
|
|
1617
|
-
target[path] = bucket;
|
|
1618
|
-
}
|
|
1619
|
-
bucket.push(entry);
|
|
1620
|
-
}
|
|
1621
|
-
}
|
|
1622
|
-
}
|
|
1623
|
-
function collectFieldIndexes(merged, part, winner, segIndex) {
|
|
1624
|
-
collectEntries(merged.fieldIndexes.numeric, part.fieldIndexes.numeric, winner, segIndex);
|
|
1625
|
-
collectEntries(merged.fieldIndexes.geopoint, part.fieldIndexes.geopoint, winner, segIndex);
|
|
1626
|
-
for (const [path, idx] of Object.entries(part.fieldIndexes.boolean)) {
|
|
1627
|
-
for (const [docIds, flag] of [
|
|
1628
|
-
[idx.trueDocs, true],
|
|
1629
|
-
[idx.falseDocs, false]
|
|
1630
|
-
]) {
|
|
1631
|
-
for (const docId of docIds) {
|
|
1632
|
-
if (!owns(winner, docId, segIndex)) continue;
|
|
1633
|
-
let target = merged.fieldIndexes.boolean[path];
|
|
1634
|
-
if (target === void 0) {
|
|
1635
|
-
target = { trueDocs: [], falseDocs: [] };
|
|
1636
|
-
merged.fieldIndexes.boolean[path] = target;
|
|
1637
|
-
}
|
|
1638
|
-
(flag ? target.trueDocs : target.falseDocs).push(docId);
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
for (const [path, values] of Object.entries(part.fieldIndexes.enum)) {
|
|
1643
|
-
for (const [value, docIds] of Object.entries(values)) {
|
|
1644
|
-
for (const docId of docIds) {
|
|
1645
|
-
if (!owns(winner, docId, segIndex)) continue;
|
|
1646
|
-
let target = merged.fieldIndexes.enum[path];
|
|
1647
|
-
if (target === void 0) {
|
|
1648
|
-
target = /* @__PURE__ */ Object.create(null);
|
|
1649
|
-
merged.fieldIndexes.enum[path] = target;
|
|
1650
|
-
}
|
|
1651
|
-
let bucket = target[value];
|
|
1652
|
-
if (bucket === void 0) {
|
|
1653
|
-
bucket = [];
|
|
1654
|
-
target[value] = bucket;
|
|
1655
|
-
}
|
|
1656
|
-
bucket.push(docId);
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
function collectSurfaceForms(merged, part) {
|
|
1662
|
-
if (!part.surfaceForms) return;
|
|
1663
|
-
let target = merged.surfaceForms;
|
|
1664
|
-
if (!target) {
|
|
1665
|
-
target = /* @__PURE__ */ Object.create(null);
|
|
1666
|
-
merged.surfaceForms = target;
|
|
1667
|
-
}
|
|
1668
|
-
for (const [surface, value] of Object.entries(part.surfaceForms)) {
|
|
1669
|
-
const count = typeof value === "number" ? value : value[0];
|
|
1670
|
-
const token = typeof value === "number" ? surface : value[1];
|
|
1671
|
-
const existing = target[surface];
|
|
1672
|
-
if (existing === void 0) {
|
|
1673
|
-
target[surface] = value;
|
|
1674
|
-
continue;
|
|
1675
|
-
}
|
|
1676
|
-
const existingCount = typeof existing === "number" ? existing : existing[0];
|
|
1677
|
-
const total = existingCount + count;
|
|
1678
|
-
target[surface] = token === surface ? total : [total, token];
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1681
|
-
function recomputeStatistics(merged) {
|
|
1682
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
1683
|
-
let totalDocuments = 0;
|
|
1684
|
-
for (const doc of Object.values(merged.documents)) {
|
|
1685
|
-
totalDocuments += 1;
|
|
1686
|
-
for (const [field, length] of Object.entries(doc.fieldLengths)) {
|
|
1687
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
1691
|
-
for (const [token, list] of Object.entries(merged.invertedIndex)) {
|
|
1692
|
-
const distinct = /* @__PURE__ */ new Set();
|
|
1693
|
-
for (const posting of list.postings) distinct.add(posting.docId);
|
|
1694
|
-
list.docFrequency = distinct.size;
|
|
1695
|
-
docFrequencies[token] = distinct.size;
|
|
1696
|
-
}
|
|
1697
|
-
const averageFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
1698
|
-
for (const [field, length] of Object.entries(totalFieldLengths)) {
|
|
1699
|
-
averageFieldLengths[field] = totalDocuments > 0 ? length / totalDocuments : 0;
|
|
1700
|
-
}
|
|
1701
|
-
merged.statistics.totalDocuments = totalDocuments;
|
|
1702
|
-
merged.statistics.totalFieldLengths = totalFieldLengths;
|
|
1703
|
-
merged.statistics.averageFieldLengths = averageFieldLengths;
|
|
1704
|
-
merged.statistics.docFrequencies = docFrequencies;
|
|
1705
|
-
merged.docCount = totalDocuments;
|
|
1706
|
-
merged.avgDocLength = Object.values(averageFieldLengths).reduce((sum, value) => sum + value, 0);
|
|
1707
|
-
}
|
|
1708
|
-
function validateMergedPartition(merged) {
|
|
1709
|
-
for (const [token, list] of Object.entries(merged.invertedIndex)) {
|
|
1710
|
-
for (const posting of list.postings) {
|
|
1711
|
-
if (!(posting.docId in merged.documents)) {
|
|
1712
|
-
throw new NarsilError(
|
|
1713
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1714
|
-
`Merged segment posting references document "${posting.docId}" that is absent from the partition`,
|
|
1715
|
-
{ token, docId: posting.docId }
|
|
1716
|
-
);
|
|
1717
|
-
}
|
|
1718
|
-
}
|
|
1719
|
-
}
|
|
1720
|
-
}
|
|
1721
|
-
function normalizeTombstones(raw) {
|
|
1722
|
-
if (raw === void 0) return [];
|
|
1723
|
-
if (!Array.isArray(raw) || !raw.every((value) => typeof value === "string")) {
|
|
1724
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment file has a malformed tombstone list");
|
|
1725
|
-
}
|
|
1726
|
-
return raw;
|
|
1727
|
-
}
|
|
1728
|
-
async function openContainer(store, key) {
|
|
1729
|
-
const bytes = await store.read(key);
|
|
1730
|
-
if (bytes === null) {
|
|
1731
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Segment file "${key}" is missing`, { key });
|
|
1732
|
-
}
|
|
1733
|
-
const { payloadBytes } = await unpackEnvelopeBytes(bytes);
|
|
1734
|
-
const raw = decode(payloadBytes);
|
|
1735
|
-
if (!(raw.payload instanceof Uint8Array)) {
|
|
1736
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "Segment file is missing its partition payload", { key });
|
|
1737
|
-
}
|
|
1738
|
-
return { payload: raw.payload, tombstones: normalizeTombstones(raw.tombstones) };
|
|
1739
|
-
}
|
|
1740
|
-
async function readEarlierSegmentContents(store, key) {
|
|
1741
|
-
const { payload, tombstones } = await openContainer(store, key);
|
|
1742
|
-
return { partition: deserializePayloadV2(payload), tombstones };
|
|
1743
|
-
}
|
|
1744
|
-
async function readEarlierSegmentDocuments(store, key) {
|
|
1745
|
-
const { payload, tombstones } = await openContainer(store, key);
|
|
1746
|
-
const stored = documentsOfPayloadV2(payload);
|
|
1747
|
-
const documents = [];
|
|
1748
|
-
for (const [docId, document] of Object.entries(stored.documents)) {
|
|
1749
|
-
documents.push({ docId, document: document.fields });
|
|
1750
|
-
}
|
|
1751
|
-
return { schema: stored.schema, documents, tombstones };
|
|
1752
|
-
}
|
|
1753
|
-
|
|
1754
|
-
// src/persistence/durability/checkpoint-count.ts
|
|
1755
|
-
var decoder = new TextDecoder();
|
|
1756
|
-
async function countCurrentLayout(store, manifest) {
|
|
1757
|
-
let total = 0;
|
|
1758
|
-
for (const partition of manifest.partitions) {
|
|
1759
|
-
const live = /* @__PURE__ */ new Set();
|
|
1760
|
-
for (const ref of partition.segments) {
|
|
1761
|
-
const contents = await readSegmentFile(store, ref);
|
|
1762
|
-
for (const docId of contents.tombstones) live.delete(docId);
|
|
1763
|
-
const ids = contents.arrays.idTable;
|
|
1764
|
-
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
1765
|
-
live.add(decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1])));
|
|
1766
|
-
}
|
|
1767
|
-
}
|
|
1768
|
-
total += live.size;
|
|
1769
|
-
}
|
|
1770
|
-
return total;
|
|
1771
|
-
}
|
|
1772
|
-
async function countCheckpointDocuments(store, indexName) {
|
|
1773
|
-
const decoded = await readSegmentManifest(store, indexName);
|
|
1774
|
-
if (decoded?.kind === "current") return countCurrentLayout(store, decoded.manifest);
|
|
1775
|
-
if (decoded?.kind === "earlier") {
|
|
1776
|
-
let total = 0;
|
|
1777
|
-
for (const partition of decodeEarlierManifest(decoded.payload).partitions) {
|
|
1778
|
-
const ordered = [];
|
|
1779
|
-
for (const segment of partition.segments) ordered.push(await readEarlierSegmentContents(store, segment.key));
|
|
1780
|
-
total += countLiveDocuments(ordered);
|
|
1781
|
-
}
|
|
1782
|
-
return total;
|
|
1783
|
-
}
|
|
1784
|
-
const bundle = await store.read(snapshotBundleKey(indexName));
|
|
1785
|
-
return bundle === null ? 0 : countSnapshotBundleDocuments(await decodeSnapshotBundle(bundle));
|
|
1786
|
-
}
|
|
1787
|
-
function removeWhatTheSegmentReplaces(manager, segment) {
|
|
1788
|
-
manager.beginBatchRemove();
|
|
1789
|
-
try {
|
|
1790
|
-
for (const docId of segment.tombstones) {
|
|
1791
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
1792
|
-
}
|
|
1793
|
-
for (const { docId } of segment.documents) {
|
|
1794
|
-
if (manager.has(docId)) manager.remove(docId);
|
|
1795
|
-
}
|
|
1796
|
-
} finally {
|
|
1797
|
-
manager.endBatchRemove();
|
|
1798
|
-
}
|
|
1799
|
-
}
|
|
1800
|
-
function attachDocuments(manager, partitionId, segment) {
|
|
1801
|
-
if (segment.documents.length === 0) return;
|
|
1802
|
-
const payload = buildSegmentPayload(
|
|
1803
|
-
segment.documents,
|
|
1804
|
-
manager.config.schema,
|
|
1805
|
-
manager.language,
|
|
1806
|
-
resolvePartitionInsertOptions(manager.config, manager.analysis),
|
|
1807
|
-
manager.config.trackPositions ?? true
|
|
1808
|
-
);
|
|
1809
|
-
const documents = segment.documents.map((entry) => entry.document);
|
|
1810
|
-
const snapshot = freezeSegmentShared(payload, documents);
|
|
1811
|
-
if (snapshot === null) {
|
|
1812
|
-
manager.mergeSegment(partitionId, payload, documents);
|
|
1813
|
-
return;
|
|
1814
|
-
}
|
|
1815
|
-
manager.attachFrozenSegment(partitionId, createSharedFrozenSegment(snapshot));
|
|
1816
|
-
}
|
|
1817
|
-
async function loadPartition(store, indexName, partition, deps) {
|
|
1818
|
-
if (deps.storedTermsAreCurrent) {
|
|
1819
|
-
const agreedSchema = {};
|
|
1820
|
-
for (const ref of partition.segments) {
|
|
1821
|
-
const segment = await readEarlierSegmentDocuments(store, ref.key);
|
|
1822
|
-
addSegmentSchema(agreedSchema, segment.schema);
|
|
1823
|
-
removeWhatTheSegmentReplaces(deps.manager, segment);
|
|
1824
|
-
attachDocuments(deps.manager, partition.partitionId, segment);
|
|
1825
|
-
}
|
|
1826
|
-
return;
|
|
1827
|
-
}
|
|
1828
|
-
const ordered = [];
|
|
1829
|
-
for (const ref of partition.segments) ordered.push(await readEarlierSegmentContents(store, ref.key));
|
|
1830
|
-
const merged = mergeTimeOrderedSegments(ordered, {
|
|
1831
|
-
indexName,
|
|
1832
|
-
partitionId: partition.partitionId,
|
|
1833
|
-
totalPartitions: deps.manager.partitionCount,
|
|
1834
|
-
language: deps.manager.language.name
|
|
1835
|
-
});
|
|
1836
|
-
deps.manager.deserializePartition(partition.partitionId, merged);
|
|
1837
|
-
}
|
|
1838
|
-
async function readGraph(store, ref) {
|
|
1839
|
-
if (ref.graphKey === null) return null;
|
|
1840
|
-
const bytes = await store.read(ref.graphKey);
|
|
1841
|
-
if (bytes === null) {
|
|
1842
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The graph file "${ref.graphKey}" is missing`, {
|
|
1843
|
-
key: ref.graphKey
|
|
1844
|
-
});
|
|
1845
|
-
}
|
|
1846
|
-
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
1847
|
-
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
1848
|
-
what: `graph file "${ref.graphKey}"`,
|
|
1849
|
-
allowsCompression: true
|
|
1850
|
-
});
|
|
1851
|
-
const graph = decodeVectorGraphPayload(decode(payload));
|
|
1852
|
-
if (graph.graphs.some((numbered) => numbered.m < MIN_M)) return null;
|
|
1853
|
-
return graphPartsOfEarlierPayload(graph);
|
|
1854
|
-
}
|
|
1855
|
-
async function loadVectorField(store, ref, vectorIndex, fileRefs) {
|
|
1856
|
-
let liveVectors = 0;
|
|
1857
|
-
for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
|
|
1858
|
-
const restore = vectorIndex.restoreCheckpoint({
|
|
1859
|
-
liveVectors,
|
|
1860
|
-
holdsGraph: ref.graphKey !== null,
|
|
1861
|
-
graph: await readGraph(store, ref)
|
|
1862
|
-
});
|
|
1863
|
-
for (const file of ref.files) {
|
|
1864
|
-
const bytes = await store.read(file.key);
|
|
1865
|
-
if (bytes === null) {
|
|
1866
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The vector file "${file.key}" is missing`, {
|
|
1867
|
-
key: file.key
|
|
1868
|
-
});
|
|
1869
|
-
}
|
|
1870
|
-
const { header, payload: payloadBytes } = await openChecksummedEnvelope(bytes, {
|
|
1871
|
-
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
1872
|
-
what: `vector file "${file.key}"`,
|
|
1873
|
-
allowsCompression: false
|
|
1874
|
-
});
|
|
1875
|
-
const payload = decodeVectorFilePayload(decode(payloadBytes));
|
|
1876
|
-
if (payload.docIds.length !== file.count) {
|
|
1877
|
-
throw new NarsilError(
|
|
1878
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1879
|
-
`The vector file "${file.key}" holds ${payload.docIds.length} vectors where the manifest counts ${file.count}`,
|
|
1880
|
-
{ key: file.key }
|
|
1881
|
-
);
|
|
1882
|
-
}
|
|
1883
|
-
fileRefs.set(file.key, { key: file.key, bytes: bytes.byteLength, checksum: header.checksum });
|
|
1884
|
-
restore.addFile({
|
|
1885
|
-
id: file.id,
|
|
1886
|
-
key: file.key,
|
|
1887
|
-
payload,
|
|
1888
|
-
dead: file.dead,
|
|
1889
|
-
location: store.pathOf === void 0 ? null : {
|
|
1890
|
-
path: await store.pathOf(file.key),
|
|
1891
|
-
vectorsOffset: HEADER_SIZE + header.payloadLength - payload.vectors.byteLength
|
|
1892
|
-
}
|
|
1893
|
-
});
|
|
1894
|
-
}
|
|
1895
|
-
restore.finish();
|
|
1896
|
-
}
|
|
1897
|
-
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
1898
|
-
let highestPartitionId = manifest.partitions.length - 1;
|
|
1899
|
-
for (const partition of manifest.partitions) {
|
|
1900
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
1901
|
-
}
|
|
1902
|
-
growPartitionsTo(deps.manager, highestPartitionId);
|
|
1903
|
-
for (const partition of manifest.partitions) {
|
|
1904
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
1905
|
-
throw new NarsilError(
|
|
1906
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1907
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
1908
|
-
{ indexName, partitionId: partition.partitionId }
|
|
1909
|
-
);
|
|
1910
|
-
}
|
|
1911
|
-
await loadPartition(store, indexName, partition, deps);
|
|
1912
|
-
}
|
|
1913
|
-
const vectorFileRefs = /* @__PURE__ */ new Map();
|
|
1914
|
-
for (const vector of manifest.vectors) {
|
|
1915
|
-
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
1916
|
-
if (vectorIndex === void 0) continue;
|
|
1917
|
-
await loadVectorField(store, vector, vectorIndex, vectorFileRefs);
|
|
1918
|
-
}
|
|
1919
|
-
return { checkpoint: manifest.checkpoint, vectorFileRefs };
|
|
1920
|
-
}
|
|
1921
|
-
|
|
1922
|
-
// src/persistence/durability/earlier-layout/upgrade.ts
|
|
1923
|
-
async function loadEarlier(deps, earlier) {
|
|
1924
|
-
const loadDeps = {
|
|
1925
|
-
manager: deps.manager,
|
|
1926
|
-
vectorIndexes: deps.vectorIndexes,
|
|
1927
|
-
storedTermsAreCurrent: deps.storedTermsAreCurrent
|
|
1928
|
-
};
|
|
1929
|
-
if (earlier.kind === "bundle") {
|
|
1930
|
-
await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
|
|
1931
|
-
return { checkpoint: await loadSnapshotBundle(earlier.bytes, loadDeps), fileRefs: /* @__PURE__ */ new Map() };
|
|
1932
|
-
}
|
|
1933
|
-
await reclaimUnlistedKeys(deps.store, deps.indexName, earlierReferencedKeys(earlier.manifest));
|
|
1934
|
-
const loaded = await loadEarlierLayout(deps.store, deps.indexName, earlier.manifest, loadDeps);
|
|
1935
|
-
return { checkpoint: loaded.checkpoint, fileRefs: loaded.vectorFileRefs };
|
|
1936
|
-
}
|
|
1937
|
-
function priorVectorsOf(earlier, fileRefs) {
|
|
1938
|
-
if (earlier.kind === "bundle") return [];
|
|
1939
|
-
return earlier.manifest.vectors.map((field) => ({
|
|
1940
|
-
fieldPath: field.fieldPath,
|
|
1941
|
-
nextFileId: field.nextFileId,
|
|
1942
|
-
graphGeneration: field.graphGeneration,
|
|
1943
|
-
graphParts: [],
|
|
1944
|
-
files: field.files.flatMap((file) => {
|
|
1945
|
-
const ref = fileRefs.get(file.key);
|
|
1946
|
-
return ref === void 0 ? [] : [{ ...file, bytes: ref.bytes, checksum: ref.checksum }];
|
|
1947
|
-
})
|
|
1948
|
-
}));
|
|
1949
|
-
}
|
|
1950
|
-
function priorPartitionsOf(earlier, partitionCount) {
|
|
1951
|
-
if (earlier.kind === "bundle") return [];
|
|
1952
|
-
return earlier.manifest.partitions.filter((partition) => partition.partitionId < partitionCount).map((partition) => ({ partitionId: partition.partitionId, nextSegmentId: partition.nextSegmentId, segments: [] }));
|
|
1953
|
-
}
|
|
1954
|
-
async function writeCurrentLayout(deps, earlier, targets, fileRefs) {
|
|
1955
|
-
const priorVectors = priorVectorsOf(earlier, fileRefs);
|
|
1956
|
-
const plans = /* @__PURE__ */ new Map();
|
|
1957
|
-
for (const [fieldPath, vectorIndex] of deps.vectorIndexes) {
|
|
1958
|
-
await vectorIndex.completeGraph();
|
|
1959
|
-
const listed = priorVectors.find((ref) => ref.fieldPath === fieldPath);
|
|
1960
|
-
plans.set(
|
|
1961
|
-
fieldPath,
|
|
1962
|
-
vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key), true)
|
|
1963
|
-
);
|
|
1964
|
-
}
|
|
1965
|
-
const written = await writeCheckpointSegments({
|
|
1966
|
-
store: deps.store,
|
|
1967
|
-
metadata: deps.metadata,
|
|
1968
|
-
targets,
|
|
1969
|
-
compactionThreshold: deps.compactionThreshold,
|
|
1970
|
-
prior: { checkpoint: [], partitions: priorPartitionsOf(earlier, deps.manager.partitionCount) },
|
|
1971
|
-
wholePartitions: targets.map((target) => partitionContentsOf(deps.manager, target.partitionId))
|
|
1972
|
-
});
|
|
1973
|
-
const vectors = await writeLiveVectors({
|
|
1974
|
-
store: deps.store,
|
|
1975
|
-
indexName: deps.indexName,
|
|
1976
|
-
plans,
|
|
1977
|
-
priorVectors,
|
|
1978
|
-
priorFileRefs: fileRefs
|
|
1979
|
-
});
|
|
1980
|
-
const committed = await commitCheckpointManifest(deps.store, deps.metadata, written, vectors.refs);
|
|
1981
|
-
recordWrittenVectors(deps.vectorIndexes, vectors.written);
|
|
1982
|
-
await adoptVectorLayouts(deps.store, deps.vectorIndexes, vectors.layouts);
|
|
1983
|
-
await removeCheckpointGarbage(
|
|
1984
|
-
deps.store,
|
|
1985
|
-
await garbageThatNoFieldStillReads(deps.store, deps.vectorIndexes, committed.garbage)
|
|
1986
|
-
);
|
|
1987
|
-
}
|
|
1988
|
-
async function upgradeEarlierLayout(deps, earlier) {
|
|
1989
|
-
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "started", layout: earlier.kind });
|
|
1990
|
-
try {
|
|
1991
|
-
const loaded = await loadEarlier(deps, earlier);
|
|
1992
|
-
const targets = await deps.replay(loaded.checkpoint);
|
|
1993
|
-
await writeCurrentLayout(deps, earlier, targets, loaded.fileRefs);
|
|
1994
|
-
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "completed", layout: earlier.kind });
|
|
1995
|
-
return { checkpoint: targets, documentCount: deps.manager.countDocuments() };
|
|
1996
|
-
} catch (error) {
|
|
1997
|
-
const failure = error instanceof Error ? error : new Error(String(error));
|
|
1998
|
-
deps.onLayoutUpgrade?.({ indexName: deps.indexName, status: "failed", layout: earlier.kind, error: failure });
|
|
1999
|
-
throw failure;
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
// src/persistence/durability/checkpoint-recovery.ts
|
|
2004
|
-
async function earlierLayoutOf(deps, decoded) {
|
|
2005
|
-
if (decoded?.kind === "earlier") return { kind: "manifest", manifest: decodeEarlierManifest(decoded.payload) };
|
|
2006
|
-
if (decoded !== null) return null;
|
|
2007
|
-
const bundle = await deps.store.read(snapshotBundleKey(deps.indexName));
|
|
2008
|
-
return bundle === null ? null : { kind: "bundle", bytes: bundle };
|
|
2009
|
-
}
|
|
2010
|
-
async function recoverCheckpoint(deps) {
|
|
2011
|
-
const decoded = await readSegmentManifest(deps.store, deps.indexName);
|
|
2012
|
-
const earlier = await earlierLayoutOf(deps, decoded);
|
|
2013
|
-
if (earlier !== null) {
|
|
2014
|
-
const upgraded = await upgradeEarlierLayout(deps, earlier);
|
|
2015
|
-
return { checkpoint: upgraded.checkpoint, upgradedDocumentCount: upgraded.documentCount };
|
|
2016
|
-
}
|
|
2017
|
-
if (decoded === null || decoded.kind !== "current") {
|
|
2018
|
-
if (!deps.keepsUnlistedFiles) await reclaimUnlistedKeys(deps.store, deps.indexName, /* @__PURE__ */ new Set());
|
|
2019
|
-
return { checkpoint: await deps.replay([]), upgradedDocumentCount: null };
|
|
2020
|
-
}
|
|
2021
|
-
await deps.store.remove(snapshotBundleKey(deps.indexName));
|
|
2022
|
-
if (!deps.keepsUnlistedFiles) {
|
|
2023
|
-
await reclaimUnlistedKeys(deps.store, deps.indexName, manifestReferencedKeys(decoded.manifest));
|
|
2024
|
-
}
|
|
2025
|
-
const checkpoint = await loadSegmentedCheckpoint(deps.store, deps.indexName, decoded.manifest, {
|
|
2026
|
-
manager: deps.manager,
|
|
2027
|
-
vectorIndexes: deps.vectorIndexes
|
|
2028
|
-
});
|
|
2029
|
-
return { checkpoint: await deps.replay(checkpoint), upgradedDocumentCount: null };
|
|
2030
|
-
}
|
|
2031
|
-
|
|
2032
|
-
// src/persistence/durability/recover-index.ts
|
|
2033
|
-
async function writeCountedMetadata(deps, indexName, documentCount) {
|
|
2034
|
-
await deps.queueMetadataWrite(indexName, async () => {
|
|
2035
|
-
const metadata = deps.hooks.buildMetadata(indexName, documentCount);
|
|
2036
|
-
if (metadata === void 0) return;
|
|
2037
|
-
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
2038
|
-
await deps.directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
2039
|
-
}).catch(() => void 0);
|
|
2040
|
-
}
|
|
2041
|
-
async function recoverPersistedIndex(deps, indexName, metadataOnly) {
|
|
2042
|
-
const { directory, hooks } = deps;
|
|
2043
|
-
const store = directoryCheckpointStore(directory);
|
|
2044
|
-
const metadata = await loadMetadata(directory, indexName);
|
|
2045
|
-
if (metadata === null) {
|
|
2046
|
-
return;
|
|
2047
|
-
}
|
|
2048
|
-
let derivedCount;
|
|
2049
|
-
if (metadataOnly && metadata.documentCount === void 0) {
|
|
2050
|
-
try {
|
|
2051
|
-
derivedCount = await countCheckpointDocuments(store, indexName);
|
|
2052
|
-
metadata.documentCount = derivedCount;
|
|
2053
|
-
} catch {
|
|
2054
|
-
derivedCount = void 0;
|
|
2055
|
-
}
|
|
2056
|
-
}
|
|
2057
|
-
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
2058
|
-
if (metadataOnly) {
|
|
2059
|
-
if (derivedCount !== void 0) await writeCountedMetadata(deps, indexName, derivedCount);
|
|
2060
|
-
return;
|
|
2061
|
-
}
|
|
2062
|
-
const manager = hooks.getManager(indexName);
|
|
2063
|
-
const current = hooks.buildMetadata(indexName);
|
|
2064
|
-
if (manager === void 0 || current === void 0) {
|
|
2065
|
-
return;
|
|
2066
|
-
}
|
|
2067
|
-
const replayDeps = {
|
|
2068
|
-
manager,
|
|
2069
|
-
vectorFieldPaths: hooks.getVectorFieldPaths(indexName),
|
|
2070
|
-
vectorIndexes: hooks.getVectorIndexes(indexName)
|
|
2071
|
-
};
|
|
2072
|
-
const replay = async (checkpoint) => {
|
|
2073
|
-
const replayed = [];
|
|
2074
|
-
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
2075
|
-
const fromSeqNo = snapshotCheckpointFor(checkpoint, partitionId);
|
|
2076
|
-
const { highestSeqNo, highestPrimaryTerm } = await replayWal(
|
|
2077
|
-
directory,
|
|
2078
|
-
indexName,
|
|
2079
|
-
partitionId,
|
|
2080
|
-
fromSeqNo,
|
|
2081
|
-
replayDeps
|
|
2082
|
-
);
|
|
2083
|
-
const primaryTerm = Math.max(
|
|
2084
|
-
checkpointPrimaryTerm(checkpoint, partitionId),
|
|
2085
|
-
highestPrimaryTerm,
|
|
2086
|
-
SINGLE_NODE_PRIMARY_TERM
|
|
2087
|
-
);
|
|
2088
|
-
replayed.push({ partitionId, lastSeqNo: highestSeqNo, primaryTerm });
|
|
2089
|
-
}
|
|
2090
|
-
return replayed;
|
|
2091
|
-
};
|
|
2092
|
-
const recovered = await recoverCheckpoint({
|
|
2093
|
-
store,
|
|
2094
|
-
indexName,
|
|
2095
|
-
metadata: current,
|
|
2096
|
-
manager,
|
|
2097
|
-
vectorIndexes: replayDeps.vectorIndexes,
|
|
2098
|
-
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
2099
|
-
compactionThreshold: deps.compactionThreshold,
|
|
2100
|
-
keepsUnlistedFiles: deps.holdsCheckpointFiles(indexName),
|
|
2101
|
-
replay,
|
|
2102
|
-
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
2103
|
-
});
|
|
2104
|
-
for (const { partitionId, lastSeqNo } of recovered.checkpoint) {
|
|
2105
|
-
deps.registerPartition(indexName, partitionId, lastSeqNo);
|
|
2106
|
-
}
|
|
2107
|
-
if (recovered.upgradedDocumentCount !== null) {
|
|
2108
|
-
await writeCountedMetadata(deps, indexName, recovered.upgradedDocumentCount);
|
|
2109
|
-
}
|
|
2110
|
-
}
|
|
2111
|
-
|
|
2112
|
-
// src/persistence/durability/group-commit.ts
|
|
2113
|
-
function createGroupCommitCoordinator(sync) {
|
|
2114
|
-
let inFlight = null;
|
|
2115
|
-
let next = null;
|
|
2116
|
-
function makeBatch() {
|
|
2117
|
-
let resolve = () => void 0;
|
|
2118
|
-
let reject = () => void 0;
|
|
2119
|
-
const promise = new Promise((res, rej) => {
|
|
2120
|
-
resolve = res;
|
|
2121
|
-
reject = rej;
|
|
2122
|
-
});
|
|
2123
|
-
return { promise, resolve, reject, waiters: 0 };
|
|
2124
|
-
}
|
|
2125
|
-
function drain() {
|
|
2126
|
-
if (inFlight !== null || next === null) {
|
|
2127
|
-
return;
|
|
2128
|
-
}
|
|
2129
|
-
const batch = next;
|
|
2130
|
-
next = null;
|
|
2131
|
-
inFlight = sync().then(() => {
|
|
2132
|
-
batch.resolve();
|
|
2133
|
-
}).catch((err) => {
|
|
2134
|
-
batch.reject(err);
|
|
2135
|
-
}).finally(() => {
|
|
2136
|
-
inFlight = null;
|
|
2137
|
-
drain();
|
|
2138
|
-
});
|
|
2139
|
-
}
|
|
2140
|
-
return {
|
|
2141
|
-
commit() {
|
|
2142
|
-
if (next === null) {
|
|
2143
|
-
next = makeBatch();
|
|
2144
|
-
}
|
|
2145
|
-
next.waiters += 1;
|
|
2146
|
-
const promise = next.promise;
|
|
2147
|
-
drain();
|
|
2148
|
-
return promise;
|
|
2149
|
-
},
|
|
2150
|
-
get pendingCount() {
|
|
2151
|
-
return next === null ? 0 : next.waiters;
|
|
2152
|
-
}
|
|
2153
|
-
};
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
// src/persistence/durability/marker-writer.ts
|
|
2157
|
-
function markerKey(indexName, partitionId) {
|
|
2158
|
-
return `${indexName}/wal/${partitionId}/commit`;
|
|
2159
|
-
}
|
|
2160
|
-
async function createMarkerWriter(directory, indexName, partitionId) {
|
|
2161
|
-
const key = markerKey(indexName, partitionId);
|
|
2162
|
-
const handle = await directory.markerHandle(key);
|
|
2163
|
-
const existing = await handle.read();
|
|
2164
|
-
const current = readCommitMarker(existing);
|
|
2165
|
-
const createdFresh = existing.length === 0;
|
|
2166
|
-
let lastSlot = current?.slotIndex ?? null;
|
|
2167
|
-
let writeSeq = current?.state.writeSeq ?? 0;
|
|
2168
|
-
return {
|
|
2169
|
-
created: createdFresh,
|
|
2170
|
-
existingHighestDurableSeqNo: current?.state.highestDurableSeqNo ?? 0,
|
|
2171
|
-
async commit(state) {
|
|
2172
|
-
const slotIndex = nextMarkerSlot(lastSlot);
|
|
2173
|
-
writeSeq += 1;
|
|
2174
|
-
const slot = encodeMarkerSlot({ ...state, writeSeq });
|
|
2175
|
-
await handle.writeSlot(markerSlotOffset(slotIndex), slot);
|
|
2176
|
-
lastSlot = slotIndex;
|
|
2177
|
-
},
|
|
2178
|
-
async close() {
|
|
2179
|
-
await handle.close();
|
|
2180
|
-
}
|
|
2181
|
-
};
|
|
2182
|
-
}
|
|
2183
|
-
|
|
2184
|
-
// src/persistence/durability/wal-writer.ts
|
|
2185
|
-
var SEGMENT_TAIL_PATTERN = /^\d{16}$/;
|
|
2186
|
-
function segmentKey(indexName, partitionId, startSeqNo) {
|
|
2187
|
-
const padded2 = startSeqNo.toString().padStart(16, "0");
|
|
2188
|
-
return `${indexName}/wal/${partitionId}/${padded2}`;
|
|
2189
|
-
}
|
|
2190
|
-
function parseSegmentStartSeqNo2(key) {
|
|
2191
|
-
const tail = key.slice(key.lastIndexOf("/") + 1);
|
|
2192
|
-
if (!SEGMENT_TAIL_PATTERN.test(tail)) {
|
|
2193
|
-
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `Malformed WAL segment key: "${key}"`, { key });
|
|
2194
|
-
}
|
|
2195
|
-
const value = Number.parseInt(tail, 10);
|
|
2196
|
-
if (!Number.isSafeInteger(value)) {
|
|
2197
|
-
throw new NarsilError(
|
|
2198
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
2199
|
-
`WAL segment start sequence number is out of range: "${key}"`,
|
|
2200
|
-
{
|
|
2201
|
-
key
|
|
2202
|
-
}
|
|
2203
|
-
);
|
|
2204
|
-
}
|
|
2205
|
-
return value;
|
|
2206
|
-
}
|
|
2207
|
-
function createWalWriter(directory, config) {
|
|
2208
|
-
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
2209
|
-
const commitKey = `${config.indexName}/wal/${config.partitionId}/commit`;
|
|
2210
|
-
let handle = null;
|
|
2211
|
-
let activeKey = null;
|
|
2212
|
-
let activeStartSeqNo = 0;
|
|
2213
|
-
let activeBytes = 0;
|
|
2214
|
-
let highestAppendedSeqNo = 0;
|
|
2215
|
-
let highestDurableSeqNo = 0;
|
|
2216
|
-
let markerWriter = null;
|
|
2217
|
-
let durabilityLock = Promise.resolve();
|
|
2218
|
-
const coordinator = createGroupCommitCoordinator(() => withDurabilityLock(flushActiveSegment));
|
|
2219
|
-
function withDurabilityLock(fn) {
|
|
2220
|
-
const result = durabilityLock.then(fn);
|
|
2221
|
-
durabilityLock = result.then(
|
|
2222
|
-
() => void 0,
|
|
2223
|
-
() => void 0
|
|
2224
|
-
);
|
|
2225
|
-
return result;
|
|
2226
|
-
}
|
|
2227
|
-
async function getMarkerWriter() {
|
|
2228
|
-
if (markerWriter === null) {
|
|
2229
|
-
const writer = await createMarkerWriter(directory, config.indexName, config.partitionId);
|
|
2230
|
-
markerWriter = writer;
|
|
2231
|
-
if (writer.existingHighestDurableSeqNo > highestDurableSeqNo) {
|
|
2232
|
-
highestDurableSeqNo = writer.existingHighestDurableSeqNo;
|
|
2233
|
-
}
|
|
2234
|
-
if (writer.created) {
|
|
2235
|
-
await directory.syncDirectoryOf(commitKey);
|
|
2236
|
-
}
|
|
2237
|
-
}
|
|
2238
|
-
return markerWriter;
|
|
2239
|
-
}
|
|
2240
|
-
async function flushActiveSegment() {
|
|
2241
|
-
if (handle === null) {
|
|
2242
|
-
return;
|
|
2243
|
-
}
|
|
2244
|
-
const writer = await getMarkerWriter();
|
|
2245
|
-
const durableByteLength = activeBytes;
|
|
2246
|
-
const activeSegmentSeqNo = activeStartSeqNo;
|
|
2247
|
-
const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
|
|
2248
|
-
await handle.sync();
|
|
2249
|
-
await writer.commit({ activeSegmentSeqNo, durableByteLength, highestDurableSeqNo: durableSeqNo });
|
|
2250
|
-
if (durableSeqNo > highestDurableSeqNo) {
|
|
2251
|
-
highestDurableSeqNo = durableSeqNo;
|
|
2252
|
-
}
|
|
2253
|
-
}
|
|
2254
|
-
async function openSegmentByKey(key) {
|
|
2255
|
-
const nextHandle = await directory.appendHandle(key);
|
|
2256
|
-
const existingSize = await nextHandle.size();
|
|
2257
|
-
let createdNew = false;
|
|
2258
|
-
if (existingSize === 0) {
|
|
2259
|
-
await nextHandle.append(writeSegmentHeader());
|
|
2260
|
-
activeBytes = SEGMENT_HEADER_SIZE;
|
|
2261
|
-
createdNew = true;
|
|
2262
|
-
} else {
|
|
2263
|
-
activeBytes = existingSize;
|
|
2264
|
-
}
|
|
2265
|
-
handle = nextHandle;
|
|
2266
|
-
activeKey = key;
|
|
2267
|
-
activeStartSeqNo = parseSegmentStartSeqNo2(key);
|
|
2268
|
-
if (createdNew) {
|
|
2269
|
-
await directory.syncDirectoryOf(key);
|
|
2270
|
-
}
|
|
2271
|
-
return createdNew;
|
|
2272
|
-
}
|
|
2273
|
-
async function findActiveSegmentKey() {
|
|
2274
|
-
const prefix = `${config.indexName}/wal/${config.partitionId}/`;
|
|
2275
|
-
const keys = (await directory.list(prefix)).filter((k) => SEGMENT_TAIL_PATTERN.test(k.slice(prefix.length))).sort(compareCodePoints);
|
|
2276
|
-
return keys.length > 0 ? keys[keys.length - 1] : null;
|
|
2277
|
-
}
|
|
2278
|
-
async function ensureSegment(seqNo) {
|
|
2279
|
-
if (handle !== null) {
|
|
2280
|
-
return;
|
|
2281
|
-
}
|
|
2282
|
-
const existing = await findActiveSegmentKey();
|
|
2283
|
-
const createdNew = await openSegmentByKey(existing ?? segmentKey(config.indexName, config.partitionId, seqNo));
|
|
2284
|
-
if (createdNew) {
|
|
2285
|
-
await withDurabilityLock(flushActiveSegment);
|
|
2286
|
-
}
|
|
2287
|
-
}
|
|
2288
|
-
async function rollToNewSegment(startSeqNo) {
|
|
2289
|
-
await withDurabilityLock(async () => {
|
|
2290
|
-
if (handle !== null) {
|
|
2291
|
-
await flushActiveSegment();
|
|
2292
|
-
await handle.close();
|
|
2293
|
-
handle = null;
|
|
2294
|
-
}
|
|
2295
|
-
await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
|
|
2296
|
-
await flushActiveSegment();
|
|
2297
|
-
});
|
|
2298
|
-
}
|
|
2299
|
-
async function maybeRoll(nextSeqNo, appendedBytes) {
|
|
2300
|
-
if (handle === null || activeBytes <= SEGMENT_HEADER_SIZE) {
|
|
2301
|
-
return;
|
|
2302
|
-
}
|
|
2303
|
-
if (activeBytes >= segmentMaxBytes || activeBytes + appendedBytes > segmentMaxBytes) {
|
|
2304
|
-
await rollToNewSegment(nextSeqNo);
|
|
2305
|
-
}
|
|
2306
|
-
}
|
|
2307
|
-
function joinFrames(frames) {
|
|
2308
|
-
if (frames.length === 1) {
|
|
2309
|
-
return frames[0];
|
|
2310
|
-
}
|
|
2311
|
-
let totalBytes = 0;
|
|
2312
|
-
for (const frame of frames) {
|
|
2313
|
-
totalBytes += frame.length;
|
|
2314
|
-
}
|
|
2315
|
-
const joined = new Uint8Array(totalBytes);
|
|
2316
|
-
let offset = 0;
|
|
2317
|
-
for (const frame of frames) {
|
|
2318
|
-
joined.set(frame, offset);
|
|
2319
|
-
offset += frame.length;
|
|
2320
|
-
}
|
|
2321
|
-
return joined;
|
|
2322
|
-
}
|
|
2323
|
-
function framesThatFit(frames, start) {
|
|
2324
|
-
const room = segmentMaxBytes - activeBytes;
|
|
2325
|
-
let end = start + 1;
|
|
2326
|
-
let bytes = frames[start].length;
|
|
2327
|
-
while (end < frames.length && bytes + frames[end].length <= room) {
|
|
2328
|
-
bytes += frames[end].length;
|
|
2329
|
-
end += 1;
|
|
2330
|
-
}
|
|
2331
|
-
return end;
|
|
2332
|
-
}
|
|
2333
|
-
async function appendGroup(frames, entries) {
|
|
2334
|
-
const activeHandle = handle;
|
|
2335
|
-
if (activeHandle === null) {
|
|
2336
|
-
throw new NarsilError(
|
|
2337
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2338
|
-
`WAL segment for "${config.indexName}" partition ${config.partitionId} is not open`,
|
|
2339
|
-
{ indexName: config.indexName, partitionId: config.partitionId }
|
|
2340
|
-
);
|
|
2341
|
-
}
|
|
2342
|
-
const joined = joinFrames([...frames]);
|
|
2343
|
-
if (activeBytes + joined.length > MAX_FILE_BYTES) {
|
|
2344
|
-
throw new NarsilError(
|
|
2345
|
-
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
2346
|
-
`A log record of ${joined.length} bytes would carry WAL segment "${activeKey}" past the file size limit of ${MAX_FILE_BYTES} bytes`,
|
|
2347
|
-
{ indexName: config.indexName, partitionId: config.partitionId, bytes: joined.length }
|
|
2348
|
-
);
|
|
2349
|
-
}
|
|
2350
|
-
await activeHandle.append(joined);
|
|
2351
|
-
activeBytes += joined.length;
|
|
2352
|
-
const lastSeqNo = entries[entries.length - 1].seqNo;
|
|
2353
|
-
if (lastSeqNo > highestAppendedSeqNo) {
|
|
2354
|
-
highestAppendedSeqNo = lastSeqNo;
|
|
2355
|
-
}
|
|
2356
|
-
}
|
|
2357
|
-
async function appendFrames(entries) {
|
|
2358
|
-
if (entries.length === 0) {
|
|
2359
|
-
return;
|
|
2360
|
-
}
|
|
2361
|
-
const frames = entries.map(frameRecord);
|
|
2362
|
-
await ensureSegment(entries[0].seqNo);
|
|
2363
|
-
let start = 0;
|
|
2364
|
-
while (start < frames.length) {
|
|
2365
|
-
await maybeRoll(entries[start].seqNo, frames[start].length);
|
|
2366
|
-
const end = framesThatFit(frames, start);
|
|
2367
|
-
await appendGroup(frames.slice(start, end), entries.slice(start, end));
|
|
2368
|
-
start = end;
|
|
2369
|
-
}
|
|
2370
|
-
}
|
|
2371
|
-
return {
|
|
2372
|
-
async appendAll(entries) {
|
|
2373
|
-
await appendFrames(entries);
|
|
2374
|
-
},
|
|
2375
|
-
async appendDurable(entry) {
|
|
2376
|
-
await appendFrames([entry]);
|
|
2377
|
-
await coordinator.commit();
|
|
2378
|
-
},
|
|
2379
|
-
async commit() {
|
|
2380
|
-
await coordinator.commit();
|
|
2381
|
-
},
|
|
2382
|
-
rollToNewSegment,
|
|
2383
|
-
async close() {
|
|
2384
|
-
await withDurabilityLock(async () => {
|
|
2385
|
-
if (handle !== null) {
|
|
2386
|
-
await flushActiveSegment();
|
|
2387
|
-
await handle.close();
|
|
2388
|
-
handle = null;
|
|
2389
|
-
}
|
|
2390
|
-
if (markerWriter !== null) {
|
|
2391
|
-
await markerWriter.close();
|
|
2392
|
-
markerWriter = null;
|
|
2393
|
-
}
|
|
2394
|
-
activeKey = null;
|
|
2395
|
-
});
|
|
2396
|
-
},
|
|
2397
|
-
get activeSegmentKey() {
|
|
2398
|
-
return activeKey;
|
|
2399
|
-
}
|
|
2400
|
-
};
|
|
2401
|
-
}
|
|
2402
|
-
|
|
2403
|
-
// src/persistence/durability/manager.ts
|
|
2404
|
-
function createDurabilityManager(config, hooks, directoryOverride) {
|
|
2405
|
-
const directory = createDurableDirectory(config.directory ?? "");
|
|
2406
|
-
const canOffloadCheckpoint = directoryOverride === void 0;
|
|
2407
|
-
const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
|
|
2408
|
-
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
2409
|
-
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
2410
|
-
const compactionThreshold = config.compactionThreshold ?? DEFAULT_COMPACTION_THRESHOLD;
|
|
2411
|
-
const mode = config.mode ?? "sync";
|
|
2412
|
-
const flushIntervalMs = config.flushIntervalMs ?? DEFAULT_ASYNC_FLUSH_INTERVAL_MS;
|
|
2413
|
-
const indexes = /* @__PURE__ */ new Map();
|
|
2414
|
-
const metadataWrites = /* @__PURE__ */ new Map();
|
|
2415
|
-
let checkpointTimer = null;
|
|
2416
|
-
let asyncFlushTimer = null;
|
|
2417
|
-
let shuttingDown = false;
|
|
2418
|
-
let fatalError = null;
|
|
2419
|
-
const heldCheckpointFiles = /* @__PURE__ */ new Map();
|
|
2420
|
-
const directoryWatch = createDirectoryWatch(directory, () => shuttingDown || fatalError !== null, markFatal);
|
|
2421
|
-
function stopTimers() {
|
|
2422
|
-
directoryWatch.stop();
|
|
2423
|
-
if (checkpointTimer !== null) clearInterval(checkpointTimer);
|
|
2424
|
-
if (asyncFlushTimer !== null) clearInterval(asyncFlushTimer);
|
|
2425
|
-
checkpointTimer = null;
|
|
2426
|
-
asyncFlushTimer = null;
|
|
2427
|
-
}
|
|
2428
|
-
function markFatal(error) {
|
|
2429
|
-
if (fatalError !== null) return;
|
|
2430
|
-
fatalError = error;
|
|
2431
|
-
stopTimers();
|
|
2432
|
-
hooks.onFatalError(error);
|
|
2433
|
-
}
|
|
2434
|
-
function toError5(err) {
|
|
2435
|
-
return err instanceof Error ? err : new Error(String(err));
|
|
2436
|
-
}
|
|
2437
|
-
async function closeWriterReportingFailure(closePromise) {
|
|
2438
|
-
try {
|
|
2439
|
-
await closePromise;
|
|
2440
|
-
} catch (err) {
|
|
2441
|
-
markFatal(toError5(err));
|
|
2442
|
-
}
|
|
2443
|
-
}
|
|
2444
|
-
function getOrCreateIndexState(indexName) {
|
|
2445
|
-
let state = indexes.get(indexName);
|
|
2446
|
-
if (state === void 0) {
|
|
2447
|
-
state = {
|
|
2448
|
-
partitions: /* @__PURE__ */ new Map(),
|
|
2449
|
-
mutationsSinceCheckpoint: 0,
|
|
2450
|
-
documentBytesSinceCheckpoint: 0,
|
|
2451
|
-
checkpointInFlight: null,
|
|
2452
|
-
unloading: false,
|
|
2453
|
-
stalledWrites: []
|
|
2454
|
-
};
|
|
2455
|
-
indexes.set(indexName, state);
|
|
2456
|
-
}
|
|
2457
|
-
return state;
|
|
2458
|
-
}
|
|
2459
|
-
function getOrCreatePartition(indexName, partitionId, startSeqNo) {
|
|
2460
|
-
const indexState = getOrCreateIndexState(indexName);
|
|
2461
|
-
let partition = indexState.partitions.get(partitionId);
|
|
2462
|
-
if (partition === void 0) {
|
|
2463
|
-
partition = {
|
|
2464
|
-
walWriter: createWalWriter(directory, { indexName, partitionId, segmentMaxBytes }),
|
|
2465
|
-
seqOwner: createSeqOwner(startSeqNo),
|
|
2466
|
-
appendChain: Promise.resolve(),
|
|
2467
|
-
appliedSeqNo: startSeqNo,
|
|
2468
|
-
failed: null
|
|
2469
|
-
};
|
|
2470
|
-
indexState.partitions.set(partitionId, partition);
|
|
2471
|
-
}
|
|
2472
|
-
return partition;
|
|
2473
|
-
}
|
|
2474
|
-
function queueMetadataWrite(indexName, write) {
|
|
2475
|
-
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
2476
|
-
const queued = previous.then(write);
|
|
2477
|
-
metadataWrites.set(
|
|
2478
|
-
indexName,
|
|
2479
|
-
queued.catch(() => void 0)
|
|
2480
|
-
);
|
|
2481
|
-
return queued;
|
|
2482
|
-
}
|
|
2483
|
-
function startCheckpointTimer() {
|
|
2484
|
-
if (checkpointTimer !== null || shuttingDown || fatalError !== null || checkpointIntervalMs <= 0) {
|
|
2485
|
-
return;
|
|
2486
|
-
}
|
|
2487
|
-
checkpointTimer = unrefInterval(() => void runScheduledCheckpoints(), checkpointIntervalMs);
|
|
2488
|
-
}
|
|
2489
|
-
function startAsyncFlushTimer() {
|
|
2490
|
-
if (mode !== "async" || asyncFlushTimer !== null || shuttingDown || fatalError !== null || flushIntervalMs <= 0) {
|
|
2491
|
-
return;
|
|
2492
|
-
}
|
|
2493
|
-
asyncFlushTimer = unrefInterval(() => void flushAllPartitions(), flushIntervalMs);
|
|
2494
|
-
}
|
|
2495
|
-
async function flushAllPartitions() {
|
|
2496
|
-
if (fatalError !== null) {
|
|
2497
|
-
return;
|
|
2498
|
-
}
|
|
2499
|
-
for (const indexState of indexes.values()) {
|
|
2500
|
-
for (const partition of indexState.partitions.values()) {
|
|
2501
|
-
if (partition.failed !== null || fatalError !== null) {
|
|
2502
|
-
continue;
|
|
2503
|
-
}
|
|
2504
|
-
try {
|
|
2505
|
-
await partition.walWriter.commit();
|
|
2506
|
-
} catch (err) {
|
|
2507
|
-
partition.failed = toError5(err);
|
|
2508
|
-
markFatal(partition.failed);
|
|
2509
|
-
}
|
|
2510
|
-
}
|
|
2511
|
-
}
|
|
2512
|
-
}
|
|
2513
|
-
async function runScheduledCheckpoints() {
|
|
2514
|
-
if (fatalError !== null) {
|
|
2515
|
-
return;
|
|
2516
|
-
}
|
|
2517
|
-
for (const indexName of [...indexes.keys()]) {
|
|
2518
|
-
try {
|
|
2519
|
-
await checkpointIndex(indexName);
|
|
2520
|
-
} catch (err) {
|
|
2521
|
-
markFatal(toError5(err));
|
|
2522
|
-
}
|
|
2523
|
-
}
|
|
2524
|
-
}
|
|
2525
|
-
function buildMutationEntry(record, seqNo) {
|
|
2526
|
-
return buildEntry({
|
|
2527
|
-
seqNo,
|
|
2528
|
-
primaryTerm: SINGLE_NODE_PRIMARY_TERM,
|
|
2529
|
-
operation: record.operation,
|
|
2530
|
-
partitionId: record.partitionId,
|
|
2531
|
-
indexName: record.indexName,
|
|
2532
|
-
documentId: record.documentId,
|
|
2533
|
-
document: record.document
|
|
2534
|
-
});
|
|
2535
|
-
}
|
|
2536
|
-
async function checkpointIndex(indexName, fromMemory = false) {
|
|
2537
|
-
const indexState = indexes.get(indexName);
|
|
2538
|
-
if (indexState === void 0 || indexState.unloading) {
|
|
2539
|
-
return;
|
|
2540
|
-
}
|
|
2541
|
-
if (indexState.checkpointInFlight !== null) {
|
|
2542
|
-
await indexState.checkpointInFlight;
|
|
2543
|
-
if (!fromMemory) {
|
|
2544
|
-
return;
|
|
2545
|
-
}
|
|
2546
|
-
}
|
|
2547
|
-
const run = performCheckpoint(indexName, indexState, fromMemory).finally(() => {
|
|
2548
|
-
indexState.checkpointInFlight = null;
|
|
2549
|
-
releaseStalledWrites(indexState);
|
|
2550
|
-
});
|
|
2551
|
-
indexState.checkpointInFlight = run;
|
|
2552
|
-
return run;
|
|
2553
|
-
}
|
|
2554
|
-
async function performCheckpoint(indexName, indexState, fromMemory = false) {
|
|
2555
|
-
await directoryWatch.verify();
|
|
2556
|
-
if (fatalError !== null) throw fatalError;
|
|
2557
|
-
await runDurableCheckpoint({
|
|
2558
|
-
directory,
|
|
2559
|
-
hooks,
|
|
2560
|
-
indexName,
|
|
2561
|
-
indexState,
|
|
2562
|
-
compactionThreshold,
|
|
2563
|
-
canOffload: canOffloadCheckpoint,
|
|
2564
|
-
fromMemory,
|
|
2565
|
-
mutationThreshold: checkpointMutationThreshold,
|
|
2566
|
-
mayContinue: () => !indexState.unloading && !shuttingDown && fatalError === null,
|
|
2567
|
-
holdsCheckpointFiles: () => heldCheckpointFiles.has(indexName),
|
|
2568
|
-
queueMetadataWrite,
|
|
2569
|
-
markFatal
|
|
2570
|
-
});
|
|
2571
|
-
}
|
|
2572
|
-
const partitionBatchDeps = {
|
|
2573
|
-
syncEachBatch: mode === "sync",
|
|
2574
|
-
fatalError: () => fatalError,
|
|
2575
|
-
markFatal,
|
|
2576
|
-
buildEntry: buildMutationEntry
|
|
2577
|
-
};
|
|
2578
|
-
async function recordMutations(records) {
|
|
2579
|
-
if (fatalError !== null) {
|
|
2580
|
-
const error = fatalError;
|
|
2581
|
-
return records.map(() => ({ ok: false, error }));
|
|
2582
|
-
}
|
|
2583
|
-
const outcomes = await recordBatchesByPartition(
|
|
2584
|
-
records,
|
|
2585
|
-
(record) => getOrCreatePartition(record.indexName, record.partitionId, 0),
|
|
2586
|
-
partitionBatchDeps
|
|
2587
|
-
);
|
|
2588
|
-
const recordedByIndex = /* @__PURE__ */ new Map();
|
|
2589
|
-
for (let i = 0; i < records.length; i++) {
|
|
2590
|
-
if (!outcomes[i].ok) continue;
|
|
2591
|
-
const recorded = recordedByIndex.get(records[i].indexName) ?? { records: 0, documentBytes: 0 };
|
|
2592
|
-
recorded.records += 1;
|
|
2593
|
-
recorded.documentBytes += records[i].document?.byteLength ?? 0;
|
|
2594
|
-
recordedByIndex.set(records[i].indexName, recorded);
|
|
2595
|
-
}
|
|
2596
|
-
for (const [indexName, recorded] of recordedByIndex) {
|
|
2597
|
-
const indexState = getOrCreateIndexState(indexName);
|
|
2598
|
-
indexState.mutationsSinceCheckpoint += recorded.records;
|
|
2599
|
-
indexState.documentBytesSinceCheckpoint += recorded.documentBytes;
|
|
2600
|
-
if (checkpointIsDue(indexState, checkpointMutationThreshold)) {
|
|
2601
|
-
void checkpointIndex(indexName).catch((err) => {
|
|
2602
|
-
markFatal(toError5(err));
|
|
2603
|
-
});
|
|
2604
|
-
}
|
|
2605
|
-
await stallWhileCheckpointsFallBehind(indexState);
|
|
2606
|
-
}
|
|
2607
|
-
if (recordedByIndex.size === 0) return outcomes;
|
|
2608
|
-
if (mode === "sync") await directoryWatch.verify();
|
|
2609
|
-
const lost = mode === "sync" ? fatalError : null;
|
|
2610
|
-
if (lost !== null) return outcomes.map((outcome) => outcome.ok ? { ok: false, error: lost } : outcome);
|
|
2611
|
-
await directoryWatch.verifyOnce();
|
|
2612
|
-
directoryWatch.start();
|
|
2613
|
-
startCheckpointTimer();
|
|
2614
|
-
startAsyncFlushTimer();
|
|
2615
|
-
return outcomes;
|
|
2616
|
-
}
|
|
2617
|
-
function recoverIndex(indexName, metadataOnly = false) {
|
|
2618
|
-
return recoverPersistedIndex(
|
|
2619
|
-
{
|
|
2620
|
-
directory,
|
|
2621
|
-
hooks,
|
|
2622
|
-
compactionThreshold,
|
|
2623
|
-
registerPartition: (name, partitionId, startSeqNo) => {
|
|
2624
|
-
getOrCreatePartition(name, partitionId, startSeqNo);
|
|
2625
|
-
},
|
|
2626
|
-
holdsCheckpointFiles: (name) => heldCheckpointFiles.has(name),
|
|
2627
|
-
queueMetadataWrite
|
|
2628
|
-
},
|
|
2629
|
-
indexName,
|
|
2630
|
-
metadataOnly
|
|
2631
|
-
);
|
|
2632
|
-
}
|
|
2633
|
-
return {
|
|
2634
|
-
isActive: () => true,
|
|
2635
|
-
async recover(metadataOnly = false) {
|
|
2636
|
-
await directoryWatch.claim();
|
|
2637
|
-
await recoverDirectoryRestores(directory);
|
|
2638
|
-
const names = await listPersistedIndexes(directory);
|
|
2639
|
-
for (const indexName of names) {
|
|
2640
|
-
await recoverIndex(indexName, metadataOnly);
|
|
2641
|
-
}
|
|
2642
|
-
await directoryWatch.verify();
|
|
2643
|
-
directoryWatch.start();
|
|
2644
|
-
startCheckpointTimer();
|
|
2645
|
-
},
|
|
2646
|
-
recoverIndex(indexName) {
|
|
2647
|
-
return recoverIndex(indexName);
|
|
2648
|
-
},
|
|
2649
|
-
highestPersistedSeqNo(indexName, partitionId) {
|
|
2650
|
-
return indexes.get(indexName)?.partitions.get(partitionId)?.appliedSeqNo ?? 0;
|
|
2651
|
-
},
|
|
2652
|
-
async recordMutation(record) {
|
|
2653
|
-
const [outcome] = await recordMutations([record]);
|
|
2654
|
-
if (!outcome.ok) {
|
|
2655
|
-
throw outcome.error;
|
|
2656
|
-
}
|
|
2657
|
-
return outcome.seqNo;
|
|
2658
|
-
},
|
|
2659
|
-
recordMutations,
|
|
2660
|
-
persistMetadata(indexName) {
|
|
2661
|
-
return queueMetadataWrite(indexName, async () => {
|
|
2662
|
-
const metadata = hooks.buildMetadata(indexName);
|
|
2663
|
-
if (metadata === void 0) {
|
|
2664
|
-
return;
|
|
2665
|
-
}
|
|
2666
|
-
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
2667
|
-
await directory.atomicWrite(`${indexName}/meta`, bytes);
|
|
2668
|
-
getOrCreateIndexState(indexName);
|
|
2669
|
-
});
|
|
2670
|
-
},
|
|
2671
|
-
checkpoint(indexName) {
|
|
2672
|
-
return checkpointIndex(indexName);
|
|
2673
|
-
},
|
|
2674
|
-
checkpointFromMemory(indexName) {
|
|
2675
|
-
return checkpointIndex(indexName, true);
|
|
2676
|
-
},
|
|
2677
|
-
async checkpointAll() {
|
|
2678
|
-
for (const indexName of [...indexes.keys()]) {
|
|
2679
|
-
await checkpointIndex(indexName);
|
|
2680
|
-
}
|
|
2681
|
-
},
|
|
2682
|
-
async removeIndex(indexName) {
|
|
2683
|
-
const indexState = indexes.get(indexName);
|
|
2684
|
-
if (indexState !== void 0) {
|
|
2685
|
-
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), closeWriterReportingFailure);
|
|
2686
|
-
indexes.delete(indexName);
|
|
2687
|
-
metadataWrites.delete(indexName);
|
|
2688
|
-
}
|
|
2689
|
-
for (const vectorIndex of hooks.getVectorIndexes(indexName).values()) await vectorIndex.releaseVectorFiles();
|
|
2690
|
-
for (const key of await directory.list(`${indexName}/`)) {
|
|
2691
|
-
await directory.remove(key);
|
|
2692
|
-
}
|
|
2693
|
-
},
|
|
2694
|
-
async unloadIndex(indexName) {
|
|
2695
|
-
const indexState = indexes.get(indexName);
|
|
2696
|
-
if (indexState === void 0) {
|
|
2697
|
-
return;
|
|
2698
|
-
}
|
|
2699
|
-
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), (close) => close);
|
|
2700
|
-
indexes.delete(indexName);
|
|
2701
|
-
metadataWrites.delete(indexName);
|
|
2702
|
-
},
|
|
2703
|
-
...createArchiveCheckpointAccess({
|
|
2704
|
-
directory,
|
|
2705
|
-
held: heldCheckpointFiles,
|
|
2706
|
-
checkpoint: (name) => checkpointIndex(name),
|
|
2707
|
-
registerPartition: (name, partitionId, startSeqNo) => getOrCreatePartition(name, partitionId, startSeqNo),
|
|
2708
|
-
markFatal
|
|
2709
|
-
}),
|
|
2710
|
-
async shutdown() {
|
|
2711
|
-
shuttingDown = true;
|
|
2712
|
-
stopTimers();
|
|
2713
|
-
for (const indexState of indexes.values()) {
|
|
2714
|
-
if (indexState.checkpointInFlight !== null) {
|
|
2715
|
-
await indexState.checkpointInFlight.catch(() => void 0);
|
|
2716
|
-
}
|
|
2717
|
-
for (const partition of indexState.partitions.values()) {
|
|
2718
|
-
await closeWriterReportingFailure(partition.walWriter.close());
|
|
2719
|
-
}
|
|
2720
|
-
}
|
|
2721
|
-
indexes.clear();
|
|
2722
|
-
terminateCheckpointWorker();
|
|
2723
|
-
await directoryWatch.release();
|
|
2724
|
-
}
|
|
2725
|
-
};
|
|
2726
|
-
}
|
|
2727
|
-
|
|
2728
|
-
// src/persistence/durability/snapshot-only/checkpoint.ts
|
|
2729
|
-
function capture(input, prior) {
|
|
2730
|
-
const listedIds = new Set((prior?.partitions ?? []).map((partition) => partition.partitionId));
|
|
2731
|
-
const partitionCountChanged = prior !== null && listedIds.size !== input.manager.partitionCount;
|
|
2732
|
-
const targets = [];
|
|
2733
|
-
const contents = [];
|
|
2734
|
-
for (let partitionId = 0; partitionId < input.manager.partitionCount; partitionId += 1) {
|
|
2735
|
-
const lastSeqNo = input.appliedSeqNoByPartition.get(partitionId) ?? 0;
|
|
2736
|
-
const unchanged = !input.rewritesEveryPartition && prior !== null && !partitionCountChanged && listedIds.has(partitionId) && checkpointLastSeqNo(prior.checkpoint, partitionId) === lastSeqNo;
|
|
2737
|
-
if (unchanged) continue;
|
|
2738
|
-
targets.push({ partitionId, lastSeqNo, primaryTerm: SINGLE_NODE_PRIMARY_TERM });
|
|
2739
|
-
contents.push(partitionContentsOf(input.manager, partitionId));
|
|
2740
|
-
}
|
|
2741
|
-
const plans = /* @__PURE__ */ new Map();
|
|
2742
|
-
for (const [fieldPath, vectorIndex] of input.vectorIndexes) {
|
|
2743
|
-
const listed = prior?.vectors.find((ref) => ref.fieldPath === fieldPath);
|
|
2744
|
-
plans.set(fieldPath, vectorIndex.planCheckpoint(listed === void 0 ? null : listed.files.map((file) => file.key)));
|
|
2745
|
-
}
|
|
2746
|
-
return { targets, plans, documentCount: input.manager.countDocuments(), contents };
|
|
2747
|
-
}
|
|
2748
|
-
async function writeCaptured(input, store, prior, captured) {
|
|
2749
|
-
const partitionCount = input.manager.partitionCount;
|
|
2750
|
-
const priorWithinCount = prior === null ? null : {
|
|
2751
|
-
checkpoint: prior.checkpoint.filter((entry) => entry.partitionId < partitionCount),
|
|
2752
|
-
partitions: prior.partitions.filter((partition) => partition.partitionId < partitionCount)
|
|
2753
|
-
};
|
|
2754
|
-
const written = await writeCheckpointSegments({
|
|
2755
|
-
store,
|
|
2756
|
-
metadata: input.metadata,
|
|
2757
|
-
targets: captured.targets,
|
|
2758
|
-
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
2759
|
-
prior: priorWithinCount,
|
|
2760
|
-
wholePartitions: captured.contents
|
|
2761
|
-
});
|
|
2762
|
-
const vectors = await writeLiveVectors({
|
|
2763
|
-
store,
|
|
2764
|
-
indexName: input.indexName,
|
|
2765
|
-
plans: captured.plans,
|
|
2766
|
-
priorVectors: prior?.vectors ?? []
|
|
2767
|
-
});
|
|
2768
|
-
const committed = await commitCheckpointManifest(store, input.metadata, written, vectors.refs);
|
|
2769
|
-
recordWrittenVectors(input.vectorIndexes, vectors.written);
|
|
2770
|
-
await removeCheckpointGarbage(
|
|
2771
|
-
store,
|
|
2772
|
-
await garbageThatNoFieldStillReads(store, input.vectorIndexes, committed.garbage)
|
|
2773
|
-
);
|
|
2774
|
-
}
|
|
2775
|
-
async function writeSnapshotOnlyCheckpoint(input) {
|
|
2776
|
-
const store = adapterCheckpointStore(input.adapter);
|
|
2777
|
-
for (const vectorIndex of input.vectorIndexes.values()) await vectorIndex.completeGraph();
|
|
2778
|
-
const prior = await readCurrentManifest(store, input.indexName);
|
|
2779
|
-
const captured = capture(input, prior);
|
|
2780
|
-
await writeCaptured(input, store, prior, captured);
|
|
2781
|
-
return captured.documentCount;
|
|
2782
|
-
}
|
|
2783
|
-
|
|
2784
|
-
// src/persistence/durability/snapshot-only/restores.ts
|
|
2785
|
-
function toError4(error) {
|
|
2786
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
2787
|
-
}
|
|
2788
|
-
function createPendingRestores(onFatalError) {
|
|
2789
|
-
const pending = /* @__PURE__ */ new Map();
|
|
2790
|
-
return {
|
|
2791
|
-
finishLater(indexName, run) {
|
|
2792
|
-
let running = null;
|
|
2793
|
-
const finish = () => {
|
|
2794
|
-
running ??= run().then(
|
|
2795
|
-
() => {
|
|
2796
|
-
if (pending.get(indexName) === finish) pending.delete(indexName);
|
|
2797
|
-
},
|
|
2798
|
-
(error) => {
|
|
2799
|
-
running = null;
|
|
2800
|
-
throw error;
|
|
2801
|
-
}
|
|
2802
|
-
);
|
|
2803
|
-
return running;
|
|
2804
|
-
};
|
|
2805
|
-
pending.set(indexName, finish);
|
|
2806
|
-
void finish().catch((error) => onFatalError(toError4(error)));
|
|
2807
|
-
return Promise.resolve();
|
|
2808
|
-
},
|
|
2809
|
-
async settle(indexName) {
|
|
2810
|
-
await pending.get(indexName)?.();
|
|
2811
|
-
},
|
|
2812
|
-
forget(indexName) {
|
|
2813
|
-
pending.delete(indexName);
|
|
2814
|
-
},
|
|
2815
|
-
async settleAll() {
|
|
2816
|
-
for (const finish of [...pending.values()]) await finish().catch(() => void 0);
|
|
2817
|
-
}
|
|
2818
|
-
};
|
|
2819
|
-
}
|
|
2820
|
-
|
|
2821
|
-
// src/persistence/durability/snapshot-only/index.ts
|
|
2822
|
-
function metadataKey(indexName) {
|
|
2823
|
-
return `${indexName}/meta`;
|
|
2824
|
-
}
|
|
2825
|
-
function createSnapshotOnlyManager(adapter, config, hooks, publisher) {
|
|
2826
|
-
const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
2827
|
-
const checkpointMutationThreshold = config.checkpointMutationThreshold ?? DEFAULT_CHECKPOINT_MUTATION_THRESHOLD;
|
|
2828
|
-
const indexes = /* @__PURE__ */ new Map();
|
|
2829
|
-
const metadataWrites = /* @__PURE__ */ new Map();
|
|
2830
|
-
const restoreStorage = adapterRestoreStorage(adapter);
|
|
2831
|
-
const restores = createPendingRestores(hooks.onFatalError);
|
|
2832
|
-
let checkpointTimer = null;
|
|
2833
|
-
let shuttingDown = false;
|
|
2834
|
-
function settleRestore(indexName) {
|
|
2835
|
-
return restores.settle(indexName);
|
|
2836
|
-
}
|
|
2837
|
-
function getOrCreateIndexState(indexName) {
|
|
2838
|
-
let state = indexes.get(indexName);
|
|
2839
|
-
if (state === void 0) {
|
|
2840
|
-
state = {
|
|
2841
|
-
mutationsSinceCheckpoint: 0,
|
|
2842
|
-
checkpointInFlight: null,
|
|
2843
|
-
applyChain: Promise.resolve(),
|
|
2844
|
-
unloading: false,
|
|
2845
|
-
appliedSeqNoByPartition: /* @__PURE__ */ new Map()
|
|
2846
|
-
};
|
|
2847
|
-
indexes.set(indexName, state);
|
|
2848
|
-
}
|
|
2849
|
-
return state;
|
|
2850
|
-
}
|
|
2851
|
-
function startCheckpointTimer() {
|
|
2852
|
-
if (checkpointTimer !== null || shuttingDown || checkpointIntervalMs <= 0) {
|
|
2853
|
-
return;
|
|
2854
|
-
}
|
|
2855
|
-
checkpointTimer = setInterval(() => {
|
|
2856
|
-
void runScheduledCheckpoints();
|
|
2857
|
-
}, checkpointIntervalMs);
|
|
2858
|
-
if (typeof checkpointTimer.unref === "function") {
|
|
2859
|
-
checkpointTimer.unref();
|
|
2860
|
-
}
|
|
2861
|
-
}
|
|
2862
|
-
async function runScheduledCheckpoints() {
|
|
2863
|
-
for (const indexName of [...indexes.keys()]) {
|
|
2864
|
-
try {
|
|
2865
|
-
await checkpointIndex(indexName);
|
|
2866
|
-
} catch (err) {
|
|
2867
|
-
hooks.onFatalError(err instanceof Error ? err : new Error(String(err)));
|
|
2868
|
-
}
|
|
2869
|
-
}
|
|
2870
|
-
}
|
|
2871
|
-
async function performCheckpoint(indexName, indexState, fromMemory) {
|
|
2872
|
-
await settleRestore(indexName);
|
|
2873
|
-
const manager = hooks.getManager(indexName);
|
|
2874
|
-
const metadata = hooks.buildMetadata(indexName);
|
|
2875
|
-
if (manager === void 0 || metadata === void 0) {
|
|
2876
|
-
return;
|
|
2877
|
-
}
|
|
2878
|
-
const documentCount = await writeSnapshotOnlyCheckpoint({
|
|
2879
|
-
adapter,
|
|
2880
|
-
indexName,
|
|
2881
|
-
metadata,
|
|
2882
|
-
manager,
|
|
2883
|
-
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
2884
|
-
appliedSeqNoByPartition: indexState.appliedSeqNoByPartition,
|
|
2885
|
-
rewritesEveryPartition: fromMemory
|
|
2886
|
-
});
|
|
2887
|
-
await queueMetadataWrite(indexName, async () => {
|
|
2888
|
-
const checkpointMetadata = hooks.buildMetadata(indexName, documentCount);
|
|
2889
|
-
if (checkpointMetadata === void 0) return;
|
|
2890
|
-
const metadataBytes = await writeMetadataEnvelope(checkpointMetadata, { checksum: true });
|
|
2891
|
-
await adapter.save(metadataKey(indexName), metadataBytes);
|
|
2892
|
-
hooks.recordCheckpoint?.(indexName, documentCount, manager.partitionCount);
|
|
2893
|
-
});
|
|
2894
|
-
indexState.mutationsSinceCheckpoint = 0;
|
|
2895
|
-
if (publisher !== void 0) {
|
|
2896
|
-
const partitionIds = [];
|
|
2897
|
-
for (let i = 0; i < manager.partitionCount; i += 1) {
|
|
2898
|
-
partitionIds.push(i);
|
|
2899
|
-
}
|
|
2900
|
-
await publisher.publishPartitions(indexName, partitionIds);
|
|
2901
|
-
}
|
|
2902
|
-
}
|
|
2903
|
-
async function checkpointIndex(indexName, fromMemory = false) {
|
|
2904
|
-
const indexState = indexes.get(indexName);
|
|
2905
|
-
if (indexState === void 0 || indexState.unloading) {
|
|
2906
|
-
return;
|
|
2907
|
-
}
|
|
2908
|
-
if (indexState.checkpointInFlight !== null) {
|
|
2909
|
-
await indexState.checkpointInFlight;
|
|
2910
|
-
if (!fromMemory) {
|
|
2911
|
-
return;
|
|
2912
|
-
}
|
|
2913
|
-
}
|
|
2914
|
-
const run = indexState.applyChain.then(() => performCheckpoint(indexName, indexState, fromMemory)).finally(() => {
|
|
2915
|
-
indexState.checkpointInFlight = null;
|
|
2916
|
-
});
|
|
2917
|
-
indexState.checkpointInFlight = run;
|
|
2918
|
-
return run;
|
|
2919
|
-
}
|
|
2920
|
-
function queueMetadataWrite(indexName, write) {
|
|
2921
|
-
const previous = metadataWrites.get(indexName) ?? Promise.resolve();
|
|
2922
|
-
const queued = previous.then(() => settleRestore(indexName)).then(write);
|
|
2923
|
-
metadataWrites.set(
|
|
2924
|
-
indexName,
|
|
2925
|
-
queued.catch(() => void 0)
|
|
2926
|
-
);
|
|
2927
|
-
return queued;
|
|
2928
|
-
}
|
|
2929
|
-
async function recoverIndex(indexName, metadataOnly = false) {
|
|
2930
|
-
await settleRestore(indexName);
|
|
2931
|
-
const metaBytes = await adapter.load(metadataKey(indexName));
|
|
2932
|
-
if (metaBytes === null) {
|
|
2933
|
-
return;
|
|
2934
|
-
}
|
|
2935
|
-
const { metadata } = await readMetadataEnvelope(metaBytes);
|
|
2936
|
-
let derivedCount;
|
|
2937
|
-
if (metadataOnly && metadata.documentCount === void 0) {
|
|
2938
|
-
try {
|
|
2939
|
-
derivedCount = await countStoredBundleDocuments(indexName);
|
|
2940
|
-
metadata.documentCount = derivedCount;
|
|
2941
|
-
} catch {
|
|
2942
|
-
derivedCount = void 0;
|
|
2943
|
-
}
|
|
2944
|
-
}
|
|
2945
|
-
await hooks.createIndexFromMetadata(metadata, !metadataOnly);
|
|
2946
|
-
if (metadataOnly) {
|
|
2947
|
-
if (derivedCount !== void 0) {
|
|
2948
|
-
await queueMetadataWrite(indexName, async () => {
|
|
2949
|
-
const upgraded = hooks.buildMetadata(indexName, derivedCount);
|
|
2950
|
-
if (upgraded === void 0) {
|
|
2951
|
-
return;
|
|
2952
|
-
}
|
|
2953
|
-
const bytes = await writeMetadataEnvelope(upgraded, { checksum: true });
|
|
2954
|
-
await adapter.save(metadataKey(indexName), bytes);
|
|
2955
|
-
}).catch(() => void 0);
|
|
2956
|
-
}
|
|
2957
|
-
return;
|
|
2958
|
-
}
|
|
2959
|
-
const manager = hooks.getManager(indexName);
|
|
2960
|
-
const current = hooks.buildMetadata(indexName);
|
|
2961
|
-
if (manager === void 0 || current === void 0) {
|
|
2962
|
-
return;
|
|
2963
|
-
}
|
|
2964
|
-
const recovered = await recoverCheckpoint({
|
|
2965
|
-
store: adapterCheckpointStore(adapter),
|
|
2966
|
-
indexName,
|
|
2967
|
-
metadata: current,
|
|
2968
|
-
manager,
|
|
2969
|
-
vectorIndexes: hooks.getVectorIndexes(indexName),
|
|
2970
|
-
storedTermsAreCurrent: metadata.analysisRevision === manager.language.revision,
|
|
2971
|
-
compactionThreshold: DEFAULT_COMPACTION_THRESHOLD,
|
|
2972
|
-
keepsUnlistedFiles: false,
|
|
2973
|
-
replay: async (checkpoint) => checkpoint,
|
|
2974
|
-
onLayoutUpgrade: hooks.onLayoutUpgrade
|
|
2975
|
-
});
|
|
2976
|
-
const indexState = getOrCreateIndexState(indexName);
|
|
2977
|
-
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId += 1) {
|
|
2978
|
-
const persistedSeqNo = snapshotCheckpointFor(recovered.checkpoint, partitionId);
|
|
2979
|
-
if (persistedSeqNo > 0) {
|
|
2980
|
-
indexState.appliedSeqNoByPartition.set(partitionId, persistedSeqNo);
|
|
2981
|
-
}
|
|
2982
|
-
}
|
|
2983
|
-
if (recovered.upgradedDocumentCount !== null) {
|
|
2984
|
-
const upgradedCount = recovered.upgradedDocumentCount;
|
|
2985
|
-
await queueMetadataWrite(indexName, async () => {
|
|
2986
|
-
const upgraded = hooks.buildMetadata(indexName, upgradedCount);
|
|
2987
|
-
if (upgraded === void 0) return;
|
|
2988
|
-
await adapter.save(metadataKey(indexName), await writeMetadataEnvelope(upgraded, { checksum: true }));
|
|
2989
|
-
}).catch(() => void 0);
|
|
2990
|
-
}
|
|
2991
|
-
}
|
|
2992
|
-
function countStoredBundleDocuments(indexName) {
|
|
2993
|
-
return countCheckpointDocuments(adapterCheckpointStore(adapter), indexName);
|
|
2994
|
-
}
|
|
2995
|
-
async function recordMutation(record) {
|
|
2996
|
-
const indexState = getOrCreateIndexState(record.indexName);
|
|
2997
|
-
const buffered = indexState.applyChain.then(async () => {
|
|
2998
|
-
await record.apply();
|
|
2999
|
-
const appliedSeqNo2 = (indexState.appliedSeqNoByPartition.get(record.partitionId) ?? 0) + 1;
|
|
3000
|
-
indexState.appliedSeqNoByPartition.set(record.partitionId, appliedSeqNo2);
|
|
3001
|
-
return appliedSeqNo2;
|
|
3002
|
-
});
|
|
3003
|
-
indexState.applyChain = buffered.then(
|
|
3004
|
-
() => void 0,
|
|
3005
|
-
() => void 0
|
|
3006
|
-
);
|
|
3007
|
-
const appliedSeqNo = await buffered;
|
|
3008
|
-
indexState.mutationsSinceCheckpoint += 1;
|
|
3009
|
-
startCheckpointTimer();
|
|
3010
|
-
if (indexState.mutationsSinceCheckpoint >= checkpointMutationThreshold) {
|
|
3011
|
-
void checkpointIndex(record.indexName).catch((err) => {
|
|
3012
|
-
hooks.onFatalError(err instanceof Error ? err : new Error(String(err)));
|
|
3013
|
-
});
|
|
3014
|
-
}
|
|
3015
|
-
return appliedSeqNo;
|
|
3016
|
-
}
|
|
3017
|
-
async function drainIndex(indexName) {
|
|
3018
|
-
const indexState = indexes.get(indexName);
|
|
3019
|
-
if (indexState !== void 0) {
|
|
3020
|
-
indexState.unloading = true;
|
|
3021
|
-
try {
|
|
3022
|
-
await indexState.applyChain;
|
|
3023
|
-
if (indexState.checkpointInFlight !== null) await indexState.checkpointInFlight;
|
|
3024
|
-
await metadataWrites.get(indexName);
|
|
3025
|
-
} catch (error) {
|
|
3026
|
-
indexState.unloading = false;
|
|
3027
|
-
throw error;
|
|
3028
|
-
}
|
|
3029
|
-
}
|
|
3030
|
-
indexes.delete(indexName);
|
|
3031
|
-
metadataWrites.delete(indexName);
|
|
3032
|
-
}
|
|
3033
|
-
return {
|
|
3034
|
-
isActive() {
|
|
3035
|
-
return true;
|
|
3036
|
-
},
|
|
3037
|
-
async recover(metadataOnly = false) {
|
|
3038
|
-
await recoverRestoreJournals(restoreStorage);
|
|
3039
|
-
const keys = await adapter.list("");
|
|
3040
|
-
for (const key of keys) {
|
|
3041
|
-
if (key.endsWith("/meta")) {
|
|
3042
|
-
await recoverIndex(key.slice(0, -"/meta".length), metadataOnly);
|
|
3043
|
-
}
|
|
3044
|
-
}
|
|
3045
|
-
startCheckpointTimer();
|
|
3046
|
-
},
|
|
3047
|
-
recoverIndex(indexName) {
|
|
3048
|
-
return recoverIndex(indexName);
|
|
3049
|
-
},
|
|
3050
|
-
highestPersistedSeqNo(indexName, partitionId) {
|
|
3051
|
-
return indexes.get(indexName)?.appliedSeqNoByPartition.get(partitionId) ?? 0;
|
|
3052
|
-
},
|
|
3053
|
-
recordMutation,
|
|
3054
|
-
async recordMutations(records) {
|
|
3055
|
-
const outcomes = [];
|
|
3056
|
-
for (const record of records) {
|
|
3057
|
-
try {
|
|
3058
|
-
outcomes.push({ ok: true, seqNo: await recordMutation(record) });
|
|
3059
|
-
} catch (error) {
|
|
3060
|
-
outcomes.push({ ok: false, error });
|
|
3061
|
-
}
|
|
3062
|
-
}
|
|
3063
|
-
return outcomes;
|
|
3064
|
-
},
|
|
3065
|
-
persistMetadata(indexName) {
|
|
3066
|
-
return queueMetadataWrite(indexName, async () => {
|
|
3067
|
-
const metadata = hooks.buildMetadata(indexName);
|
|
3068
|
-
if (metadata === void 0) {
|
|
3069
|
-
return;
|
|
3070
|
-
}
|
|
3071
|
-
const bytes = await writeMetadataEnvelope(metadata, { checksum: true });
|
|
3072
|
-
await adapter.save(metadataKey(indexName), bytes);
|
|
3073
|
-
getOrCreateIndexState(indexName);
|
|
3074
|
-
});
|
|
3075
|
-
},
|
|
3076
|
-
checkpoint(indexName) {
|
|
3077
|
-
return checkpointIndex(indexName);
|
|
3078
|
-
},
|
|
3079
|
-
checkpointFromMemory(indexName) {
|
|
3080
|
-
return checkpointIndex(indexName, true);
|
|
3081
|
-
},
|
|
3082
|
-
async checkpointAll() {
|
|
3083
|
-
for (const indexName of [...indexes.keys()]) {
|
|
3084
|
-
await checkpointIndex(indexName);
|
|
3085
|
-
}
|
|
3086
|
-
},
|
|
3087
|
-
stageRestore(indexName) {
|
|
3088
|
-
return createRestoreStaging(restoreStorage, indexName, {
|
|
3089
|
-
finish: (run) => restores.finishLater(indexName, run),
|
|
3090
|
-
onStorageLost: hooks.onFatalError
|
|
3091
|
-
});
|
|
3092
|
-
},
|
|
3093
|
-
adoptRestoredCheckpoint(indexName, checkpoint) {
|
|
3094
|
-
const indexState = getOrCreateIndexState(indexName);
|
|
3095
|
-
indexState.appliedSeqNoByPartition.clear();
|
|
3096
|
-
for (const { partitionId, lastSeqNo } of checkpoint) {
|
|
3097
|
-
if (lastSeqNo > 0) indexState.appliedSeqNoByPartition.set(partitionId, lastSeqNo);
|
|
3098
|
-
}
|
|
3099
|
-
},
|
|
3100
|
-
async removeIndex(indexName) {
|
|
3101
|
-
await settleRestore(indexName).catch(() => void 0);
|
|
3102
|
-
restores.forget(indexName);
|
|
3103
|
-
await restoreStorage.remove(journalKey(indexName));
|
|
3104
|
-
await drainIndex(indexName);
|
|
3105
|
-
for (const key of await adapter.list(`${indexName}/`)) {
|
|
3106
|
-
await adapter.delete(key);
|
|
3107
|
-
}
|
|
3108
|
-
},
|
|
3109
|
-
async unloadIndex(indexName) {
|
|
3110
|
-
await settleRestore(indexName);
|
|
3111
|
-
await drainIndex(indexName);
|
|
3112
|
-
},
|
|
3113
|
-
async reloadIndex(indexName) {
|
|
3114
|
-
const indexState = getOrCreateIndexState(indexName);
|
|
3115
|
-
const reload = indexState.applyChain.then(() => recoverIndex(indexName));
|
|
3116
|
-
indexState.applyChain = reload.catch(() => void 0);
|
|
3117
|
-
await reload;
|
|
3118
|
-
},
|
|
3119
|
-
async shutdown() {
|
|
3120
|
-
shuttingDown = true;
|
|
3121
|
-
if (checkpointTimer !== null) {
|
|
3122
|
-
clearInterval(checkpointTimer);
|
|
3123
|
-
checkpointTimer = null;
|
|
3124
|
-
}
|
|
3125
|
-
await restores.settleAll();
|
|
3126
|
-
for (const indexState of indexes.values()) {
|
|
3127
|
-
if (indexState.checkpointInFlight !== null) {
|
|
3128
|
-
await indexState.checkpointInFlight.catch(() => void 0);
|
|
3129
|
-
}
|
|
3130
|
-
}
|
|
3131
|
-
indexes.clear();
|
|
3132
|
-
}
|
|
3133
|
-
};
|
|
3134
|
-
}
|
|
3135
|
-
|
|
3136
|
-
// src/engine/durability-integration.ts
|
|
3137
|
-
function buildMetadata(indexName, hooks, documentCount) {
|
|
3138
|
-
const config = hooks.getIndexConfig(indexName);
|
|
3139
|
-
if (config === void 0) {
|
|
3140
|
-
return void 0;
|
|
3141
|
-
}
|
|
3142
|
-
return metadataFromConfig(indexName, config, hooks.getManager(indexName)?.partitionCount, documentCount);
|
|
3143
|
-
}
|
|
3144
|
-
function createDurabilityIntegration(tier, hooks) {
|
|
3145
|
-
const managerHooks = {
|
|
3146
|
-
getManager: hooks.getManager,
|
|
3147
|
-
getVectorFieldPaths: hooks.getVectorFieldPaths,
|
|
3148
|
-
getVectorIndexes: hooks.getVectorIndexes,
|
|
3149
|
-
buildMetadata: (indexName, documentCount) => buildMetadata(indexName, hooks, documentCount),
|
|
3150
|
-
createIndexFromMetadata: hooks.createIndexFromMetadata,
|
|
3151
|
-
recordCheckpoint: hooks.recordCheckpoint,
|
|
3152
|
-
onLayoutUpgrade: hooks.onLayoutUpgrade,
|
|
3153
|
-
onFatalError: hooks.onFatalError
|
|
3154
|
-
};
|
|
3155
|
-
const manager = tier.kind === "wal" ? createDurabilityManager(tier.config, managerHooks) : createSnapshotOnlyManager(tier.adapter, tier.config, managerHooks, hooks.checkpointPublisher);
|
|
3156
|
-
function partitionFor(indexName, docId) {
|
|
3157
|
-
const partitionManager = hooks.getManager(indexName);
|
|
3158
|
-
const partitionCount = partitionManager?.partitionCount ?? 1;
|
|
3159
|
-
return fnv1a(docId) % Math.max(1, partitionCount);
|
|
3160
|
-
}
|
|
3161
|
-
async function recordMutation(indexName, docId, operation, document, apply) {
|
|
3162
|
-
const partitionId = partitionFor(indexName, docId);
|
|
3163
|
-
const seqNo = await manager.recordMutation({
|
|
3164
|
-
indexName,
|
|
3165
|
-
partitionId,
|
|
3166
|
-
operation,
|
|
3167
|
-
documentId: docId,
|
|
3168
|
-
document,
|
|
3169
|
-
apply
|
|
3170
|
-
});
|
|
3171
|
-
return { indexName, partitionId, seqNo };
|
|
3172
|
-
}
|
|
3173
|
-
return {
|
|
3174
|
-
manager,
|
|
3175
|
-
recordInsertOrUpdate(indexName, docId, document, apply) {
|
|
3176
|
-
return recordMutation(indexName, docId, "INDEX", encodeDocument(document), apply);
|
|
3177
|
-
},
|
|
3178
|
-
async recordMutationBatch(indexName, mutations) {
|
|
3179
|
-
const records = [];
|
|
3180
|
-
const outcomes = new Array(mutations.length);
|
|
3181
|
-
const recordPositions = [];
|
|
3182
|
-
for (let i = 0; i < mutations.length; i++) {
|
|
3183
|
-
const mutation = mutations[i];
|
|
3184
|
-
try {
|
|
3185
|
-
records.push({
|
|
3186
|
-
indexName,
|
|
3187
|
-
partitionId: partitionFor(indexName, mutation.docId),
|
|
3188
|
-
operation: mutation.document === null ? "DELETE" : "INDEX",
|
|
3189
|
-
documentId: mutation.docId,
|
|
3190
|
-
document: mutation.document === null ? null : encodeDocument(mutation.document),
|
|
3191
|
-
apply: mutation.apply
|
|
3192
|
-
});
|
|
3193
|
-
recordPositions.push(i);
|
|
3194
|
-
} catch (error) {
|
|
3195
|
-
outcomes[i] = { ok: false, error };
|
|
3196
|
-
}
|
|
3197
|
-
}
|
|
3198
|
-
const recorded = await manager.recordMutations(records);
|
|
3199
|
-
for (let i = 0; i < recorded.length; i++) {
|
|
3200
|
-
const outcome = recorded[i];
|
|
3201
|
-
outcomes[recordPositions[i]] = outcome.ok ? { ok: true, write: { indexName, partitionId: records[i].partitionId, seqNo: outcome.seqNo } } : outcome;
|
|
3202
|
-
}
|
|
3203
|
-
return outcomes;
|
|
3204
|
-
},
|
|
3205
|
-
recordRemove(indexName, docId, apply) {
|
|
3206
|
-
return recordMutation(indexName, docId, "DELETE", null, apply);
|
|
3207
|
-
}
|
|
3208
|
-
};
|
|
3209
|
-
}
|
|
3210
|
-
|
|
3211
|
-
// src/engine/durability-wiring.ts
|
|
3212
|
-
function createDurabilityFromTier(tier, wiring) {
|
|
3213
|
-
return createDurabilityIntegration(tier, {
|
|
3214
|
-
checkpointPublisher: { publishPartitions: wiring.publishCheckpointedPartitions },
|
|
3215
|
-
getManager: (indexName) => wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName) : void 0,
|
|
3216
|
-
getVectorFieldPaths: (indexName) => wiring.indexRegistry.get(indexName)?.vectorFieldPaths ?? /* @__PURE__ */ new Set(),
|
|
3217
|
-
getVectorIndexes: (indexName) => (wiring.indexRegistry.has(indexName) ? wiring.getManager(indexName)?.getVectorIndexes() : void 0) ?? /* @__PURE__ */ new Map(),
|
|
3218
|
-
getIndexConfig: (indexName) => {
|
|
3219
|
-
const entry = wiring.indexRegistry.get(indexName);
|
|
3220
|
-
return entry === void 0 ? void 0 : metadataConfigOf(entry);
|
|
3221
|
-
},
|
|
3222
|
-
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
3223
|
-
recordCheckpoint: wiring.recordCheckpoint,
|
|
3224
|
-
onLayoutUpgrade: wiring.emitLayoutUpgrade,
|
|
3225
|
-
onFatalError: wiring.emitFatalError
|
|
3226
|
-
});
|
|
3227
|
-
}
|
|
3228
|
-
|
|
3229
|
-
export { createDurabilityFromTier };
|